From 5af55e789d73361d4e20df579c34aca153388ca5 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 30 Jul 2026 16:15:59 -0600 Subject: [PATCH 1/5] feat(analyst): trace analysts run official DSPy RLMs; retire the Ax stack - clients/python dspy_rlm_bridge: official dspy.RLM in a child process, seven allowlisted trace tools, strict JSON I/O (14/14 tests) - src/analyst engine contract (engine.ts, dspy-rlm-engine.ts, trace-tool-callback.ts); callLlmJson demoted to an explicit non-recursive baseline - store-backed citation verification: cited trace/span ids must resolve in the trace store, encoded ids are rejected - external-optimizer model proxy fails on zero-usage 200 responses and on over-reserved output after recording the actual charge - delete ax-service, ax-cost-service, trace-analyst/loop, structure-findings, AxGepaSteeringOptimizer (no external callers) - repin analyst benchmark implementation and dependency-lock digests for the changed fingerprint set --- README.md | 3 +- clients/python/README.md | 31 +- clients/python/pyproject.toml | 1 + .../src/agent_eval_rpc/dspy_rlm_bridge.py | 730 ++++++++++++++++ clients/python/tests/test_dspy_rlm_bridge.py | 540 ++++++++++++ clients/python/uv.lock | 15 + docs/trace-analysis.md | 654 ++++----------- package.json | 1 - pnpm-lock.yaml | 18 - ...check-analyst-benchmark-implementation.mjs | 18 +- scripts/verify-package-exports.mjs | 4 +- src/analyst/ax-cost-service.test.ts | 522 ------------ src/analyst/ax-cost-service.ts | 270 ------ src/analyst/ax-service.test.ts | 56 -- src/analyst/ax-service.ts | 66 -- .../benchmark-command-artifact.test.ts | 14 +- src/analyst/benchmark-command-artifact.ts | 2 +- .../benchmark-command-persistence.test.ts | 120 +-- src/analyst/benchmark-command-persistence.ts | 4 +- src/analyst/benchmark-command-result.ts | 2 +- src/analyst/benchmark-command.test.ts | 96 +-- src/analyst/benchmark-command.ts | 30 +- src/analyst/benchmark-implementation.test.ts | 24 +- src/analyst/benchmark-implementation.ts | 23 +- src/analyst/benchmark-public-errors.test.ts | 26 + src/analyst/benchmark-public-errors.ts | 8 +- src/analyst/benchmark-public-model.ts | 11 +- src/analyst/benchmark-public-rlm.ts | 238 ++++++ src/analyst/benchmark-public-types.ts | 14 +- src/analyst/benchmark-real-model.test.ts | 26 +- src/analyst/benchmark-real-model.ts | 3 +- src/analyst/default-registry.test.ts | 16 +- src/analyst/default-registry.ts | 45 +- src/analyst/define.test.ts | 53 +- src/analyst/define.ts | 52 +- src/analyst/dspy-rlm-engine.test.ts | 160 ++++ src/analyst/dspy-rlm-engine.ts | 247 ++++++ src/analyst/engine.ts | 77 ++ src/analyst/finding-signature.ts | 32 +- src/analyst/finding-subject.test.ts | 2 +- src/analyst/index.ts | 33 +- src/analyst/kind-factory-ax-contract.test.ts | 781 ------------------ src/analyst/kind-factory.test.ts | 343 ++++++++ src/analyst/kind-factory.ts | 694 +++++++--------- src/analyst/kinds/failure-mode.ts | 25 +- src/analyst/kinds/improvement.ts | 30 +- src/analyst/kinds/index.ts | 4 +- src/analyst/kinds/kinds.test.ts | 72 +- src/analyst/kinds/knowledge-gap.ts | 24 +- src/analyst/kinds/knowledge-poisoning.ts | 26 +- src/analyst/structure-findings.test.ts | 177 ---- src/analyst/structure-findings.ts | 144 ---- src/analyst/tool-groups.ts | 18 +- src/analyst/trace-tool-callback.test.ts | 61 ++ src/analyst/trace-tool-callback.ts | 178 ++++ src/analyst/types.ts | 2 +- src/campaign/analyst-surface.test.ts | 2 +- src/campaign/analyst-surface.ts | 4 +- .../external-optimizer-model-proxy.ts | 65 +- src/campaign/external-optimizer-subprocess.ts | 9 +- src/index.ts | 28 +- src/steering-optimizer.ts | 201 +---- src/trace-analyst/analyst.test.ts | 381 --------- src/trace-analyst/analyst.ts | 244 ++---- src/trace-analyst/index.ts | 4 +- src/trace-analyst/loop.test.ts | 267 ------ src/trace-analyst/loop.ts | 276 ------- src/trace-analyst/prompts.ts | 59 +- src/trace-analyst/tools.test.ts | 225 ++--- src/trace-analyst/tools.ts | 23 +- src/traced-analyst.ts | 85 +- tests/analyst-tracing.test.ts | 55 +- .../external-optimizer-process.test.ts | 115 ++- tests/steering-optimizer.test.ts | 14 +- 74 files changed, 3968 insertions(+), 4955 deletions(-) create mode 100644 clients/python/src/agent_eval_rpc/dspy_rlm_bridge.py create mode 100644 clients/python/tests/test_dspy_rlm_bridge.py delete mode 100644 src/analyst/ax-cost-service.test.ts delete mode 100644 src/analyst/ax-cost-service.ts delete mode 100644 src/analyst/ax-service.test.ts delete mode 100644 src/analyst/ax-service.ts create mode 100644 src/analyst/benchmark-public-errors.test.ts create mode 100644 src/analyst/benchmark-public-rlm.ts create mode 100644 src/analyst/dspy-rlm-engine.test.ts create mode 100644 src/analyst/dspy-rlm-engine.ts create mode 100644 src/analyst/engine.ts delete mode 100644 src/analyst/kind-factory-ax-contract.test.ts create mode 100644 src/analyst/kind-factory.test.ts delete mode 100644 src/analyst/structure-findings.test.ts delete mode 100644 src/analyst/structure-findings.ts create mode 100644 src/analyst/trace-tool-callback.test.ts create mode 100644 src/analyst/trace-tool-callback.ts delete mode 100644 src/trace-analyst/analyst.test.ts delete mode 100644 src/trace-analyst/loop.test.ts delete mode 100644 src/trace-analyst/loop.ts diff --git a/README.md b/README.md index a0dce60e..47512137 100644 --- a/README.md +++ b/README.md @@ -357,7 +357,8 @@ pnpm tsx examples/selfimprove-quickstart/index.ts You do not need a runnable agent to analyze data you already captured. Use `analyzeRuns()` for `RunRecord[]`. For traces, run a registry of built-in or custom analysts, measure it on labeled issues and exact span locations, then turn only reviewed findings into eval data. -For a public quality check, convert CodeTraceBench with `traces import-codetracebench`, then run `agent-eval analyst-benchmark` against the pinned labels and a real model. +For a public quality check, convert CodeTraceBench with `traces import-codetracebench`, then run `agent-eval analyst-benchmark` against pinned labels. +The command compares an empty baseline with the official DSPy `RLM` trace analyst and records its trace reads, model calls, tokens, cost, runtime, and cited findings. See [concepts](./docs/concepts.md), [customer paths](./docs/customer-journeys.md), and [trace analysis](./docs/trace-analysis.md). diff --git a/clients/python/README.md b/clients/python/README.md index 5a9bc94a..940ad00f 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -217,8 +217,7 @@ Missing provider usage fails the run instead of assuming zero cost. ### DSPy -DSPy programs should use DSPy's official optimizers directly. -Install DSPy 3.2.1 and the Agent Eval metric adapter with: +Install DSPy 3.2.1 and the Agent Eval adapters with: ```sh python -m pip install "agent-eval-rpc[dspy]" @@ -249,6 +248,34 @@ Identical calls share one judge result, including concurrent calls. `DspyJudgeMetric` rejects DSPy's default unrestricted disk-cache pickle handling. Configure the official restricted cache as shown above, or call `dspy.configure_cache(enable_disk_cache=False)` before creating the metric. +DSPy programs should use DSPy's official optimizers directly. +Agent Eval also runs the official `dspy.RLM` for recursive trace analysis. +The TypeScript API starts the Python bridge, provides authenticated trace tools, keeps provider credentials in Node, enforces model and trace-read limits, and validates cited findings. +The Python bridge owns no trace storage or model credentials. + +```ts +import { createDspyRlmTraceEngine } from '@tangle-network/agent-eval/analyst' +import { analyzeTraces } from '@tangle-network/agent-eval/traces' + +const engine = createDspyRlmTraceEngine({ + baseUrl: process.env.LLM_BASE_URL!, + apiKey: process.env.LLM_API_KEY!, + model: process.env.LLM_MODEL!, + pricing: { + inputUsdPerMillion: 3, + outputUsdPerMillion: 15, + }, + runner: { command: '.venv/bin/python' }, +}) + +const result = await analyzeTraces( + { question: 'What first caused this run to fail?' }, + { source: 'run.otlp.jsonl', engine, toolGroup: 'singleTrace' }, +) +``` + +See [Trace Analysis](../../docs/trace-analysis.md) for custom definitions, limits, result fields, and the public quality benchmark. + DSPy 3.2.1 pins GEPA 0.0.27. The general Optimize Anything bridge uses GEPA 0.1.4, so repository checks install them in separate environments: diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 425e5062..f5accc7b 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -35,6 +35,7 @@ dev = [ "ruff>=0.6", ] dspy = [ + "deno==2.7.14", "dspy==3.2.1", ] diff --git a/clients/python/src/agent_eval_rpc/dspy_rlm_bridge.py b/clients/python/src/agent_eval_rpc/dspy_rlm_bridge.py new file mode 100644 index 00000000..e7a69c4e --- /dev/null +++ b/clients/python/src/agent_eval_rpc/dspy_rlm_bridge.py @@ -0,0 +1,730 @@ +"""Run an official DSPy RLM against trace tools owned by the Node process.""" + +from __future__ import annotations + +import argparse +import hashlib +import inspect +import json +import math +import os +import platform +import shutil +import subprocess +import sys +from collections.abc import Callable, Mapping +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +import httpx + +from agent_eval_rpc.optimizer_bridge_common import atomic_write_json + +_MAX_INPUT_BYTES = 4 * 1024 * 1024 +_TOOL_CALLBACK_TIMEOUT_SECONDS = 60.0 +_SEVERITIES = frozenset({"critical", "high", "medium", "low", "info"}) +_FINDING_KEYS = frozenset( + { + "severity", + "claim", + "subject", + "confidence", + "rationale", + "recommended_action", + "evidence", + } +) +_REQUIRED_FINDING_KEYS = frozenset({"severity", "claim", "confidence", "evidence"}) +_EVIDENCE_KEYS = frozenset({"uri", "excerpt"}) +_TRACE_ANALYSIS_PROMPT = """ +Investigate the supplied question using the enabled trace tools and Python for aggregation. +Follow analyst_instructions as task-specific policy. +Use llm_query or llm_query_batched when a focused model subquery is useful. +Do not claim that you inspected data you did not retrieve. + +Return: +1. answer: a concise answer supported by the trace data. +2. findings_json: a JSON array, without Markdown fences. Each element must contain only: + - severity: "critical", "high", "medium", "low", or "info" + - claim: a non-empty string + - subject: an optional non-empty string + - confidence: a finite number from 0 through 1 + - rationale: an optional string + - recommended_action: an optional string + - evidence: a non-empty array of objects containing only uri and optional excerpt + +Every evidence URI must identify exact trace or span IDs returned by an enabled tool. +For span evidence, use trace:///span/. +Percent encoding leaves letters and digits unchanged. Never use base64. +Never invent, approximate, or silently normalize an identifier. +Return [] when no finding has citable evidence. +""".strip() + + +@dataclass(frozen=True) +class _ToolCallback: + url: str + token: str + + +_ACTIVE_TOOL_CALLBACK: ContextVar[_ToolCallback | None] = ContextVar( + "dspy_rlm_tool_callback", + default=None, +) + + +def getDatasetOverview(filters: dict[str, Any] | None = None) -> Any: + """Return a bounded summary of the trace dataset.""" + + return _call_trace_tool("getDatasetOverview", _without_none(filters=filters)) + + +def queryTraces( + limit: int, + filters: dict[str, Any] | None = None, + offset: int | None = None, +) -> Any: + """Return one bounded page of trace summaries.""" + + return _call_trace_tool( + "queryTraces", + _without_none(filters=filters, limit=limit, offset=offset), + ) + + +def countTraces(filters: dict[str, Any] | None = None) -> Any: + """Count traces that match optional filters.""" + + return _call_trace_tool("countTraces", _without_none(filters=filters)) + + +def viewTrace(trace_id: str) -> Any: + """Read a bounded projection of one trace.""" + + return _call_trace_tool("viewTrace", {"trace_id": trace_id}) + + +def viewSpans(trace_id: str, span_ids: list[str]) -> Any: + """Read a bounded set of spans from one trace.""" + + return _call_trace_tool("viewSpans", {"trace_id": trace_id, "span_ids": span_ids}) + + +def searchTrace( + trace_id: str, + regex_pattern: str, + max_matches: int = 50, +) -> Any: + """Search one trace with a bounded regular expression query.""" + + return _call_trace_tool( + "searchTrace", + { + "trace_id": trace_id, + "regex_pattern": regex_pattern, + "max_matches": max_matches, + }, + ) + + +def searchSpan( + trace_id: str, + span_id: str, + regex_pattern: str, + max_matches: int = 50, +) -> Any: + """Search one span with a bounded regular expression query.""" + + return _call_trace_tool( + "searchSpan", + { + "trace_id": trace_id, + "span_id": span_id, + "regex_pattern": regex_pattern, + "max_matches": max_matches, + }, + ) + + +_TRACE_TOOL_FUNCTIONS: dict[str, Callable[..., Any]] = { + function.__name__: function + for function in ( + getDatasetOverview, + queryTraces, + countTraces, + viewTrace, + viewSpans, + searchTrace, + searchSpan, + ) +} +_TRACE_TOOL_ARGUMENTS = { + "getDatasetOverview": frozenset({"filters"}), + "queryTraces": frozenset({"filters", "limit", "offset"}), + "countTraces": frozenset({"filters"}), + "viewTrace": frozenset({"trace_id"}), + "viewSpans": frozenset({"trace_id", "span_ids"}), + "searchTrace": frozenset({"trace_id", "regex_pattern", "max_matches"}), + "searchSpan": frozenset({"trace_id", "span_id", "regex_pattern", "max_matches"}), +} +_TRACE_TOOL_REQUIRED_ARGUMENTS = { + "getDatasetOverview": frozenset(), + "queryTraces": frozenset({"limit"}), + "countTraces": frozenset(), + "viewTrace": frozenset({"trace_id"}), + "viewSpans": frozenset({"trace_id", "span_ids"}), + "searchTrace": frozenset({"trace_id", "regex_pattern", "max_matches"}), + "searchSpan": frozenset({"trace_id", "span_id", "regex_pattern", "max_matches"}), +} + + +def main() -> None: + previous_umask = os.umask(0o077) + try: + _main() + finally: + os.umask(previous_umask) + + +def _main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--input", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + input_value = _read_json(Path(args.input)) + operation = input_value.get("operation") + if operation == "inspect": + _require_exact_keys(input_value, {"operation"}, "inspect input") + dspy = _load_dspy() + with _probed_interpreter(dspy) as (_, deno_command): + output = {"runtime": _runtime_identity(deno_command)} + elif operation == "analyze": + validated = _validate_analyze_input(input_value) + output = _analyze(validated) + else: + raise ValueError("DSPy RLM bridge operation must be inspect or analyze") + atomic_write_json(Path(args.output), output) + + +def _analyze(input_value: dict[str, Any]) -> dict[str, Any]: + dspy = _load_dspy() + model_proxy = input_value["modelProxy"] + limits = input_value["limits"] + with _probed_interpreter(dspy) as (interpreter, deno_command): + runtime = _runtime_identity(deno_command) + lm = dspy.LM( + f"openai/{model_proxy['model']}", + api_base=model_proxy["baseUrl"], + api_key=model_proxy["apiKey"], + cache=False, + num_retries=0, + max_tokens=model_proxy["maxOutputTokens"], + ) + tools = _build_dspy_tools(dspy, input_value["toolSpecs"]) + program = dspy.RLM( + _build_signature(dspy), + max_iterations=limits["maxIterations"], + max_llm_calls=limits["maxLlmCalls"], + max_output_chars=limits["maxOutputChars"], + tools=tools, + sub_lm=lm, + interpreter=interpreter, + ) + + history_before = _lm_history_length(lm) + callback = input_value["toolCallback"] + with _tool_callback(callback["url"], callback["token"]): + with dspy.context(lm=lm): + prediction = program( + question=input_value["question"], + analyst_instructions=input_value["instructions"], + ) + model_calls = _lm_history_length(lm) - history_before + if model_calls < 0: + raise RuntimeError("DSPy LM history shrank during trace analysis") + + answer = _prediction_string(prediction, "answer") + findings = _parse_findings_json(_prediction_string(prediction, "findings_json")) + trajectory = _prediction_field(prediction, "trajectory") + if not isinstance(trajectory, list): + raise ValueError("DSPy RLM prediction trajectory must be an array") + _require_finite_json(trajectory, "DSPy RLM prediction trajectory") + + return { + "answer": answer, + "findings": findings, + "trajectory": trajectory, + "modelCalls": model_calls, + "runtime": runtime, + } + + +def _build_signature(dspy: Any) -> Any: + return dspy.Signature( + { + "question": ( + str, + dspy.InputField(desc="The trace-analysis question to answer."), + ), + "analyst_instructions": ( + str, + dspy.InputField(desc="Task-specific investigation and reporting instructions."), + ), + "answer": ( + str, + dspy.OutputField(desc="A concise, evidence-supported answer."), + ), + "findings_json": ( + str, + dspy.OutputField( + desc="A strict JSON array of cited findings matching the required schema." + ), + ), + }, + _TRACE_ANALYSIS_PROMPT, + ) + + +def _build_dspy_tools(dspy: Any, tool_specs: list[dict[str, Any]]) -> list[Any]: + tools = [] + for spec in tool_specs: + properties = spec["parameters"]["properties"] + tools.append( + dspy.Tool( + _TRACE_TOOL_FUNCTIONS[spec["name"]], + name=spec["name"], + desc=spec["description"], + args=properties, + ) + ) + return tools + + +@contextmanager +def _probed_interpreter(dspy: Any) -> Any: + deno_command = _build_deno_command(dspy) + interpreter = dspy.PythonInterpreter(deno_command=deno_command) + try: + try: + probe_output = interpreter.execute("print(1+1)") + except Exception as error: + raise RuntimeError( + "DSPy RLM sandbox startup probe failed before any model call" + ) from error + if not isinstance(probe_output, str) or probe_output.strip() != "2": + raise RuntimeError( + "DSPy RLM sandbox startup probe returned " + f"{probe_output!r}, expected '2', before any model call" + ) + yield interpreter, deno_command + finally: + interpreter.shutdown() + + +def _build_deno_command(dspy: Any) -> list[str]: + deno_executable = _find_deno_executable() + runner_path = _dspy_runner_path(dspy) + deno_cache_dir = _deno_cache_dir(deno_executable) + for label, path in (("DSPy runner", runner_path), ("Deno cache", deno_cache_dir)): + if "," in str(path): + raise RuntimeError(f"{label} path cannot contain a comma: {path}") + return [ + str(deno_executable), + "run", + "--no-config", + "--node-modules-dir=none", + f"--allow-read={runner_path},{deno_cache_dir}", + str(runner_path), + ] + + +def _find_deno_executable() -> Path: + executable_name = "deno.exe" if os.name == "nt" else "deno" + beside_python = Path(sys.executable).parent / executable_name + if beside_python.is_file() and os.access(beside_python, os.X_OK): + return beside_python.resolve() + + from_path = shutil.which(executable_name) + if from_path is not None: + return Path(from_path).resolve() + + try: + from deno import find_deno_bin + + packaged = Path(find_deno_bin()) + except (FileNotFoundError, ImportError) as error: + raise RuntimeError( + "DSPy RLM requires the deno==2.7.14 executable beside the Python interpreter" + ) from error + if not packaged.is_file() or not os.access(packaged, os.X_OK): + raise RuntimeError(f"DSPy RLM Deno executable is not executable: {packaged}") + return packaged.resolve() + + +def _dspy_runner_path(dspy: Any) -> Path: + runner_path = Path(inspect.getfile(dspy.PythonInterpreter)).with_name("runner.js") + if not runner_path.is_file(): + raise RuntimeError(f"DSPy PythonInterpreter runner is unavailable: {runner_path}") + return runner_path.resolve() + + +def _deno_cache_dir(deno_executable: Path) -> Path: + configured = os.environ.get("DENO_DIR") + if configured: + return Path(configured).expanduser().resolve() + try: + completed = subprocess.run( + [str(deno_executable), "info", "--json", "--no-config"], + capture_output=True, + check=False, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError) as error: + raise RuntimeError("DSPy RLM could not inspect the Deno cache directory") from error + if completed.returncode != 0: + detail = completed.stderr.strip() or f"exit code {completed.returncode}" + raise RuntimeError(f"DSPy RLM could not inspect the Deno cache directory: {detail}") + try: + info = json.loads(completed.stdout, parse_constant=_reject_json_constant) + except (json.JSONDecodeError, ValueError) as error: + raise RuntimeError("Deno info returned invalid JSON") from error + deno_dir = info.get("denoDir") if isinstance(info, dict) else None + if not isinstance(deno_dir, str) or not deno_dir.strip(): + raise RuntimeError("Deno info did not report denoDir") + return Path(deno_dir).expanduser().resolve() + + +@contextmanager +def _tool_callback(url: str, token: str) -> Any: + reset_token = _ACTIVE_TOOL_CALLBACK.set(_ToolCallback(url=url, token=token)) + try: + yield + finally: + _ACTIVE_TOOL_CALLBACK.reset(reset_token) + + +def _call_trace_tool(name: str, args: dict[str, Any]) -> Any: + callback = _ACTIVE_TOOL_CALLBACK.get() + if callback is None: + raise RuntimeError(f"{name} called outside an active DSPy RLM analysis") + response = httpx.post( + callback.url, + headers={"Authorization": f"Bearer {callback.token}"}, + json={"name": name, "args": args}, + timeout=_TOOL_CALLBACK_TIMEOUT_SECONDS, + ) + response.raise_for_status() + try: + payload = response.json() + except ValueError as error: + raise ValueError(f"trace tool callback returned invalid JSON for {name}") from error + if not isinstance(payload, dict) or set(payload) != {"result"}: + raise ValueError(f"trace tool callback returned an invalid envelope for {name}") + result = payload["result"] + _require_finite_json(result, f"trace tool callback result for {name}") + return result + + +def _validate_analyze_input(value: dict[str, Any]) -> dict[str, Any]: + _require_exact_keys( + value, + { + "operation", + "question", + "instructions", + "modelProxy", + "toolCallback", + "limits", + "toolSpecs", + }, + "analyze input", + ) + if value["operation"] != "analyze": + raise ValueError("analyze input operation must be analyze") + _require_non_empty_string(value["question"], "question") + _require_non_empty_string(value["instructions"], "instructions") + + model_proxy = _require_object(value["modelProxy"], "modelProxy") + _require_exact_keys( + model_proxy, + {"baseUrl", "apiKey", "model", "maxOutputTokens"}, + "modelProxy", + ) + _require_http_url(model_proxy["baseUrl"], "modelProxy.baseUrl") + _require_non_empty_string(model_proxy["apiKey"], "modelProxy.apiKey") + _require_non_empty_string(model_proxy["model"], "modelProxy.model") + _require_positive_integer(model_proxy["maxOutputTokens"], "modelProxy.maxOutputTokens") + + callback = _require_object(value["toolCallback"], "toolCallback") + _require_exact_keys(callback, {"url", "token"}, "toolCallback") + _require_http_url(callback["url"], "toolCallback.url") + _require_non_empty_string(callback["token"], "toolCallback.token") + + limits = _require_object(value["limits"], "limits") + _require_exact_keys( + limits, + {"maxIterations", "maxLlmCalls", "maxOutputChars"}, + "limits", + ) + for key in ("maxIterations", "maxLlmCalls", "maxOutputChars"): + _require_positive_integer(limits[key], f"limits.{key}") + + tool_specs = value["toolSpecs"] + if not isinstance(tool_specs, list): + raise ValueError("toolSpecs must be an array") + seen: set[str] = set() + for index, raw_spec in enumerate(tool_specs): + spec = _require_object(raw_spec, f"toolSpecs[{index}]") + _require_exact_keys( + spec, + {"name", "description", "parameters"}, + f"toolSpecs[{index}]", + ) + name = _require_non_empty_string(spec["name"], f"toolSpecs[{index}].name") + if name not in _TRACE_TOOL_FUNCTIONS: + raise ValueError(f"toolSpecs[{index}].name is not a supported trace tool") + if name in seen: + raise ValueError(f"toolSpecs contains duplicate tool {name}") + seen.add(name) + _require_non_empty_string( + spec["description"], + f"toolSpecs[{index}].description", + ) + parameters = _require_object( + spec["parameters"], + f"toolSpecs[{index}].parameters", + ) + if parameters.get("type") != "object": + raise ValueError(f"toolSpecs[{index}].parameters.type must be object") + properties = parameters.get("properties") + if not isinstance(properties, dict): + raise ValueError(f"toolSpecs[{index}].parameters.properties must be an object") + if set(properties) != _TRACE_TOOL_ARGUMENTS[name]: + raise ValueError(f"toolSpecs[{index}].parameters.properties do not match {name}") + required = parameters.get("required", []) + if ( + not isinstance(required, list) + or not all(isinstance(item, str) for item in required) + or len(set(required)) != len(required) + ): + raise ValueError(f"toolSpecs[{index}].parameters.required must be unique strings") + if set(required) != _TRACE_TOOL_REQUIRED_ARGUMENTS[name]: + raise ValueError(f"toolSpecs[{index}].parameters.required do not match {name}") + _require_finite_json(parameters, f"toolSpecs[{index}].parameters") + return value + + +def _parse_findings_json(value: str) -> list[dict[str, Any]]: + try: + parsed = json.loads(value, parse_constant=_reject_json_constant) + except json.JSONDecodeError as error: + raise ValueError("DSPy RLM findings_json must be valid JSON") from error + if not isinstance(parsed, list): + raise ValueError("DSPy RLM findings_json must be a JSON array") + return [_validate_finding(row, index) for index, row in enumerate(parsed)] + + +def _validate_finding(value: Any, index: int) -> dict[str, Any]: + finding = _require_object(value, f"findings[{index}]") + keys = set(finding) + if not _REQUIRED_FINDING_KEYS <= keys or not keys <= _FINDING_KEYS: + raise ValueError(f"findings[{index}] has missing or unknown fields") + if finding["severity"] not in _SEVERITIES: + raise ValueError(f"findings[{index}].severity is invalid") + _require_bounded_string(finding["claim"], f"findings[{index}].claim", 2_000) + if "subject" in finding: + _require_bounded_string(finding["subject"], f"findings[{index}].subject", 400) + confidence = finding["confidence"] + if ( + isinstance(confidence, bool) + or not isinstance(confidence, (int, float)) + or not math.isfinite(confidence) + or confidence < 0 + or confidence > 1 + ): + raise ValueError(f"findings[{index}].confidence must be a finite number from 0 to 1") + for key, maximum in (("rationale", 4_000), ("recommended_action", 2_000)): + if key in finding: + _require_optional_bounded_string( + finding[key], + f"findings[{index}].{key}", + maximum, + ) + evidence = finding["evidence"] + if not isinstance(evidence, list) or not evidence: + raise ValueError(f"findings[{index}].evidence must be a non-empty array") + for evidence_index, raw_evidence in enumerate(evidence): + item = _require_object( + raw_evidence, + f"findings[{index}].evidence[{evidence_index}]", + ) + if set(item) not in ({"uri"}, _EVIDENCE_KEYS): + raise ValueError( + f"findings[{index}].evidence[{evidence_index}] has missing or unknown fields" + ) + _require_bounded_string( + item["uri"], + f"findings[{index}].evidence[{evidence_index}].uri", + 2_000, + ) + if "excerpt" in item: + _require_optional_bounded_string( + item["excerpt"], + f"findings[{index}].evidence[{evidence_index}].excerpt", + 2_000, + ) + return finding + + +def _runtime_identity(deno_command: list[str]) -> dict[str, Any]: + return { + "engine": "dspy-rlm", + "packages": { + "agent-eval-rpc": _package_version("agent-eval-rpc"), + "dspy": _package_version("dspy"), + "deno": _package_version("deno"), + }, + "python": { + "implementation": sys_implementation_name(), + "version": platform.python_version(), + }, + "bridge": { + "sourceSha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + }, + "sandbox": { + "probeOutput": "2", + "runtime": "deno-pyodide", + "config": "none", + "nodeModulesDir": "none", + }, + } + + +def sys_implementation_name() -> str: + return platform.python_implementation().lower() + + +def _package_version(name: str) -> str: + try: + return version(name) + except PackageNotFoundError as error: + raise RuntimeError( + f"{name} is required for the DSPy RLM bridge; install agent-eval-rpc[dspy]" + ) from error + + +def _load_dspy() -> Any: + try: + import dspy + except ImportError as error: + raise RuntimeError( + "DSPy is required for this bridge; install agent-eval-rpc[dspy]" + ) from error + return dspy + + +def _lm_history_length(lm: Any) -> int: + history = getattr(lm, "history", None) + if not isinstance(history, list): + raise RuntimeError("DSPy LM does not expose model-call history") + return len(history) + + +def _prediction_field(prediction: Any, name: str) -> Any: + if isinstance(prediction, Mapping) and name in prediction: + return prediction[name] + if hasattr(prediction, name): + return getattr(prediction, name) + raise ValueError(f"DSPy RLM prediction is missing {name}") + + +def _prediction_string(prediction: Any, name: str) -> str: + value = _prediction_field(prediction, name) + return _require_non_empty_string(value, f"DSPy RLM prediction {name}") + + +def _read_json(path: Path) -> dict[str, Any]: + raw = path.read_bytes() + if len(raw) > _MAX_INPUT_BYTES: + raise ValueError(f"DSPy RLM bridge input exceeds {_MAX_INPUT_BYTES} bytes") + try: + value = json.loads(raw, parse_constant=_reject_json_constant) + except json.JSONDecodeError as error: + raise ValueError("DSPy RLM bridge input must be valid JSON") from error + if not isinstance(value, dict): + raise ValueError("DSPy RLM bridge input must be a JSON object") + return value + + +def _reject_json_constant(value: str) -> Any: + raise ValueError(f"non-finite JSON constant {value} is not allowed") + + +def _require_exact_keys(value: Mapping[str, Any], expected: set[str], label: str) -> None: + if set(value) != expected: + raise ValueError(f"{label} must contain exactly {sorted(expected)}") + + +def _require_object(value: Any, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise ValueError(f"{label} must be an object") + return value + + +def _require_non_empty_string(value: Any, label: str) -> str: + if not isinstance(value, str) or not value.strip() or value != value.strip(): + raise ValueError(f"{label} must be a non-empty trimmed string") + return value + + +def _require_bounded_string(value: Any, label: str, maximum: int) -> str: + result = _require_non_empty_string(value, label) + if len(result) > maximum: + raise ValueError(f"{label} exceeds {maximum} characters") + return result + + +def _require_optional_bounded_string(value: Any, label: str, maximum: int) -> str: + if not isinstance(value, str): + raise ValueError(f"{label} must be a string") + if len(value) > maximum: + raise ValueError(f"{label} exceeds {maximum} characters") + return value + + +def _require_positive_integer(value: Any, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{label} must be a positive integer") + return value + + +def _require_http_url(value: Any, label: str) -> str: + result = _require_non_empty_string(value, label) + parsed = urlparse(result) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError(f"{label} must be an HTTP(S) URL") + if parsed.username is not None or parsed.password is not None: + raise ValueError(f"{label} must not contain credentials") + return result + + +def _require_finite_json(value: Any, label: str) -> None: + try: + json.dumps(value, ensure_ascii=False, allow_nan=False) + except (TypeError, ValueError) as error: + raise ValueError(f"{label} must be finite JSON") from error + + +def _without_none(**values: Any) -> dict[str, Any]: + return {key: value for key, value in values.items() if value is not None} + + +if __name__ == "__main__": + main() diff --git a/clients/python/tests/test_dspy_rlm_bridge.py b/clients/python/tests/test_dspy_rlm_bridge.py new file mode 100644 index 00000000..2ee45f4e --- /dev/null +++ b/clients/python/tests/test_dspy_rlm_bridge.py @@ -0,0 +1,540 @@ +from __future__ import annotations + +import hashlib +import json +import os +import stat +import sys +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import httpx +import pytest + +from agent_eval_rpc import dspy_rlm_bridge + +DENO_COMMAND = [ + "/venv/bin/deno", + "run", + "--no-config", + "--node-modules-dir=none", + "--allow-read=/venv/dspy/runner.js,/cache/deno", + "/venv/dspy/runner.js", +] +RUNTIME = { + "engine": "dspy-rlm", + "packages": { + "agent-eval-rpc": "0.137.0", + "dspy": "3.2.1", + "deno": "2.7.14", + }, + "python": {"implementation": "cpython", "version": "3.13.0"}, + "bridge": {"sourceSha256": "a" * 64}, + "sandbox": { + "probeOutput": "2", + "runtime": "deno-pyodide", + "config": "none", + "nodeModulesDir": "none", + }, +} + + +def test_inspect_reports_pinned_runtime_and_private_atomic_output( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + input_path = tmp_path / "input.json" + output_path = tmp_path / "output.json" + input_path.write_text(json.dumps({"operation": "inspect"})) + calls: dict[str, Any] = {} + monkeypatch.setattr(dspy_rlm_bridge, "_load_dspy", lambda: _fake_dspy(calls)) + monkeypatch.setattr( + dspy_rlm_bridge, + "_build_deno_command", + lambda _dspy: DENO_COMMAND, + ) + monkeypatch.setattr( + dspy_rlm_bridge, + "_package_version", + lambda name: { + "agent-eval-rpc": "0.137.0", + "dspy": "3.2.1", + "deno": "2.7.14", + }[name], + ) + monkeypatch.setattr(dspy_rlm_bridge.platform, "python_version", lambda: "3.13.0") + monkeypatch.setattr(dspy_rlm_bridge, "sys_implementation_name", lambda: "cpython") + monkeypatch.setattr( + sys, + "argv", + ["dspy-rlm-bridge", "--input", str(input_path), "--output", str(output_path)], + ) + previous_umask = os.umask(0) + os.umask(previous_umask) + + dspy_rlm_bridge.main() + + output = json.loads(output_path.read_text()) + assert output == { + "runtime": { + "engine": "dspy-rlm", + "packages": { + "agent-eval-rpc": "0.137.0", + "dspy": "3.2.1", + "deno": "2.7.14", + }, + "python": {"implementation": "cpython", "version": "3.13.0"}, + "bridge": { + "sourceSha256": hashlib.sha256( + Path(dspy_rlm_bridge.__file__).read_bytes() + ).hexdigest() + }, + "sandbox": { + "probeOutput": "2", + "runtime": "deno-pyodide", + "config": "none", + "nodeModulesDir": "none", + }, + } + } + assert calls["probe_code"] == "print(1+1)" + assert calls["interpreter_shutdown"] is True + assert stat.S_IMODE(output_path.stat().st_mode) == 0o600 + observed_umask = os.umask(previous_umask) + os.umask(observed_umask) + assert observed_umask == previous_umask + + +def test_analyze_uses_official_rlm_contract_and_enabled_node_tool_specs( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + calls: dict[str, Any] = {"http": []} + fake_dspy = _fake_dspy(calls) + monkeypatch.setattr(dspy_rlm_bridge, "_load_dspy", lambda: fake_dspy) + monkeypatch.setattr( + dspy_rlm_bridge, + "_build_deno_command", + lambda _dspy: DENO_COMMAND, + ) + monkeypatch.setattr(dspy_rlm_bridge, "_runtime_identity", lambda _command: RUNTIME) + monkeypatch.setenv("OPENAI_API_KEY", "must-not-be-read") + + def fake_post(url: str, **kwargs: Any) -> httpx.Response: + calls["http"].append({"url": url, **kwargs}) + return httpx.Response( + 200, + json={"result": {"trace_id": "trace-1", "spans": []}}, + request=httpx.Request("POST", url), + ) + + monkeypatch.setattr(httpx, "post", fake_post) + input_path, output_path = _write_analyze_input(tmp_path) + monkeypatch.setattr( + sys, + "argv", + ["dspy-rlm-bridge", "--input", str(input_path), "--output", str(output_path)], + ) + + dspy_rlm_bridge.main() + + assert calls["lm"] == { + "model": "openai/analysis-model", + "api_base": "http://127.0.0.1:8123/v1", + "api_key": "proxy-key", + "cache": False, + "num_retries": 0, + "max_tokens": 700, + } + assert calls["context_lm"] is calls["lm_instance"] + assert calls["rlm"]["sub_lm"] is calls["lm_instance"] + assert calls["rlm"]["max_iterations"] == 4 + assert calls["rlm"]["max_llm_calls"] == 6 + assert calls["rlm"]["max_output_chars"] == 8_000 + assert calls["rlm"]["interpreter"] is calls["interpreter"] + assert [tool.name for tool in calls["rlm"]["tools"]] == ["viewTrace"] + tool = calls["rlm"]["tools"][0] + assert tool.desc == "Read one trace." + assert tool.args == {"trace_id": {"type": "string", "minLength": 1}} + assert calls["program_input"] == { + "question": "Why did this run fail?", + "analyst_instructions": "Find the earliest causal failure.", + } + assert calls["http"] == [ + { + "url": "http://127.0.0.1:8124/call", + "headers": {"Authorization": "Bearer callback-token"}, + "json": {"name": "viewTrace", "args": {"trace_id": "trace-1"}}, + "timeout": 60.0, + } + ] + assert calls["order"][:3] == ["interpreter-init", "probe", "lm-init"] + assert calls["interpreter_shutdown"] is True + assert json.loads(output_path.read_text()) == { + "answer": "The first failing operation is step 2.", + "findings": [ + { + "severity": "high", + "claim": "Step 2 failed before recovery.", + "subject": "step:2", + "confidence": 0.95, + "rationale": "The span has the first error.", + "recommended_action": "Fix step 2.", + "evidence": [ + { + "uri": "trace://trace-1/span/step-2", + "excerpt": "status=ERROR", + } + ], + } + ], + "trajectory": [{"iteration": 1, "tool": "viewTrace"}], + "modelCalls": 3, + "runtime": RUNTIME, + } + + +def test_analyze_accepts_all_canonical_node_tool_schemas(tmp_path: Path) -> None: + input_path, _ = _write_analyze_input(tmp_path) + input_value = json.loads(input_path.read_text()) + arguments = { + "getDatasetOverview": ["filters"], + "queryTraces": ["filters", "limit", "offset"], + "countTraces": ["filters"], + "viewTrace": ["trace_id"], + "viewSpans": ["trace_id", "span_ids"], + "searchTrace": ["trace_id", "regex_pattern", "max_matches"], + "searchSpan": ["trace_id", "span_id", "regex_pattern", "max_matches"], + } + required = { + "getDatasetOverview": [], + "queryTraces": ["limit"], + "countTraces": [], + "viewTrace": ["trace_id"], + "viewSpans": ["trace_id", "span_ids"], + "searchTrace": ["trace_id", "regex_pattern", "max_matches"], + "searchSpan": ["trace_id", "span_id", "regex_pattern", "max_matches"], + } + input_value["toolSpecs"] = [ + { + "name": name, + "description": f"Node descriptor for {name}.", + "parameters": { + "type": "object", + "properties": {argument: {"type": "string"} for argument in tool_arguments}, + "required": required[name], + "additionalProperties": False, + }, + } + for name, tool_arguments in arguments.items() + ] + + assert dspy_rlm_bridge._validate_analyze_input(input_value) is input_value + + +def test_deno_command_ignores_repo_configuration_and_node_modules( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + bin_dir = tmp_path / "venv" / "bin" + bin_dir.mkdir(parents=True) + python = bin_dir / "python" + deno = bin_dir / "deno" + python.touch() + deno.touch(mode=0o700) + runner = tmp_path / "dspy" / "runner.js" + runner.parent.mkdir() + runner.touch() + cache = tmp_path / "deno-cache" + cache.mkdir() + monkeypatch.setattr(dspy_rlm_bridge.sys, "executable", str(python)) + monkeypatch.setattr(dspy_rlm_bridge, "_dspy_runner_path", lambda _dspy: runner) + monkeypatch.setattr(dspy_rlm_bridge, "_deno_cache_dir", lambda _deno: cache) + + command = dspy_rlm_bridge._build_deno_command(SimpleNamespace()) + + assert command == [ + str(deno.resolve()), + "run", + "--no-config", + "--node-modules-dir=none", + f"--allow-read={runner},{cache}", + str(runner), + ] + assert "--node-modules-dir=auto" not in command + + +def test_broken_sandbox_fails_before_lm_construction( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + calls: dict[str, Any] = {} + fake_dspy = _fake_dspy(calls, probe_output="sandbox error") + monkeypatch.setattr(dspy_rlm_bridge, "_load_dspy", lambda: fake_dspy) + monkeypatch.setattr( + dspy_rlm_bridge, + "_build_deno_command", + lambda _dspy: DENO_COMMAND, + ) + input_path, output_path = _write_analyze_input(tmp_path) + monkeypatch.setattr( + sys, + "argv", + ["dspy-rlm-bridge", "--input", str(input_path), "--output", str(output_path)], + ) + + with pytest.raises(RuntimeError, match="before any model call"): + dspy_rlm_bridge.main() + + assert "lm" not in calls + assert calls["interpreter_shutdown"] is True + assert not output_path.exists() + + +@pytest.mark.parametrize( + ("function_name", "kwargs", "expected_args"), + [ + ("getDatasetOverview", {}, {}), + ("queryTraces", {"limit": 5, "offset": 2}, {"limit": 5, "offset": 2}), + ("countTraces", {"filters": {"has_errors": True}}, {"filters": {"has_errors": True}}), + ("viewTrace", {"trace_id": "t1"}, {"trace_id": "t1"}), + ( + "viewSpans", + {"trace_id": "t1", "span_ids": ["s1", "s2"]}, + {"trace_id": "t1", "span_ids": ["s1", "s2"]}, + ), + ( + "searchTrace", + {"trace_id": "t1", "regex_pattern": "error"}, + {"trace_id": "t1", "regex_pattern": "error", "max_matches": 50}, + ), + ( + "searchSpan", + {"trace_id": "t1", "span_id": "s1", "regex_pattern": "error", "max_matches": 3}, + { + "trace_id": "t1", + "span_id": "s1", + "regex_pattern": "error", + "max_matches": 3, + }, + ), + ], +) +def test_fixed_trace_tools_send_authenticated_callback_payloads( + monkeypatch: pytest.MonkeyPatch, + function_name: str, + kwargs: dict[str, Any], + expected_args: dict[str, Any], +) -> None: + observed: list[dict[str, Any]] = [] + + def fake_post(url: str, **request: Any) -> httpx.Response: + observed.append({"url": url, **request}) + return httpx.Response( + 200, + json={"result": {"ok": True}}, + request=httpx.Request("POST", url), + ) + + monkeypatch.setattr(httpx, "post", fake_post) + function = getattr(dspy_rlm_bridge, function_name) + with dspy_rlm_bridge._tool_callback("http://127.0.0.1:9000/call", "secret-token"): + assert function(**kwargs) == {"ok": True} + + assert observed == [ + { + "url": "http://127.0.0.1:9000/call", + "headers": {"Authorization": "Bearer secret-token"}, + "json": {"name": function_name, "args": expected_args}, + "timeout": 60.0, + } + ] + + +@pytest.mark.parametrize( + "findings_json", + [ + "not json", + json.dumps( + [ + { + "severity": "high", + "claim": "Invalid extra field.", + "confidence": 0.8, + "evidence": [{"uri": "trace://t/span/s"}], + "extra": True, + } + ] + ), + ], +) +def test_analyze_rejects_malformed_findings( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + findings_json: str, +) -> None: + calls: dict[str, Any] = {} + fake_dspy = _fake_dspy(calls, findings_json=findings_json, invoke_tool=False) + monkeypatch.setattr(dspy_rlm_bridge, "_load_dspy", lambda: fake_dspy) + monkeypatch.setattr( + dspy_rlm_bridge, + "_build_deno_command", + lambda _dspy: DENO_COMMAND, + ) + monkeypatch.setattr(dspy_rlm_bridge, "_runtime_identity", lambda _command: RUNTIME) + input_path, output_path = _write_analyze_input(tmp_path) + monkeypatch.setattr( + sys, + "argv", + ["dspy-rlm-bridge", "--input", str(input_path), "--output", str(output_path)], + ) + + with pytest.raises(ValueError, match="findings"): + dspy_rlm_bridge.main() + + assert not output_path.exists() + + +def _write_analyze_input(tmp_path: Path) -> tuple[Path, Path]: + input_path = tmp_path / "input.json" + output_path = tmp_path / "output.json" + input_path.write_text( + json.dumps( + { + "operation": "analyze", + "question": "Why did this run fail?", + "instructions": "Find the earliest causal failure.", + "modelProxy": { + "baseUrl": "http://127.0.0.1:8123/v1", + "apiKey": "proxy-key", + "model": "analysis-model", + "maxOutputTokens": 700, + }, + "toolCallback": { + "url": "http://127.0.0.1:8124/call", + "token": "callback-token", + }, + "limits": { + "maxIterations": 4, + "maxLlmCalls": 6, + "maxOutputChars": 8_000, + }, + "toolSpecs": [ + { + "name": "viewTrace", + "description": "Read one trace.", + "parameters": { + "type": "object", + "properties": {"trace_id": {"type": "string", "minLength": 1}}, + "required": ["trace_id"], + "additionalProperties": False, + }, + } + ], + } + ) + ) + return input_path, output_path + + +def _fake_dspy( + calls: dict[str, Any], + *, + findings_json: str | None = None, + invoke_tool: bool = True, + probe_output: str = "2\n", +) -> Any: + valid_findings = json.dumps( + [ + { + "severity": "high", + "claim": "Step 2 failed before recovery.", + "subject": "step:2", + "confidence": 0.95, + "rationale": "The span has the first error.", + "recommended_action": "Fix step 2.", + "evidence": [ + { + "uri": "trace://trace-1/span/step-2", + "excerpt": "status=ERROR", + } + ], + } + ] + ) + + class FakePythonInterpreter: + def __init__(self, *, deno_command: list[str]) -> None: + calls.setdefault("order", []).append("interpreter-init") + calls["interpreter"] = self + calls["deno_command"] = deno_command + + def execute(self, code: str) -> str: + calls["order"].append("probe") + calls["probe_code"] = code + return probe_output + + def shutdown(self) -> None: + calls["interpreter_shutdown"] = True + + class FakeLm: + def __init__(self, model: str, **kwargs: Any) -> None: + calls.setdefault("order", []).append("lm-init") + self.history: list[dict[str, Any]] = [] + calls["lm"] = {"model": model, **kwargs} + calls["lm_instance"] = self + + class FakeTool: + def __init__( + self, + function: Any, + *, + name: str, + desc: str, + args: dict[str, Any], + ) -> None: + self.func = function + self.name = name + self.desc = desc + self.args = args + + class FakeRlm: + def __init__(self, signature: Any, **kwargs: Any) -> None: + calls["signature"] = signature + calls["rlm"] = kwargs + + def __call__(self, **kwargs: Any) -> Any: + calls["program_input"] = kwargs + lm = calls["lm_instance"] + lm.history.extend([{"call": 1}, {"call": 2}, {"call": 3}]) + if invoke_tool: + assert calls["rlm"]["tools"][0].func(trace_id="trace-1") == { + "trace_id": "trace-1", + "spans": [], + } + return SimpleNamespace( + answer="The first failing operation is step 2.", + findings_json=findings_json if findings_json is not None else valid_findings, + trajectory=[{"iteration": 1, "tool": "viewTrace"}], + ) + + @contextmanager + def fake_context(*, lm: Any) -> Any: + calls["context_lm"] = lm + yield + + return SimpleNamespace( + LM=FakeLm, + PythonInterpreter=FakePythonInterpreter, + Tool=FakeTool, + RLM=FakeRlm, + Signature=lambda fields, instructions: { + "fields": fields, + "instructions": instructions, + }, + InputField=lambda **kwargs: {"kind": "input", **kwargs}, + OutputField=lambda **kwargs: {"kind": "output", **kwargs}, + context=fake_context, + ) diff --git a/clients/python/uv.lock b/clients/python/uv.lock index 5e67694e..1518ef1b 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -49,6 +49,7 @@ dev = [ { name = "ruff" }, ] dspy = [ + { name = "deno" }, { name = "dspy" }, ] @@ -65,6 +66,7 @@ skillopt-source = [ [package.metadata] requires-dist = [ + { name = "deno", marker = "extra == 'dspy'", specifier = "==2.7.14" }, { name = "dspy", marker = "extra == 'dspy'", specifier = "==3.2.1" }, { name = "filelock", specifier = ">=3.16" }, { name = "httpx", specifier = ">=0.27" }, @@ -950,6 +952,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/05/66/73034ad30b59f13439b75e620989dacba4c047256e358ba7c2e9ec98ea22/datasets-5.0.0-py3-none-any.whl", hash = "sha256:7dd34927a0fd7046e98aad5cb9430e699c373238a15befa7b9bf22b991a7fee6", size = 555084, upload-time = "2026-06-05T13:18:24.435Z" }, ] +[[package]] +name = "deno" +version = "2.7.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/e4/adbeb37143551a4e575049e8b291bedc5f57191753ade31f7cf6665b1d52/deno-2.7.14.tar.gz", hash = "sha256:63a79caa105368813e434593f2c059d4cc3e2562ec57f606348e9321a525d582", size = 8168, upload-time = "2026-04-28T17:34:10.315Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/a1/82951e37de914ca9445e3dd99a5f06e36e857000f9d34beb3f9fc5ff0971/deno-2.7.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3860efe76d9c8491ee847256f31ff5f1b1d1871137daab91eb4f28c0543eeb4d", size = 47841490, upload-time = "2026-04-28T17:33:53.968Z" }, + { url = "https://files.pythonhosted.org/packages/fe/1f/809cac7474d86ad41f2938b4713cf846b261bba7a1fdb5ab27b3fabc435e/deno-2.7.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:194560abc29021d3b5b4be9bfd628f2b21152a3bae031bdcd30e41584ba4b9d8", size = 44600497, upload-time = "2026-04-28T17:33:57.426Z" }, + { url = "https://files.pythonhosted.org/packages/52/51/c60816c902e5fa9daf4bab5869a5895e58748f05739eab399c1ae6d0ee37/deno-2.7.14-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:f47511d0c8080c2bdb0ee58f2b96d657b1e5f2ede088b1bb9a98bfb80362af22", size = 48267415, upload-time = "2026-04-28T17:34:00.553Z" }, + { url = "https://files.pythonhosted.org/packages/79/a8/ea52407ebe29caadc67b6899437898880aad8d207ed889dd4f9ba7d9ddf7/deno-2.7.14-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a92040900311849a812d47e29e1494b1a4f4c7431536721888356c7a3662d22", size = 50283833, upload-time = "2026-04-28T17:34:04.094Z" }, + { url = "https://files.pythonhosted.org/packages/98/0e/2581a39773004081f40ef4811059fe867953d86166b3f7fb77abb5d68757/deno-2.7.14-py3-none-win_amd64.whl", hash = "sha256:1646dbcd447465bc6eb9a9e25eef0283f74dd68ad5bd9c50bcaec641737eb730", size = 49278963, upload-time = "2026-04-28T17:34:07.671Z" }, +] + [[package]] name = "dill" version = "0.4.1" diff --git a/docs/trace-analysis.md b/docs/trace-analysis.md index d5940953..8115b89a 100644 --- a/docs/trace-analysis.md +++ b/docs/trace-analysis.md @@ -1,558 +1,244 @@ # Trace Analysis -Trace analysis answers three different questions: +A trace analyst is a recursive research program that answers a question by choosing which trace data to inspect. +It is not a single prompt over a prebuilt trace dump. -1. What happened in the run? -2. Which exact steps support a suspected problem? -3. Does an analyst find labeled problems reliably enough to use? +Agent Eval separates five concerns: -Keep those answers separate. -A generated finding is a review request, not training truth. +| Part | Responsibility | +|---|---| +| Analyst definition | The question, investigation policy, allowed tools, and limits | +| Analysis engine | Runs the recursive investigation | +| Trace store | Provides bounded reads over OTLP traces | +| Finding | Records one structured claim with exact evidence | +| Analysis result | Returns the prose answer, findings, investigation steps, model calls, tool calls, and runtime | -## Run The Built-In Analysts +The built-in model-backed analysts use the official DSPy `RLM`. +DSPy runs the research loop and a sandboxed Python interpreter. +Agent Eval owns trace access, credentials, cancellation, cost accounting, and output validation. -The default registry always includes deterministic checks. -Model-assisted analysts are added only when you provide a model client. +`callLlmJson()` and `createPublicBenchmarkDirectRunner()` are direct-call baselines. +They are not trace analysts. -```ts -import { - buildDefaultAnalystRegistry, -} from '@tangle-network/agent-eval/analyst' -import { OtlpFileTraceStore } from '@tangle-network/agent-eval/traces' - -const traceStore = new OtlpFileTraceStore({ path: 'traces.otlp.jsonl' }) -const analysts = buildDefaultAnalystRegistry() +## Install -const result = await analysts.run('release-42', { traceStore }) +Install the TypeScript package and the matching Python package with DSPy: -for (const finding of result.findings) { - console.log(finding.claim, finding.evidence_refs) -} +```sh +pnpm add @tangle-network/agent-eval +python -m venv .venv +.venv/bin/pip install "agent-eval-rpc[dspy]" ``` -Use `result.per_analyst` to inspect failures, latency, calls, tokens, and cost. -An analyst failure is recorded separately from an agent failure. +The Python extra pins the tested stable DSPy and Deno versions. -Products can implement `TraceAnalysisStore`; they do not need to use the file store in production. -Custom stores provide `hasTrace` and batched `hasSpans` alongside the seven reads, and accept a `TraceAnalysisStoreContext` so cancellation reaches storage and scans. -The binding validates every custom-store result. -Missing fields, undeclared fields, inconsistent counts, and false continuation flags throw `TraceAnalysisStoreContractError` with code `backend_integrity`. -The analyst runs one Ax executor loop and accepts only an explicit structured `final(task, { report, findings })` result; max-turn fallback text fails loud. - -### Bind the same reads into another agent environment - -`buildTraceAnalysisToolDescriptors()` is the canonical definition of the analyst's seven bounded read operations and does not expose Ax types. -Each descriptor carries the stable `traces` namespace, function name, description, JSON input schema in `parameters`, and a handler already bound to the supplied `TraceAnalysisStore`. -`buildTraceAnalystTools()` adapts those descriptors into Ax functions; it does not define a second tool surface. -The bound handlers wrap custom stores with `createBoundedTraceAnalysisStore()`, so page limits, byte ceilings, not-found errors, and cancellation do not depend on the transport or adapter. +## Answer One Question ```ts import { - buildTraceAnalysisToolDescriptors, - type TraceAnalysisStore, -} from '@tangle-network/agent-eval/traces' - -declare const store: TraceAnalysisStore -declare function qualifyToolName(namespace: string, name: string): string - -const tools = buildTraceAnalysisToolDescriptors({ store }).map( - ({ namespace, name, description, parameters, handler }) => ({ - name: qualifyToolName(namespace, name), - description, - inputSchema: parameters, - handler, - }), -) -``` - -Map these fields into the host's existing tool transport. -The host owns namespace encoding; use its existing convention instead of inventing one here. -Do not copy the schemas or reimplement the handlers in an MCP, Runtime, or provider adapter. - -`queryTraces.limit`, `viewSpans.span_ids`, and search `max_matches` caps are present in the JSON Schemas and enforced before store calls. -Invalid arguments throw `TraceAnalysisValidationError` with code `validation`; responses that cannot fit their byte ceiling throw `TraceAnalysisLimitError` with code `limit_exceeded`. -Search patterns use RE2 syntax, which rejects backreferences and lookaround instead of allowing exponential-time expressions. -Search results return `hits` and an exact `has_more` flag; they do not invent a total after a capped scan. -`viewSpans` partitions every requested id across `spans`, `missing_span_ids`, and `omitted_span_ids`; `has_more` is true when omitted ids must be retried. -Attribute and match text shortening includes a deterministic marker. -Trace pages set `has_more`, and the overview returns every error cluster or fails explicitly when the configured response limit is too small. - -### Analyze captured tool spans in memory - -Use `toolSpansToTraceAnalysisStore()` when a live worker already returns canonical `ToolSpan[]` records. -The function snapshots the records immediately, groups them by `runId`, and exposes the same bounded reads and searches as the file-backed store. - -```ts + createDspyRlmTraceEngine, +} from '@tangle-network/agent-eval/analyst' import { analyzeTraces, - toolSpansToTraceAnalysisStore, - type ToolSpan, - ToolTraceMissingError, } from '@tangle-network/agent-eval/traces' -declare const capturedToolSpans: ToolSpan[] | undefined - -if (!capturedToolSpans?.length) throw new ToolTraceMissingError() -const source = toolSpansToTraceAnalysisStore(capturedToolSpans) -const result = await analyzeTraces({ question: 'Why are tools failing?' }, { source, ai, model }) -``` +const engine = createDspyRlmTraceEngine({ + baseUrl: process.env.LLM_BASE_URL!, + apiKey: process.env.LLM_API_KEY!, + model: process.env.LLM_MODEL!, + pricing: { + inputUsdPerMillion: 3, + outputUsdPerMillion: 15, + }, + maxCostUsd: 0.50, + runner: { command: '.venv/bin/python' }, +}) -`undefined`, `null`, and an empty array throw `ToolTraceMissingError` with code `capture_integrity`. -An empty tool list cannot distinguish a real tool-free run from broken capture, so the adapter never reports it as a clean trace set. -Use a complete OTLP trace source when proving that a run executed successfully without tools. +const result = await analyzeTraces( + { question: 'What first caused this run to fail?' }, + { + source: 'run.otlp.jsonl', + engine, + toolGroup: 'singleTrace', + limits: { + maxIterations: 8, + maxLlmCalls: 4, + maxToolCalls: 32, + }, + }, +) -## Deterministic failure coverage (no LLM) +console.log(result.answer) +console.log(result.findings) +console.log(result.trajectory) +``` -Before (or alongside) the LLM analyst, `OtlpFileTraceStore.getOverview()` returns a -`DatasetOverview` whose `error_clusters` are computed deterministically: error -spans are grouped by a normalized failure signature (uuids / hex ids / numbers / -absolute paths / durations collapsed), each cluster carrying its prevalence, -exemplar `trace_id`/`span_id`, and a verbatim sample. This is a zero-LLM, -reproducible failure checklist the analyst then explains and closes: +Omit `pricing` only when Agent Eval already recognizes the exact model or model family. +Unknown pricing fails before the first model call. -```ts -const overview = await store.getOverview() -for (const c of overview.error_clusters) { - console.log(`${c.trace_count}× ${c.signature}: e.g. trace ${c.exemplar_trace_ids[0]}`) -} -``` +The provider key remains in the Node process. +The Python process receives an authenticated loopback model endpoint with an ephemeral credential. +Each trace read also crosses an authenticated loopback callback and counts against `maxToolCalls`. -## Recursive control integrity (no LLM) +## Define A Reusable Analyst -`CONTROL_INTEGRITY_ANALYST` checks the existing `SupervisorRunSources` or `SupervisorRunTree` directly. -It does not define another run format. -Register it as a custom-input analyst and pass the existing value under its stable id: +An analyst definition contains no model, credentials, or execution state. +The same definition can run with DSPy or another engine that implements `TraceAnalysisEngine`. ```ts import { - AnalystRegistry, - CONTROL_INTEGRITY_ANALYST, + defineTraceAnalyst, + runTraceAnalyst, } from '@tangle-network/agent-eval/analyst' import { - readLoopsSupervisorRun, -} from '@tangle-network/agent-eval/supervisor-run' - -const sources = await readLoopsSupervisorRun(runDir) -const registry = new AnalystRegistry() -registry.register(CONTROL_INTEGRITY_ANALYST) + OtlpFileTraceStore, +} from '@tangle-network/agent-eval/traces' -const result = await registry.run('run-123', { - custom: { 'control-integrity': sources }, +const repeatedFailure = defineTraceAnalyst({ + id: 'repeated-tool-failure', + description: 'Finds the repeated tool failure with the largest impact.', + area: 'tool-use', + version: '1.0.0', + question: 'Which repeated tool failure should we fix first?', + instructions: [ + 'Compare frequency and downstream impact.', + 'Cite the failing span and at least one affected downstream span.', + 'Return no finding when the traces do not support a repeated failure.', + ].join('\n'), + toolGroup: 'all', + minimumEvidenceCitations: 2, + limits: { + maxIterations: 10, + maxLlmCalls: 6, + maxToolCalls: 40, + }, }) -``` -Pass `SupervisorRunSources` when it is available. -A `SupervisorRunTree` does not retain raw journal multiplicity or worker request and acknowledgement rows, so tree input explicitly reports those checks as unavailable. +const store = new OtlpFileTraceStore({ path: 'runs.otlp.jsonl' }) +await store.ensureIndexed() -The deterministic pass can prove only facts represented by these two existing surfaces. - -| Question | Current evidence | What the analyst can say | -|---|---|---| -| Is every invocation attached to one unambiguous tree? | `rootId`, `rollout_id`, `parent_rollout_id`, `run_id` | Duplicate ids, missing parents, extra parentless roots, cross-run edges, and ancestry cycles are violations with exact field references. | -| Did invocation roles survive capture? | Explicit journal and `RolloutLine.role` values | The root must remain `supervisor`; non-root roles are consumed as recorded, and workers may spawn workers. | -| Is the causal order possible? | `outcome.metrics.spawned_at`, `started_at`, `settled_at`, `completed_at`, `finished_at` when present | A child before its parent, a child after its parent closed, or a close before a start is a violation; absent timestamps produce no timing claim. | -| Did a queued steer reach the worker? | `SupervisorRunSources.workers[].inbox` and `.events` | Requests and acknowledgements are joined by request id, not compared as totals. Missing, malformed, duplicate, or uncorrelated rows make the affected count unavailable. | -| Can behavior be attributed to an exact profile? | `policy.agent_profile_cell_id` | An absent id is reported as unavailable. | -| Can action authorship or reasoning be inspected? | `messages[]` | Empty gap rows are reported as unavailable. | +const result = await runTraceAnalyst({ + definition: repeatedFailure, + engine, + store, + context: { runId: 'release-42' }, +}) +``` -An empty finding list means only that no implemented rule fired on the captured fields. -It does not certify that an agent chose the action, that the action was authorized, that a budget or depth limit was enforced, or that a finding caused a later decision. -Those claims require upstream action-decision events carrying `action_id`, `actor_rollout_id`, `target_rollout_id`, `action_kind`, `authority_snapshot_id`, requested and granted resource/depth values, the authorization result, and any `finding_id` or evidence references that caused the action. -Resume integrity additionally requires an explicit prior-session id and resumed-session id rather than a prose summary. +Keep product policy in the definition. +Keep model transport, recursion mechanics, secrets, and accounting in the engine. +Keep exact trace facts in deterministic tools or checks. -Malformed source rows are excluded from structural claims. -Their count is retained in `SupervisorRunTree.gaps`, so analyzing a projected tree later cannot turn an unreadable parent row into a missing-parent violation. +## Run The Built-In Set -## Add A Custom Analyst +The default registry always includes deterministic behavior checks. +It adds the four recursive analysts only when an engine is supplied: -`defineTraceAnalyst()` fills the fixed registry fields. -The custom function receives the same bounded `TraceAnalysisStore` used by the built-ins. +- `failure-mode` +- `knowledge-gap` +- `knowledge-poisoning` +- `improvement` ```ts import { - AnalystRegistry, - defineTraceAnalyst, - makeFinding, + buildDefaultAnalystRegistry, } from '@tangle-network/agent-eval/analyst' -const analysts = new AnalystRegistry() - -analysts.register(defineTraceAnalyst({ - id: 'repeated-tool-errors', - description: 'Reports the largest repeated tool error cluster.', - cost: { kind: 'deterministic' }, - async analyze(store) { - const overview = await store.getOverview({ has_errors: true }) - const cluster = overview.error_clusters[0] - if (!cluster) return [] - - return [makeFinding({ - analyst_id: 'repeated-tool-errors', - area: 'tool-use', - subject: cluster.signature, - claim: `${cluster.span_count} failed spans share one error`, - severity: 'high', - confidence: 1, - evidence_refs: [{ - kind: 'span', - uri: `trace://${encodeURIComponent(cluster.exemplar_trace_ids[0])}/span/${encodeURIComponent(cluster.exemplar_span_ids[0])}`, - excerpt: cluster.status_message_sample, - }], - recommended_action: 'Fix the highest-frequency tool error before changing prompts.', - validation_plan: 'Run fresh cases and require this error signature to disappear.', - })] - }, -})) +const registry = buildDefaultAnalystRegistry({ engine }) +const run = await registry.run('release-42', { traceStore: store }) + +for (const finding of run.findings) { + console.log(finding.analyst_id, finding.claim, finding.evidence_refs) +} ``` -Use code for exact facts such as exit codes, missing fields, and repeated calls. -Use model-assisted analysts for semantic questions such as whether a response ignored user intent. +`run.per_analyst` records each analyst's status, latency, calls, tokens, and cost. +One analyst failure does not become an agent failure. -## Measure An Analyst +Pass `definitions` to replace the four built-ins. +Omit `engine` for deterministic-only analysis. -Do not judge an analyst by persuasive prose. -Label the issue identity and exact evidence locations, then run the same cases through every implementation. +## Trace Tools -```ts -import { - compareAnalystRunners, - registryBenchmarkRunner, - renderAnalystBenchmarkMarkdown, - runAnalystBenchmark, - traceStoreEvidenceResolver, -} from '@tangle-network/agent-eval/analyst' +The engine receives only the tool group declared by the analyst: -const benchmark = await runAnalystBenchmark({ - cases: [{ - id: 'failed-command', - clusterId: 'incident-42', - labelState: 'positive', - input: { traceStore }, - expectedIssues: [{ - id: 'repeated-command', - subjects: ['failure-mode:repeated-command'], - evidence: [{ kind: 'span', uri: 'trace://run-1/span/tool-3' }], - criticalEvidence: [{ kind: 'span', uri: 'trace://run-1/span/tool-1' }], - }], - labeledEvidence: [ - { kind: 'span', uri: 'trace://run-1/span/tool-1' }, - { kind: 'span', uri: 'trace://run-1/span/tool-3' }, - ], - }], - runners: [registryBenchmarkRunner({ id: 'built-in', registry: analysts })], - repetitions: 3, - resolveEvidence: traceStoreEvidenceResolver((input) => input.traceStore), - benchmark: { - id: 'failure-localization', - dataset: { - id: 'my-team/trace-failures', - revision: 'git-sha-or-content-digest', - split: 'test', - }, - }, -}) - -console.log(renderAnalystBenchmarkMarkdown(benchmark)) -``` +| Group | Use | +|---|---| +| `singleTrace` | Inspect one known trajectory | +| `discoveryAndSearch` | Find relevant traces and spans across a dataset | +| `all` | Use every bounded trace operation | -The result reports: +The canonical operations are: -- issue recall and finding precision, -- first bad step accuracy, -- citation coverage, exact source-quote coverage, agreement with labeled locations, and actual location resolution, -- false positives and failures on trusted-negative cases, -- predictions and failures on unlabeled cases, -- matched-label agreement and full-prediction agreement, -- failed runs, -- latency, calls, every reported token counter, and known or missing cost, -- dataset revision, case tags, case metadata, and runner metadata. +- `getDatasetOverview` +- `queryTraces` +- `countTraces` +- `viewTrace` +- `viewSpans` +- `searchTrace` +- `searchSpan` -Use `compareAnalystRunners()` for paired differences between two implementations. -Repetitions are averaged within each case before comparison. -Treat its interval as inferential only with at least 20 independent cases. +Use `buildTraceAnalysisToolDescriptors({ store })` to bind the same operations into another runtime. +Each descriptor includes its name, namespace, description, JSON input schema, and bounded handler. +Do not copy the schemas or reimplement the handlers in another adapter. -## Load Public Trace Labels +Custom stores implement `TraceAnalysisStore`. +`OtlpFileTraceStore`, `otlpTextToTraceAnalysisStore()`, and `toolSpansToTraceAnalysisStore()` provide common adapters. +Store results are checked for missing fields, undeclared fields, inconsistent counts, invalid continuation flags, oversized responses, and unsafe search patterns. -Use the published label adapters with `@tangle-network/traces` or your own trajectory loader. -Agent Eval does not download datasets or own trace capture. -Load public data at an immutable commit and record that commit in `benchmark.dataset.revision`. +## Result Contract -```ts -import { - agentRxBenchmarkCase, - agentRxPredictionsToFindings, - codeTraceBenchCase, - codeTracerPredictionsToFindings, -} from '@tangle-network/agent-eval/analyst' -import { otlpTextToTraceAnalysisStore } from '@tangle-network/agent-eval/traces' -import { chatTrajectoryToSpans, serializeSpans } from '@tangle-network/traces' +Every recursive run returns: -const codeSpans = chatTrajectoryToSpans(codeTraceTrajectory, { - traceId: codeTraceRow.traj_id, -}) -const codeCase = codeTraceBenchCase(codeTraceRow, { - traceStore: otlpTextToTraceAnalysisStore(serializeSpans(codeSpans)), -}) +| Field | Meaning | +|---|---| +| `answer` | Direct answer to the analyst question | +| `findings` | Valid cited claims accepted by the analyst policy | +| `trajectory` | DSPy RLM investigation steps | +| `modelCalls` | Successful provider completions used by the engine | +| `toolCalls` | Trace reads admitted for execution, including a read that later fails | +| `runtime` | Engine, package, sandbox identity, provider attempts, and successful completions | -const agentRxSpans = chatTrajectoryToSpans(agentRxMessages, { - traceId: String(agentRxRow.trajectory_id), - stepMode: 'message', -}) -const rootCauseCase = agentRxBenchmarkCase(agentRxRow, { - traceStore: otlpTextToTraceAnalysisStore(serializeSpans(agentRxSpans)), -}, { - stepCount: agentRxMessages.length, -}) +Each finding requires exact `trace://` or `finding://` evidence. +Trace citations must resolve to an existing span. +When a citation includes an excerpt, the exact text must occur in that span or referenced finding. +Unknown, transformed, or fabricated identifiers are rejected. +An empty findings array means no submitted claim passed the evidence rules. +It does not prove the run was correct. -const codeTracerFindings = codeTracerPredictionsToFindings( - codeTraceRow.traj_id, - codetracerLabels, - { stepCount: codeTraceRow.step_count }, -) -const agentRxFindings = agentRxPredictionsToFindings( - agentRxRow.trajectory_id, - agentRxJudgeOutput, - { stepCount: agentRxMessages.length }, -) -``` +## Measure Analyst Quality -`codeTraceBenchCase()` accepts the public [CodeTraceBench](https://huggingface.co/datasets/NJU-LINK/CodeTraceBench) JSONL format. -It scores the published incorrect-step task by default, including clean trajectories. -Pass `labelSet: 'incorrect-and-unuseful'` to both the case and prediction adapters only for an explicitly combined experiment. -Every cited step is checked against `step_count`. - -`agentRxBenchmarkCase()` accepts the public [AgentRx](https://huggingface.co/datasets/microsoft/AgentRx) label format. -AgentRx category quality and root-step accuracy are scored independently. -`traceAnalystQualityJudge` averages them when a root-step label exists. -Pass `target: 'all-failures'` only when the analyst is designed to identify every annotated failure. - -Both adapters emit `trace:///span/step-` evidence by default. -`@tangle-network/traces` uses the same IDs when converting chat trajectories. -Pass `stepUri` when your trace store uses another URI scheme. -`codeTracerPredictionsToFindings()` and `agentRxPredictionsToFindings()` translate the maintained upstream engines' native outputs into the same evidence and category shape. -The CodeTracer adapter accepts the published `stage_id` format and the flat or grouped step-label formats emitted by CodeTracer 0.2. -AgentRx `Report.to_dict()` judge votes reduce to the upstream majority failure type and Python-rounded mean step, and direct `failures` arrays use the same reduction. -`failure_case: 0` produces no finding, which scores as a missed root cause on AgentRx's failed trajectories. -External runners can return `observedLatencyMs`, `usage`, `metadata`, and `error` together. -This records an upstream failure without discarding work already performed. -Set `observedLatencyMs: null` when an imported run did not record duration. -The report keeps it unknown instead of timing the import code. - -## Run A Real-Model Public Benchmark - -`agent-eval analyst-benchmark` runs the existing label adapters and `runAnalystBenchmark()` with two runners: an empty-finding baseline and a benchmark-specific model analyst. -The CodeTraceBench runner emits one prediction per incorrect step, including wrong actions that the agent later recovers from. -Final task success is evidence about the final state, not proof that every earlier action was correct. -Unuseful but correct exploration is a separate CodeTraceBench label and is not scored by the default run. -The AgentRx runner emits one taxonomy label and one root-cause step. -The generic `failure-mode` analyst is not used because its `failure-mode` area does not match either public task. - -Convert CodeTraceBench trajectories with the maintained importer. -It writes one OTLP JSONL file per trajectory, preserves assistant step ids, and produces a receipt with source and output hashes. -Each label row's `traj_id` or `trajectory_id` must exactly equal the OTLP `trace_id`. -Use a domain-qualified ID when an upstream dataset reuses local trajectory numbers. -Keep the extracted CodeTraceBench artifact tree intact because each row's `source_relpath` locates its final test output. - -```bash -traces import-codetracebench \ - .artifacts/bench_manifest.verified.jsonl \ - --trajectory-dir .artifacts/codetrace-normalized \ - --out .artifacts/codetrace-otlp \ - --revision aa213b84ffb6690fc37ca15766d6ca174ec36d4d \ - --concurrency 8 -``` +Measure the analyst on labeled traces before using its findings for automated changes. +At minimum, report issue recall, finding precision, exact evidence accuracy, trusted-negative false positives, repeat agreement, failures, calls, tokens, cost, and latency. -Run a bounded comparison through any OpenAI-compatible endpoint. -The key is read from the named environment variable and is not written to the result. +`runAnalystBenchmark()` compares any `AnalystBenchmarkRunner` implementations. +`agent-eval analyst-benchmark` runs the public AgentRx or CodeTraceBench adapters with: -```bash -export CLI_BRIDGE_BEARER="" +1. an empty-finding baseline, +2. the actual DSPy RLM trace analyst. +```sh agent-eval analyst-benchmark \ --dataset codetracebench \ - --labels .artifacts/bench_manifest.verified.jsonl \ - --trace-dir .artifacts/codetrace-otlp \ - --artifact-dir .artifacts/codetrace-extracted \ - --out .artifacts/codetrace-glm52 \ + --labels .artifacts/manifest.jsonl \ + --trace-dir .artifacts/traces \ + --artifact-dir .artifacts/results \ + --out .artifacts/analyst-run \ --revision aa213b84ffb6690fc37ca15766d6ca174ec36d4d \ --split verified \ --base-url http://127.0.0.1:3355/v1 \ --api-key-env CLI_BRIDGE_BEARER \ - --model opencode/zai-coding-plan/glm-5.2 \ + --model claude-code/sonnet \ + --python .venv/bin/python \ --limit 20 \ --seed 7 \ - --concurrency 4 \ + --concurrency 1 \ --max-cost-usd 5 ``` -The trace directory may use any filenames, but every JSONL file must contain exactly one trace. -For CodeTraceBench, the artifact loader reads available final test output and structured result files. -It adds raw final-test text and one parsed pass, fail, or unavailable outcome as searchable `EVALUATOR` spans on the same case. -Raw result JSON is hashed and parsed but is not sent to the model. -`--artifact-dir` may be a shared extraction root with one directory per `traj_id`, or the extraction root for one archive. -It prefers `panes/post-test.txt` over the duplicate `sessions/tests.log`. -It also recognizes `test_output.txt`, `results.json`, `result.json`, `report.json`, `*_result.json`, and `*_metrics.json`. -Each discovered file records its role, path, byte count, and SHA-256. -Files exposed as spans also record their span ids. -Missing evidence roles are explicit. -Known Terminal-Bench, SWE-Bench, and SWE-Multi result formats become one explicit passed or failed outcome span. -A missing or unparseable result becomes `unavailable`; it is never inferred from the trajectory or raw test text. -Raw test output is optional because some public cases retain only structured results. - -Before the first model call, the command also checks that every selected label has a matching `step-` span. -It refuses a missing dataset revision, implicit all-case run, duplicate trajectory id, multi-trace file, missing step, oversized evidence, or existing `result.json`. -The model selects positive integer assistant step ids. -The runner constructs each canonical trace URI and exact action excerpt from the selected span. -A missing, non-assistant, or empty step fails that model run while preserving its raw output and usage. -The command consumes already-downloaded labels, normalized traces, and extracted artifacts. -Dataset download, archive extraction, and trajectory conversion remain separate import steps. - -The output directory contains: - -- `manifest.json` with the immutable dataset, model, case, and input identity. -- `initialization-complete.json` written only after every initial file is durable. -- `observations.jsonl` with one fsynced, hash-chained row per completed case and runner. -- `cost-ledger.jsonl` with durable run-wide model reservations and receipts. -- `model-responses/` with one strict, content-hashed response and receipt per paid call for crash recovery. -- `result.json` with every observation, summary metric, comparison, error, latency, token counter, measured or estimated cost, explicit unknown cost, selected case id, source digest, dependency-lock digest, artifact digest, analyst protocol digest, implementation digest, and case distribution. -- `report.md` with the same run and selection distribution rendered for review. -- `run.local.json` with machine-local paths, endpoint, and command, kept out of the shareable result. - -When active calls temporarily hold the remaining money limit, later calls wait for their final receipts. -The command rejects new work only when committed spend plus the next enforced maximum cannot fit. - -Resume only the exact same run after an interruption: - -```bash -agent-eval analyst-benchmark --resume -``` - -Resume rejects changed labels, traces, artifacts, model settings, case selection, local paths, or endpoint. -It runs only missing case and runner pairs. -If the provider response was saved before the process stopped, resume settles that exact response without another provider call. -If the call stopped before a response was saved, resume reuses the same provider request id. -An already complete run is read and checked without another model call. - -The command exits `2` when any model analyst fails. -Its failed row still records latency, calls, available token counters, known spend, and the analyst error. -See the [32-case GLM-5.2 reference run](https://github.com/tangle-network/agent-eval/tree/main/benchmarks/trace-analysis/codetracebench-glm52-20260730) for a complete measured result and its stated limits. - -`--limit` uses deterministic hash selection, not stratified sampling. -Limited runs are marked `representativeOfInput: false`. -The result compares source and selected distributions for label class, agent, model, difficulty, and solved state. -CodeTraceBench label classes distinguish `positive`, `trusted-negative`, `unlabeled-failure`, and `unlabeled-unknown`. -Micro precision, recall, and F1 pool all labeled steps and predictions. -Macro precision, recall, and F1 average per-case scores over issue-bearing cases, matching step-localization papers that report per-trajectory means. -The all-row result remains intact for comparison with published work. -The additional calibrated view measures labeled positives against solved, label-empty negatives and reports failed, label-empty rows separately instead of calling them clean. -The report separately counts final-result files and passed, failed, or unavailable outcomes. -Only a full census of the supplied input is marked representative. - -AgentRx uses the same command with `--dataset agentrx` after obtaining its contact-gated label and trajectory files. - -## Use Upstream Scorers - -Agent Eval adapts upstream evaluators instead of copying them. - -```ts -import { createEvaluator } from '@arizeai/phoenix-evals' -import { ExactMatch } from 'autoevals' -import { - autoevalsScorerJudge, - phoenixEvaluatorJudge, -} from '@tangle-network/agent-eval/campaign' - -const phoenix = createEvaluator( - ({ output, expected }) => output === expected ? 1 : 0, - { name: 'exact-match', kind: 'CODE', telemetry: { isEnabled: false } }, -) - -const phoenixJudge = phoenixEvaluatorJudge(phoenix, { - mapInput: ({ artifact, scenario }) => ({ output: artifact, expected: scenario.expected }), -}) - -const autoevalsJudge = autoevalsScorerJudge(ExactMatch, { - name: 'exact-match', - kind: 'CODE', - mapInput: ({ artifact, scenario }) => ({ output: artifact, expected: scenario.expected }), -}) -``` - -These adapters do not install either upstream package for consumers. -Install only the scorer package you use. -Missing or non-finite scores throw instead of becoming passes. -Phoenix evaluators marked `MINIMIZE` or `NEUTRAL` require `toComposite` so candidate selection never assumes the wrong direction. -Mark model-backed evaluators as `kind: 'LLM'` and provide `paidCall` with the model and a receipt mapper. -The campaign then passes its cancellation signal and cost ledger through the adapter. -An LLM evaluator is rejected before execution when either cost capture or the campaign ledger is missing. - -## Turn Reviewed Findings Into Eval Data - -Generated findings can populate a review queue. -They cannot promote themselves into learning data. - -```ts -import { - analystFindingDigest, - analystRunDigest, - analystRunToFeedbackTrajectory, - analystRunToReviewRequests, -} from '@tangle-network/agent-eval' - -const runDigest = analystRunDigest(result) -const requests = analystRunToReviewRequests(result) -await reviewQueue.add(requests) - -declare const acceptedFindingIds: ReadonlySet - -const trajectory = analystRunToFeedbackTrajectory(result, { - task: { intent: 'Find why the command failed.' }, - reviewRequests: requests, - reviewDecisions: [ - ...result.findings.map((finding) => ({ - runDigest, - findingId: finding.finding_id, - findingDigest: analystFindingDigest(finding), - verdict: acceptedFindingIds.has(finding.finding_id) ? 'confirmed' as const : 'rejected' as const, - source: 'user' as const, - reviewerId: 'reviewer-42', - reviewId: 'trace-review-918', - reason: 'Reviewed against the cited span.', - decidedAt: new Date().toISOString(), - })), - { - runDigest, - verdict: 'completeness_assessed', - missedIssues: [], - source: 'user', - reviewerId: 'reviewer-42', - reviewId: 'trace-review-918', - reason: 'Reviewed the full run for omitted findings.', - decidedAt: new Date().toISOString(), - }, - ], - trace: { artifactUri: 'traces.otlp.jsonl', traceIds: ['run-1'] }, -}) -``` - -`analystRunToFeedbackTrajectory()` stores review requests separately from labels. -It can archive an unreviewed run. -`feedbackTrajectoryToOptimizerRow()` requires every decision to match the complete run digest, every finding decision to match the finding digest, and one independent completeness assessment. -Its score is F1 over confirmed findings and independently identified misses. -Generic labels and run-level outcomes do not satisfy these requirements. - -## Required Trace Data - -Useful analysis needs: - -- stable run, trace, and span IDs, -- parent-child links and ordered timestamps, -- model, prompt, and configuration identity, -- complete tool names, arguments, results, and error codes, -- token, cost, and latency data when available, -- retrieval source IDs and scores when retrieval is involved, -- final environment outcomes such as tests, task completion, or policy blocks. - -Do not include secrets, raw OAuth tokens, or unredacted personal data. +The command writes every observation before producing `result.json` and `report.md`. +It can resume without repeating completed cases. +Dataset revisions, selected case IDs, input hashes, trace hashes, result artifacts, usage, errors, and comparison settings remain in the output. -Use [`@tangle-network/traces`](https://github.com/tangle-network/traces) to normalize coding-agent sessions, run HALO as an external report engine, or run Hodoscope as a behavior-discovery engine. +Use fresh development cases while changing questions or instructions. +Report final quality only on an untouched holdout. diff --git a/package.json b/package.json index 2dc536fe..d6b9250d 100644 --- a/package.json +++ b/package.json @@ -167,7 +167,6 @@ }, "dependencies": { "@asteasolutions/zod-to-openapi": "^9.1.0", - "@ax-llm/ax": "^23.0.5", "@hono/node-server": "^2.0.12", "@tangle-network/agent-core": "0.4.28", "@tangle-network/agent-interface": "0.39.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0816c6b9..ce2a0a89 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,9 +18,6 @@ importers: '@asteasolutions/zod-to-openapi': specifier: ^9.1.0 version: 9.1.0(zod@4.4.3) - '@ax-llm/ax': - specifier: ^23.0.5 - version: 23.0.5(zod@4.4.3) '@hono/node-server': specifier: ^2.0.12 version: 2.0.12(hono@4.12.32) @@ -144,15 +141,6 @@ packages: peerDependencies: zod: ^4.0.0 - '@ax-llm/ax@23.0.5': - resolution: {integrity: sha512-Kgs+P4hPHMyppOnlEQTSFUJTuuLVWfvvV4QakHwLv3VRlw2T98okZkiwicdN0AKGH/uTf0ktF0Kbq1s9c7AxcA==} - hasBin: true - peerDependencies: - zod: ^3.24.0 || ^4.0.0 - peerDependenciesMeta: - zod: - optional: true - '@biomejs/biome@2.5.5': resolution: {integrity: sha512-r1S8nFsAG1MY+vJFZALzIvwXAJv6ejDQ0mxP21Tgr9YK3ZFtjrvbBwDdNhx1rUqvccEIeNg20cYCNzl6Cr69pQ==} engines: {node: '>=14.21.3'} @@ -1904,12 +1892,6 @@ snapshots: openapi3-ts: 4.6.0 zod: 4.4.3 - '@ax-llm/ax@23.0.5(zod@4.4.3)': - dependencies: - '@opentelemetry/api': 1.9.1 - optionalDependencies: - zod: 4.4.3 - '@biomejs/biome@2.5.5': optionalDependencies: '@biomejs/cli-darwin-arm64': 2.5.5 diff --git a/scripts/check-analyst-benchmark-implementation.mjs b/scripts/check-analyst-benchmark-implementation.mjs index 43a67e1b..1081a418 100644 --- a/scripts/check-analyst-benchmark-implementation.mjs +++ b/scripts/check-analyst-benchmark-implementation.mjs @@ -13,6 +13,10 @@ const IMPLEMENTATION_MODULE_PATH = resolve( ) const DIGEST_DOMAIN = 'agent-eval/public-analyst-benchmark/implementation' const DEPENDENCY_LOCK_DIGEST_DOMAIN = 'agent-eval/public-analyst-benchmark/dependency-lock' +const NON_TYPESCRIPT_IMPLEMENTATION_FILES = [ + 'clients/python/src/agent_eval_rpc/dspy_rlm_bridge.py', + 'clients/python/src/agent_eval_rpc/optimizer_bridge_common.py', +] const UTF8 = new TextDecoder('utf-8', { fatal: true }) const implementation = await loadImplementationModule() @@ -80,11 +84,15 @@ async function discoverImplementationFiles(sourceRoot) { external: ['./benchmark-implementation'], logLevel: 'silent', }) - const files = Object.keys(result.metafile.inputs) - .map((path) => path.replaceAll('\\', '/')) + const files = [ + ...Object.keys(result.metafile.inputs).map((path) => path.replaceAll('\\', '/')), + ...NON_TYPESCRIPT_IMPLEMENTATION_FILES, + ] .sort() for (const file of files) { - if (!file.startsWith('src/') || !file.endsWith('.ts')) { + const supportedTypescript = file.startsWith('src/') && file.endsWith('.ts') + const supportedPython = NON_TYPESCRIPT_IMPLEMENTATION_FILES.includes(file) + if (!supportedTypescript && !supportedPython) { throw new Error(`unsupported public analyst benchmark implementation input: ${file}`) } } @@ -147,7 +155,7 @@ function assertImplementationModule(value) { file.startsWith('/') || file.includes('\\') || file.split('/').includes('..') || - !file.endsWith('.ts') + (!file.endsWith('.ts') && !file.endsWith('.py')) ) { throw new Error(`invalid public analyst benchmark implementation source path: ${file}`) } @@ -168,6 +176,8 @@ function assertImplementationModule(value) { throw new Error('unsupported public analyst benchmark dependency lock digest algorithm') } assertFileManifest(value.ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES, [ + 'clients/python/pyproject.toml', + 'clients/python/uv.lock', 'package.json', 'pnpm-lock.yaml', ]) diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index b45a14fc..2aaff20e 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -336,8 +336,8 @@ try { type RemovedCanonicalFindingSchema = typeof import('@tangle-network/agent-eval/analyst').CanonicalRawAnalystFindingSchema // @ts-expect-error analyst cost is reported through the usage receipt type RemovedAnalystCostAlias = import('@tangle-network/agent-eval').AnalystRunSummary['cost_usd'] - // @ts-expect-error recovery has one plural-evidence processRow callback - type RemovedCanonicalProcessRow = import('@tangle-network/agent-eval/analyst').StructureFindingsOptions['processCanonicalRow'] + // @ts-expect-error one-shot finding structuring was removed; use a recursive analysis engine + type RemovedStructureFindingsOptions = import('@tangle-network/agent-eval/analyst').StructureFindingsOptions // @ts-expect-error comparison partitions use explicit train, selection, and test fields type RemovedComparisonHoldout = CompareOptimizationMethodsOptions['holdoutScenarios'] // @ts-expect-error SurfaceProposer is the only candidate-proposal contract diff --git a/src/analyst/ax-cost-service.test.ts b/src/analyst/ax-cost-service.test.ts deleted file mode 100644 index 71472924..00000000 --- a/src/analyst/ax-cost-service.test.ts +++ /dev/null @@ -1,522 +0,0 @@ -import { - AxAIGoogleGeminiModel, - AxAIOpenAIModel, - type AxAIService, - type AxChatRequest, - type AxChatResponse, - ai as createAxAi, -} from '@ax-llm/ax' -import { describe, expect, it, vi } from 'vitest' -import { CostCeilingReachedError, CostLedger } from '../cost-ledger' -import { maximumChargeForAxChatRequest, meterAxChatService } from './ax-cost-service' -import { usageReceiptFromCostLedger } from './usage-receipt' - -function request(model = 'gpt-4o-mini'): AxChatRequest { - return { - model, - chatPrompt: [{ role: 'user', content: 'inspect this trace' }], - } -} - -function fakeAi( - chat: ( - request: Readonly, - options?: Readonly>, - ) => Promise, -): AxAIService { - return { chat } as AxAIService -} - -describe('meterAxChatService', () => { - it('records every chat receipt independently of analyst findings', async () => { - let received: Readonly> | undefined - let receivedOptions: Readonly> | undefined - const ai = fakeAi(async (input, callOptions) => { - received = input - receivedOptions = callOptions - return { - results: [{ index: 0, content: 'done' }], - modelUsage: { - ai: 'openai', - model: 'gpt-4o-mini', - tokens: { - promptTokens: 100, - completionTokens: 20, - totalTokens: 120, - cacheReadTokens: 8, - }, - }, - } - }) - const ledger = new CostLedger(1) - const metered = meterAxChatService(ai, { - ledger, - actor: 'failure-mode', - maxOutputTokens: 64, - }) - - await metered.chat(request()) - - expect(received?.modelConfig?.maxTokens).toBe(64) - expect(received?.modelConfig?.n).toBe(1) - expect(received?.functionCall).toBe('none') - expect(receivedOptions).toMatchObject({ - retry: { maxRetries: 0 }, - stream: false, - showThoughts: false, - thinkingTokenBudget: 'none', - }) - expect(ledger.list()).toHaveLength(1) - expect(usageReceiptFromCostLedger(ledger)).toEqual({ - calls: 1, - tokens: { input: 100, output: 20, cached: 8 }, - cost: { kind: 'estimated', usd: expect.any(Number) }, - }) - }) - - it('preserves native function-call policy when the request declares functions', async () => { - const received: AxChatRequest[] = [] - const metered = meterAxChatService( - fakeAi(async (input) => { - received.push(input as AxChatRequest) - return { - results: [{ index: 0, content: 'done' }], - modelUsage: { - ai: 'openai', - model: 'gpt-4o-mini', - tokens: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, - }, - } - }), - { - ledger: new CostLedger(), - actor: 'failure-mode', - maxOutputTokens: 64, - }, - ) - const functions = [ - { - name: 'inspect', - description: 'Inspect one trace.', - parameters: { type: 'object' as const, properties: {} }, - }, - ] - - await metered.chat({ ...request(), functions }) - await metered.chat({ ...request(), functions, functionCall: 'required' }) - - expect(received[0]?.functionCall).toBeUndefined() - expect(received[1]?.functionCall).toBe('required') - }) - - it('bounds a caller-supplied output limit without increasing it', async () => { - let maxTokens: number | undefined - const ai = fakeAi(async (input) => { - maxTokens = input.modelConfig?.maxTokens - return { - results: [{ index: 0, content: 'done' }], - modelUsage: { - ai: 'openai', - model: 'gpt-4o-mini', - tokens: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, - }, - } - }) - const metered = meterAxChatService(ai, { - ledger: new CostLedger(), - actor: 'failure-mode', - maxOutputTokens: 64, - }) - - await metered.chat({ ...request(), modelConfig: { maxTokens: 16 } }) - - expect(maxTokens).toBe(16) - }) - - it('rejects before the provider call when the hard budget is exhausted', async () => { - const chatResponse = vi.fn(async () => ({ - results: [{ index: 0, content: 'never called' }], - })) - const metered = meterAxChatService(fakeAi(chatResponse), { - ledger: new CostLedger(0), - actor: 'failure-mode', - maxOutputTokens: 64, - }) - - await expect(metered.chat(request())).rejects.toBeInstanceOf(CostCeilingReachedError) - expect(chatResponse).not.toHaveBeenCalled() - }) - - it('rejects an unpriced model before spending a capped run', async () => { - const chatResponse = vi.fn(async () => ({ - results: [{ index: 0, content: 'never called' }], - })) - const metered = meterAxChatService(fakeAi(chatResponse), { - ledger: new CostLedger(1), - actor: 'failure-mode', - maxOutputTokens: 64, - }) - - await expect(metered.chat(request('private-model-without-pricing'))).rejects.toThrow( - /cannot reserve unpriced model/, - ) - expect(chatResponse).not.toHaveBeenCalled() - }) - - it('uses the configured service model when Ax omits a request override', async () => { - const chatResponse = vi.fn(async () => ({ - results: [{ index: 0, content: 'done' }], - modelUsage: { - ai: 'openai', - model: 'gpt-4o-mini', - tokens: { promptTokens: 2, completionTokens: 1, totalTokens: 3 }, - }, - })) - const ledger = new CostLedger(1) - const metered = meterAxChatService(fakeAi(chatResponse), { - ledger, - actor: 'failure-mode', - maxOutputTokens: 64, - defaultModel: 'gpt-4o-mini', - }) - - await metered.chat({ chatPrompt: [{ role: 'user', content: 'inspect' }] }) - - expect(chatResponse).toHaveBeenCalledOnce() - expect(ledger.list()[0]?.model).toBe('gpt-4o-mini') - }) - - it('reconciles cache and reasoning tokens whether Ax reports them inside or outside totals', async () => { - const ai = fakeAi(async () => ({ - results: [{ index: 0, content: 'done' }], - modelUsage: { - ai: 'anthropic', - model: 'gpt-4o-mini', - tokens: { - promptTokens: 100, - completionTokens: 20, - totalTokens: 133, - reasoningTokens: 5, - cacheCreationTokens: 5, - cacheReadTokens: 8, - }, - }, - })) - const ledger = new CostLedger() - const metered = meterAxChatService(ai, { - ledger, - actor: 'failure-mode', - maxOutputTokens: 64, - }) - - await metered.chat(request()) - - expect(usageReceiptFromCostLedger(ledger).tokens).toEqual({ - input: 100, - output: 20, - reasoning: 5, - cached: 8, - cacheWrite: 5, - }) - }) - - it('preserves thoughts reported outside provider totals', async () => { - const ai = fakeAi(async () => ({ - results: [{ index: 0, content: 'done' }], - modelUsage: { - ai: 'anthropic', - model: 'gpt-4o-mini', - tokens: { - promptTokens: 20, - completionTokens: 10, - totalTokens: 30, - thoughtsTokens: 100, - }, - }, - })) - const ledger = new CostLedger() - const metered = meterAxChatService(ai, { - ledger, - actor: 'failure-mode', - maxOutputTokens: 128, - }) - - await metered.chat(request()) - - expect(usageReceiptFromCostLedger(ledger).tokens).toEqual({ - input: 20, - output: 110, - reasoning: 100, - }) - }) - - it('preserves reasoning reported separately from visible completion tokens', async () => { - const ai = fakeAi(async () => ({ - results: [{ index: 0, content: 'done' }], - modelUsage: { - ai: 'openai', - model: 'gpt-4o-mini', - tokens: { - promptTokens: 100, - completionTokens: 10, - reasoningTokens: 50, - totalTokens: 160, - }, - }, - })) - const ledger = new CostLedger() - const metered = meterAxChatService(ai, { - ledger, - actor: 'failure-mode', - maxOutputTokens: 128, - }) - - await metered.chat(request()) - - expect(usageReceiptFromCostLedger(ledger).tokens).toEqual({ - input: 100, - output: 60, - reasoning: 50, - }) - }) - - it('reserves the full output bound for multi-completion requests', () => { - const input = { ...request(), modelConfig: { maxTokens: 64, n: 3 } } - const requestBytes = new TextEncoder().encode(JSON.stringify(input)).byteLength - - expect(maximumChargeForAxChatRequest(input)).toMatchObject({ - inputTokens: requestBytes * 3, - outputTokens: 192, - }) - }) - - it('refuses to claim a hard maximum before completion count is explicit', () => { - expect( - maximumChargeForAxChatRequest({ ...request(), modelConfig: { maxTokens: 64 } }), - ).toBeUndefined() - }) - - it('rejects invalid completion counts before calling the provider', async () => { - const chatResponse = vi.fn(async () => ({ results: [{ index: 0, content: 'never called' }] })) - const metered = meterAxChatService(fakeAi(chatResponse), { - ledger: new CostLedger(1), - actor: 'failure-mode', - maxOutputTokens: 64, - }) - - await expect(metered.chat({ ...request(), modelConfig: { n: 0 } })).rejects.toThrow( - /modelConfig\.n/, - ) - expect(chatResponse).not.toHaveBeenCalled() - }) - - it('marks contradictory provider totals as uncaptured instead of undercounting', async () => { - const ai = fakeAi(async () => ({ - results: [{ index: 0, content: 'done' }], - modelUsage: { - ai: 'openai', - model: 'gpt-4o-mini', - tokens: { promptTokens: 10, completionTokens: 4, totalTokens: 12 }, - }, - })) - const ledger = new CostLedger() - const metered = meterAxChatService(ai, { - ledger, - actor: 'failure-mode', - maxOutputTokens: 64, - }) - - await metered.chat(request()) - - expect(usageReceiptFromCostLedger(ledger)).toEqual({ - calls: 1, - tokens: null, - cost: { kind: 'uncaptured', usd: null }, - knownCostUsd: 0, - }) - }) - - it('keeps an observed provider charge when only token usage is missing', async () => { - const ledger = new CostLedger() - - await ledger.runPaidCall({ - channel: 'analyst', - phase: 'analyst.test', - actor: 'failure-mode', - model: 'gpt-4o-mini', - execute: async () => undefined, - receipt: () => ({ - model: 'gpt-4o-mini', - inputTokens: 0, - outputTokens: 0, - usageUnknown: true, - actualCostUsd: 0.25, - }), - }) - - expect(usageReceiptFromCostLedger(ledger)).toEqual({ - calls: 1, - tokens: null, - cost: { kind: 'observed', usd: 0.25 }, - }) - }) - - it('rejects every cache breakpoint and non-text input before a capped call', async () => { - const chatResponse = vi.fn(async () => ({ results: [{ index: 0, content: 'never called' }] })) - const metered = meterAxChatService(fakeAi(chatResponse), { - ledger: new CostLedger(1), - actor: 'failure-mode', - maxOutputTokens: 64, - }) - - await expect( - metered.chat({ - model: 'gpt-4o-mini', - chatPrompt: [{ role: 'user', content: 'inspect', cache: true }], - }), - ).rejects.toThrow(/hard maximumCharge/) - await expect( - metered.chat({ - model: 'gpt-4o-mini', - chatPrompt: [{ role: 'user', content: [{ type: 'text', text: 'inspect', cache: true }] }], - }), - ).rejects.toThrow(/hard maximumCharge/) - await expect( - metered.chat({ - model: 'gpt-4o-mini', - chatPrompt: [ - { - role: 'user', - content: [{ type: 'image', mimeType: 'image/png', image: 'aGVsbG8=' }], - }, - ], - }), - ).rejects.toThrow(/hard maximumCharge/) - expect(chatResponse).not.toHaveBeenCalled() - }) - - it('retains known spend when a later provider call has no receipt', async () => { - let call = 0 - const ai = fakeAi(async () => { - call += 1 - if (call === 2) throw new Error('provider disconnected') - return { - results: [{ index: 0, content: 'done' }], - modelUsage: { - ai: 'openai', - model: 'gpt-4o-mini', - tokens: { promptTokens: 10, completionTokens: 4, totalTokens: 14 }, - }, - } - }) - const ledger = new CostLedger() - const metered = meterAxChatService(ai, { - ledger, - actor: 'failure-mode', - maxOutputTokens: 64, - }) - - await metered.chat(request()) - await expect(metered.chat(request())).rejects.toThrow('provider disconnected') - - const usage = usageReceiptFromCostLedger(ledger) - expect(usage.cost).toEqual({ kind: 'uncaptured', usd: null }) - expect(usage.knownCostUsd).toBeGreaterThan(0) - }) - - it('disables Ax provider retries so each reservation covers one HTTP call', async () => { - const fetchImpl = vi.fn(async () => new Response('temporary failure', { status: 500 })) - const ai = createAxAi({ - name: 'openai', - apiKey: 'test-key', - apiURL: 'https://provider.invalid/v1', - config: { model: AxAIOpenAIModel.GPT4OMini }, - options: { fetch: fetchImpl }, - }) - const metered = meterAxChatService(ai, { - ledger: new CostLedger(), - actor: 'failure-mode', - maxOutputTokens: 64, - }) - - await expect(metered.chat(request(), { retry: { maxRetries: 4 } })).rejects.toThrow() - expect(fetchImpl).toHaveBeenCalledOnce() - }) - - it('keeps Gemini 3 output bounded without Ax thinking-level conflicts', async () => { - const fetchImpl = vi.fn( - async (_input: unknown, _init?: RequestInit) => - new Response('temporary failure', { status: 500 }), - ) - const ai = createAxAi({ - name: 'google-gemini', - apiKey: 'test-key', - config: { model: AxAIGoogleGeminiModel.Gemini3Flash }, - options: { fetch: fetchImpl }, - }) - const metered = meterAxChatService(ai, { - ledger: new CostLedger(), - actor: 'failure-mode', - maxOutputTokens: 64, - }) - - await expect( - metered.chat(request('models/gemini-3-flash-preview'), { thinkingTokenBudget: 'none' }), - ).rejects.toThrow() - - expect(fetchImpl).toHaveBeenCalledOnce() - const init = fetchImpl.mock.calls[0]?.[1] as RequestInit | undefined - const body = JSON.parse(String(init?.body)) as { - generationConfig?: { - maxOutputTokens?: number - thinkingConfig?: { - includeThoughts?: boolean - thinkingBudget?: number - thinkingLevel?: string - } - } - } - expect(body.generationConfig?.maxOutputTokens).toBe(64) - expect(body.generationConfig?.thinkingConfig).toEqual({ includeThoughts: false }) - }) - - it('combines cancellation signals on every supported Node 20 release', async () => { - const anyDescriptor = Object.getOwnPropertyDescriptor(AbortSignal, 'any') - Object.defineProperty(AbortSignal, 'any', { - configurable: true, - value: undefined, - }) - const runController = new AbortController() - const callController = new AbortController() - let providerSignal: AbortSignal | undefined - const ai = fakeAi( - async (_request, callOptions) => - new Promise((_resolve, reject) => { - providerSignal = callOptions?.abortSignal as AbortSignal | undefined - providerSignal?.addEventListener( - 'abort', - () => reject(providerSignal?.reason ?? new DOMException('aborted', 'AbortError')), - { once: true }, - ) - }), - ) - const metered = meterAxChatService(ai, { - ledger: new CostLedger(), - actor: 'failure-mode', - maxOutputTokens: 64, - signal: runController.signal, - }) - - try { - const run = metered.chat(request(), { abortSignal: callController.signal }) - await Promise.resolve() - callController.abort(new DOMException('cancelled', 'AbortError')) - - await expect(run).rejects.toMatchObject({ name: 'AbortError' }) - expect(providerSignal?.aborted).toBe(true) - } finally { - if (anyDescriptor) Object.defineProperty(AbortSignal, 'any', anyDescriptor) - else Reflect.deleteProperty(AbortSignal, 'any') - } - }) -}) diff --git a/src/analyst/ax-cost-service.ts b/src/analyst/ax-cost-service.ts deleted file mode 100644 index 017f89b4..00000000 --- a/src/analyst/ax-cost-service.ts +++ /dev/null @@ -1,270 +0,0 @@ -import type { AxAIService, AxChatRequest, AxChatResponse } from '@ax-llm/ax' -import type { CostLedgerHandle, CostReceiptInput, MaximumCharge } from '../cost-ledger' - -interface MeterAxChatServiceOptions { - ledger: CostLedgerHandle - actor: string - /** Hard output limit applied to every Ax chat call. */ - maxOutputTokens: number - /** Model configured on the Ax service when requests omit an override. */ - defaultModel?: string - phase?: string - tags?: Record - signal?: AbortSignal -} - -interface AxChatCallOptions { - abortSignal?: AbortSignal - stream?: boolean - showThoughts?: boolean - thinkingTokenBudget?: 'minimal' | 'low' | 'medium' | 'high' | 'highest' | 'none' - retry?: { maxRetries?: number; [key: string]: unknown } - [key: string]: unknown -} - -interface AxChatService { - chat( - request: Readonly, - options?: Readonly, - ): Promise> -} - -/** - * Meter every chat call an Ax program makes through the shared paid-call ledger. - * The wrapper disables provider streaming because a stream has no complete usage - * receipt until it is consumed, while Ax's analyst output is not streamed to users. - */ -export function meterAxChatService( - ai: AxAIService, - options: MeterAxChatServiceOptions, -): AxAIService & AxChatService { - assertPositiveInteger(options.maxOutputTokens, 'maxOutputTokens') - const source = ai as AxAIService & Partial - if (typeof source.chat !== 'function') { - throw new TypeError('meterAxChatService: Ax service must implement chat()') - } - const providerChat = source.chat.bind(ai) - - const chat: AxChatService['chat'] = async (request, callOptions = {}) => { - const boundedRequest = boundOutputTokens(request, options.maxOutputTokens) - const model = modelName(boundedRequest.model) || options.defaultModel || '' - const canTurnOffThinking = canDisableThinking(ai, model) - const combined = combineSignals(options.signal, callOptions.abortSignal) - try { - const paid = await options.ledger.runPaidCall({ - channel: 'analyst', - phase: options.phase ?? 'analyst.ax.chat', - actor: options.actor, - model, - tags: options.tags, - signal: combined.signal, - maximumCharge: maximumChargeForAxChatRequest(boundedRequest, model), - execute: async (executionSignal) => { - const providerOptions: AxChatCallOptions = { - ...callOptions, - abortSignal: executionSignal, - retry: { ...callOptions.retry, maxRetries: 0 }, - stream: false, - showThoughts: false, - } - if (canTurnOffThinking) providerOptions.thinkingTokenBudget = 'none' - else Reflect.deleteProperty(providerOptions, 'thinkingTokenBudget') - const response = await providerChat(boundedRequest, providerOptions) - if (response instanceof ReadableStream) { - throw new Error('meterAxChatService: provider returned a stream after stream:false') - } - return response - }, - receipt: (response) => costReceiptFromAxResponse(response, model), - }) - if (!paid.succeeded) throw paid.error - return paid.value - } finally { - combined.dispose() - } - } - - return new Proxy(ai as AxAIService & AxChatService, { - get(target, property) { - if (property === 'chat') return chat - const value = Reflect.get(target, property, target) - return typeof value === 'function' ? value.bind(target) : value - }, - }) -} - -/** Conservative priced bound for one Ax text chat request. */ -export function maximumChargeForAxChatRequest( - request: Readonly>, - defaultModel?: string, -): MaximumCharge | undefined { - const model = modelName(request.model) || defaultModel || '' - const maxTokens = request.modelConfig?.maxTokens - const completions = request.modelConfig?.n - if (!model || maxTokens === undefined || completions === undefined) return undefined - assertPositiveInteger(maxTokens, 'request.modelConfig.maxTokens') - assertPositiveInteger(completions, 'request.modelConfig.n') - const maximumOutputTokens = maxTokens * completions - assertPositiveInteger(maximumOutputTokens, 'maximum output tokens') - if (containsUnboundedOrCacheableContent(request)) return undefined - let inputTokens: number - try { - const pricedRequest = request.model === undefined ? { ...request, model } : request - inputTokens = new TextEncoder().encode(JSON.stringify(pricedRequest)).byteLength * completions - } catch { - return undefined - } - assertPositiveInteger(inputTokens, 'maximum input tokens') - return { model, inputTokens, outputTokens: maximumOutputTokens } -} - -function boundOutputTokens( - request: Readonly>, - limit: number, -): AxChatRequest { - const requested = request.modelConfig?.maxTokens - if (requested !== undefined) { - assertPositiveInteger(requested, 'request.modelConfig.maxTokens') - } - const completions = request.modelConfig?.n ?? 1 - assertPositiveInteger(completions, 'request.modelConfig.n') - const maxTokens = requested === undefined ? limit : Math.min(requested, limit) - return { - ...request, - ...(request.functionCall === undefined && !request.functions?.length - ? { functionCall: 'none' as const } - : {}), - modelConfig: { ...request.modelConfig, maxTokens, n: completions }, - } -} - -function costReceiptFromAxResponse( - response: AxChatResponse, - fallbackModel: string, -): CostReceiptInput { - const usage = response.modelUsage - const tokens = usage?.tokens - const model = usage?.model || fallbackModel - if ( - !tokens || - !validUsage(tokens.promptTokens) || - !validUsage(tokens.completionTokens) || - !validUsage(tokens.totalTokens) || - tokens.totalTokens < tokens.promptTokens + tokens.completionTokens || - !validOptionalUsage(tokens.cacheReadTokens) || - !validOptionalUsage(tokens.cacheCreationTokens) || - !validOptionalUsage(tokens.reasoningTokens) || - !validOptionalUsage(tokens.thoughtsTokens) - ) { - return { - model, - inputTokens: 0, - outputTokens: 0, - usageUnknown: true, - } - } - const cacheReadTokens = validUsage(tokens.cacheReadTokens) ? tokens.cacheReadTokens : 0 - const cacheCreationTokens = validUsage(tokens.cacheCreationTokens) - ? tokens.cacheCreationTokens - : 0 - const totalCacheTokens = cacheReadTokens + cacheCreationTokens - const thoughtsTokens = validUsage(tokens.thoughtsTokens) ? tokens.thoughtsTokens : 0 - const reasoningTokens = Math.max( - validUsage(tokens.reasoningTokens) ? tokens.reasoningTokens : 0, - thoughtsTokens, - ) - // OpenAI includes reasoning in completionTokens, while other Ax providers may - // report hidden output separately. Only add reasoning when it cannot be a - // subset of completionTokens; thoughts are always separately billed output. - const separateReasoningTokens = reasoningTokens > tokens.completionTokens ? reasoningTokens : 0 - const additionalOutputTokens = Math.max(thoughtsTokens, separateReasoningTokens) - const outputTokens = tokens.completionTokens + additionalOutputTokens - const directTotal = tokens.promptTokens + tokens.completionTokens - const validTotals = new Set([ - directTotal, - directTotal + totalCacheTokens, - directTotal + additionalOutputTokens, - directTotal + totalCacheTokens + additionalOutputTokens, - ]) - if (!validTotals.has(tokens.totalTokens)) { - return { - model, - inputTokens: 0, - outputTokens: 0, - usageUnknown: true, - } - } - return { - model, - inputTokens: tokens.promptTokens, - outputTokens, - ...(reasoningTokens > 0 ? { reasoningTokens } : {}), - ...(cacheReadTokens > 0 ? { cachedTokens: cacheReadTokens } : {}), - ...(cacheCreationTokens > 0 ? { cacheWriteTokens: cacheCreationTokens } : {}), - } -} - -function combineSignals( - first: AbortSignal | undefined, - second: AbortSignal | undefined, -): { signal: AbortSignal | undefined; dispose(): void } { - if (!first) return { signal: second, dispose: () => {} } - if (!second || first === second) return { signal: first, dispose: () => {} } - if (typeof AbortSignal.any === 'function') { - return { signal: AbortSignal.any([first, second]), dispose: () => {} } - } - - const controller = new AbortController() - const dispose = (): void => { - first.removeEventListener('abort', abortFromFirst) - second.removeEventListener('abort', abortFromSecond) - } - const abortFrom = (source: AbortSignal): void => { - if (!controller.signal.aborted) controller.abort(source.reason) - dispose() - } - const abortFromFirst = (): void => abortFrom(first) - const abortFromSecond = (): void => abortFrom(second) - if (first.aborted) abortFrom(first) - else if (second.aborted) abortFrom(second) - else { - first.addEventListener('abort', abortFromFirst, { once: true }) - second.addEventListener('abort', abortFromSecond, { once: true }) - } - return { signal: controller.signal, dispose } -} - -function containsUnboundedOrCacheableContent(request: Readonly>): boolean { - if (request.functions?.some((fn) => fn.cache === true)) return true - return request.chatPrompt.some((message) => { - if (message.cache === true) return true - if (!('content' in message) || !Array.isArray(message.content)) return false - return message.content.some((part) => part.cache === true || part.type !== 'text') - }) -} - -function modelName(value: unknown): string { - return typeof value === 'string' ? value : '' -} - -function canDisableThinking(ai: AxAIService, model: string): boolean { - // Ax maps "none" to a Gemini 3 thinking level, which cannot coexist with maxTokens. - const namedAi = ai as { getName?: () => string } - const serviceName = typeof namedAi.getName === 'function' ? namedAi.getName() : '' - const modelId = model.slice(model.lastIndexOf('/') + 1) - return serviceName !== 'GoogleGeminiAI' || !/^gemini-3(?:[.-]|$)/i.test(modelId) -} - -function validUsage(value: unknown): value is number { - return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 -} - -function validOptionalUsage(value: unknown): boolean { - return value === undefined || validUsage(value) -} - -function assertPositiveInteger(value: number, field: string): void { - if (!Number.isSafeInteger(value) || value <= 0) { - throw new RangeError(`meterAxChatService: ${field} must be a positive integer`) - } -} diff --git a/src/analyst/ax-service.test.ts b/src/analyst/ax-service.test.ts deleted file mode 100644 index e3dd823c..00000000 --- a/src/analyst/ax-service.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { createAnalystAi, getConfiguredAnalystModel } from './ax-service' - -const axMock = vi.hoisted(() => ({ - ai: vi.fn(() => ({ chat: vi.fn() })), -})) - -vi.mock('@ax-llm/ax', () => ({ ai: axMock.ai })) - -describe('createAnalystAi', () => { - beforeEach(() => axMock.ai.mockClear()) - - it('retains the configured model for pre-call spend estimation', () => { - const service = createAnalystAi({ - apiKey: 'test', - baseUrl: 'https://example.test/v1', - model: 'gpt-4o-mini', - }) - - expect(getConfiguredAnalystModel(service)).toBe('gpt-4o-mini') - expect(axMock.ai).toHaveBeenCalledWith({ - name: 'openai', - apiKey: 'test', - apiURL: 'https://example.test/v1', - config: { model: 'gpt-4o-mini' }, - }) - }) - - it('rejects a blank model before constructing a provider service', () => { - expect(() => - createAnalystAi({ - apiKey: 'test', - baseUrl: 'https://example.test/v1', - model: ' ', - }), - ).toThrow(/model must be a non-empty string/) - expect(axMock.ai).not.toHaveBeenCalled() - }) - - it('forwards gateway policy headers through the shared constructor', () => { - createAnalystAi({ - apiKey: 'test', - baseUrl: 'http://127.0.0.1:3355/v1', - headers: { 'X-Bridge-Mode': 'hosted-safe' }, - model: 'claude-code/sonnet', - }) - - expect(axMock.ai).toHaveBeenCalledWith({ - name: 'openai', - apiKey: 'test', - apiURL: 'http://127.0.0.1:3355/v1', - headers: { 'X-Bridge-Mode': 'hosted-safe' }, - config: { model: 'claude-code/sonnet' }, - }) - }) -}) diff --git a/src/analyst/ax-service.ts b/src/analyst/ax-service.ts deleted file mode 100644 index db1d0d20..00000000 --- a/src/analyst/ax-service.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { AxAIArgs, AxAIService } from '@ax-llm/ax' -import { ai } from '@ax-llm/ax' - -const configuredModels = new WeakMap() - -export interface CreateAnalystAiConfig { - /** OpenAI-compatible API key forwarded as `Authorization: Bearer`. - * cli-bridge ignores the value on loopback but Ax requires a non-empty string. */ - apiKey: string - /** OpenAI-compatible base URL — e.g. `https://router.tangle.tools/v1` or a - * cli-bridge loopback. */ - baseUrl?: string - /** Additional headers required by the gateway, such as tenant or execution policy. */ - headers?: Record - /** Model id forwarded to analyst calls. */ - model: string - /** Ax provider name. Defaults to the OpenAI-compatible client. */ - provider?: AxAIArgs['name'] -} - -/** - * Construct the `AxAIService` an analyst kind calls through - * (`createTraceAnalystKind({ ai })`). - * - * Ax's `ai()` pins `config.model` to the OpenAI catalog enum, but every - * OpenAI-compatible router an analyst points at (router.tangle.tools, - * cli-bridge) accepts arbitrary model ids (claude-code/sonnet, openai/gpt-5.4, - * …). Consumers were each re-rolling `ai({ name, apiKey, apiURL, config })` - * behind an `as (a: any) => any` cast to dodge the enum; this is the one - * canonical constructor so they don't have to — and don't take a direct - * `@ax-llm/ax` dependency for it. - */ -export function createAnalystAi(config: CreateAnalystAiConfig): AxAIService { - const model = config.model.trim() - if (!model) throw new TypeError('createAnalystAi: model must be a non-empty string') - const args = { - name: config.provider ?? 'openai', - apiKey: config.apiKey, - ...(config.baseUrl ? { apiURL: config.baseUrl } : {}), - ...(config.headers ? { headers: config.headers } : {}), - config: { model }, - } as unknown as AxAIArgs - const service = ai(args) - configuredModels.set(service as object, model) - return service -} - -export function getConfiguredAnalystModel(service: AxAIService): string | undefined { - return configuredModels.get(service as object) -} - -/** Resolve the model before paid work so every request can be bounded and attributed. */ -export function resolveAnalystModel(service: AxAIService, override?: string): string { - if (override !== undefined) { - const model = override.trim() - if (!model) throw new TypeError('createTraceAnalystKind: model must be a non-empty string') - return model - } - const model = getConfiguredAnalystModel(service)?.trim() - if (!model) { - throw new TypeError( - 'createTraceAnalystKind: model is required for Ax services not created by createAnalystAi()', - ) - } - return model -} diff --git a/src/analyst/benchmark-command-artifact.test.ts b/src/analyst/benchmark-command-artifact.test.ts index 8a8674a5..f9f8cb47 100644 --- a/src/analyst/benchmark-command-artifact.test.ts +++ b/src/analyst/benchmark-command-artifact.test.ts @@ -17,8 +17,8 @@ describe('analyst benchmark artifact reader', () => { agentRxCommandArgs(fixture), { TEST_ANALYST_KEY: 'unused' }, { - createModelRunner: () => ({ - id: 'model', + createAnalystRunner: () => ({ + id: 'dspy-rlm', analyze: () => ({ findings: [], usage: UNKNOWN_USAGE }), }), }, @@ -78,7 +78,7 @@ describe('analyst benchmark artifact reader', () => { { schema_version: '1.0.0', finding_id: 'finding-1', - analyst_id: 'model', + analyst_id: 'dspy-rlm', produced_at: '2026-07-30T00:00:00.000Z', severity: 'high', area: 'tool-use', @@ -163,8 +163,8 @@ describe('analyst benchmark artifact reader', () => { agentRxCommandArgs(fixture), { TEST_ANALYST_KEY: 'unused' }, { - createModelRunner: () => ({ - id: 'model', + createAnalystRunner: () => ({ + id: 'dspy-rlm', analyze: () => ({ findings: [], usage: UNKNOWN_USAGE }), }), }, @@ -189,8 +189,8 @@ async function completedResultPath(dataset: 'agentrx' | 'codetracebench'): Promi args, { TEST_ANALYST_KEY: 'unused' }, { - createModelRunner: () => ({ - id: 'model', + createAnalystRunner: () => ({ + id: 'dspy-rlm', analyze: () => ({ findings: [], usage: UNKNOWN_USAGE }), }), }, diff --git a/src/analyst/benchmark-command-artifact.ts b/src/analyst/benchmark-command-artifact.ts index dac09f82..bb91a41e 100644 --- a/src/analyst/benchmark-command-artifact.ts +++ b/src/analyst/benchmark-command-artifact.ts @@ -78,7 +78,7 @@ export interface AnalystBenchmarkRunIdentity { analystProtocolSha256: string implementationSha256: string dependencyLockSha256: string - runnerIds: readonly ['empty', 'model'] + runnerIds: readonly ['empty', 'dspy-rlm'] } inputs: { labelsSha256: string diff --git a/src/analyst/benchmark-command-persistence.test.ts b/src/analyst/benchmark-command-persistence.test.ts index 8ac76982..745154e7 100644 --- a/src/analyst/benchmark-command-persistence.test.ts +++ b/src/analyst/benchmark-command-persistence.test.ts @@ -23,8 +23,8 @@ describe('analyst benchmark persistence and resume', () => { const first = await agentRxFixture() const second = await agentRxFixture() const dependencies = { - createModelRunner: () => ({ - id: 'model', + createAnalystRunner: () => ({ + id: 'dspy-rlm', analyze: () => ({ findings: [], usage: UNKNOWN_USAGE }), }), } @@ -60,7 +60,7 @@ describe('analyst benchmark persistence and resume', () => { const fixture = await agentRxFixture() const analyze = vi.fn(() => ({ findings: [], usage: UNKNOWN_USAGE })) const dependencies = { - createModelRunner: () => ({ id: 'model', analyze }), + createAnalystRunner: () => ({ id: 'dspy-rlm', analyze }), } const attempts = await Promise.allSettled([ @@ -94,7 +94,7 @@ describe('analyst benchmark persistence and resume', () => { args, { TEST_ANALYST_KEY: 'unused' }, { - createModelRunner: () => { + createAnalystRunner: () => { throw new Error('stop after initialization') }, }, @@ -113,8 +113,8 @@ describe('analyst benchmark persistence and resume', () => { [...args, '--resume'], { TEST_ANALYST_KEY: 'unused' }, { - createModelRunner: () => ({ - id: 'model', + createAnalystRunner: () => ({ + id: 'dspy-rlm', async analyze() { markAnalysisStarted() await analysisReleased @@ -131,8 +131,8 @@ describe('analyst benchmark persistence and resume', () => { [...args, '--resume'], { TEST_ANALYST_KEY: 'unused' }, { - createModelRunner: () => ({ - id: 'model', + createAnalystRunner: () => ({ + id: 'dspy-rlm', analyze: () => ({ findings: [], usage: UNKNOWN_USAGE }), }), }, @@ -153,7 +153,7 @@ describe('analyst benchmark persistence and resume', () => { args, { TEST_ANALYST_KEY: 'unused' }, { - createModelRunner: () => { + createAnalystRunner: () => { throw new Error('stop after initialization') }, }, @@ -170,7 +170,7 @@ describe('analyst benchmark persistence and resume', () => { runAnalystBenchmarkCommand( [...args, '--resume'], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner: () => ({ id: 'model', analyze }) }, + { createAnalystRunner: () => ({ id: 'dspy-rlm', analyze }) }, ), ).resolves.toBe(0) @@ -196,7 +196,7 @@ describe('analyst benchmark persistence and resume', () => { args, { TEST_ANALYST_KEY: 'unused' }, { - createModelRunner: () => { + createAnalystRunner: () => { throw new Error('stop after initialization') }, }, @@ -208,16 +208,16 @@ describe('analyst benchmark persistence and resume', () => { const localReceipt = JSON.parse(await readFile(localReceiptPath, 'utf8')) localReceipt.command = 'conflicting command' await writeFile(localReceiptPath, `${JSON.stringify(localReceipt, null, 2)}\n`) - const createModelRunner = vi.fn() + const createAnalystRunner = vi.fn() await expect( runAnalystBenchmarkCommand( [...args, '--resume'], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/local run receipt does not exactly match interrupted initialization/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() await expect(access(markerPath)).rejects.toThrow() }) @@ -226,16 +226,16 @@ describe('analyst benchmark persistence and resume', () => { await mkdir(fixture.outDir) await writeFile(join(fixture.outDir, 'report.md'), 'unrelated report\n') await unlink(fixture.labelsPath) - const createModelRunner = vi.fn() + const createAnalystRunner = vi.fn() await expect( runAnalystBenchmarkCommand( agentRxCommandArgs(fixture), { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/existing benchmark output directory/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() }) it('rejects symbolic-link output directories and observation logs', async () => { @@ -243,26 +243,26 @@ describe('analyst benchmark persistence and resume', () => { const targetDirectory = join(outputFixture.root, 'target-output') await mkdir(targetDirectory) await symlink(targetDirectory, outputFixture.outDir, 'dir') - const createModelRunner = vi.fn() + const createAnalystRunner = vi.fn() await expect( runAnalystBenchmarkCommand( [...agentRxCommandArgs(outputFixture), '--resume'], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/output must be a real directory/) const logFixture = await agentRxFixture() const args = agentRxCommandArgs(logFixture) const runner = { - id: 'model', + id: 'dspy-rlm', analyze: () => ({ findings: [], usage: UNKNOWN_USAGE }), } await runAnalystBenchmarkCommand( args, { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner: () => runner }, + { createAnalystRunner: () => runner }, ) await unlink(join(logFixture.outDir, 'result.json')) await unlink(join(logFixture.outDir, 'report.md')) @@ -276,16 +276,16 @@ describe('analyst benchmark persistence and resume', () => { runAnalystBenchmarkCommand( [...args, '--resume'], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/observation log must be a real file/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() }) it('persists completed observation rows when finalization is interrupted', async () => { const fixture = await agentRxFixture() const modelRunner: AnalystBenchmarkRunner = { - id: 'model', + id: 'dspy-rlm', async analyze() { await writeFile(join(fixture.outDir, 'result.json'), 'interrupted finalization\n') return { findings: [], usage: UNKNOWN_USAGE } @@ -296,14 +296,14 @@ describe('analyst benchmark persistence and resume', () => { runAnalystBenchmarkCommand( agentRxCommandArgs(fixture), { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner: () => modelRunner }, + { createAnalystRunner: () => modelRunner }, ), ).rejects.toThrow(/refusing to replace existing benchmark artifact/) const rows = await progressRows(fixture.outDir) expect(rows).toHaveLength(2) expect(rows.map((row) => row.sequence)).toEqual([0, 1]) - expect(rows.map((row) => row.observation.runnerId).sort()).toEqual(['empty', 'model']) + expect(rows.map((row) => row.observation.runnerId).sort()).toEqual(['dspy-rlm', 'empty']) await expect(readFile(join(fixture.outDir, 'report.md'), 'utf8')).rejects.toThrow() }) @@ -312,7 +312,7 @@ describe('analyst benchmark persistence and resume', () => { const args = [...agentRxCommandArgs(fixture), '--repetitions', '3', '--concurrency', '1'] const firstCalls: string[] = [] const firstRunner: AnalystBenchmarkRunner = { - id: 'model', + id: 'dspy-rlm', analyze(_input, context) { firstCalls.push(`${context.caseId}/${context.repetition}`) return { findings: [], usage: UNKNOWN_USAGE } @@ -321,15 +321,15 @@ describe('analyst benchmark persistence and resume', () => { await runAnalystBenchmarkCommand( args, { TEST_ANALYST_KEY: 'first-secret' }, - { createModelRunner: () => firstRunner }, + { createAnalystRunner: () => firstRunner }, ) expect(firstCalls).toHaveLength(3) const allRows = await progressRows(fixture.outDir) - const firstModelRow = allRows.findIndex((row) => row.observation.runnerId === 'model') + const firstModelRow = allRows.findIndex((row) => row.observation.runnerId === 'dspy-rlm') expect(firstModelRow).toBeGreaterThanOrEqual(0) const retainedRows = allRows.slice(0, firstModelRow + 1) - expect(retainedRows.filter((row) => row.observation.runnerId === 'model')).toHaveLength(1) + expect(retainedRows.filter((row) => row.observation.runnerId === 'dspy-rlm')).toHaveLength(1) await writeFile( join(fixture.outDir, ANALYST_BENCHMARK_OBSERVATIONS_FILE), `${retainedRows.map((row) => JSON.stringify(row)).join('\n')}\n`, @@ -339,12 +339,12 @@ describe('analyst benchmark persistence and resume', () => { const retainedModelKeys = new Set( retainedRows - .filter((row) => row.observation.runnerId === 'model') + .filter((row) => row.observation.runnerId === 'dspy-rlm') .map((row) => `${row.observation.caseId}/${row.observation.repetition}`), ) const resumedCalls: string[] = [] const resumedRunner: AnalystBenchmarkRunner = { - id: 'model', + id: 'dspy-rlm', analyze(_input, context) { resumedCalls.push(`${context.caseId}/${context.repetition}`) return { findings: [], usage: UNKNOWN_USAGE } @@ -353,7 +353,7 @@ describe('analyst benchmark persistence and resume', () => { await runAnalystBenchmarkCommand( [...args, '--resume'], { TEST_ANALYST_KEY: 'second-secret' }, - { createModelRunner: () => resumedRunner }, + { createAnalystRunner: () => resumedRunner }, ) expect(resumedCalls).toHaveLength(2) @@ -382,17 +382,17 @@ describe('analyst benchmark persistence and resume', () => { const fixture = await agentRxFixture() const args = agentRxCommandArgs(fixture) const analyze = vi.fn(() => ({ findings: [], usage: UNKNOWN_USAGE })) - const createModelRunner = vi.fn(() => ({ id: 'model', analyze })) - await runAnalystBenchmarkCommand(args, { TEST_ANALYST_KEY: 'unused' }, { createModelRunner }) + const createAnalystRunner = vi.fn(() => ({ id: 'dspy-rlm', analyze })) + await runAnalystBenchmarkCommand(args, { TEST_ANALYST_KEY: 'unused' }, { createAnalystRunner }) await expect( runAnalystBenchmarkCommand( [...args, '--resume'], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).resolves.toBe(0) - expect(createModelRunner).toHaveBeenCalledOnce() + expect(createAnalystRunner).toHaveBeenCalledOnce() expect(analyze).toHaveBeenCalledOnce() }) @@ -403,8 +403,8 @@ describe('analyst benchmark persistence and resume', () => { args, { TEST_ANALYST_KEY: 'unused' }, { - createModelRunner: () => ({ - id: 'model', + createAnalystRunner: () => ({ + id: 'dspy-rlm', analyze: () => ({ findings: [], usage: UNKNOWN_USAGE }), }), }, @@ -413,16 +413,16 @@ describe('analyst benchmark persistence and resume', () => { await unlink(join(fixture.outDir, 'report.md')) const changedArgs = [...args] changedArgs[changedArgs.indexOf('--model') + 1] = 'another/model' - const createModelRunner = vi.fn() + const createAnalystRunner = vi.fn() await expect( runAnalystBenchmarkCommand( [...changedArgs, '--resume'], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/resume configuration or inputs do not match/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() const changedEndpointArgs = [...args] changedEndpointArgs[changedEndpointArgs.indexOf('--base-url') + 1] = 'http://127.0.0.1:4455/v1' @@ -430,10 +430,10 @@ describe('analyst benchmark persistence and resume', () => { runAnalystBenchmarkCommand( [...changedEndpointArgs, '--resume'], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/resume configuration or inputs do not match/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() const alternateLabelsPath = join(fixture.root, 'same-labels-another-path.json') await writeFile(alternateLabelsPath, await readFile(fixture.labelsPath, 'utf8')) @@ -443,39 +443,39 @@ describe('analyst benchmark persistence and resume', () => { runAnalystBenchmarkCommand( [...changedPathArgs, '--resume'], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/resume configuration or inputs do not match/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() }) it('rejects malformed and duplicate observation log rows before model calls', async () => { const fixture = await agentRxFixture() const args = agentRxCommandArgs(fixture) const runner = { - id: 'model', + id: 'dspy-rlm', analyze: () => ({ findings: [], usage: UNKNOWN_USAGE }), } await runAnalystBenchmarkCommand( args, { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner: () => runner }, + { createAnalystRunner: () => runner }, ) await unlink(join(fixture.outDir, 'result.json')) await unlink(join(fixture.outDir, 'report.md')) const observationPath = join(fixture.outDir, ANALYST_BENCHMARK_OBSERVATIONS_FILE) const valid = await readFile(observationPath, 'utf8') - const createModelRunner = vi.fn(() => runner) + const createAnalystRunner = vi.fn(() => runner) await writeFile(observationPath, '{not json}\n') await expect( runAnalystBenchmarkCommand( [...args, '--resume'], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/invalid JSON/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() const tamperedRows = valid .trim() @@ -490,10 +490,10 @@ describe('analyst benchmark persistence and resume', () => { runAnalystBenchmarkCommand( [...args, '--resume'], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/digest does not match its contents/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() const [firstLine] = valid.trim().split('\n') await writeFile(observationPath, `${valid}${firstLine}\n`) @@ -501,20 +501,20 @@ describe('analyst benchmark persistence and resume', () => { runAnalystBenchmarkCommand( [...args, '--resume'], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/duplicate benchmark observation/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() }) it('rejects completed results whose derived metrics were changed', async () => { const fixture = await agentRxFixture() const args = agentRxCommandArgs(fixture) - const createModelRunner = vi.fn(() => ({ - id: 'model', + const createAnalystRunner = vi.fn(() => ({ + id: 'dspy-rlm', analyze: () => ({ findings: [], usage: UNKNOWN_USAGE }), })) - await runAnalystBenchmarkCommand(args, { TEST_ANALYST_KEY: 'unused' }, { createModelRunner }) + await runAnalystBenchmarkCommand(args, { TEST_ANALYST_KEY: 'unused' }, { createAnalystRunner }) const resultPath = join(fixture.outDir, 'result.json') const artifact = JSON.parse(await readFile(resultPath, 'utf8')) artifact.result.summaries[1].failedRuns = 1 @@ -524,9 +524,9 @@ describe('analyst benchmark persistence and resume', () => { runAnalystBenchmarkCommand( [...args, '--resume'], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/summaries do not match durable observations/) - expect(createModelRunner).toHaveBeenCalledOnce() + expect(createAnalystRunner).toHaveBeenCalledOnce() }) }) diff --git a/src/analyst/benchmark-command-persistence.ts b/src/analyst/benchmark-command-persistence.ts index 2368856c..0a7bd0ce 100644 --- a/src/analyst/benchmark-command-persistence.ts +++ b/src/analyst/benchmark-command-persistence.ts @@ -156,7 +156,7 @@ export function createRunIdentity( analystProtocolSha256: publicBenchmarkProtocolSha256(config.dataset), implementationSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, - runnerIds: ['empty', 'model'], + runnerIds: ['empty', 'dspy-rlm'], }, inputs: { labelsSha256: prepared.labelsSha256, @@ -535,7 +535,7 @@ export async function readProgress( } if ( !allowedCases.has(observation.caseId) || - (observation.runnerId !== 'empty' && observation.runnerId !== 'model') || + (observation.runnerId !== 'empty' && observation.runnerId !== 'dspy-rlm') || observation.repetition >= repetitions || observation.executionIndex >= plannedObservationCount ) { diff --git a/src/analyst/benchmark-command-result.ts b/src/analyst/benchmark-command-result.ts index 4d7133d1..89d39a49 100644 --- a/src/analyst/benchmark-command-result.ts +++ b/src/analyst/benchmark-command-result.ts @@ -147,7 +147,7 @@ export function assertCompletedArtifactMatchesRun( const expectedComparisons = [ compareAnalystRunners(artifact.result, { baselineRunnerId: 'empty', - candidateRunnerId: 'model', + candidateRunnerId: 'dspy-rlm', seed: config.seed, }), ] diff --git a/src/analyst/benchmark-command.test.ts b/src/analyst/benchmark-command.test.ts index 84c983b9..c56d6c77 100644 --- a/src/analyst/benchmark-command.test.ts +++ b/src/analyst/benchmark-command.test.ts @@ -26,7 +26,7 @@ describe('runAnalystBenchmarkCommand', () => { it('writes complete paired results and preserves unknown model cost', async () => { const fixture = await codeTraceFixture() const modelRunner: AnalystBenchmarkRunner = { - id: 'model', + id: 'dspy-rlm', async analyze(input, context) { const manifest = JSON.parse( await readFile(join(fixture.outDir, ANALYST_BENCHMARK_MANIFEST_FILE), 'utf8'), @@ -53,7 +53,7 @@ describe('runAnalystBenchmarkCommand', () => { return { findings: [ makeFinding({ - analyst_id: 'model', + analyst_id: 'dspy-rlm', area: 'incorrect', claim: 'Step 2 changes the wrong file.', severity: 'high', @@ -72,7 +72,7 @@ describe('runAnalystBenchmarkCommand', () => { commandArgs(fixture), { TEST_ANALYST_KEY: 'do-not-persist-this-key' }, { - createModelRunner: (_dataset, config) => { + createAnalystRunner: (_dataset, config) => { expect(config.durability).toEqual({ runIdentitySha256: expect.stringMatching(/^[a-f0-9]{64}$/), responseCacheDir: join(fixture.outDir, 'model-responses'), @@ -95,7 +95,7 @@ describe('runAnalystBenchmarkCommand', () => { )) as unknown as Record expect(artifact.result.provenance).toMatchObject({ caseCount: 1, - runnerIds: ['empty', 'model'], + runnerIds: ['empty', 'dspy-rlm'], runnerOrderSeed: 7, metadata: { protocolSha256: expect.stringMatching(/^[a-f0-9]{64}$/), @@ -107,7 +107,7 @@ describe('runAnalystBenchmarkCommand', () => { expect(artifact.result.summaries).toEqual([ expect.objectContaining({ runnerId: 'empty', issueRecall: 0, knownCostUsd: 0 }), expect.objectContaining({ - runnerId: 'model', + runnerId: 'dspy-rlm', issueRecall: 1, costUnknownRuns: 1, knownCostUsd: 0, @@ -181,7 +181,7 @@ describe('runAnalystBenchmarkCommand', () => { }) expect(artifact.comparisons[0]).toMatchObject({ baselineRunnerId: 'empty', - candidateRunnerId: 'model', + candidateRunnerId: 'dspy-rlm', }) expect( artifact.comparisons[0].metrics.every((metric: { inferenceLimitations: string[] }) => @@ -193,7 +193,7 @@ describe('runAnalystBenchmarkCommand', () => { runners: [ expect.objectContaining({ runnerId: 'empty', positiveRuns: 1 }), expect.objectContaining({ - runnerId: 'model', + runnerId: 'dspy-rlm', positiveRuns: 1, matchedIncorrectSteps: 1, precision: 1, @@ -245,14 +245,14 @@ describe('runAnalystBenchmarkCommand', () => { const fixture = await agentRxFixture() const args = agentRxCommandArgs(fixture) const modelRunner: AnalystBenchmarkRunner = { - id: 'model', + id: 'dspy-rlm', analyze: () => ({ findings: [], usage: UNKNOWN_USAGE }), } await runAnalystBenchmarkCommand( args, { TEST_ANALYST_KEY: 'unused' }, { - createModelRunner: () => modelRunner, + createAnalystRunner: () => modelRunner, }, ) await appendFile( @@ -271,36 +271,36 @@ describe('runAnalystBenchmarkCommand', () => { }, })}\n`, ) - const createModelRunner = vi.fn(() => modelRunner) + const createAnalystRunner = vi.fn(() => modelRunner) await expect( runAnalystBenchmarkCommand( [...args, '--resume'], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/pending or incomplete cost entries/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() }) it('rejects missing labeled spans before constructing or calling a model runner', async () => { const fixture = await codeTraceFixture({ labeledStep: 3 }) - const createModelRunner = vi.fn() + const createAnalystRunner = vi.fn() await expect( runAnalystBenchmarkCommand( commandArgs(fixture), { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/missing labeled span step-3/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() }) it('runs trajectory-only cases with an explicit unavailable outcome', async () => { const fixture = await codeTraceFixture({ withVerificationArtifacts: false }) const modelRunner: AnalystBenchmarkRunner = { - id: 'model', + id: 'dspy-rlm', async analyze(input) { const trace = await input.traceStore?.viewTrace({ trace_id: 'trace-1' }) expect(trace?.spans).toEqual( @@ -314,16 +314,16 @@ describe('runAnalystBenchmarkCommand', () => { return { findings: [], usage: UNKNOWN_USAGE } }, } - const createModelRunner = vi.fn(() => modelRunner) + const createAnalystRunner = vi.fn(() => modelRunner) await expect( runAnalystBenchmarkCommand( commandArgs(fixture), { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).resolves.toBe(0) - expect(createModelRunner).toHaveBeenCalledOnce() + expect(createAnalystRunner).toHaveBeenCalledOnce() const artifact = JSON.parse(await readFile(join(fixture.outDir, 'result.json'), 'utf8')) expect(artifact.inputs.verificationArtifacts[0]).toMatchObject({ status: 'missing', @@ -347,66 +347,66 @@ describe('runAnalystBenchmarkCommand', () => { incorrect_step_ids: [2], }), ) - const createModelRunner = vi.fn() + const createAnalystRunner = vi.fn() await expect( runAnalystBenchmarkCommand( commandArgs(fixture), { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/label key 'incorrect_step_ids'/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() }) it('refuses credential-bearing base URLs before constructing the model runner', async () => { const fixture = await codeTraceFixture() const args = commandArgs(fixture) args[args.indexOf('--base-url') + 1] = 'http://secret@127.0.0.1:3355/v1' - const createModelRunner = vi.fn() + const createAnalystRunner = vi.fn() await expect( - runAnalystBenchmarkCommand(args, { TEST_ANALYST_KEY: 'unused' }, { createModelRunner }), + runAnalystBenchmarkCommand(args, { TEST_ANALYST_KEY: 'unused' }, { createAnalystRunner }), ).rejects.toThrow(/without credentials/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() }) it('refuses remote plain HTTP before constructing the model runner', async () => { const fixture = await codeTraceFixture() const args = commandArgs(fixture) args[args.indexOf('--base-url') + 1] = 'http://provider.example/v1' - const createModelRunner = vi.fn() + const createAnalystRunner = vi.fn() await expect( - runAnalystBenchmarkCommand(args, { TEST_ANALYST_KEY: 'unused' }, { createModelRunner }), + runAnalystBenchmarkCommand(args, { TEST_ANALYST_KEY: 'unused' }, { createAnalystRunner }), ).rejects.toThrow(/must use HTTPS unless/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() }) it('rejects branch names and short revision hashes before constructing a runner', async () => { const fixture = await codeTraceFixture() - const createModelRunner = vi.fn() + const createAnalystRunner = vi.fn() for (const revision of ['main', 'abc123']) { const args = commandArgs(fixture) args[args.indexOf('--revision') + 1] = revision await expect( - runAnalystBenchmarkCommand(args, { TEST_ANALYST_KEY: 'unused' }, { createModelRunner }), + runAnalystBenchmarkCommand(args, { TEST_ANALYST_KEY: 'unused' }, { createAnalystRunner }), ).rejects.toThrow(/full 40- or 64-character hexadecimal digest/) } - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() }) it('runs AgentRx without imposing CodeTraceBench artifact requirements', async () => { const fixture = await agentRxFixture() const modelRunner: AnalystBenchmarkRunner = { - id: 'model', + id: 'dspy-rlm', analyze() { return { findings: [ makeFinding({ - analyst_id: 'model', + analyst_id: 'dspy-rlm', area: 'system-failure', claim: 'The worker lost its provider at the root step.', severity: 'high', @@ -424,7 +424,7 @@ describe('runAnalystBenchmarkCommand', () => { runAnalystBenchmarkCommand( agentRxCommandArgs(fixture), { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner: () => modelRunner }, + { createAnalystRunner: () => modelRunner }, ), ).resolves.toBe(0) @@ -441,7 +441,7 @@ describe('runAnalystBenchmarkCommand', () => { }) expect( artifact.result.summaries.map((summary: { runnerId: string }) => summary.runnerId), - ).toEqual(['empty', 'model']) + ).toEqual(['empty', 'dspy-rlm']) expect(artifact.agentRxCalibration).toMatchObject({ protocol: 'official-agentrx-root-cause', upstreamRevision: AGENT_RX_UPSTREAM_REVISION, @@ -452,7 +452,7 @@ describe('runAnalystBenchmarkCommand', () => { exactStepAccuracy: 0, }), expect.objectContaining({ - runnerId: 'model', + runnerId: 'dspy-rlm', predictedRuns: 1, exactStepAccuracy: 1, rootCauseCategoryAccuracy: 1, @@ -473,8 +473,8 @@ describe('runAnalystBenchmarkCommand', () => { args, { TEST_ANALYST_KEY: 'unused' }, { - createModelRunner: () => ({ - id: 'model', + createAnalystRunner: () => ({ + id: 'dspy-rlm', analyze: () => ({ findings: [], usage: UNKNOWN_USAGE }), }), }, @@ -501,7 +501,7 @@ describe('runAnalystBenchmarkCommand', () => { it('returns a failing exit code without discarding failed model usage', async () => { const fixture = await agentRxFixture() const modelRunner: AnalystBenchmarkRunner = { - id: 'model', + id: 'dspy-rlm', analyze() { return { findings: [], @@ -523,7 +523,7 @@ describe('runAnalystBenchmarkCommand', () => { runAnalystBenchmarkCommand( agentRxCommandArgs(fixture), { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner: () => modelRunner }, + { createAnalystRunner: () => modelRunner }, ), ).resolves.toBe(2) @@ -531,7 +531,7 @@ describe('runAnalystBenchmarkCommand', () => { await readFile(join(fixture.outDir, 'result.json'), 'utf8'), ) as Record expect(artifact.result.summaries[1]).toMatchObject({ - runnerId: 'model', + runnerId: 'dspy-rlm', completedRuns: 0, failedRuns: 1, calls: 8, @@ -556,8 +556,8 @@ describe('runAnalystBenchmarkCommand', () => { [...agentRxCommandArgs(fixture), '--max-cost-usd', '0.25'], { TEST_ANALYST_KEY: 'unused' }, { - createModelRunner: (_dataset, config) => ({ - id: 'model', + createAnalystRunner: (_dataset, config) => ({ + id: 'dspy-rlm', async analyze() { const denied = await config.costLedger!.runPaidCall({ channel: 'analyst', @@ -605,8 +605,8 @@ describe('runAnalystBenchmarkCommand', () => { agentRxCommandArgs(fixture), { TEST_ANALYST_KEY: 'unused' }, { - createModelRunner: (_dataset, config) => ({ - id: 'model', + createAnalystRunner: (_dataset, config) => ({ + id: 'dspy-rlm', async analyze(_input, context) { const paid = await config.costLedger!.runPaidCall({ channel: 'analyst', @@ -656,16 +656,16 @@ describe('runAnalystBenchmarkCommand', () => { 'rejects invalid run-wide spend limit %s before model construction', async (value) => { const fixture = await agentRxFixture() - const createModelRunner = vi.fn() + const createAnalystRunner = vi.fn() await expect( runAnalystBenchmarkCommand( [...agentRxCommandArgs(fixture), '--max-cost-usd', value], { TEST_ANALYST_KEY: 'unused' }, - { createModelRunner }, + { createAnalystRunner }, ), ).rejects.toThrow(/max-cost-usd must be a positive finite number/) - expect(createModelRunner).not.toHaveBeenCalled() + expect(createAnalystRunner).not.toHaveBeenCalled() }, ) }) diff --git a/src/analyst/benchmark-command.ts b/src/analyst/benchmark-command.ts index 5c5f5720..121bc58c 100644 --- a/src/analyst/benchmark-command.ts +++ b/src/analyst/benchmark-command.ts @@ -46,8 +46,8 @@ import { renderCodeTraceCalibrationMarkdown, summarizeCodeTraceCalibration, } from './benchmark-public-calibration' +import { createPublicBenchmarkRlmRunner } from './benchmark-public-rlm' import { - createPublicBenchmarkModelRunner, emptyPublicBenchmarkRunner, type PublicAnalystBenchmarkDataset, type PublicAnalystBenchmarkModelConfig, @@ -77,7 +77,7 @@ export { export { readAnalystBenchmarkArtifact } from './benchmark-command-result' export interface AnalystBenchmarkCommandDependencies { - createModelRunner?: ( + createAnalystRunner?: ( dataset: PublicAnalystBenchmarkDataset, config: PublicAnalystBenchmarkModelConfig, ) => AnalystBenchmarkRunner @@ -180,13 +180,13 @@ async function executeAnalystBenchmarkCommand( ) } - const createModelRunner = - dependencies.createModelRunner ?? + const createAnalystRunner = + dependencies.createAnalystRunner ?? ((dataset: PublicAnalystBenchmarkDataset, model: PublicAnalystBenchmarkModelConfig) => - createPublicBenchmarkModelRunner(dataset, model)) + createPublicBenchmarkRlmRunner(dataset, model)) const runners = [ emptyPublicBenchmarkRunner(), - createModelRunner(config.dataset, { + createAnalystRunner(config.dataset, { ...config.model, costLedger, durability: { @@ -269,7 +269,7 @@ async function executeAnalystBenchmarkCommand( const comparisons = [ compareAnalystRunners(result, { baselineRunnerId: 'empty', - candidateRunnerId: 'model', + candidateRunnerId: 'dspy-rlm', seed: config.seed, }), ] @@ -341,7 +341,7 @@ function assertObservationAccountingComplete( `Analyst benchmark stopped before scoring: ${observation.error.message}`, ) } - if (observation.runnerId !== 'model') return + if (observation.runnerId !== 'dspy-rlm') return const summary = costLedger.summary({ channel: 'analyst', tags: { @@ -350,7 +350,7 @@ function assertObservationAccountingComplete( }, }) if (!summary.accountingComplete || summary.pendingCalls > 0 || summary.unresolvedCalls > 0) { - throw accountingError(costLedger, 'the model observation has incomplete cost accounting', { + throw accountingError(costLedger, 'the recursive analyst has incomplete cost accounting', { channel: 'analyst', tags: { benchmarkCaseId: observation.caseId, @@ -381,7 +381,7 @@ function accountingError( export const ANALYST_BENCHMARK_HELP = `agent-eval analyst-benchmark -Run a real-model trace analyst against public AgentRx or CodeTraceBench labels. +Run the recursive DSPy trace analyst against public AgentRx or CodeTraceBench labels. Required: --dataset agentrx|codetracebench @@ -402,6 +402,7 @@ Controls: --concurrency Parallel benchmark jobs. Default: 1 --repetitions Runs per case and runner. Default: 1 --max-output-tokens Model output limit per call. Default: 4096 + --python Python with agent-eval-rpc[dspy]. Default: python --timeout-ms Model analyst deadline per case. Default: 300000 --max-cost-usd Run-wide spend limit. Default: 5 --max-artifact-bytes Final evidence bytes per case. Default: ${DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES} @@ -437,6 +438,8 @@ function parseCommandConfig( throw new Error(`--api-key-env points to an empty or missing variable: ${apiKeyEnv}`) } + const maxCostUsd = positiveFiniteFlag(flags, 'max-cost-usd', 5) + const python = flags.get('python')?.trim() return { dataset, labelsPath: requiredFlag(flags, 'labels'), @@ -451,12 +454,14 @@ function parseCommandConfig( model: requiredFlag(flags, 'model'), maxOutputTokens: positiveFlag(flags, 'max-output-tokens', 4_096), timeoutMs: positiveFlag(flags, 'timeout-ms', 300_000), + maxCostUsdPerAnalysis: maxCostUsd, + ...(python ? { dspyRlm: { runner: { command: python } } } : {}), }, limit: positiveFlag(flags, 'limit'), seed: integerFlag(flags, 'seed', 0), concurrency: positiveFlag(flags, 'concurrency', 1), repetitions: positiveFlag(flags, 'repetitions', 1), - maxCostUsd: positiveFiniteFlag(flags, 'max-cost-usd', 5), + maxCostUsd, maxArtifactBytes: positiveFlag( flags, 'max-artifact-bytes', @@ -510,6 +515,7 @@ const KNOWN_FLAGS = new Set([ 'concurrency', 'repetitions', 'max-output-tokens', + 'python', 'timeout-ms', 'max-cost-usd', 'max-artifact-bytes', @@ -678,7 +684,7 @@ function renderArtifactMarkdown(artifact: AnalystBenchmarkArtifact): string { } function benchmarkExitCode(result: AnalystBenchmarkResult): number { - return result.summaries.find((summary) => summary.runnerId === 'model')?.failedRuns ? 2 : 0 + return result.summaries.find((summary) => summary.runnerId === 'dspy-rlm')?.failedRuns ? 2 : 0 } function printSuccessSummary( diff --git a/src/analyst/benchmark-implementation.test.ts b/src/analyst/benchmark-implementation.test.ts index c6335351..f10fc9d4 100644 --- a/src/analyst/benchmark-implementation.test.ts +++ b/src/analyst/benchmark-implementation.test.ts @@ -19,6 +19,8 @@ const REPOSITORY_ROOT = fileURLToPath(new URL('../../', import.meta.url)) const CHECKER = join(REPOSITORY_ROOT, 'scripts/check-analyst-benchmark-implementation.mjs') const EXPECTED_FILES = [ + 'clients/python/src/agent_eval_rpc/dspy_rlm_bridge.py', + 'clients/python/src/agent_eval_rpc/optimizer_bridge_common.py', 'src/analyst/benchmark-agentrx-calibration.ts', 'src/analyst/benchmark-command-artifact.ts', 'src/analyst/benchmark-command-persistence.ts', @@ -36,6 +38,7 @@ const EXPECTED_FILES = [ 'src/analyst/benchmark-public-data.ts', 'src/analyst/benchmark-public-errors.ts', 'src/analyst/benchmark-public-model.ts', + 'src/analyst/benchmark-public-rlm.ts', 'src/analyst/benchmark-public-types.ts', 'src/analyst/benchmark-real-model.ts', 'src/analyst/benchmark-report.ts', @@ -45,8 +48,21 @@ const EXPECTED_FILES = [ 'src/analyst/benchmark-verification-artifacts.ts', 'src/analyst/benchmark-verification-outcome.ts', 'src/analyst/benchmark.ts', + 'src/analyst/dspy-rlm-engine.ts', + 'src/analyst/engine.ts', + 'src/analyst/finding-signature.ts', + 'src/analyst/finding-subject.ts', + 'src/analyst/kind-factory.ts', + 'src/analyst/parse-tolerant.ts', + 'src/analyst/tool-groups.ts', + 'src/analyst/trace-tool-callback.ts', 'src/analyst/types.ts', 'src/analyst/usage-receipt.ts', + 'src/campaign/external-optimizer-contracts.ts', + 'src/campaign/external-optimizer-http.ts', + 'src/campaign/external-optimizer-model-proxy.ts', + 'src/campaign/external-optimizer-resources.ts', + 'src/campaign/external-optimizer-subprocess.ts', 'src/campaign/search-ledger-errors.ts', 'src/campaign/search-ledger-file.ts', 'src/campaign/single-run-lock.ts', @@ -76,6 +92,7 @@ const EXPECTED_FILES = [ 'src/trace-analyst/store-otlp.ts', 'src/trace-analyst/store-schemas.ts', 'src/trace-analyst/store.ts', + 'src/trace-analyst/tools.ts', 'src/trace-analyst/types.ts', 'src/trace/attribute-vocabulary.ts', 'src/trace/otlp-attributes.ts', @@ -96,7 +113,12 @@ describe('public analyst benchmark implementation digest', () => { expect(ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM).toBe( 'sha256-canonical-file-manifest', ) - expect(ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES).toEqual(['package.json', 'pnpm-lock.yaml']) + expect(ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES).toEqual([ + 'clients/python/pyproject.toml', + 'clients/python/uv.lock', + 'package.json', + 'pnpm-lock.yaml', + ]) expect(analystBenchmarkDependencyLockDigest()).toBe(ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256) }) diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index 175eafa0..ccf6fc76 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -3,14 +3,18 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM = 'sha256-canonic export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM = 'sha256-canonical-file-manifest' export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ + 'clients/python/pyproject.toml', + 'clients/python/uv.lock', 'package.json', 'pnpm-lock.yaml', ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - '1e03f2daed356d60316aabefb407ec1e437ac94d408d61eea4ae096e9c6fbb5b' + 'b00ffeede06a5034d2d18aa33988379e5575e5e61d8f7d8fda6693969e04cac4' export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ + 'clients/python/src/agent_eval_rpc/dspy_rlm_bridge.py', + 'clients/python/src/agent_eval_rpc/optimizer_bridge_common.py', 'src/analyst/benchmark-agentrx-calibration.ts', 'src/analyst/benchmark-command-artifact.ts', 'src/analyst/benchmark-command-persistence.ts', @@ -28,6 +32,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ 'src/analyst/benchmark-public-data.ts', 'src/analyst/benchmark-public-errors.ts', 'src/analyst/benchmark-public-model.ts', + 'src/analyst/benchmark-public-rlm.ts', 'src/analyst/benchmark-public-types.ts', 'src/analyst/benchmark-real-model.ts', 'src/analyst/benchmark-report.ts', @@ -37,8 +42,21 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ 'src/analyst/benchmark-verification-artifacts.ts', 'src/analyst/benchmark-verification-outcome.ts', 'src/analyst/benchmark.ts', + 'src/analyst/dspy-rlm-engine.ts', + 'src/analyst/engine.ts', + 'src/analyst/finding-signature.ts', + 'src/analyst/finding-subject.ts', + 'src/analyst/kind-factory.ts', + 'src/analyst/parse-tolerant.ts', + 'src/analyst/tool-groups.ts', + 'src/analyst/trace-tool-callback.ts', 'src/analyst/types.ts', 'src/analyst/usage-receipt.ts', + 'src/campaign/external-optimizer-contracts.ts', + 'src/campaign/external-optimizer-http.ts', + 'src/campaign/external-optimizer-model-proxy.ts', + 'src/campaign/external-optimizer-resources.ts', + 'src/campaign/external-optimizer-subprocess.ts', 'src/campaign/search-ledger-errors.ts', 'src/campaign/search-ledger-file.ts', 'src/campaign/single-run-lock.ts', @@ -68,6 +86,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ 'src/trace-analyst/store-otlp.ts', 'src/trace-analyst/store-schemas.ts', 'src/trace-analyst/store.ts', + 'src/trace-analyst/tools.ts', 'src/trace-analyst/types.ts', 'src/trace/attribute-vocabulary.ts', 'src/trace/otlp-attributes.ts', @@ -75,7 +94,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = - '4dba263b6256a30d56c7fdb2d992d3a953c0035d731f359b704db806f68f75ac' + '610fb8555731e8a4eceaedfa98c47ea37b040341aa684791c3f55ded9dc50357' export function analystBenchmarkImplementationDigest() { return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 diff --git a/src/analyst/benchmark-public-errors.test.ts b/src/analyst/benchmark-public-errors.test.ts new file mode 100644 index 00000000..0e103414 --- /dev/null +++ b/src/analyst/benchmark-public-errors.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'vitest' +import { publicBenchmarkError } from './benchmark-public-errors' + +describe('publicBenchmarkError', () => { + it('keeps the beginning and final exception from long errors', () => { + const error = publicBenchmarkError(new Error(`START ${'x'.repeat(800)} FINAL_EXCEPTION`)) + + expect(error.message).toContain('START') + expect(error.message).toContain('chars omitted') + expect(error.message).toContain('FINAL_EXCEPTION') + expect(error.message.length).toBeLessThanOrEqual(500) + }) + + it('redacts explicit and patterned secrets before truncating', () => { + const secret = 'provider-secret-value' + const error = publicBenchmarkError( + new Error(`Bearer bearer-token ${secret} ${'x'.repeat(800)} api_key=tail-secret`), + [secret], + ) + + expect(error.message).not.toContain(secret) + expect(error.message).not.toContain('bearer-token') + expect(error.message).not.toContain('tail-secret') + expect(error.message.match(/\[REDACTED\]/g)?.length).toBeGreaterThanOrEqual(3) + }) +}) diff --git a/src/analyst/benchmark-public-errors.ts b/src/analyst/benchmark-public-errors.ts index 1809cff4..d83056aa 100644 --- a/src/analyst/benchmark-public-errors.ts +++ b/src/analyst/benchmark-public-errors.ts @@ -91,11 +91,15 @@ function redactSensitiveText(value: string, secrets: readonly string[]): string for (const secret of secrets) { if (secret) redacted = redacted.replaceAll(secret, '[REDACTED]') } - return redacted + redacted = redacted .replace(/\bBearer\s+[^\s"',;]+/gi, 'Bearer [REDACTED]') .replace( /\b(api[_-]?key|access[_-]?token|refresh[_-]?token|password|secret)\b\s*[:=]\s*[^\s"',;]+/gi, '$1=[REDACTED]', ) - .slice(0, 500) + if (redacted.length <= 500) return redacted + const head = redacted.slice(0, 180) + const omitted = redacted.length - 460 + const marker = `...[${omitted} chars omitted]...` + return `${head}${marker}${redacted.slice(-(500 - head.length - marker.length))}` } diff --git a/src/analyst/benchmark-public-model.ts b/src/analyst/benchmark-public-model.ts index 76be1f31..4e3225ef 100644 --- a/src/analyst/benchmark-public-model.ts +++ b/src/analyst/benchmark-public-model.ts @@ -48,7 +48,8 @@ import { usageReceiptFromCostLedger } from './usage-receipt' const TRACE_PROJECTION_ATTRIBUTE_BYTE_CAPS = [4_096, 2_048, 1_024, 512, 256, 128, 64] as const -export function createPublicBenchmarkModelRunner( +/** One-shot JSON baseline. This is not a recursive trace analyst. */ +export function createPublicBenchmarkDirectRunner( dataset: PublicAnalystBenchmarkDataset, config: PublicAnalystBenchmarkModelConfig, ): AnalystBenchmarkRunner { @@ -85,7 +86,7 @@ export function createPublicBenchmarkModelRunner( } return { - id: 'model', + id: 'direct', async analyze(input, context) { const trajectoryId = trajectoryIdFromCaseId(dataset, context.caseId) const costTags = { @@ -98,7 +99,7 @@ export function createPublicBenchmarkModelRunner( let providerModel = model let producedAt: string | undefined let modelMetadata: Record = { - analysisMode: 'single-pass', + analysisMode: 'direct-baseline', outputAdapter, protocolSha256: publicBenchmarkProtocolSha256(dataset), } @@ -244,7 +245,7 @@ export function createPublicBenchmarkModelRunner( trajectoryId, predictions: rawPredictions, store: input.traceStore, - analystId: 'model', + analystId: 'direct', providerModel, producedAt: requiredString(producedAt ?? '', 'finding producedAt'), ...(context.signal ? { signal: context.signal } : {}), @@ -523,7 +524,7 @@ async function publicBenchmarkPredictionsToFindings(options: { evidence_refs: [evidenceByStep.get(step)!], recommended_action: prediction.recommended_action, metadata: { - analysis_mode: 'single-pass', + analysis_mode: 'direct-baseline', model: options.providerModel, }, produced_at: options.producedAt, diff --git a/src/analyst/benchmark-public-rlm.ts b/src/analyst/benchmark-public-rlm.ts new file mode 100644 index 00000000..0aa92ad9 --- /dev/null +++ b/src/analyst/benchmark-public-rlm.ts @@ -0,0 +1,238 @@ +import { + CostAccountingIncompleteError, + CostCallConflictError, + CostCeilingReachedError, + CostLedger, + CostLedgerPersistenceError, + CostReceiptCaptureError, + CostReservationExceededError, + type CustomTokenPricing, +} from '../cost-ledger' +import { resolveModelPricing } from '../metrics' +import type { AnalystBenchmarkRunner } from './benchmark' +import { validateCodeTraceFindingEvidence } from './benchmark-evidence-validation' +import { adaptPublicBenchmarkFindings } from './benchmark-public-adapters' +import { publicBenchmarkError } from './benchmark-public-errors' +import { + CODE_TRACE_BENCH_ANALYST_PROMPT, + publicBenchmarkProtocolSha256, +} from './benchmark-public-model' +import type { + PublicAnalystBenchmarkDataset, + PublicAnalystBenchmarkModelConfig, +} from './benchmark-public-types' +import { createDspyRlmTraceEngine, type DspyRlmTraceEngineOptions } from './dspy-rlm-engine' +import { evidenceRefsFromRawFinding } from './finding-signature' +import { runTraceAnalyst, type TraceAnalystDefinition } from './kind-factory' +import type { AnalystFinding, AnalystRunInputs, AnalystUsageReceipt } from './types' +import { makeFinding } from './types' + +const AGENT_RX_RLM_INSTRUCTIONS = `Analyze exactly one failed agent trajectory. +Find the first unrecoverable critical failure, not every later symptom. +Use the trace tools to inspect the action and its following observation. +Emit zero findings only when evidence does not support a root cause. +Otherwise emit exactly one finding whose subject is exactly one of: +instruction-plan-adherence-failure +invention-of-new-information +invalid-invocation +misinterpretation-of-tool-output-handoff-failure +intent-plan-misalignment +underspecified-user-intent +intent-not-supported +guardrails-triggered +system-failure +inconclusive +Cite exactly one assistant span named step- as trace:///span/step-. +The excerpt must quote the assistant action exactly.` + +const CODE_TRACE_RLM_INSTRUCTIONS = `${CODE_TRACE_BENCH_ANALYST_PROMPT} +Use the trace tools rather than asking for the whole trajectory in the prompt. +Keep retrieved trace objects in Python variables. +Never print an entire trace, full source file, or more than 12000 characters in one iteration. +Build a compact table of assistant step ids, actions, following observations, and final verification. +Inspect suspicious steps with viewSpans or searchSpan instead of repeatedly printing the table. +Submit as soon as every state-changing assistant step has a supported verdict. +For each incorrect step, set subject to incorrect-step-. +Cite the assistant span as trace:///span/step-. +The evidence excerpt must be an exact quote from that span's action content. +Return no finding for a clean trajectory.` + +/** Public benchmark candidate that runs the actual recursive trace analyst. */ +export function createPublicBenchmarkRlmRunner( + dataset: PublicAnalystBenchmarkDataset, + config: PublicAnalystBenchmarkModelConfig, +): AnalystBenchmarkRunner { + const costLedger = config.costLedger ?? new CostLedger() + const limits = { + maxIterations: config.dspyRlm?.maxIterations ?? 8, + maxLlmCalls: config.dspyRlm?.maxLlmCalls ?? 4, + maxToolCalls: config.dspyRlm?.maxToolCalls ?? 32, + maxOutputChars: config.dspyRlm?.maxOutputChars ?? 8_000, + } + const pricing = config.pricing ?? pricingForModel(config.model) + const engine = createDspyRlmTraceEngine({ + baseUrl: config.baseUrl, + apiKey: config.apiKey, + model: config.model, + maxOutputTokens: config.maxOutputTokens, + timeoutMs: config.timeoutMs, + maxCostUsd: config.maxCostUsdPerAnalysis ?? 1, + pricing, + ...(config.dspyRlm?.runner ? { runner: config.dspyRlm.runner } : {}), + } satisfies DspyRlmTraceEngineOptions) + const definition = publicBenchmarkDefinition(dataset, limits) + + return { + id: 'dspy-rlm', + async analyze(input, context) { + const trajectoryId = trajectoryIdFromCaseId(dataset, context.caseId) + const tags = { + benchmarkCaseId: context.caseId, + benchmarkRepetition: String(context.repetition), + } + let usage: AnalystUsageReceipt | undefined + let rawFindings: AnalystFinding[] = [] + try { + if (!input.traceStore) { + throw new Error(`${dataset} DSPy RLM runner requires a trace store`) + } + const completed = await runTraceAnalyst({ + definition, + engine, + store: input.traceStore, + context: { + runId: context.caseId, + correlationId: `${context.caseId}:${context.repetition}`, + costLedger, + costPhase: 'analyst.public-benchmark.dspy-rlm', + tags, + recordUsage: (receipt) => { + usage = receipt + }, + signal: context.signal, + }, + }) + const producedAt = new Date().toISOString() + rawFindings = completed.findings.map((finding) => + makeFinding({ + analyst_id: 'dspy-rlm', + area: dataset === 'agentrx' ? 'root-cause' : 'incorrect', + subject: finding.subject, + claim: finding.claim, + rationale: finding.rationale, + severity: finding.severity, + confidence: finding.confidence, + evidence_refs: evidenceRefsFromRawFinding(finding), + recommended_action: finding.recommended_action, + metadata: { + analysis_mode: 'recursive', + engine: 'dspy-rlm', + model: config.model, + }, + produced_at: producedAt, + }), + ) + const findings = adaptPublicBenchmarkFindings( + dataset, + trajectoryId, + rawFindings, + 'dspy-rlm', + ) + if (dataset === 'codetracebench') { + await validateCodeTraceFindingEvidence({ + trajectoryId, + findings, + store: input.traceStore, + ...(context.signal ? { signal: context.signal } : {}), + }) + } + return { + findings, + usage, + metadata: { + analysisMode: 'recursive', + engine: 'dspy-rlm', + protocolSha256: publicBenchmarkProtocolSha256(dataset), + answer: completed.answer, + trajectory: completed.trajectory, + modelCalls: completed.modelCalls, + toolCalls: completed.toolCalls, + runtime: completed.runtime, + }, + } + } catch (error) { + if (context.signal?.aborted) throw error + if (isPaidCallControlError(error)) throw error + return { + findings: [], + usage, + error: publicBenchmarkError(error, [config.apiKey]), + metadata: { + analysisMode: 'recursive', + engine: 'dspy-rlm', + rawFindings, + }, + } + } + }, + } +} + +function publicBenchmarkDefinition( + dataset: PublicAnalystBenchmarkDataset, + limits: { + maxIterations: number + maxLlmCalls: number + maxToolCalls: number + maxOutputChars: number + }, +): TraceAnalystDefinition { + return { + id: dataset === 'agentrx' ? 'agentrx-dspy-rlm' : 'codetracebench-dspy-rlm', + description: + dataset === 'agentrx' + ? 'Localizes the first unrecoverable root-cause step.' + : 'Localizes every incorrect state-changing assistant step.', + area: dataset === 'agentrx' ? 'root-cause' : 'incorrect', + version: '1.0.0', + question: + dataset === 'agentrx' + ? 'What is the first unrecoverable root cause in this failed trajectory?' + : 'Which assistant steps are incorrect under the CodeTraceBench definition?', + instructions: dataset === 'agentrx' ? AGENT_RX_RLM_INSTRUCTIONS : CODE_TRACE_RLM_INSTRUCTIONS, + toolGroup: 'singleTrace', + limits, + } +} + +function pricingForModel(model: string): CustomTokenPricing { + const pricing = resolveModelPricing(model) + if (!pricing) { + throw new Error( + `no pricing is configured for '${model}'; provide PublicAnalystBenchmarkModelConfig.pricing`, + ) + } + return { + inputUsdPerMillion: pricing.input * 1_000, + outputUsdPerMillion: pricing.output * 1_000, + } +} + +function trajectoryIdFromCaseId(dataset: PublicAnalystBenchmarkDataset, caseId: string): string { + const prefix = dataset === 'agentrx' ? 'agentrx:' : 'codetrace:' + if (!caseId.startsWith(prefix) || caseId.length === prefix.length) { + throw new Error(`unexpected ${dataset} benchmark case id '${caseId}'`) + } + return caseId.slice(prefix.length) +} + +function isPaidCallControlError(error: unknown): boolean { + return ( + error instanceof CostAccountingIncompleteError || + error instanceof CostCallConflictError || + error instanceof CostCeilingReachedError || + error instanceof CostLedgerPersistenceError || + error instanceof CostReceiptCaptureError || + error instanceof CostReservationExceededError + ) +} diff --git a/src/analyst/benchmark-public-types.ts b/src/analyst/benchmark-public-types.ts index f0d1a339..9e8b6637 100644 --- a/src/analyst/benchmark-public-types.ts +++ b/src/analyst/benchmark-public-types.ts @@ -1,4 +1,5 @@ -import type { CostLedgerHandle } from '../cost-ledger' +import type { ExternalOptimizerRunnerCommand } from '../campaign/external-optimizer-contracts' +import type { CostLedgerHandle, CustomTokenPricing } from '../cost-ledger' import type { AnalystBenchmarkCase } from './benchmark' import type { VerificationArtifactManifest } from './benchmark-verification-artifacts' import type { AnalystRunInputs } from './types' @@ -11,6 +12,17 @@ export interface PublicAnalystBenchmarkModelConfig { model: string maxOutputTokens: number timeoutMs: number + /** Required when the model is absent from agent-eval's pricing table. */ + pricing?: CustomTokenPricing + /** Independent per-case recursive-engine spend limit. Default: 1 USD. */ + maxCostUsdPerAnalysis?: number + dspyRlm?: { + runner?: ExternalOptimizerRunnerCommand + maxIterations?: number + maxLlmCalls?: number + maxToolCalls?: number + maxOutputChars?: number + } costLedger?: CostLedgerHandle durability?: { runIdentitySha256: string diff --git a/src/analyst/benchmark-real-model.test.ts b/src/analyst/benchmark-real-model.test.ts index 5405d234..cb56d773 100644 --- a/src/analyst/benchmark-real-model.test.ts +++ b/src/analyst/benchmark-real-model.test.ts @@ -6,12 +6,12 @@ import { CostCallConflictError, CostLedger, type CostLedgerPersistence } from '. import type { TraceAnalysisStore } from '../trace-analyst/store' import type { TraceAnalystSpan } from '../trace-analyst/types' import { - createPublicBenchmarkModelRunner, + createPublicBenchmarkDirectRunner, publicBenchmarkProtocolSha256, } from './benchmark-real-model' import { buildTraceToolsForGroup } from './tool-groups' -describe('createPublicBenchmarkModelRunner', () => { +describe('createPublicBenchmarkDirectRunner', () => { it('makes one bounded structured call and validates the exact cited action', async () => { const traceId = 'trace-1' const action = 'writeFile("broken configuration")' @@ -44,7 +44,7 @@ describe('createPublicBenchmarkModelRunner', () => { ], }) }) as typeof fetch - const runner = createPublicBenchmarkModelRunner('codetracebench', { + const runner = createPublicBenchmarkDirectRunner('codetracebench', { baseUrl: 'https://provider.invalid/v1', apiKey: 'test', model: 'glm-5.2', @@ -68,7 +68,7 @@ describe('createPublicBenchmarkModelRunner', () => { expect(output.error).toBeUndefined() expect(output.findings).toHaveLength(1) expect(output.findings[0]).toMatchObject({ - analyst_id: 'model', + analyst_id: 'direct', subject: 'incorrect-step-2', evidence_refs: [ { @@ -82,7 +82,7 @@ describe('createPublicBenchmarkModelRunner', () => { tokens: { input: 100, output: 50 }, }) expect(output.metadata).toMatchObject({ - analysisMode: 'single-pass', + analysisMode: 'direct-baseline', providerModel: 'glm-5.2', outputAdapter: 'codetracebench-incorrect-step', }) @@ -105,7 +105,7 @@ describe('createPublicBenchmarkModelRunner', () => { ], }), ) as typeof fetch - const runner = createPublicBenchmarkModelRunner('agentrx', { + const runner = createPublicBenchmarkDirectRunner('agentrx', { baseUrl: 'https://provider.invalid/v1', apiKey: 'test', model: 'glm-5.2', @@ -188,17 +188,17 @@ describe('createPublicBenchmarkModelRunner', () => { const context = { caseId: `codetrace:${traceId}`, repetition: 0 } await expect( - createPublicBenchmarkModelRunner('codetracebench', config).analyze(input, context), + createPublicBenchmarkDirectRunner('codetracebench', config).analyze(input, context), ).rejects.toBeInstanceOf(CostCallConflictError) expect(fetchImpl).toHaveBeenCalledTimes(1) expect(ledger.listPending?.()).toHaveLength(1) state.rejectSettlement = false - const resumed = await createPublicBenchmarkModelRunner('codetracebench', config).analyze( + const resumed = await createPublicBenchmarkDirectRunner('codetracebench', config).analyze( input, context, ) - const replayed = await createPublicBenchmarkModelRunner('codetracebench', config).analyze( + const replayed = await createPublicBenchmarkDirectRunner('codetracebench', config).analyze( input, context, ) @@ -255,7 +255,7 @@ describe('createPublicBenchmarkModelRunner', () => { markStarted() return new Promise(() => {}) }) as typeof fetch - const firstRunner = createPublicBenchmarkModelRunner('codetracebench', { + const firstRunner = createPublicBenchmarkDirectRunner('codetracebench', { baseUrl: 'https://provider.invalid/v1', apiKey: 'test', model: 'glm-5.2', @@ -278,7 +278,7 @@ describe('createPublicBenchmarkModelRunner', () => { }) }) as typeof fetch await expect( - createPublicBenchmarkModelRunner('codetracebench', { + createPublicBenchmarkDirectRunner('codetracebench', { baseUrl: 'https://provider.invalid/v1', apiKey: 'test', model: 'glm-5.2', @@ -297,7 +297,7 @@ describe('createPublicBenchmarkModelRunner', () => { it.each([401, 429])('persists HTTP %i without the provider body or bearer', async (status) => { const secret = 'AUDIT_SECRET_456' - const runner = createPublicBenchmarkModelRunner('codetracebench', { + const runner = createPublicBenchmarkDirectRunner('codetracebench', { baseUrl: 'https://provider.invalid/v1', apiKey: secret, model: 'glm-5.2', @@ -331,7 +331,7 @@ describe('createPublicBenchmarkModelRunner', () => { response: () => Promise, spans = [span('trace-1', 'step-1', 'A')], ) => - createPublicBenchmarkModelRunner('codetracebench', { + createPublicBenchmarkDirectRunner('codetracebench', { baseUrl: 'https://provider.invalid/v1', apiKey: 'test', model: 'glm-5.2', diff --git a/src/analyst/benchmark-real-model.ts b/src/analyst/benchmark-real-model.ts index 7b4cfff7..aef951d3 100644 --- a/src/analyst/benchmark-real-model.ts +++ b/src/analyst/benchmark-real-model.ts @@ -11,9 +11,10 @@ export { } from './benchmark-public-data' export { CODE_TRACE_BENCH_ANALYST_PROMPT, - createPublicBenchmarkModelRunner, + createPublicBenchmarkDirectRunner, publicBenchmarkProtocolSha256, } from './benchmark-public-model' +export { createPublicBenchmarkRlmRunner } from './benchmark-public-rlm' export type { PreparedPublicAnalystBenchmark, PublicAnalystBenchmarkDataset, diff --git a/src/analyst/default-registry.test.ts b/src/analyst/default-registry.test.ts index 75b7d8e0..fab84c7e 100644 --- a/src/analyst/default-registry.test.ts +++ b/src/analyst/default-registry.test.ts @@ -3,6 +3,7 @@ import type { TraceAnalysisStore } from '../trace-analyst/store' import type { TraceAnalystSpan } from '../trace-analyst/types' import { behavioralAnalyst } from './behavioral-analyst' import { buildDefaultAnalystRegistry } from './default-registry' +import type { TraceAnalysisEngine } from './engine' function span(over: Partial & { span_id: string }): TraceAnalystSpan { return { @@ -65,8 +66,15 @@ function traceStore( const fakeStore = traceStore(['t1'], async () => ({ trace_id: 't1', spans: SPANS })) -function stubAi() { - return {} as never +function stubEngine(): TraceAnalysisEngine { + return { + id: 'test-engine', + description: 'test', + model: 'test-model', + analyze: async () => { + throw new Error('not called') + }, + } } describe('buildDefaultAnalystRegistry', () => { @@ -77,8 +85,8 @@ describe('buildDefaultAnalystRegistry', () => { expect(ids).toContain('efficiency-behavioral') }) - it('registers the agentic RLM kinds when an ai service is supplied', () => { - const ids = buildDefaultAnalystRegistry({ ai: stubAi(), model: 'test-model' }) + it('registers recursive analysts when an engine is supplied', () => { + const ids = buildDefaultAnalystRegistry({ engine: stubEngine() }) .list() .map((a) => a.id) expect(ids).toContain('efficiency-behavioral') diff --git a/src/analyst/default-registry.ts b/src/analyst/default-registry.ts index e59de0ed..3da077b2 100644 --- a/src/analyst/default-registry.ts +++ b/src/analyst/default-registry.ts @@ -1,46 +1,31 @@ -/** - * `buildDefaultAnalystRegistry` — the canonical analyst suite, so consumers - * stop hand-wiring `new AnalystRegistry()` + per-kind `createTraceAnalystKind`. - * - * The deterministic `behavioralAnalyst` is ALWAYS registered (it needs no - * model and is model-agnostic by construction). The agentic RLM kinds are - * registered only when an `ai` service is supplied — so a caller with no LLM - * still gets the full behavioral/efficiency diagnosis, and the substrate's - * "any model (including no model)" guarantee holds at the suite level. - */ - -import type { AxAIService } from '@ax-llm/ax' import { type BehavioralAnalystOptions, behavioralAnalyst } from './behavioral-analyst' -import { createTraceAnalystKind, type TraceAnalystKindSpec } from './kind-factory' +import type { TraceAnalysisEngine } from './engine' +import { createTraceAnalyst, type TraceAnalystDefinition } from './kind-factory' import { DEFAULT_TRACE_ANALYST_KINDS } from './kinds' import { AnalystRegistry, type AnalystRegistryOptions } from './registry' export interface DefaultAnalystRegistryOptions { - /** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */ - ai?: AxAIService - /** Required unless `ai` was created by `createAnalystAi`. */ - model?: string - /** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */ - kinds?: readonly TraceAnalystKindSpec[] - /** Set false to omit the deterministic behavioral analyst (default: include). */ + /** + * Recursive engine for model-backed trace analysts. When omitted, the + * registry contains only deterministic analysts. + */ + engine?: TraceAnalysisEngine + definitions?: readonly TraceAnalystDefinition[] includeBehavioral?: boolean - /** Bound deterministic trace scanning without changing the store. */ behavioral?: BehavioralAnalystOptions - /** Forwarded to the AnalystRegistry constructor (signal, tags, priorFindings). */ registry?: AnalystRegistryOptions } export function buildDefaultAnalystRegistry( - opts: DefaultAnalystRegistryOptions = {}, + options: DefaultAnalystRegistryOptions = {}, ): AnalystRegistry { - const registry = new AnalystRegistry(opts.registry) - if (opts.includeBehavioral !== false) { - registry.register(behavioralAnalyst(opts.behavioral)) + const registry = new AnalystRegistry(options.registry) + if (options.includeBehavioral !== false) { + registry.register(behavioralAnalyst(options.behavioral)) } - if (opts.ai) { - const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS - for (const spec of kinds) { - registry.register(createTraceAnalystKind(spec, { ai: opts.ai, model: opts.model })) + if (options.engine) { + for (const definition of options.definitions ?? DEFAULT_TRACE_ANALYST_KINDS) { + registry.register(createTraceAnalyst(definition, { engine: options.engine })) } } return registry diff --git a/src/analyst/define.test.ts b/src/analyst/define.test.ts index b0705255..332f74e8 100644 --- a/src/analyst/define.test.ts +++ b/src/analyst/define.test.ts @@ -2,48 +2,39 @@ import { describe, expect, it } from 'vitest' import { defineTraceAnalyst } from './define' describe('defineTraceAnalyst', () => { - it('fills the fixed trace-store fields and preserves the declared cost', () => { - const analyst = defineTraceAnalyst({ + it('defines an engine-independent research question', () => { + const definition = defineTraceAnalyst({ id: 'failed-tools', description: 'Find failed tool calls.', - cost: { kind: 'deterministic' }, - async analyze() { - return [] - }, + area: 'tool-use', + version: '1.0.0', + question: 'Why are tools failing?', + instructions: 'Inspect failures and cite exact spans.', + toolGroup: 'discoveryAndSearch', + limits: { maxIterations: 6 }, }) - expect(analyst).toMatchObject({ + + expect(definition).toMatchObject({ id: 'failed-tools', - inputKind: 'trace-store', - version: '1.0.0', - cost: { kind: 'deterministic' }, + area: 'tool-use', + question: 'Why are tools failing?', + toolGroup: 'discoveryAndSearch', + limits: { maxIterations: 6 }, }) + expect(definition).not.toHaveProperty('engine') + expect(definition).not.toHaveProperty('cost') }) - it('rejects empty identity fields before registration', () => { + it('rejects empty identity fields', () => { expect(() => defineTraceAnalyst({ id: '', description: 'x', - cost: { kind: 'deterministic' }, - async analyze() { - return [] - }, + area: 'x', + version: '1.0.0', + instructions: 'x', + toolGroup: 'all', }), - ).toThrow(/id must not be empty/) - }) - - it('rejects a missing cost declaration from JavaScript callers', () => { - const missingCost = { - id: 'model-backed', - description: 'Calls a model.', - async analyze() { - return [] - }, - } - - expect(() => { - // @ts-expect-error JavaScript callers can omit a TypeScript-required property. - defineTraceAnalyst(missingCost) - }).toThrow(/cost must be declared/) + ).toThrow(/id must be a non-empty string/) }) }) diff --git a/src/analyst/define.ts b/src/analyst/define.ts index 7aaab949..70e3fa46 100644 --- a/src/analyst/define.ts +++ b/src/analyst/define.ts @@ -1,36 +1,26 @@ -import type { TraceAnalysisStore } from '../trace-analyst/store' -import type { Analyst, AnalystCost, AnalystFinding } from './types' +import type { TraceAnalystDefinition } from './kind-factory' -export interface DefineTraceAnalystOptions { - id: string - description: string - version?: string - cost: AnalystCost - analyze: Analyst['analyze'] -} - -/** Define a custom trace analyst without repeating fixed registry fields. */ -export function defineTraceAnalyst( - options: DefineTraceAnalystOptions, -): Analyst { - if (!options.id.trim()) throw new TypeError('defineTraceAnalyst: id must not be empty') - if (!options.description.trim()) { - throw new TypeError('defineTraceAnalyst: description must not be empty') - } - if (options.cost === undefined) { - throw new TypeError('defineTraceAnalyst: cost must be declared') +/** + * Define a reusable trace-research question. + * + * The returned value contains no model, credentials, or execution state. Bind + * it to any TraceAnalysisEngine with `runTraceAnalyst` or + * `createTraceAnalyst`. + */ +export function defineTraceAnalyst(definition: TraceAnalystDefinition): TraceAnalystDefinition { + for (const [name, value] of [ + ['id', definition.id], + ['description', definition.description], + ['area', definition.area], + ['version', definition.version], + ['instructions', definition.instructions], + ] as const) { + if (typeof value !== 'string' || !value.trim()) { + throw new TypeError(`defineTraceAnalyst: ${name} must be a non-empty string`) + } } return { - id: options.id, - description: options.description, - version: options.version ?? '1.0.0', - inputKind: 'trace-store', - cost: options.cost, - analyze: options.analyze, + ...definition, + limits: definition.limits ? { ...definition.limits } : undefined, } } - -export type TraceAnalystAnalyze = ( - store: TraceAnalysisStore, - context: Parameters['analyze']>[1], -) => Promise diff --git a/src/analyst/dspy-rlm-engine.test.ts b/src/analyst/dspy-rlm-engine.test.ts new file mode 100644 index 00000000..a5144cce --- /dev/null +++ b/src/analyst/dspy-rlm-engine.test.ts @@ -0,0 +1,160 @@ +import { createServer } from 'node:http' +import { describe, expect, it } from 'vitest' +import { CostLedger } from '../cost-ledger' +import type { TraceAnalysisToolDescriptor } from '../trace-analyst/tools' +import { createDspyRlmTraceEngine } from './dspy-rlm-engine' + +const CHILD_SCRIPT = ` +const fs = require('node:fs') +const inputPath = process.argv[process.argv.indexOf('--input') + 1] +const outputPath = process.argv[process.argv.indexOf('--output') + 1] +const input = JSON.parse(fs.readFileSync(inputPath, 'utf8')) +async function main() { + const model = await fetch(input.modelProxy.baseUrl + '/chat/completions', { + method: 'POST', + headers: { + authorization: 'Bearer ' + input.modelProxy.apiKey, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + model: input.modelProxy.model, + messages: [{ role: 'user', content: input.question }], + max_tokens: input.modelProxy.maxOutputTokens, + }), + }) + if (!model.ok) throw new Error('model proxy failed: ' + await model.text()) + await model.json() + const tool = await fetch(input.toolCallback.url, { + method: 'POST', + headers: { + authorization: 'Bearer ' + input.toolCallback.token, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name: 'getDatasetOverview', args: {} }), + }) + if (!tool.ok) throw new Error('trace callback failed: ' + await tool.text()) + const toolResult = await tool.json() + fs.writeFileSync(outputPath, JSON.stringify({ + answer: 'The run failed after one inspected trace.', + findings: [], + trajectory: [{ tool: 'getDatasetOverview', result: toolResult.result }], + modelCalls: 1, + runtime: { engine: 'test-bridge' }, + })) +} +main().catch((error) => { + console.error(error) + process.exit(1) +}) +` + +describe('createDspyRlmTraceEngine', () => { + it('runs a child through authenticated model and trace callbacks', async () => { + let providerAuthorization = '' + const provider = createServer((request, response) => { + providerAuthorization = request.headers.authorization ?? '' + response.writeHead(200, { 'content-type': 'application/json' }) + response.end( + JSON.stringify({ + choices: [{ message: { role: 'assistant', content: 'investigate' } }], + usage: { prompt_tokens: 3, completion_tokens: 2 }, + }), + ) + }) + await new Promise((resolve, reject) => { + provider.once('error', reject) + provider.listen(0, '127.0.0.1', () => { + provider.off('error', reject) + resolve() + }) + }) + const address = provider.address() + if (!address || typeof address === 'string') throw new Error('provider did not bind') + + const ledger = new CostLedger() + let toolExecutions = 0 + const tool: TraceAnalysisToolDescriptor = { + namespace: 'traces', + name: 'getDatasetOverview', + description: 'Return a summary.', + parameters: { type: 'object', properties: {}, additionalProperties: false }, + handler: async () => { + toolExecutions += 1 + return { total_traces: 1 } + }, + } + const engine = createDspyRlmTraceEngine({ + baseUrl: `http://127.0.0.1:${address.port}/v1`, + apiKey: 'provider-secret', + model: 'model-a', + pricing: { + inputUsdPerMillion: 1, + outputUsdPerMillion: 2, + }, + maxCostUsd: 0.1, + maxOutputTokens: 64, + runner: { + command: process.execPath, + args: ['-e', CHILD_SCRIPT, '--'], + }, + timeoutMs: 5_000, + }) + + try { + const result = await engine.analyze({ + analystId: 'failure-mode', + question: 'What failed?', + instructions: 'Inspect the trace.', + tools: [tool], + limits: { + maxIterations: 2, + maxLlmCalls: 1, + maxToolCalls: 1, + maxOutputChars: 1_000, + }, + costLedger: ledger, + costPhase: 'analyst.test', + }) + + expect(result).toMatchObject({ + answer: 'The run failed after one inspected trace.', + findings: [], + modelCalls: 1, + toolCalls: 1, + runtime: { + engine: 'test-bridge', + modelRequestAttempts: 1, + modelSuccessfulCompletions: 1, + }, + }) + expect(toolExecutions).toBe(1) + expect(providerAuthorization).toBe('Bearer provider-secret') + expect(ledger.list()).toEqual([ + expect.objectContaining({ + channel: 'analyst', + phase: 'analyst.test', + actor: 'failure-mode', + model: 'model-a', + inputTokens: 3, + outputTokens: 2, + costUnknown: false, + usageUnknown: false, + }), + ]) + } finally { + await new Promise((resolve, reject) => { + provider.close((error) => (error ? reject(error) : resolve())) + }) + } + }) + + it('requires explicit pricing for an unknown model', () => { + expect(() => + createDspyRlmTraceEngine({ + baseUrl: 'https://provider.example/v1', + apiKey: 'secret', + model: 'unknown-model-with-no-pricing', + }), + ).toThrow("no pricing is configured for 'unknown-model-with-no-pricing'") + }) +}) diff --git a/src/analyst/dspy-rlm-engine.ts b/src/analyst/dspy-rlm-engine.ts new file mode 100644 index 00000000..26f392c7 --- /dev/null +++ b/src/analyst/dspy-rlm-engine.ts @@ -0,0 +1,247 @@ +import { + type ExternalOptimizerModelProxy, + type ExternalOptimizerRunnerCommand, + removeCredentialEnvironment, +} from '../campaign/external-optimizer-contracts' +import { startExternalOptimizerModelProxy } from '../campaign/external-optimizer-model-proxy' +import { runWithCleanup } from '../campaign/external-optimizer-resources' +import { runExternalOptimizerProcess } from '../campaign/external-optimizer-subprocess' +import type { CustomTokenPricing } from '../cost-ledger' +import { resolveModelPricing } from '../metrics' +import type { TraceAnalysisEngine, TraceAnalysisEngineResult } from './engine' +import { type RawAnalystFinding, RawAnalystFindingSchema } from './finding-signature' +import { startTraceToolCallback } from './trace-tool-callback' + +const DEFAULT_TIMEOUT_MS = 10 * 60_000 +const DEFAULT_MODEL_OUTPUT_TOKENS = 4_096 +const DEFAULT_MAX_COST_USD = 1 +const MAX_MODEL_REQUEST_BYTES = 16 * 1024 * 1024 +const MAX_MODEL_RESPONSE_BYTES = 4 * 1024 * 1024 +const BRIDGE_MODULE = 'agent_eval_rpc.dspy_rlm_bridge' + +export interface DspyRlmTraceEngineOptions { + baseUrl: string + apiKey: string + model: string + /** Exact provider rates. Required when the model is absent from the pricing table. */ + pricing?: CustomTokenPricing + /** Maximum provider spend for one investigation. Default: 1 USD. */ + maxCostUsd?: number + /** Controller response cap. Default: 4096. */ + maxOutputTokens?: number + /** Python command used to load agent-eval-rpc[dspy]. Default: python. */ + runner?: ExternalOptimizerRunnerCommand + /** Whole investigation deadline. Default: 10 minutes. */ + timeoutMs?: number +} + +/** Use the official DSPy RLM as a bounded recursive trace-analysis engine. */ +export function createDspyRlmTraceEngine(options: DspyRlmTraceEngineOptions): TraceAnalysisEngine { + assertOptions(options) + const maxOutputTokens = options.maxOutputTokens ?? DEFAULT_MODEL_OUTPUT_TOKENS + if (!Number.isSafeInteger(maxOutputTokens) || maxOutputTokens <= 0) { + throw new TypeError('DSPy RLM maxOutputTokens must be a positive safe integer') + } + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { + throw new TypeError('DSPy RLM timeoutMs must be a positive safe integer') + } + const maxCostUsd = options.maxCostUsd ?? DEFAULT_MAX_COST_USD + if (!Number.isFinite(maxCostUsd) || maxCostUsd <= 0) { + throw new TypeError('DSPy RLM maxCostUsd must be positive and finite') + } + const pricing = options.pricing ?? pricingForModel(options.model) + + return { + id: 'dspy-rlm', + description: 'Official DSPy RLM with bounded trace tools and metered model calls.', + model: options.model, + async analyze(request) { + const callback = await startTraceToolCallback({ + tools: request.tools, + maxCalls: request.limits.maxToolCalls, + ...(request.signal ? { signal: request.signal } : {}), + }) + let modelProxy: ExternalOptimizerModelProxy | undefined + const result = await runWithCleanup({ + label: 'DSPy RLM trace-analysis resources', + run: async () => { + modelProxy = await startExternalOptimizerModelProxy({ + upstreamBaseUrl: options.baseUrl, + upstreamApiKey: options.apiKey, + model: options.model, + budget: { + maxCostUsd, + maxRequests: request.limits.maxIterations + request.limits.maxLlmCalls + 1, + maxRequestBytes: MAX_MODEL_REQUEST_BYTES, + maxResponseBytes: MAX_MODEL_RESPONSE_BYTES, + maxOutputTokensPerRequest: maxOutputTokens, + requestTimeoutMs: timeoutMs, + pricing, + }, + costLedger: request.costLedger, + channel: 'analyst', + phase: request.costPhase, + actor: request.analystId, + ...(request.costTags ? { tags: request.costTags } : {}), + ...(request.signal ? { signal: request.signal } : {}), + }) + request.log?.('trace analyst engine started', { + engine: 'dspy-rlm', + model: options.model, + tools: request.tools.map((tool) => tool.name), + limits: request.limits, + }) + const runner = sanitizedRunner(options.runner) + const raw = await runExternalOptimizerProcess({ + label: 'DSPy RLM trace analysis', + tempPrefix: 'agent-eval-dspy-rlm-', + module: BRIDGE_MODULE, + input: { + operation: 'analyze', + question: request.question, + instructions: request.instructions, + modelProxy: { + baseUrl: modelProxy.baseUrl, + apiKey: modelProxy.apiKey, + model: options.model, + maxOutputTokens, + }, + toolCallback: { + url: callback.url, + token: callback.token, + }, + toolSpecs: request.tools.map(({ name, description, parameters }) => ({ + name, + description, + parameters, + })), + limits: { + maxIterations: request.limits.maxIterations, + maxLlmCalls: request.limits.maxLlmCalls, + maxOutputChars: request.limits.maxOutputChars, + }, + }, + ...(runner ? { runner } : {}), + timeoutMs, + ...(request.signal ? { signal: request.signal } : {}), + }) + const parsed = parseBridgeOutput(raw) + const successfulCompletions = modelProxy.successfulCompletions() + const requestAttempts = modelProxy.requestAttempts() + if (parsed.modelCalls !== successfulCompletions) { + throw new Error( + `DSPy RLM reported ${parsed.modelCalls} model calls, but the provider proxy recorded ${successfulCompletions}`, + ) + } + return { + ...parsed, + toolCalls: callback.calls(), + runtime: { + ...parsed.runtime, + modelRequestAttempts: requestAttempts, + modelSuccessfulCompletions: successfulCompletions, + }, + } satisfies TraceAnalysisEngineResult + }, + cleanup: async () => { + const results = await Promise.allSettled([ + ...(modelProxy ? [modelProxy.close()] : []), + callback.close(), + ]) + const errors = results.flatMap((entry) => + entry.status === 'rejected' ? [entry.reason] : [], + ) + if (errors.length > 0) { + throw new AggregateError(errors, 'DSPy RLM resource cleanup failed') + } + }, + }) + request.log?.('trace analyst engine completed', { + engine: 'dspy-rlm', + model_calls: result.modelCalls, + model_request_attempts: result.runtime.modelRequestAttempts, + tool_calls: result.toolCalls, + findings: result.findings.length, + }) + return result + }, + } +} + +function parseBridgeOutput(value: unknown): Omit { + if (!isRecord(value)) throw new Error('DSPy RLM bridge output must be an object') + if (typeof value.answer !== 'string' || !value.answer.trim()) { + throw new Error('DSPy RLM bridge returned no answer') + } + if (!Array.isArray(value.findings)) { + throw new Error('DSPy RLM bridge findings must be an array') + } + const findings: RawAnalystFinding[] = value.findings.map((finding, index) => { + const parsed = RawAnalystFindingSchema.safeParse(finding) + if (!parsed.success) { + throw new Error( + `DSPy RLM bridge finding ${index} is invalid: ${parsed.error.issues + .map((issue) => `${issue.path.join('.')}: ${issue.message}`) + .join('; ')}`, + ) + } + return parsed.data + }) + if (!Array.isArray(value.trajectory)) { + throw new Error('DSPy RLM bridge trajectory must be an array') + } + if (!Number.isSafeInteger(value.modelCalls) || (value.modelCalls as number) <= 0) { + throw new Error('DSPy RLM bridge modelCalls must be a positive safe integer') + } + if (!isRecord(value.runtime)) { + throw new Error('DSPy RLM bridge runtime must be an object') + } + return { + answer: value.answer, + findings, + trajectory: value.trajectory, + modelCalls: value.modelCalls as number, + runtime: value.runtime, + } +} + +function assertOptions(options: DspyRlmTraceEngineOptions): void { + for (const [name, value] of [ + ['baseUrl', options.baseUrl], + ['apiKey', options.apiKey], + ['model', options.model], + ] as const) { + if (typeof value !== 'string' || !value.trim()) { + throw new TypeError(`DSPy RLM ${name} must be a non-empty string`) + } + } +} + +function pricingForModel(model: string): CustomTokenPricing { + const pricing = resolveModelPricing(model) + if (!pricing) { + throw new Error( + `no pricing is configured for '${model}'; provide DspyRlmTraceEngineOptions.pricing`, + ) + } + return { + inputUsdPerMillion: pricing.input * 1_000, + outputUsdPerMillion: pricing.output * 1_000, + } +} + +function sanitizedRunner( + runner: ExternalOptimizerRunnerCommand | undefined, +): ExternalOptimizerRunnerCommand | undefined { + if (!runner) return undefined + return { + ...(runner.command ? { command: runner.command } : {}), + ...(runner.args ? { args: runner.args } : {}), + ...(runner.env ? { env: removeCredentialEnvironment(runner.env) } : {}), + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/src/analyst/engine.ts b/src/analyst/engine.ts new file mode 100644 index 00000000..7544816c --- /dev/null +++ b/src/analyst/engine.ts @@ -0,0 +1,77 @@ +import type { CostLedgerHandle } from '../cost-ledger' +import type { TraceAnalysisToolDescriptor } from '../trace-analyst/tools' +import type { RawAnalystFinding } from './finding-signature' + +/** Hard limits for one recursive trace investigation. */ +export interface TraceAnalystLimits { + /** Maximum controller iterations. */ + maxIterations: number + /** Maximum recursive model queries issued from the controller program. */ + maxLlmCalls: number + /** Maximum trace-store reads. */ + maxToolCalls: number + /** Maximum controller output retained between iterations. */ + maxOutputChars: number +} + +export const DEFAULT_TRACE_ANALYST_LIMITS: Readonly = { + maxIterations: 12, + maxLlmCalls: 8, + maxToolCalls: 48, + maxOutputChars: 10_000, +} + +export interface TraceAnalysisEngineRequest { + analystId: string + question: string + instructions: string + tools: readonly TraceAnalysisToolDescriptor[] + limits: TraceAnalystLimits + costLedger: CostLedgerHandle + costPhase: string + costTags?: Record + signal?: AbortSignal + log?: (message: string, fields?: Record) => void +} + +/** Complete result of one recursive investigation, before registry adaptation. */ +export interface TraceAnalysisEngineResult { + answer: string + findings: RawAnalystFinding[] + /** Engine-native steps retained for audit and debugging. */ + trajectory: readonly unknown[] + /** + * Successful model completions used by the engine. + * The usage receipt may contain more provider attempts when a request fails. + */ + modelCalls: number + /** Trace-tool requests admitted for execution, including a tool that later fails. */ + toolCalls: number + runtime: Record +} + +/** + * A recursive research engine that can inspect traces with bounded tools. + * + * Implementations may use DSPy RLM, HALO, another upstream engine, or a + * caller-owned implementation. The analyst definition and finding contract do + * not change when the engine changes. + */ +export interface TraceAnalysisEngine { + readonly id: string + readonly description: string + readonly model?: string + analyze(request: TraceAnalysisEngineRequest): Promise +} + +export function resolveTraceAnalystLimits( + limits: Partial | undefined, +): TraceAnalystLimits { + const resolved = { ...DEFAULT_TRACE_ANALYST_LIMITS, ...limits } + for (const [name, value] of Object.entries(resolved)) { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new TypeError(`trace analyst ${name} must be a positive safe integer`) + } + } + return resolved +} diff --git a/src/analyst/finding-signature.ts b/src/analyst/finding-signature.ts index 0905a8a5..7a633585 100644 --- a/src/analyst/finding-signature.ts +++ b/src/analyst/finding-signature.ts @@ -1,15 +1,8 @@ /** - * Typed Ax output for analyst findings. + * Engine-neutral structured output for trace-analyst findings. * - * Ax binds the field as `findings:json[]` so the provider emits native - * structured output. At the kind-factory boundary every row is validated - * before it becomes an `AnalystFinding`. - * - * Why not `f.object().array()` directly in the signature? The Ax - * signature string `question:string -> findings:json[]` already lets - * the provider emit JSON arrays. A Zod boundary is required either - * way (the provider can return any JSON), and Zod gives us a single - * validation surface independent of which Ax version is installed. + * Every recursive engine returns this shape. The TypeScript boundary validates + * it before a finding can enter a registry or benchmark. */ import { z } from 'zod' @@ -61,7 +54,7 @@ export const RAW_FINDING_SCHEMA_PROMPT = `Each finding MUST be a strict JSON obj - severity: "critical" | "high" | "medium" | "low" | "info" - claim: one-sentence statement (max 2000 chars) - subject?: one exact subject form listed by this kind; omit rather than guess - - evidence: REQUIRED non-empty array of {"uri": string, "excerpt"?: string}. Use real identifiers with span://, event://, artifact://, metric://, or finding://. Include a short exact quote in excerpt when available. If nothing is citable, do not emit the finding. + - evidence: REQUIRED non-empty array of {"uri": string, "excerpt"?: string}. Use trace:///span/ for trace evidence or finding:// for supplied prior findings. URL encoding means percent encoding, never base64. Include a short exact quote in excerpt when available. If nothing is citable, do not emit the finding. - confidence: number 0..1 (0.9+ exact evidence; 0.6-0.8 inferred pattern; <0.5 speculative) - rationale?: one or two reasoning sentences - recommended_action?: concrete imperative change; omit for descriptive findings @@ -111,10 +104,19 @@ function parseFindingWithSchema( } function evidenceKindFromUri(uri: string): EvidenceRef['kind'] { - if (uri.startsWith('span://')) return 'span' - if (/^trace:\/\/[^/]+\/span\/[^/]+$/.test(uri)) return 'span' - if (uri.startsWith('event://')) return 'event' + if (parseTraceSpanEvidenceUri(uri)) return 'span' if (uri.startsWith('finding://')) return 'finding' - if (uri.startsWith('metric://')) return 'metric' return 'artifact' } + +export function parseTraceSpanEvidenceUri(uri: string): { traceId: string; spanId: string } | null { + const match = /^trace:\/\/([^/]+)\/span\/([^/]+)$/.exec(uri) + if (!match) return null + try { + const traceId = decodeURIComponent(match[1]!) + const spanId = decodeURIComponent(match[2]!) + return traceId && spanId ? { traceId, spanId } : null + } catch { + return null + } +} diff --git a/src/analyst/finding-subject.test.ts b/src/analyst/finding-subject.test.ts index 5f6e3aa6..f8408b40 100644 --- a/src/analyst/finding-subject.test.ts +++ b/src/analyst/finding-subject.test.ts @@ -339,7 +339,7 @@ describe('KIND_EXPECTED_SUBJECTS', () => { ['knowledge-poisoning', KNOWLEDGE_POISONING_KIND_SPEC], ])('%s actor prompt embeds its complete allowed grammar', (kindId, spec) => { const grammar = findingSubjectGrammarPromptFor(kindId) - expect(spec.actorDescription).toContain(grammar) + expect(spec.instructions).toContain(grammar) for (const subjectKind of KIND_EXPECTED_SUBJECTS[kindId] ?? []) { expect(grammar).toContain(FINDING_SUBJECT_SYNTAX[subjectKind]) } diff --git a/src/analyst/index.ts b/src/analyst/index.ts index f7347b43..0f141cd1 100644 --- a/src/analyst/index.ts +++ b/src/analyst/index.ts @@ -19,7 +19,6 @@ export { createVerifierAdapter, liftSeverity, } from './adapters' -export { type CreateAnalystAiConfig, createAnalystAi } from './ax-service' export type { BehavioralAnalystOptions } from './behavioral-analyst' export { behavioralAnalyst, deriveEfficiencyFindings } from './behavioral-analyst' export type { @@ -139,7 +138,8 @@ export type { export { adaptPublicBenchmarkFindings, CODE_TRACE_BENCH_ANALYST_PROMPT, - createPublicBenchmarkModelRunner, + createPublicBenchmarkDirectRunner, + createPublicBenchmarkRlmRunner, emptyPublicBenchmarkRunner, loadPublicBenchmarkRows, preparePublicAnalystBenchmark, @@ -186,8 +186,19 @@ export { buildDefaultAnalystRegistry, type DefaultAnalystRegistryOptions, } from './default-registry' -export type { DefineTraceAnalystOptions, TraceAnalystAnalyze } from './define' export { defineTraceAnalyst } from './define' +export type { DspyRlmTraceEngineOptions } from './dspy-rlm-engine' +export { createDspyRlmTraceEngine } from './dspy-rlm-engine' +export type { + TraceAnalysisEngine, + TraceAnalysisEngineRequest, + TraceAnalysisEngineResult, + TraceAnalystLimits, +} from './engine' +export { + DEFAULT_TRACE_ANALYST_LIMITS, + resolveTraceAnalystLimits, +} from './engine' export type { RawAnalystEvidence, RawAnalystFinding, @@ -214,10 +225,15 @@ export { export type { DiffPolicy, FindingsDiff, PersistedFinding } from './findings-store' export { defaultIsMaterial, diffFindings, FindingsStore } from './findings-store' export type { - CreateTraceAnalystKindOpts, - TraceAnalystKindSpec, + CreateTraceAnalystOptions, + TraceAnalystDefinition, +} from './kind-factory' +export { + createTraceAnalyst, + renderPriorFindings, + renderUpstreamFindings, + runTraceAnalyst, } from './kind-factory' -export { createTraceAnalystKind, renderPriorFindings, renderUpstreamFindings } from './kind-factory' export { DEFAULT_TRACE_ANALYST_KINDS, FAILURE_MODE_KIND_SPEC, @@ -250,11 +266,6 @@ export type { RegistryRunOpts, } from './registry' export { AnalystRegistry } from './registry' -export { - type StructureFindingsOptions, - type StructureFindingsResult, - structureFindings, -} from './structure-findings' export type { TraceToolGroupName } from './tool-groups' export { buildTraceToolsForGroup } from './tool-groups' export type { diff --git a/src/analyst/kind-factory-ax-contract.test.ts b/src/analyst/kind-factory-ax-contract.test.ts deleted file mode 100644 index e1801e83..00000000 --- a/src/analyst/kind-factory-ax-contract.test.ts +++ /dev/null @@ -1,781 +0,0 @@ -import type { AxAIService } from '@ax-llm/ax' -import { afterEach, describe, expect, it, vi } from 'vitest' -import { CostLedger } from '../cost-ledger' -import { createAnalystAi } from './ax-service' -import { RAW_FINDING_SCHEMA_PROMPT, RawAnalystFindingSchema } from './finding-signature' -import { createTraceAnalystKind, type TraceAnalystKindSpec } from './kind-factory' -import { DEFAULT_TRACE_ANALYST_KINDS } from './kinds' -import { IMPROVEMENT_KIND_SPEC } from './kinds/improvement' -import { AnalystRegistry } from './registry' -import { type AnalystUsageReceipt, makeFinding } from './types' - -const axMock = vi.hoisted(() => ({ - agentCalls: [] as Array<{ signature: string; options: Record }>, - forwardResult: { report: '', findings: [] as unknown[] } as unknown, - events: [] as string[], -})) - -vi.mock('@ax-llm/ax', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - AxJSRuntime: class { - readonly options: unknown - constructor(options?: unknown) { - this.options = options - } - }, - agent: (signature: string, options: Record) => { - if ( - options.functions !== undefined && - !( - typeof (options.functions as { [Symbol.iterator]?: unknown })[Symbol.iterator] === - 'function' - ) - ) { - throw new TypeError('functions must be iterable') - } - axMock.agentCalls.push({ signature, options }) - return { - async forward(ai: { chat(request: unknown): Promise }) { - axMock.events.push('run') - await ai.chat({ - model: 'gpt-4o-mini', - chatPrompt: [{ role: 'user', content: 'actor' }], - }) - return axMock.forwardResult - }, - getUsage() { - return { actor: [], responder: [] } - }, - getChatLog() { - return [] - }, - } - }, - } -}) - -afterEach(() => { - axMock.forwardResult = { report: '', findings: [] } -}) - -function testAi( - chat = vi.fn(async () => ({ - results: [], - modelUsage: { - ai: 'openai', - model: 'gpt-4o-mini', - tokens: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, - }, - })), -): AxAIService { - const service = createAnalystAi({ - apiKey: 'test', - baseUrl: 'https://provider.invalid/v1', - model: 'gpt-4o-mini', - }) as unknown as { chat: typeof chat } - service.chat = chat - return service as unknown as AxAIService -} - -describe('createTraceAnalystKind Ax contract', () => { - it.each([ - ['recursion', { maxDepth: 2 }, 'subqueries'], - ['responderDescription', 'old prompt', 'actorDescription'], - ['maxDepth', 2, 'subqueries'], - ['maxParallelSubagents', 3, 'subqueries.maxParallel'], - ['subagentDescription', 'old prompt', 'actorDescription'], - ])( - 'rejects removed kind option %s instead of silently using defaults', - (name, value, replacement) => { - expect(() => - createTraceAnalystKind({ ...testSpec(), [name]: value } as never, { ai: testAi() }), - ).toThrow(`'${name}' is unsupported; use '${replacement}'`) - expect(axMock.agentCalls).toHaveLength(0) - }, - ) - - it('requires an explicit model for opaque Ax services before paid work', () => { - const chat = vi.fn() - - expect(() => createTraceAnalystKind(testSpec(), { ai: { chat } as never })).toThrow( - /model is required.*createAnalystAi/, - ) - expect(chat).not.toHaveBeenCalled() - }) - - it('passes trace tools as the iterable Ax functions shape', async () => { - axMock.agentCalls.length = 0 - const tool = { namespace: 'traces', name: 'getDatasetOverview' } - const spec: TraceAnalystKindSpec = { - id: 'failure-mode', - description: 'Find trace failures', - area: 'failure-mode', - version: '0.0.1', - actorDescription: 'Return findings.', - buildTools: () => [tool as never], - cost: { kind: 'llm' }, - } - const analyst = createTraceAnalystKind(spec, { ai: testAi() }) - - let receipt: AnalystUsageReceipt | undefined - const findings = await analyst.analyze( - {} as never, - { tags: {}, recordUsage: (usage: AnalystUsageReceipt) => (receipt = usage) } as never, - ) - - expect(findings).toEqual([]) - expect(receipt).toEqual({ - calls: 1, - tokens: { input: 1, output: 1 }, - cost: { kind: 'estimated', usd: expect.any(Number) }, - }) - expect(axMock.agentCalls).toHaveLength(1) - const functions = Array.from( - axMock.agentCalls[0]!.options.functions as Iterable>, - ) - expect(functions).toHaveLength(1) - expect(functions[0]).toBe(tool) - }) - - it('uses prepared context instead of exposing tools to the model', async () => { - axMock.agentCalls.length = 0 - const buildTools = vi.fn(() => [{ namespace: 'traces', name: 'getDatasetOverview' } as never]) - const analyst = createTraceAnalystKind( - { - ...testSpec(), - buildTools, - prepareContext: async () => - JSON.stringify({ trace_id: 'trace-1', spans: [{ span_id: 'step-2' }] }), - }, - { ai: testAi() }, - ) - - await analyst.analyze({} as never, { tags: {} } as never) - - expect(buildTools).not.toHaveBeenCalled() - expect(axMock.agentCalls).toHaveLength(1) - expect(axMock.agentCalls[0]).toMatchObject({ - signature: 'context:string, question:string -> report:string, findings:json[]', - options: { - contextFields: ['context'], - directResponse: 'auto', - functions: [], - }, - }) - }) - - it('writes provider calls to a shared ledger without counting unrelated calls', async () => { - axMock.forwardResult = { report: '', findings: [] } - const costLedger = new CostLedger() - await costLedger.runPaidCall({ - channel: 'analyst', - phase: 'search.proposal', - actor: 'other-analyst', - model: 'gpt-4o-mini', - tags: { analystId: 'other', analystRunId: 'ar_other' }, - execute: async () => 'ok', - receipt: () => ({ model: 'gpt-4o-mini', inputTokens: 10, outputTokens: 2 }), - }) - const analyst = createTraceAnalystKind(testSpec(), { ai: testAi() }) - let receipt: AnalystUsageReceipt | undefined - - await analyst.analyze( - {} as never, - { - correlationId: 'ar_shared', - costLedger, - costPhase: 'search.proposal', - recordUsage: (usage: AnalystUsageReceipt) => (receipt = usage), - } as never, - ) - - expect(receipt).toMatchObject({ calls: 1, tokens: { input: 1, output: 1 } }) - expect(costLedger.list({ tags: { analystRunId: 'ar_shared' } })).toHaveLength(1) - expect(costLedger.list()).toHaveLength(2) - }) - - it('assembles one canonical output contract for every shipped kind', async () => { - axMock.agentCalls.length = 0 - axMock.forwardResult = { report: '', findings: [] } - - for (const shipped of DEFAULT_TRACE_ANALYST_KINDS) { - const analyst = createTraceAnalystKind({ ...shipped, buildTools: () => [] }, { ai: testAi() }) - await analyst.analyze({} as never, { tags: {} } as never) - } - - expect(axMock.agentCalls).toHaveLength(DEFAULT_TRACE_ANALYST_KINDS.length) - for (const call of axMock.agentCalls) { - expect(call.signature).toBe('question:string -> report:string, findings:json[]') - const actor = call.options.executorOptions as { description: string } - const responder = call.options.responderOptions as { description: string } - expect(actor.description.split(RAW_FINDING_SCHEMA_PROMPT)).toHaveLength(2) - expect(actor.description).toContain('call final(task, evidence)') - expect(responder.description.split(RAW_FINDING_SCHEMA_PROMPT)).toHaveLength(2) - expect(responder.description).toContain('Use only evidence produced by the analysis stage') - expect(call.options.directResponse).toBe('off') - expect(actor.description).not.toMatch(/`area`\s*=/) - expect(actor.description).not.toContain('evidence_uri') - expect(actor.description).not.toContain('submitAnalysis') - } - const poisoningActor = axMock.agentCalls[2]!.options.executorOptions as { - description: string - } - expect(poisoningActor.description).toContain( - 'requires at least 2 evidence citations per finding', - ) - }) - it('lifts every citation and derives finding/event evidence kinds', async () => { - axMock.agentCalls.length = 0 - axMock.forwardResult = { - report: '', - findings: [ - { - severity: 'high', - claim: 'upstream diagnosis and trace event jointly support the change', - evidence: [ - { uri: 'finding://failure-123', excerpt: 'failure cluster' }, - { uri: 'event://trace-1/event-8', excerpt: 'runtime contradiction' }, - { uri: 'trace://trace-1/span/step-4', excerpt: 'benchmark step' }, - ], - confidence: 0.9, - }, - ], - } - const analyst = createTraceAnalystKind(testSpec(), { ai: testAi() }) - - const findings = await analyst.analyze({} as never, { tags: {} } as never) - - expect(findings).toHaveLength(1) - expect(findings[0]!.evidence_refs).toEqual([ - { kind: 'finding', uri: 'finding://failure-123', excerpt: 'failure cluster' }, - { kind: 'event', uri: 'event://trace-1/event-8', excerpt: 'runtime contradiction' }, - { kind: 'span', uri: 'trace://trace-1/span/step-4', excerpt: 'benchmark step' }, - ]) - }) - - it('passes the full evidence list to postProcess', async () => { - axMock.agentCalls.length = 0 - axMock.forwardResult = { - report: '', - findings: [ - { - severity: 'high', - claim: 'original claim', - evidence: [ - { uri: 'span://trace/action', excerpt: 'primary' }, - { uri: 'event://trace/result', excerpt: 'corroborating' }, - ], - confidence: 0.9, - }, - ], - } - const analyst = createTraceAnalystKind( - { - ...testSpec(), - postProcess: (row) => { - const exactRow = RawAnalystFindingSchema.parse(row) - return { - ...exactRow, - claim: `${exactRow.claim}; checked ${exactRow.evidence[0]?.uri}`, - } - }, - }, - { ai: testAi() }, - ) - - const findings = await analyst.analyze({} as never, { tags: {} } as never) - - expect(findings).toHaveLength(1) - expect(findings[0]).toMatchObject({ - claim: 'original claim; checked span://trace/action', - evidence_refs: [ - { kind: 'span', uri: 'span://trace/action', excerpt: 'primary' }, - { kind: 'event', uri: 'event://trace/result', excerpt: 'corroborating' }, - ], - }) - }) - - it('lets postProcess replace one citation without losing the rest', async () => { - axMock.forwardResult = { - report: '', - findings: [ - { - severity: 'high', - claim: 'original claim', - evidence: [ - { uri: 'span://trace/action', excerpt: 'primary' }, - { uri: 'event://trace/result', excerpt: 'corroborating' }, - ], - confidence: 0.9, - }, - ], - } - const analyst = createTraceAnalystKind( - { - ...testSpec(), - postProcess: (row) => ({ - ...row, - evidence: [ - { ...row.evidence[0]!, uri: 'span://trace/reclassified' }, - ...row.evidence.slice(1), - ], - }), - }, - { ai: testAi() }, - ) - - const findings = await analyst.analyze({} as never, { tags: {} } as never) - - expect(findings[0]?.evidence_refs).toEqual([ - { kind: 'span', uri: 'span://trace/reclassified', excerpt: 'primary' }, - { kind: 'event', uri: 'event://trace/result', excerpt: 'corroborating' }, - ]) - }) - - it('rejects rows below a kind minimum-citation contract', async () => { - axMock.agentCalls.length = 0 - axMock.forwardResult = { - report: '', - findings: [ - { - severity: 'high', - claim: 'poisoning with only the action half', - evidence: [{ uri: 'span://trace/action' }], - confidence: 0.8, - }, - ], - } - const log = vi.fn() - const analyst = createTraceAnalystKind( - { ...testSpec(), minimumEvidenceCitations: 2 }, - { ai: testAi() }, - ) - - const findings = await analyst.analyze({} as never, { tags: {}, log } as never) - - expect(findings).toEqual([]) - expect(log).toHaveBeenCalledWith( - 'finding rejected: insufficient evidence citations', - expect.objectContaining({ required: 2, received: 1 }), - ) - }) - - it('does not count duplicate evidence as independent citations', async () => { - axMock.agentCalls.length = 0 - axMock.forwardResult = { - report: '', - findings: [ - { - severity: 'high', - claim: 'duplicate support', - evidence: [ - { uri: 'span://trace/action' }, - { uri: 'span://trace/action', excerpt: 'same span again' }, - ], - confidence: 0.8, - }, - ], - } - const log = vi.fn() - const analyst = createTraceAnalystKind( - { ...testSpec(), minimumEvidenceCitations: 2 }, - { ai: testAi() }, - ) - - const findings = await analyst.analyze({} as never, { tags: {}, log } as never) - - expect(findings).toEqual([]) - expect(log).toHaveBeenCalledWith( - 'finding rejected: insufficient evidence citations', - expect.objectContaining({ required: 2, received: 2, distinct: 1 }), - ) - }) - - it('applies the minimum-citation contract to prose-recovery findings', async () => { - axMock.agentCalls.length = 0 - axMock.forwardResult = { - report: 'A'.repeat(220), - findings: [], - } - const log = vi.fn() - const fetchImpl = vi.fn( - async () => - new Response( - JSON.stringify({ - choices: [ - { - message: { - content: JSON.stringify([ - { - severity: 'high', - claim: 'one-sided poisoning claim', - evidence: [{ uri: 'span://trace/action' }], - confidence: 0.8, - }, - ]), - }, - }, - ], - }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ), - ) as unknown as typeof fetch - const analyst = createTraceAnalystKind( - { ...testSpec(), minimumEvidenceCitations: 2 }, - { - ai: testAi(), - recovery: { baseUrl: 'https://example.test/v1', model: 'test', fetchImpl }, - }, - ) - - await expect(analyst.analyze({} as never, { tags: {}, log } as never)).rejects.toThrow( - /no finding satisfied its acceptance rules/, - ) - expect(log).toHaveBeenCalledWith( - 'analyst.kind test-kind recovery', - expect.objectContaining({ recovered: 0, rejected_insufficient_evidence: 2 }), - ) - }) - - it('applies the kind postProcess rule to recovered and fallback rows', async () => { - axMock.agentCalls.length = 0 - axMock.forwardResult = { report: 'A'.repeat(220), findings: [] } - const fetchImpl = stubRecoveryFetch([ - { - severity: 'high', - claim: 'kind-specific false positive', - evidence: [{ uri: 'span://trace/span' }], - confidence: 0.8, - }, - ]) - const analyst = createTraceAnalystKind( - { ...testSpec(), postProcess: () => null }, - { - ai: testAi(), - recovery: { baseUrl: 'https://example.test/v1', model: 'test', fetchImpl }, - }, - ) - - await expect(analyst.analyze({} as never, { tags: {} } as never)).rejects.toThrow( - /no finding satisfied its acceptance rules/, - ) - }) - - it('rejects a recovered subject that belongs to another analyst kind', async () => { - axMock.agentCalls.length = 0 - axMock.forwardResult = { report: 'A'.repeat(220), findings: [] } - const log = vi.fn() - const fetchImpl = stubRecoveryFetch([ - { - severity: 'high', - claim: 'prompt locus emitted by the failure classifier', - subject: 'system-prompt:instructions', - evidence: [{ uri: 'span://trace/span' }], - confidence: 0.8, - }, - ]) - const analyst = createTraceAnalystKind( - { ...testSpec(), id: 'failure-mode' }, - { - ai: testAi(), - recovery: { baseUrl: 'https://example.test/v1', model: 'test', fetchImpl }, - }, - ) - - const findings = await analyst.analyze({} as never, { tags: {}, log } as never) - - expect(findings).toHaveLength(1) - expect(findings[0]).toMatchObject({ - claim: 'Analyst produced a diagnosis but no structured findings — see report.', - metadata: { kind_version: '0.0.1', outcome: 'extraction_failed' }, - }) - expect(log).toHaveBeenCalledWith( - 'analyst.kind failure-mode recovery', - expect.objectContaining({ recovered: 0, rejected_wrong_subject: 2 }), - ) - }) - - it('fails when the Ax pipeline returns a malformed response', async () => { - axMock.agentCalls.length = 0 - axMock.forwardResult = 'Actor stopped without a structured response.' - const analyst = createTraceAnalystKind(testSpec(), { ai: testAi() }) - - await expect(analyst.analyze({} as never, { tags: {} } as never)).rejects.toThrow( - 'Trace analyst response must contain report and findings', - ) - }) - - it('rejects a report-only result when the kind requires structured findings', async () => { - axMock.forwardResult = { report: 'A'.repeat(220), findings: [] } - const analyst = createTraceAnalystKind( - { ...testSpec(), requireStructuredFindings: true }, - { ai: testAi() }, - ) - - await expect(analyst.analyze({} as never, { tags: {} } as never)).rejects.toThrow( - /produced no valid structured findings after 0 turns/, - ) - }) - - it('uses the exact structured Ax pipeline response', async () => { - axMock.forwardResult = { - report: 'Captured evidence-backed report.', - findings: [ - { - severity: 'high', - confidence: 0.9, - claim: 'Captured finding', - evidence: [{ uri: 'trace://run-1/span-1' }], - recommended_action: 'Use the captured result.', - }, - ], - } - const analyst = createTraceAnalystKind(testSpec(), { ai: testAi() }) - - const findings = await analyst.analyze({} as never, { tags: {} } as never) - - expect(findings).toHaveLength(1) - expect(findings[0]?.claim).toBe('Captured finding') - }) - - it('fails loud when optional recovery cannot complete', async () => { - axMock.agentCalls.length = 0 - axMock.forwardResult = { report: 'A'.repeat(220), findings: [] } - const log = vi.fn() - const fetchImpl = vi.fn(async () => { - throw new Error('recovery provider unavailable') - }) as unknown as typeof fetch - const analyst = createTraceAnalystKind(testSpec(), { - ai: testAi(), - recovery: { baseUrl: 'https://example.test/v1', model: 'test', fetchImpl }, - }) - - await expect(analyst.analyze({} as never, { tags: {}, log } as never)).rejects.toThrow( - 'recovery provider unavailable', - ) - }) - - it('honors a kind postProcess null rejection', async () => { - axMock.agentCalls.length = 0 - axMock.forwardResult = { - report: '', - findings: [ - { - severity: 'medium', - claim: 'kind-specific false positive', - evidence: [{ uri: 'span://trace/span' }], - confidence: 0.7, - }, - ], - } - const analyst = createTraceAnalystKind( - { ...testSpec(), postProcess: () => null }, - { ai: testAi() }, - ) - - const findings = await analyst.analyze({} as never, { tags: {} } as never) - - expect(findings).toEqual([]) - }) - - it('enforces the allocated analyst budget before a provider call', async () => { - axMock.forwardResult = { report: '', findings: [] } - const providerChat = vi.fn(async () => ({ - results: [], - modelUsage: { - ai: 'openai', - model: 'gpt-4o-mini', - tokens: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, - }, - })) - const analyst = createTraceAnalystKind( - { ...testSpec(), maxOutputTokens: 64 }, - { ai: testAi(providerChat) }, - ) - - await expect(analyst.analyze({} as never, { budgetUsd: 0 } as never)).rejects.toThrow( - /would exceed ceiling 0/, - ) - expect(providerChat).not.toHaveBeenCalled() - }) - - it('waits for a cancelled provider call to settle before reporting usage', async () => { - axMock.forwardResult = { report: '', findings: [] } - const controller = new AbortController() - type TestAxResponse = { - results: never[] - modelUsage: { - ai: string - model: string - tokens: { promptTokens: number; completionTokens: number; totalTokens: number } - } - } - let markStarted!: () => void - let finishProvider!: (value: TestAxResponse) => void - const started = new Promise((resolve) => { - markStarted = resolve - }) - const provider = new Promise((resolve) => { - finishProvider = resolve - }) - const providerChat = vi.fn(async () => { - markStarted() - return provider - }) - const analyst = createTraceAnalystKind(testSpec(), { ai: testAi(providerChat) }) - let receipt: AnalystUsageReceipt | undefined - - const run = analyst.analyze( - {} as never, - { - signal: controller.signal, - recordUsage: (usage: AnalystUsageReceipt) => (receipt = usage), - } as never, - ) - await started - controller.abort(new DOMException('cancelled', 'AbortError')) - await Promise.resolve() - expect(receipt).toBeUndefined() - finishProvider({ - results: [], - modelUsage: { - ai: 'openai', - model: 'gpt-4o-mini', - tokens: { promptTokens: 10, completionTokens: 4, totalTokens: 14 }, - }, - }) - - await expect(run).rejects.toMatchObject({ name: 'AbortError' }) - expect(receipt).toMatchObject({ - calls: 1, - tokens: { input: 10, output: 4 }, - cost: { kind: 'estimated', usd: expect.any(Number) }, - }) - }) - - it('reports a stuck cancelled provider call without hanging', async () => { - axMock.forwardResult = { report: '', findings: [] } - const controller = new AbortController() - let markStarted!: () => void - const started = new Promise((resolve) => { - markStarted = resolve - }) - const providerChat = vi.fn(async () => { - markStarted() - return new Promise(() => {}) - }) - const analyst = createTraceAnalystKind(testSpec(), { - ai: testAi(providerChat), - settlementTimeoutMs: 1, - }) - const log = vi.fn() - let receipt: AnalystUsageReceipt | undefined - - const run = analyst.analyze( - {} as never, - { - signal: controller.signal, - log, - recordUsage: (usage: AnalystUsageReceipt) => (receipt = usage), - } as never, - ) - await started - controller.abort(new DOMException('cancelled', 'AbortError')) - - await expect(run).rejects.toMatchObject({ name: 'AbortError' }) - expect(receipt).toEqual({ - calls: 1, - tokens: null, - cost: { kind: 'uncaptured', usd: null }, - knownCostUsd: 0, - }) - expect(log).toHaveBeenCalledWith('analyst.kind test-kind provider settlement timed out', { - pending_calls: 1, - timeout_ms: 1, - }) - }) - - it('renders same-run findings as dependency context, not prior-run memory', async () => { - axMock.agentCalls.length = 0 - axMock.forwardResult = { report: '', findings: [] } - const upstream = makeFinding({ - analyst_id: 'failure-mode', - area: 'failure-mode', - claim: 'agent repeated the same failed tool call', - severity: 'high', - confidence: 0.9, - evidence_refs: [{ kind: 'span', uri: 'span://trace/tool-call' }], - recommended_action: 'deduplicate identical calls', - }) - const analyst = createTraceAnalystKind(testSpec(), { ai: testAi() }) - - await analyst.analyze({} as never, { upstreamFindings: [upstream] } as never) - - const actor = ( - axMock.agentCalls[0]?.options.executorOptions as { description?: string } | undefined - )?.description - expect(actor).toContain('UPSTREAM FINDINGS (produced earlier in this same registry run)') - expect(actor).toContain(`id=${upstream.finding_id} source=failure-mode high`) - expect(actor).toContain('claim=agent repeated the same failed tool call') - expect(actor).toContain('action=deduplicate identical calls') - expect(actor).toContain('evidence=span://trace/tool-call') - expect(actor).not.toContain('PRIOR FINDINGS (from a previous run') - }) - - it('renders registry-forwarded prior findings into the improvement actor input', async () => { - axMock.agentCalls.length = 0 - axMock.forwardResult = { report: '', findings: [] } - const prior = makeFinding({ - analyst_id: 'failure-mode', - area: 'failure-mode', - claim: 'the worker stopped after executing only the first JavaScript block', - severity: 'high', - confidence: 0.99, - evidence_refs: [{ kind: 'span', uri: 'span://trace/worker-turn-7' }], - }) - const registry = new AnalystRegistry() - registry.register( - createTraceAnalystKind( - { ...IMPROVEMENT_KIND_SPEC, buildTools: () => [], subqueries: { maxCalls: 0 } }, - { ai: testAi() }, - ), - ) - - await registry.run( - 'prior-finding-render', - { traceStore: {} as never }, - { priorFindings: { '*': [prior] } }, - ) - - const actor = ( - axMock.agentCalls[0]?.options.executorOptions as { description?: string } | undefined - )?.description - expect(actor).toContain('PRIOR FINDINGS (from a previous run on related data)') - expect(actor).toContain(`id=${prior.finding_id} high`) - expect(actor).toContain('the worker stopped after executing only the first JavaScript block') - }) -}) - -function testSpec(): TraceAnalystKindSpec { - return { - id: 'test-kind', - description: 'test', - area: 'test', - version: '0.0.1', - actorDescription: 'Analyze the supplied traces.', - buildTools: () => [], - cost: { kind: 'llm' }, - } -} - -function stubRecoveryFetch(findings: unknown[]): typeof fetch { - return vi.fn( - async () => - new Response( - JSON.stringify({ choices: [{ message: { content: JSON.stringify(findings) } }] }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ), - ) as unknown as typeof fetch -} diff --git a/src/analyst/kind-factory.test.ts b/src/analyst/kind-factory.test.ts new file mode 100644 index 00000000..2c2f0571 --- /dev/null +++ b/src/analyst/kind-factory.test.ts @@ -0,0 +1,343 @@ +import { describe, expect, it, vi } from 'vitest' +import type { TraceAnalysisStore } from '../trace-analyst/store' +import type { TraceAnalystSpan } from '../trace-analyst/types' +import type { TraceAnalysisEngine, TraceAnalysisEngineRequest } from './engine' +import { createTraceAnalyst, runTraceAnalyst, type TraceAnalystDefinition } from './kind-factory' +import { makeFinding } from './types' + +const spans: TraceAnalystSpan[] = [ + traceSpan('step-1', 'candidate'), + traceSpan('step-2', '401 unauthorized'), +] +const store = traceStore(spans) +const definition: TraceAnalystDefinition = { + id: 'test-research', + description: 'Find the causal failure.', + area: 'failure-mode', + version: '1.0.0', + question: 'What caused the failure?', + instructions: 'Inspect exact spans and cite them.', + toolGroup: 'discovery', + limits: { + maxIterations: 6, + maxLlmCalls: 3, + maxToolCalls: 12, + maxOutputChars: 2_000, + }, +} + +function context() { + return { + runId: 'run-1', + correlationId: 'analysis-1', + tags: { suite: 'unit' }, + } +} + +describe('runTraceAnalyst', () => { + it('binds a definition to an engine and retains the investigation record', async () => { + let request: TraceAnalysisEngineRequest | undefined + const engine = fakeEngine(async (input) => { + request = input + return { + answer: 'The first tool call failed authentication.', + findings: [ + { + severity: 'high', + claim: 'Authentication failed before useful work began.', + subject: 'auth-failure', + confidence: 0.95, + evidence: [ + { + uri: 'trace://run-1/span/step-2', + excerpt: '401 unauthorized', + }, + ], + }, + ], + trajectory: [{ code: 'overview = getDatasetOverview()' }], + modelCalls: 2, + toolCalls: 3, + runtime: { engine: 'test' }, + } + }) + + const result = await runTraceAnalyst({ + definition, + engine, + store, + context: context(), + }) + + expect(request).toMatchObject({ + analystId: 'test-research', + question: 'What caused the failure?', + limits: { + maxIterations: 6, + maxLlmCalls: 3, + maxToolCalls: 12, + maxOutputChars: 2_000, + }, + costPhase: 'trace-analysis', + }) + expect(request?.tools.map((tool) => tool.name)).toEqual([ + 'getDatasetOverview', + 'queryTraces', + 'countTraces', + ]) + expect(request?.instructions).toContain('strict findings array') + expect(result.findings).toHaveLength(1) + expect(result.trajectory).toHaveLength(1) + }) + + it('applies a null post-processor as a rejection', async () => { + const result = await runTraceAnalyst({ + definition: { ...definition, postProcess: () => null }, + engine: fakeEngine(async () => ({ + answer: 'A candidate was investigated.', + findings: [ + { + severity: 'low', + claim: 'Rejected candidate.', + confidence: 0.5, + evidence: [{ uri: 'trace://run-1/span/step-1' }], + }, + ], + trajectory: [], + modelCalls: 1, + toolCalls: 1, + runtime: {}, + })), + store, + context: context(), + }) + + expect(result.findings).toEqual([]) + }) + + it('fails when a definition requires findings and the engine abstains', async () => { + await expect( + runTraceAnalyst({ + definition: { ...definition, requireStructuredFindings: true }, + engine: fakeEngine(async () => ({ + answer: 'No supported issue was found.', + findings: [], + trajectory: [], + modelCalls: 1, + toolCalls: 1, + runtime: {}, + })), + store, + context: context(), + }), + ).rejects.toThrow(/returned no valid structured findings/) + }) + + it('rejects transformed trace and span identifiers', async () => { + const log = vi.fn() + const result = await runTraceAnalyst({ + definition, + engine: findingEngine({ + uri: 'trace://cnVuLTE=/span/c3RlcC0y', + excerpt: '401 unauthorized', + }), + store, + context: { ...context(), log }, + }) + + expect(result.findings).toEqual([]) + expect(log).toHaveBeenCalledWith('finding rejected: unresolved evidence', { + uri: 'trace://cnVuLTE=/span/c3RlcC0y', + reason: 'trace span does not exist', + }) + }) + + it('rejects a nonexistent trace span', async () => { + const log = vi.fn() + const result = await runTraceAnalyst({ + definition, + engine: findingEngine({ uri: 'trace://run-1/span/step-99' }), + store, + context: { ...context(), log }, + }) + + expect(result.findings).toEqual([]) + expect(log).toHaveBeenCalledWith('finding rejected: unresolved evidence', { + uri: 'trace://run-1/span/step-99', + reason: 'trace span does not exist', + }) + }) + + it('rejects an excerpt that is absent from the cited span', async () => { + const log = vi.fn() + const result = await runTraceAnalyst({ + definition, + engine: findingEngine({ + uri: 'trace://run-1/span/step-2', + excerpt: 'authentication succeeded', + }), + store, + context: { ...context(), log }, + }) + + expect(result.findings).toEqual([]) + expect(log).toHaveBeenCalledWith('finding rejected: unresolved evidence', { + uri: 'trace://run-1/span/step-2', + reason: 'excerpt is not present in the cited span', + }) + }) + + it('accepts a supplied finding citation and checks its excerpt', async () => { + const prior = makeFinding({ + analyst_id: 'prior-analyst', + area: 'failure-mode', + claim: 'Authentication failed before the first useful tool call.', + severity: 'high', + confidence: 0.9, + evidence_refs: [{ kind: 'span', uri: 'trace://run-1/span/step-2' }], + }) + const result = await runTraceAnalyst({ + definition, + engine: findingEngine({ + uri: `finding://${encodeURIComponent(prior.finding_id)}`, + excerpt: 'first useful tool call', + }), + store, + context: { ...context(), priorFindings: [prior] }, + }) + + expect(result.findings).toHaveLength(1) + }) +}) + +describe('createTraceAnalyst', () => { + it('maps engine findings into registry findings with execution metadata', async () => { + const analyst = createTraceAnalyst(definition, { + engine: fakeEngine(async () => ({ + answer: 'The tool failed.', + findings: [ + { + severity: 'high', + claim: 'The tool failed.', + confidence: 0.9, + evidence: [{ uri: 'trace://run-1/span/step-2' }], + }, + ], + trajectory: [{ code: 'viewSpans(...)' }], + modelCalls: 3, + toolCalls: 4, + runtime: { dspy: '3.2.1' }, + })), + versionSuffix: 'optimized', + }) + const findings = await analyst.analyze(store, context()) + + expect(analyst.version).toBe('1.0.0+optimized') + expect(analyst.cost.models).toEqual(['test-model']) + expect(findings[0]).toMatchObject({ + analyst_id: 'test-research', + area: 'failure-mode', + claim: 'The tool failed.', + metadata: { + analysis_engine: 'test-engine', + analysis_model: 'test-model', + analysis_model_calls: 3, + analysis_tool_calls: 4, + analysis_runtime: { dspy: '3.2.1' }, + }, + }) + }) + + it('records an empty usage receipt without inventing model tokens', async () => { + const recordUsage = vi.fn() + const analyst = createTraceAnalyst(definition, { + engine: fakeEngine(async () => ({ + answer: 'No issue.', + findings: [], + trajectory: [], + modelCalls: 1, + toolCalls: 1, + runtime: {}, + })), + }) + + await analyst.analyze(store, { ...context(), recordUsage }) + + expect(recordUsage).toHaveBeenCalledWith({ + calls: 0, + tokens: { input: 0, output: 0 }, + cost: { kind: 'observed', usd: 0 }, + }) + }) +}) + +function fakeEngine(analyze: TraceAnalysisEngine['analyze']): TraceAnalysisEngine { + return { + id: 'test-engine', + description: 'test', + model: 'test-model', + analyze, + } +} + +function findingEngine(evidence: { uri: string; excerpt?: string }): TraceAnalysisEngine { + return fakeEngine(async () => ({ + answer: 'Authentication failed.', + findings: [ + { + severity: 'high', + claim: 'Authentication failed before useful work began.', + confidence: 0.95, + evidence: [evidence], + }, + ], + trajectory: [], + modelCalls: 1, + toolCalls: 1, + runtime: {}, + })) +} + +function traceSpan(spanId: string, content: string): TraceAnalystSpan { + return { + trace_id: 'run-1', + span_id: spanId, + parent_span_id: null, + name: 'message.assistant', + kind: 'LLM', + start_time: '2026-07-30T00:00:00.000Z', + end_time: '2026-07-30T00:00:01.000Z', + duration_ms: 1_000, + status: 'OK', + service_name: 'test', + agent_name: 'test-agent', + model_name: 'test-model', + tool_name: null, + attributes: { content }, + } +} + +function traceStore(rows: TraceAnalystSpan[]): TraceAnalysisStore { + const store: Pick = { + async hasSpans({ trace_id, span_ids }) { + return rows + .filter((span) => span.trace_id === trace_id && span_ids.includes(span.span_id)) + .map((span) => span.span_id) + }, + async viewSpans({ trace_id, span_ids }) { + const found = rows.filter( + (span) => span.trace_id === trace_id && span_ids.includes(span.span_id), + ) + const foundIds = new Set(found.map((span) => span.span_id)) + return { + trace_id, + spans: found, + missing_span_ids: span_ids.filter((spanId) => !foundIds.has(spanId)), + omitted_span_ids: [], + has_more: false, + truncated_attribute_count: 0, + } + }, + } + return store as unknown as TraceAnalysisStore +} diff --git a/src/analyst/kind-factory.ts b/src/analyst/kind-factory.ts index bd146640..0c8d31ae 100644 --- a/src/analyst/kind-factory.ts +++ b/src/analyst/kind-factory.ts @@ -1,389 +1,338 @@ -/** - * Analyst-kind factory — the typed way to define trace analysts. - * - * A "kind" is a specialized analyst whose actor prompt, tool subset, - * and bounded Ax subqueries target one failure-mode lens (failure-mode - * classification, knowledge gap discovery, knowledge poisoning, - * self-improvement, ...). Kinds emit findings in the typed - * `RawAnalystFinding` shape via a JSON-array Ax output; the factory - * validates each row with Zod and lifts it into `AnalystFinding[]`. - * - * Composition rules: - * - Each kind owns its actor description. No generic "answer this - * question" prompt — the prompt names the failure lens. - * - Each kind picks a narrow tool subset from `ANALYST_TOOL_GROUPS`. - * A kind that never needs full-trace dumps can drop `viewTrace` / - * `viewSpans` and stay cheap. - * - Each kind declares its subquery + parallelism budget. Discovery-heavy - * kinds can fan out more bounded semantic questions than narrow lenses. - * - * Evaluate prompt changes through `runAnalystBenchmark`, where labels include - * exact evidence locations and clean cases instead of output-string overlap. - */ - -import type { AxAIService, AxFunction } from '@ax-llm/ax' import { CostLedger } from '../cost-ledger' -import { runTraceAnalysisLoop } from '../trace-analyst/loop' import type { TraceAnalysisStore } from '../trace-analyst/store' -import { meterAxChatService } from './ax-cost-service' -import { resolveAnalystModel } from './ax-service' +import type { TraceAnalysisEngine, TraceAnalysisEngineResult, TraceAnalystLimits } from './engine' +import { resolveTraceAnalystLimits } from './engine' import { evidenceRefsFromRawFinding, parseRawFinding, + parseTraceSpanEvidenceUri, RAW_FINDING_SCHEMA_PROMPT, type RawAnalystFinding, } from './finding-signature' import { KIND_EXPECTED_SUBJECTS, parseFindingSubject } from './finding-subject' -import { structureFindings } from './structure-findings' -import type { Analyst, AnalystContext, AnalystCost, AnalystFinding } from './types' +import { buildTraceToolsForGroup, type TraceToolGroupName } from './tool-groups' +import type { Analyst, AnalystContext, AnalystFinding } from './types' import { makeFinding } from './types' import { settleUsageReceiptFromCostLedger, validateUsageSettlementTimeout } from './usage-receipt' -/** - * Per-kind specification. The factory turns this into a regular - * `Analyst` ready for `AnalystRegistry.register()`. - */ -export interface TraceAnalystKindSpec { - /** Stable id. Appears in finding_id, telemetry, and registry exclusions. */ +/** A named research question independent of the recursive engine that runs it. */ +export interface TraceAnalystDefinition { id: string - /** One-sentence description shown in `registry.list()`. */ description: string - /** Coarse classification stamped on every emitted finding (`failure-mode`, `knowledge-gap`, ...). */ area: string - /** Bump on any breaking change to the actor prompt or output schema. */ version: string - /** Actor system prompt. Must instruct the LLM to emit `findings` per the schema. */ - actorDescription: string - /** Tool functions the actor may call. Pick narrow subsets via `ANALYST_TOOL_GROUPS`. */ - buildTools: (store: TraceAnalysisStore) => AxFunction[] - /** - * Deterministically prepare one bounded context value before model work. - * Returning a string uses Ax's long-context path without tools. Returning - * undefined keeps the normal tool-driven path. - */ + /** User-facing question. A function may derive it from run context. */ + question?: string | ((context: AnalystContext) => string) + /** Research policy, evidence rules, and domain-specific instructions. */ + instructions: string + /** Smallest trace-tool set that can answer the question. */ + toolGroup: TraceToolGroupName + /** Optional bounded context assembled deterministically before the investigation. */ prepareContext?: ( store: TraceAnalysisStore, - ctx: AnalystContext, + context: AnalystContext, ) => string | undefined | Promise - /** Bounded semantic subqueries. `maxCalls: 0` disables model fan-out. */ - subqueries?: { maxCalls: number; maxParallel?: number } - /** Actor turn cap. Default 12. */ - maxTurns?: number - /** Runtime char cap. Default 6000. */ - maxRuntimeChars?: number - /** Maximum output tokens for every actor and subquery model call. Default 4096. */ - maxOutputTokens?: number - /** Cost classification surfaced in `registry.list()` and budget enforcement. */ - cost: AnalystCost - /** Per-finding-row hook — kinds may reject / rewrite before lifting. */ - postProcess?: (row: RawAnalystFinding, ctx: AnalystContext) => RawAnalystFinding | null - /** Minimum citations per finding. Default 1; rows below it are rejected. */ + limits?: Partial + postProcess?: (row: RawAnalystFinding, context: AnalystContext) => RawAnalystFinding | null minimumEvidenceCitations?: number - /** Reject a substantive report when no structured finding survives validation. */ requireStructuredFindings?: boolean } -export interface CreateTraceAnalystKindOpts { - /** AxAIService bound at registration time. */ - ai: AxAIService - /** Required unless `ai` was created by {@link createAnalystAi}. */ - model?: string - /** Override the spec's `version` (e.g. when an optimizer has fitted a new prompt). */ +export interface CreateTraceAnalystOptions { + engine: TraceAnalysisEngine versionSuffix?: string - /** - * Optional two-phase recovery: when the agentic harvest is empty but the - * actor produced a substantive free-form `report`, extract findings from that - * prose via a tolerant chat-completions pass (`structureFindings`) — no - * strict-emission contract, so it works on weak models. Omit to leave the - * actor's harvest as-is (the report is still surfaced fail-loud either way). - */ - recovery?: { baseUrl: string; apiKey?: string; model?: string; fetchImpl?: typeof fetch } - /** Maximum post-cancellation wait for a provider receipt. Default 5 seconds. */ settlementTimeoutMs?: number } -/** - * Build an `Analyst` from a kind spec. - * - * Lifts the Ax pipeline once at registration time so the registry - * gets a stateless analyst. The Ax agent is freshly constructed per - * `analyze()` call (the agent carries chat-log + usage state we don't - * want shared across analyst runs). - */ -export function createTraceAnalystKind( - spec: TraceAnalystKindSpec, - opts: CreateTraceAnalystKindOpts, -): Analyst { - rejectRemovedKindOptions(spec) - const version = opts.versionSuffix ? `${spec.version}+${opts.versionSuffix}` : spec.version - const model = resolveAnalystModel(opts.ai, opts.model) - const minimumEvidenceCitations = spec.minimumEvidenceCitations ?? 1 - if (!Number.isInteger(minimumEvidenceCitations) || minimumEvidenceCitations < 1) { - throw new TypeError('minimumEvidenceCitations must be a positive integer') +/** Run one definition and retain its answer, findings, and full investigation record. */ +export async function runTraceAnalyst(args: { + definition: TraceAnalystDefinition + engine: TraceAnalysisEngine + store: TraceAnalysisStore + context: AnalystContext + settlementTimeoutMs?: number +}): Promise { + const { definition, context } = args + validateDefinition(definition) + const minimumEvidenceCitations = definition.minimumEvidenceCitations ?? 1 + const settlementTimeoutMs = validateUsageSettlementTimeout(args.settlementTimeoutMs) + const costLedger = context.costLedger ?? new CostLedger(context.budgetUsd) + const costTags = { + ...(context.tags ?? {}), + analystId: definition.id, + ...(context.correlationId ? { analystRunId: context.correlationId } : {}), } - const settlementTimeoutMs = validateUsageSettlementTimeout(opts.settlementTimeoutMs) + + try { + const preparedContext = await definition.prepareContext?.(args.store, context) + if (preparedContext !== undefined && typeof preparedContext !== 'string') { + throw new TypeError(`trace analyst '${definition.id}' prepareContext must return a string`) + } + const instructions = [ + definition.instructions.trim(), + renderPriorFindings(context.priorFindings), + renderUpstreamFindings(context.upstreamFindings), + RAW_FINDING_SCHEMA_PROMPT, + minimumEvidenceCitations > 1 + ? `Every finding requires at least ${minimumEvidenceCitations} distinct evidence citations.` + : '', + preparedContext ? `PREPARED CONTEXT:\n${preparedContext}` : '', + 'Return a direct prose answer and a strict findings array. Use trace tools to investigate. Do not infer trace facts from the question alone.', + ] + .filter(Boolean) + .join('\n\n') + + const completed = await args.engine.analyze({ + analystId: definition.id, + question: deriveQuestion(context, definition), + instructions, + tools: buildTraceToolsForGroup(definition.toolGroup, args.store), + limits: resolveTraceAnalystLimits(definition.limits), + costLedger, + costPhase: context.costPhase ?? 'trace-analysis', + costTags, + ...(context.signal ? { signal: context.signal } : {}), + ...(context.log ? { log: context.log } : {}), + }) + const findings = await acceptFindings( + definition, + completed.findings, + args.store, + context, + minimumEvidenceCitations, + ) + if (definition.requireStructuredFindings && findings.length === 0) { + throw new Error( + `trace analyst '${definition.id}' returned no valid structured findings: ${truncateForContext(completed.answer, 600)}`, + ) + } + context.log?.(`trace analyst ${definition.id} completed`, { + engine: args.engine.id, + model_calls: completed.modelCalls, + tool_calls: completed.toolCalls, + submitted_findings: completed.findings.length, + accepted_findings: findings.length, + }) + return { ...completed, findings } + } finally { + const usage = await settleUsageReceiptFromCostLedger(costLedger, { + channel: 'analyst', + tags: costTags, + timeoutMs: settlementTimeoutMs, + }) + if (!usage.settled) { + context.log?.(`trace analyst ${definition.id} provider settlement timed out`, { + pending_calls: usage.pendingCalls, + timeout_ms: settlementTimeoutMs, + }) + } + context.recordUsage?.(usage.receipt) + } +} + +/** Adapt a research definition to the common Analyst registry contract. */ +export function createTraceAnalyst( + definition: TraceAnalystDefinition, + options: CreateTraceAnalystOptions, +): Analyst { + validateDefinition(definition) + const version = options.versionSuffix + ? `${definition.version}+${options.versionSuffix}` + : definition.version return { - id: spec.id, - description: spec.description, + id: definition.id, + description: definition.description, inputKind: 'trace-store', - cost: { ...spec.cost, settlement_timeout_ms: settlementTimeoutMs }, + cost: { + kind: 'llm', + ...(options.engine.model ? { models: [options.engine.model] } : {}), + settlement_timeout_ms: validateUsageSettlementTimeout(options.settlementTimeoutMs), + }, version, - async analyze(store, ctx) { - const maxOutputTokens = spec.maxOutputTokens ?? 4096 - const costLedger = ctx.costLedger ?? new CostLedger(ctx.budgetUsd) - const costTags = { - ...(ctx.tags ?? {}), - analystId: spec.id, - ...(ctx.correlationId ? { analystRunId: ctx.correlationId } : {}), - } - const meteredAi = meterAxChatService(opts.ai, { - ledger: costLedger, - actor: spec.id, - maxOutputTokens, - defaultModel: model, - phase: ctx.costPhase, - signal: ctx.signal, - tags: costTags, + async analyze(store, context) { + const completed = await runTraceAnalyst({ + definition, + engine: options.engine, + store, + context, + settlementTimeoutMs: options.settlementTimeoutMs, }) - try { - const preparedContext = await spec.prepareContext?.(store, ctx) - if (preparedContext !== undefined && typeof preparedContext !== 'string') { - throw new TypeError(`Trace analyst '${spec.id}' prepareContext must return a string`) - } - const tools = preparedContext === undefined ? spec.buildTools(store) : [] - const analysisMode = preparedContext === undefined ? 'tool-loop' : 'prepared-context' - const maxSubqueries = spec.subqueries?.maxCalls ?? 0 - const maxParallel = spec.subqueries?.maxParallel ?? 2 - const priorContext = renderPriorFindings(ctx.priorFindings) - const upstreamContext = renderUpstreamFindings(ctx.upstreamFindings) - const actorDescription = - spec.actorDescription.trim() + - priorContext + - upstreamContext + - '\n\n' + - RAW_FINDING_SCHEMA_PROMPT + - (minimumEvidenceCitations > 1 - ? `\n\nThis kind requires at least ${minimumEvidenceCitations} evidence citations per finding; rows with fewer are rejected.` - : '') + - '\n\nFirst write `report`: a concise free-form prose diagnosis of what ' + - 'the traces show — what succeeded, what was suboptimal or failed — with ' + - 'concrete trace ids and numbers. THEN return the structured `findings` ' + - 'array (it MAY be empty when there is nothing to report).' - - ctx.log?.(`analyst.kind ${spec.id} forward`, { - max_subqueries: maxSubqueries, - tool_count: tools.length, - analysis_mode: analysisMode, - prepared_context_chars: preparedContext?.length ?? 0, - tags: ctx.tags, - }) - - const completed = await runTraceAnalysisLoop({ - id: spec.id, - description: spec.description, - prompt: actorDescription, - question: deriveQuestion(ctx, spec), - ai: meteredAi, - model, - tools, - findingType: 'object', - maxSubqueries, - maxParallelSubqueries: maxParallel, - maxTurns: spec.maxTurns ?? 12, - maxRuntimeChars: spec.maxRuntimeChars ?? 6000, - ...(preparedContext !== undefined ? { context: preparedContext } : {}), - ...(ctx.signal ? { signal: ctx.signal } : {}), - }) - const { report, findings: submittedFindings } = completed - - const expectedSubjects = KIND_EXPECTED_SUBJECTS[spec.id] - const out: AnalystFinding[] = [] - const rawRows = submittedFindings - let rejectedWrongKind = 0 - let rejectedInsufficientEvidence = 0 - const processRow = (parsed: RawAnalystFinding): RawAnalystFinding | null => { - const callbackResult = spec.postProcess ? spec.postProcess(parsed, ctx) : parsed - if (!callbackResult) return null - const postProcessed = parseRawFinding(callbackResult, ctx.log) - if (!postProcessed) return null - if (expectedSubjects && postProcessed.subject !== undefined) { - const parsedSubject = parseFindingSubject(postProcessed.subject) - if (parsedSubject === null) { - ctx.log?.('finding rejected: subject failed to parse', { - kind: spec.id, - subject: postProcessed.subject, - }) - rejectedWrongKind += 1 - return null - } - if (!expectedSubjects.includes(parsedSubject.kind)) { - ctx.log?.('finding rejected: subject variant not allowed for this kind', { - kind: spec.id, - subject_kind: parsedSubject.kind, - subject: postProcessed.subject, - allowed: expectedSubjects, - }) - rejectedWrongKind += 1 - return null - } - } - const distinctEvidenceCitations = new Set( - postProcessed.evidence.map((citation) => citation.uri.trim()), - ).size - if (distinctEvidenceCitations < minimumEvidenceCitations) { - ctx.log?.('finding rejected: insufficient evidence citations', { - kind: spec.id, - required: minimumEvidenceCitations, - received: postProcessed.evidence.length, - distinct: distinctEvidenceCitations, - }) - rejectedInsufficientEvidence += 1 - return null - } - return postProcessed - } - for (const row of rawRows) { - const parsed = parseRawFinding(row, ctx.log) - if (!parsed) continue - const postProcessed = processRow(parsed) - if (!postProcessed) continue - out.push( - toAnalystFinding(spec, version, postProcessed, { - analysis_mode: analysisMode, - analysis_turn_count: completed.turnCount, - }), - ) - } + return completed.findings.map((finding) => + toAnalystFinding(definition, version, finding, { + analysis_engine: options.engine.id, + analysis_model: options.engine.model, + analysis_model_calls: completed.modelCalls, + analysis_tool_calls: completed.toolCalls, + analysis_runtime: completed.runtime, + }), + ) + }, + } +} - ctx.log?.(`analyst.kind ${spec.id} done`, { - emitted: rawRows.length, - accepted: out.length, - rejected_wrong_subject: rejectedWrongKind, - rejected_insufficient_evidence: rejectedInsufficientEvidence, +async function acceptFindings( + definition: TraceAnalystDefinition, + submitted: readonly unknown[], + store: TraceAnalysisStore, + context: AnalystContext, + minimumEvidenceCitations: number, +): Promise { + const expectedSubjects = KIND_EXPECTED_SUBJECTS[definition.id] + const accepted: RawAnalystFinding[] = [] + for (const row of submitted) { + const parsed = parseRawFinding(row, context.log) + if (!parsed) continue + const processed = definition.postProcess ? definition.postProcess(parsed, context) : parsed + if (!processed) continue + const validated = parseRawFinding(processed, context.log) + if (!validated) continue + if (expectedSubjects && validated.subject !== undefined) { + const subject = parseFindingSubject(validated.subject) + if (subject === null || !expectedSubjects.includes(subject.kind)) { + context.log?.('finding rejected: subject is not valid for analyst', { + analyst_id: definition.id, + subject: validated.subject, + allowed: expectedSubjects, }) + continue + } + } + const distinctEvidence = new Set(validated.evidence.map((citation) => citation.uri.trim())).size + if (distinctEvidence < minimumEvidenceCitations) { + context.log?.('finding rejected: insufficient evidence citations', { + analyst_id: definition.id, + required: minimumEvidenceCitations, + distinct: distinctEvidence, + }) + continue + } + if (!(await evidenceIsResolvable(validated, store, context))) continue + accepted.push(validated) + } + return accepted +} - // Two-phase recovery / fail-loud. The actor reasons free-form (the - // `report`); a weak model often produces a sound diagnosis but fails the - // strict findings emission (or the rows get rejected). If the harvest is - // empty but the report is substantive, recover findings from the prose - // via the tolerant structuring pass (opt-in), and — either way — surface - // the report as a visible info finding so an empty harvest is never a - // silent zero. A genuinely empty diagnosis (short/no report) stays empty. - if (out.length === 0 && report.trim().length >= 200) { - if (opts.recovery) { - const wrongKindBefore = rejectedWrongKind - const insufficientEvidenceBefore = rejectedInsufficientEvidence - const recovered = await structureFindings({ - report, - analystId: spec.id, - area: spec.area, - model: opts.recovery.model ?? model, - baseUrl: opts.recovery.baseUrl, - apiKey: opts.recovery.apiKey, - fetchImpl: opts.recovery.fetchImpl, - costLedger, - costPhase: ctx.costPhase, - costTags, - signal: ctx.signal, - maxTokens: Math.min(maxOutputTokens, 2_000), - processRow, - findingMetadata: { kind_version: version }, - }) - out.push(...recovered.findings) - ctx.log?.(`analyst.kind ${spec.id} recovery`, { - outcome: recovered.outcome, - recovered: recovered.findings.length, - rejected_wrong_subject: rejectedWrongKind - wrongKindBefore, - rejected_insufficient_evidence: - rejectedInsufficientEvidence - insufficientEvidenceBefore, - }) - } - if (out.length === 0) { - if (spec.requireStructuredFindings) { - throw new Error( - `Trace analyst '${spec.id}' produced no valid structured findings after ${completed.turnCount} turns: ${truncateForContext(report, 600)}`, - ) - } - const fallback = processRow({ - claim: 'Analyst produced a diagnosis but no structured findings — see report.', - rationale: report.slice(0, 1500), - severity: 'info', - confidence: 0.3, - evidence: [{ uri: 'report://summary', excerpt: report.slice(0, 2000) }], - }) - if (fallback) { - out.push( - toAnalystFinding(spec, version, fallback, { - analysis_mode: analysisMode, - analysis_turn_count: completed.turnCount, - outcome: 'extraction_failed', - }), - ) - } else { - throw new Error( - `Trace analyst '${spec.id}' produced a substantive report, but no finding satisfied its acceptance rules`, - ) - } - } - } - return out - } finally { - const usage = await settleUsageReceiptFromCostLedger(costLedger, { - tags: { - analystId: spec.id, - ...(ctx.correlationId ? { analystRunId: ctx.correlationId } : {}), +async function evidenceIsResolvable( + finding: RawAnalystFinding, + store: TraceAnalysisStore, + context: AnalystContext, +): Promise { + const knownFindings = new Map( + [...(context.priorFindings ?? []), ...(context.upstreamFindings ?? [])].map((entry) => [ + entry.finding_id, + entry, + ]), + ) + for (const citation of finding.evidence) { + const traceLocation = parseTraceSpanEvidenceUri(citation.uri) + if (traceLocation) { + const storeContext = context.signal ? { signal: context.signal } : undefined + const existing = await store.hasSpans( + { + trace_id: traceLocation.traceId, + span_ids: [traceLocation.spanId], + }, + storeContext, + ) + if (!existing.includes(traceLocation.spanId)) { + rejectEvidence(context, citation.uri, 'trace span does not exist') + return false + } + if (citation.excerpt !== undefined) { + const viewed = await store.viewSpans( + { + trace_id: traceLocation.traceId, + span_ids: [traceLocation.spanId], }, - timeoutMs: settlementTimeoutMs, - }) - if (!usage.settled) { - ctx.log?.(`analyst.kind ${spec.id} provider settlement timed out`, { - pending_calls: usage.pendingCalls, - timeout_ms: settlementTimeoutMs, - }) + storeContext, + ) + const span = viewed.spans.find((entry) => entry.span_id === traceLocation.spanId) + if (!span || !containsExactText(span, citation.excerpt)) { + rejectEvidence(context, citation.uri, 'excerpt is not present in the cited span') + return false } - ctx.recordUsage?.(usage.receipt) } - }, + continue + } + + const findingId = parseFindingEvidenceUri(citation.uri) + const referenced = findingId ? knownFindings.get(findingId) : undefined + if (!referenced) { + rejectEvidence(context, citation.uri, 'citation is not a supplied finding or trace span') + return false + } + if (citation.excerpt !== undefined && !containsExactText(referenced, citation.excerpt)) { + rejectEvidence(context, citation.uri, 'excerpt is not present in the cited finding') + return false + } } + return true } -function rejectRemovedKindOptions(spec: TraceAnalystKindSpec): void { - const supplied = spec as unknown as Record - const migrations = [ - ['recursion', 'subqueries'], - ['responderDescription', 'actorDescription'], - ['maxDepth', 'subqueries'], - ['maxParallelSubagents', 'subqueries.maxParallel'], - ['subagentDescription', 'actorDescription'], - ] as const - for (const [removed, replacement] of migrations) { - if (removed in supplied) { - throw new TypeError( - `createTraceAnalystKind: '${removed}' is unsupported; use '${replacement}'`, - ) +function parseFindingEvidenceUri(uri: string): string | null { + const match = /^finding:\/\/([^/?#]+)$/.exec(uri) + if (!match) return null + try { + return decodeURIComponent(match[1]!) || null + } catch { + return null + } +} + +function containsExactText(value: unknown, expected: string, depth = 0): boolean { + if (!expected || depth > 20) return false + if (typeof value === 'string') return value.includes(expected) + if (Array.isArray(value)) { + return value.some((entry) => containsExactText(entry, expected, depth + 1)) + } + if (typeof value === 'object' && value !== null) { + return Object.values(value).some((entry) => containsExactText(entry, expected, depth + 1)) + } + return false +} + +function rejectEvidence(context: AnalystContext, uri: string, reason: string): void { + context.log?.('finding rejected: unresolved evidence', { uri, reason }) +} + +function validateDefinition(definition: TraceAnalystDefinition): void { + for (const [name, value] of [ + ['id', definition.id], + ['description', definition.description], + ['area', definition.area], + ['version', definition.version], + ['instructions', definition.instructions], + ] as const) { + if (typeof value !== 'string' || !value.trim()) { + throw new TypeError(`trace analyst ${name} must be a non-empty string`) } } + const minimumEvidenceCitations = definition.minimumEvidenceCitations ?? 1 + if (!Number.isSafeInteger(minimumEvidenceCitations) || minimumEvidenceCitations < 1) { + throw new TypeError('minimumEvidenceCitations must be a positive safe integer') + } + resolveTraceAnalystLimits(definition.limits) } -function deriveQuestion(ctx: AnalystContext, spec: TraceAnalystKindSpec): string { - // The actor's user message must orient it at the task, not echo the kind id. - // A bare id like "failure-mode" gives the actor nothing to act on, so it - // spends turns inspecting the input instead of reading traces. Operators can - // still steer with `tags.focus = "leaf-X"`, appended to the task directive. - const focus = ctx.tags?.focus?.trim() - const task = `Analyze this trace dataset with the available tools and report ${spec.area} findings. ${spec.description}` - return focus ? `${task} Focus: ${focus}.` : task +function deriveQuestion(context: AnalystContext, definition: TraceAnalystDefinition): string { + const supplied = + typeof definition.question === 'function' ? definition.question(context) : definition.question + const base = + supplied?.trim() || + `Analyze this trace dataset and report ${definition.area} findings. ${definition.description}` + const focus = context.tags?.focus?.trim() + return focus ? `${base}\nFocus: ${focus}` : base } function toAnalystFinding( - spec: TraceAnalystKindSpec, + definition: TraceAnalystDefinition, version: string, raw: RawAnalystFinding, - metadata: Record = {}, + metadata: Record, ): AnalystFinding { return makeFinding({ - analyst_id: spec.id, - area: spec.area, + analyst_id: definition.id, + area: definition.area, subject: raw.subject, claim: raw.claim, rationale: raw.rationale, @@ -391,79 +340,46 @@ function toAnalystFinding( confidence: raw.confidence, evidence_refs: evidenceRefsFromRawFinding(raw), recommended_action: raw.recommended_action, - metadata: { kind_version: version, ...metadata }, + metadata: { definition_version: version, ...metadata }, }) } -/** - * Render a compact prior-findings block the actor reads alongside its - * brief. Each row is one line so the actor can scan dozens cheaply. - * The kind's prompt instructs the actor to (a) check whether a new - * cluster matches a prior `finding_id` (carry the id forward via - * `id_basis` to keep diffs stable) and (b) raise severity / confidence - * when a prior finding has reappeared without remediation. - * - * Returns the empty string when there are no prior findings — most - * runs are "first-of-its-kind" and the prompt stays unchanged. - * - * Exported for tests + for consumers that build their own actor - * prompts (e.g. specialized analysts living outside the default kinds). - */ export function renderPriorFindings(prior: AnalystContext['priorFindings']): string { if (!prior || prior.length === 0) return '' - const MAX_ROWS = 40 // keep the block under ~2KB; older history is summarized externally - const rows = prior.slice(0, MAX_ROWS).map((f) => { - const subject = f.subject ? ` [${f.subject}]` : '' - return ` - id=${f.finding_id} ${f.severity}${subject} ${truncateForContext(f.claim, 160)}` + const maxRows = 40 + const rows = prior.slice(0, maxRows).map((finding) => { + const subject = finding.subject ? ` [${finding.subject}]` : '' + return `- id=${finding.finding_id} ${finding.severity}${subject} ${truncateForContext(finding.claim, 160)}` }) - const overflow = - prior.length > MAX_ROWS - ? `\n ... +${prior.length - MAX_ROWS} more prior findings (older history truncated)` - : '' + if (prior.length > maxRows) rows.push(`- ${prior.length - maxRows} older findings omitted`) return [ - '', - '', - 'PRIOR FINDINGS (from a previous run on related data):', - 'When the work you do now matches a row below, REUSE the `finding_id` (pass it as `id_basis`) so the cross-run diff stays stable.', - 'A finding that reappears with no remediation evidence SHOULD raise its `confidence` and may justify a higher `severity`.', + 'PRIOR FINDINGS:', + 'Reuse a matching finding id through id_basis and raise confidence only when current evidence confirms recurrence.', ...rows, - overflow, - ] - .filter(Boolean) - .join('\n') + ].join('\n') } -/** Render findings produced earlier in this same registry run. */ export function renderUpstreamFindings(upstream: AnalystContext['upstreamFindings']): string { if (!upstream || upstream.length === 0) return '' - const MAX_ROWS = 40 - const rows = upstream.slice(0, MAX_ROWS).map((finding) => { + const maxRows = 40 + const rows = upstream.slice(0, maxRows).map((finding) => { const subject = finding.subject ? ` [${finding.subject}]` : '' - const action = finding.recommended_action - ? ` action=${truncateForContext(finding.recommended_action, 120)}` - : '' - const evidence = finding.evidence_refs[0] + const evidence = finding.evidence_refs[0]?.uri ? ` evidence=${truncateForContext(finding.evidence_refs[0].uri, 120)}` : '' - return ` - id=${finding.finding_id} source=${finding.analyst_id} ${finding.severity}${subject} claim=${truncateForContext(finding.claim, 160)}${action}${evidence}` + return `- id=${finding.finding_id} source=${finding.analyst_id} ${finding.severity}${subject} claim=${truncateForContext(finding.claim, 160)}${evidence}` }) - const overflow = - upstream.length > MAX_ROWS - ? `\n ... +${upstream.length - MAX_ROWS} more upstream findings (truncated)` - : '' + if (upstream.length > maxRows) { + rows.push(`- ${upstream.length - maxRows} additional upstream findings omitted`) + } return [ - '', - '', - 'UPSTREAM FINDINGS (produced earlier in this same registry run):', - 'Use these as intermediate evidence. Build on them instead of repeating the same diagnosis, and cite a dependency with `finding://`.', + 'UPSTREAM FINDINGS:', + 'Build on these findings instead of repeating them. Cite dependencies as finding://.', ...rows, - overflow, - ] - .filter(Boolean) - .join('\n') + ].join('\n') } -function truncateForContext(s: string, max: number): string { - if (s.length <= max) return s - return `${s.slice(0, max - 1).trimEnd()}…` +function truncateForContext(value: string, max: number): string { + if (value.length <= max) return value + return `${value.slice(0, max - 3).trimEnd()}...` } diff --git a/src/analyst/kinds/failure-mode.ts b/src/analyst/kinds/failure-mode.ts index aed5ce3d..51b4d3e8 100644 --- a/src/analyst/kinds/failure-mode.ts +++ b/src/analyst/kinds/failure-mode.ts @@ -11,8 +11,7 @@ */ import { findingSubjectGrammarPromptFor } from '../finding-subject' -import type { TraceAnalystKindSpec } from '../kind-factory' -import { buildTraceToolsForGroup } from '../tool-groups' +import type { TraceAnalystDefinition } from '../kind-factory' const subjectGrammar = findingSubjectGrammarPromptFor('failure-mode') @@ -22,29 +21,23 @@ ${subjectGrammar} DISCOVERY → CLUSTER → CITE protocol: -1. Call \`traces.getDatasetOverview({})\` first. Use \`has_errors\`, \`models\`, \`agent_names\`, \`tools\`, and \`sample_trace_ids\` to size the failure surface. -2. Use \`traces.queryTraces({ filters: { has_errors: true }, limit })\` to pull error-bearing traces. Combine with \`traces.countTraces\` to see what fraction of the dataset failed. -3. For each candidate failure cluster, use \`traces.searchTrace\` with regex like \`STATUS_CODE_ERROR\`, \`MaxTurnsExceeded\`, \`assertion\`, \`unauthorized\`, \`timeout\`, \`429\`, \`5\\d\\d\`, the agent's specific error strings, or the names of its tools. Pull one or two representative traces per cluster, **not all** of them. +1. Call \`getDatasetOverview({})\` first. Use \`has_errors\`, \`models\`, \`agent_names\`, \`tools\`, and \`sample_trace_ids\` to size the failure surface. +2. Use \`queryTraces(filters={"has_errors": true}, limit=...)\` to pull error-bearing traces. Combine with \`countTraces\` to see what fraction of the dataset failed. +3. For each candidate failure cluster, use \`searchTrace\` with regex like \`STATUS_CODE_ERROR\`, \`MaxTurnsExceeded\`, \`assertion\`, \`unauthorized\`, \`timeout\`, \`429\`, \`5\\d\\d\`, the agent's specific error strings, or the names of its tools. Pull one or two representative traces per cluster, **not all** of them. 4. **Cluster, do not enumerate.** Two failures with the same root cause should be ONE finding citing both traces, not two findings. The point of this analyst is to compress N runs into K modes. 5. For each defensible cluster, emit ONE finding. Use a lowercase cluster label matching the subject grammar ("tool-call-loop", "auth-revoked-mid-run", ...). Rate it critical when it blocks the run, high when the run finishes degraded, and medium when it slows convergence. Cite representative spans and include exact error, payload, or contradictory-output quotes. Use confidence 0.85+ when multiple traces show the same shape, 0.6-0.8 for a single-trace inference, and <0.5 for speculation. Keep the imperative fix idea short; the improvement analyst expands it. If the dataset has no failures, return an empty findings array — do NOT pad with low-confidence speculation. -**Use subqueries over loaded evidence.** After the first scan, load representative span excerpts for each candidate cluster. Then send one bounded \`llmQuery\` per cluster in one batch, including the exact excerpts and asking it to classify the root cause. Subqueries cannot call trace tools. Merge or split clusters yourself from their classifications and the cited source evidence. +**Use subqueries over loaded evidence.** After the first scan, load representative span excerpts for each candidate cluster. Then send one bounded \`llm_query\` per cluster in one batch, including the exact excerpts and asking it to classify the root cause. Subqueries cannot call trace tools. Merge or split clusters yourself from their classifications and the cited source evidence.` -OBSERVABILITY rules: -- Each non-final turn must emit at least one \`console.log\` for evidence. -- Reuse runtime variables across turns; don't recompute.` - -export const FAILURE_MODE_KIND_SPEC: TraceAnalystKindSpec = { +export const FAILURE_MODE_KIND_SPEC: TraceAnalystDefinition = { id: 'failure-mode', description: 'Clusters trace-dataset failures into distinct failure modes with cited evidence and a short recommended action.', area: 'failure-mode', version: '1.2.0', - actorDescription: ACTOR_PROMPT, - buildTools: (store) => buildTraceToolsForGroup('all', store), - subqueries: { maxCalls: 8, maxParallel: 4 }, - maxTurns: 24, - cost: { kind: 'llm' }, + instructions: ACTOR_PROMPT, + toolGroup: 'all', + limits: { maxLlmCalls: 8, maxIterations: 24, maxToolCalls: 64 }, } diff --git a/src/analyst/kinds/improvement.ts b/src/analyst/kinds/improvement.ts index a4570ece..33cdf98c 100644 --- a/src/analyst/kinds/improvement.ts +++ b/src/analyst/kinds/improvement.ts @@ -17,8 +17,7 @@ */ import { findingSubjectGrammarPromptFor } from '../finding-subject' -import type { TraceAnalystKindSpec } from '../kind-factory' -import { buildTraceToolsForGroup } from '../tool-groups' +import type { TraceAnalystDefinition } from '../kind-factory' const subjectGrammar = findingSubjectGrammarPromptFor('improvement') @@ -30,7 +29,7 @@ ${subjectGrammar} DISCOVERY → CANDIDATE-FIXES → COMPETE → CITE protocol: -1. \`traces.getDatasetOverview({})\` first. Note the agents, tools, and any system-prompt fingerprints (look for the prompt text echoed in early spans). +1. \`getDatasetOverview({})\` first. Note the agents, tools, and any system-prompt fingerprints (look for the prompt text echoed in early spans). 2. For each high-severity failure pattern, generate 2-3 candidate fixes. Real candidate axes: - **System-prompt edit** — add an instruction, remove a misleading one, restructure precedence - **Tool description edit** — rewrite a tool's description so the agent picks it correctly / passes valid args @@ -42,29 +41,28 @@ DISCOVERY → CANDIDATE-FIXES → COMPETE → CITE protocol: - **Skill / MCP / hook / subagent** — change the reusable profile component responsible for the behavior - **Workflow / rollout policy** — change orchestration, budget, sampling, or stopping behavior - **Code** — change an implementation path when profile edits cannot repair the behavior -3. **Compare candidate fixes with bounded subqueries.** Load the representative failure excerpts, then send one \`llmQuery\` per candidate-fix axis the same evidence. Ask for likely effect, side effects, and implementation scope. Subqueries cannot call trace tools; trace ids alone are insufficient context. +3. **Compare candidate fixes with bounded subqueries.** Load the representative failure excerpts, then send one \`llm_query\` per candidate-fix axis the same evidence. Ask for likely effect, side effects, and implementation scope. Subqueries cannot call trace tools; trace ids alone are insufficient context. 4. After the comparisons return, **pick the winning candidate per cluster** based on expected effect and risk, then emit ONE finding. Keep the alternatives and rejection reasons in the rationale so the recommendation is auditable. 5. **Cross-reference upstream findings.** Cite prior failure-mode or knowledge-gap findings as \`finding://\`. This builds the dependency graph that lets the dashboard show "fix #X resolves failure modes A, B, C." For each winning recommendation, emit ONE finding. Use one exact locus from the subject grammar and state the edit in one sentence. Match leverage to the source failure's severity; use medium for quality-of-life changes and info for cleanup with no behavioral effect. Cite the targeted \`finding://\` when available and the most representative span when useful. Quote the problem being fixed. Use confidence 0.85+ for a mechanical fix to a well-evidenced failure, 0.6-0.8 when judgment is required, and <0.5 for speculation. Explain in at most two sentences why this candidate beat its alternatives. The recommended action must be the literal diff, quoted replacement, tool description, or setting change. -If no upstream failure findings exist in this run, derive your own from the trace dataset using the failure-mode protocol inline (\`searchTrace\` for STATUS_CODE_ERROR / MaxTurnsExceeded / etc.). But prefer to consume upstream findings when present — the kinds are designed to chain. +If no upstream failure findings exist in this run, derive your own from the trace dataset using the failure-mode protocol inline (\`searchTrace\` for STATUS_CODE_ERROR / MaxTurnsExceeded / etc.). Prefer upstream findings when present because the analysts are designed to chain. -Do NOT propose a fix you cannot defend with evidence. "Tighten the prompt" is not a finding; "Add 'When the user asks for X, always Y' to the system prompt section "request-classification"" is. +Do NOT propose a fix you cannot defend with evidence. "Tighten the prompt" is not a finding; "Add 'When the user asks for X, always Y' to the system prompt section "request-classification"" is.` -OBSERVABILITY rules: -- Each non-final turn must emit at least one \`console.log\` for evidence.` - -export const IMPROVEMENT_KIND_SPEC: TraceAnalystKindSpec = { +export const IMPROVEMENT_KIND_SPEC: TraceAnalystDefinition = { id: 'improvement', description: 'Converts upstream failure / gap / poisoning findings into concrete locus-named edits (prompt, tool-doc, RAG, scaffolding) with leverage grades.', area: 'improvement', version: '1.2.0', - actorDescription: ACTOR_PROMPT, - buildTools: (store) => buildTraceToolsForGroup('all', store), - subqueries: { maxCalls: 8, maxParallel: 4 }, - maxTurns: 30, - maxRuntimeChars: 12000, - cost: { kind: 'llm' }, + instructions: ACTOR_PROMPT, + toolGroup: 'all', + limits: { + maxLlmCalls: 8, + maxIterations: 30, + maxToolCalls: 80, + maxOutputChars: 12_000, + }, } diff --git a/src/analyst/kinds/index.ts b/src/analyst/kinds/index.ts index f6a40861..e07d556f 100644 --- a/src/analyst/kinds/index.ts +++ b/src/analyst/kinds/index.ts @@ -19,7 +19,7 @@ export { IMPROVEMENT_KIND_SPEC } from './improvement' export { KNOWLEDGE_GAP_KIND_SPEC } from './knowledge-gap' export { KNOWLEDGE_POISONING_KIND_SPEC } from './knowledge-poisoning' -import type { TraceAnalystKindSpec } from '../kind-factory' +import type { TraceAnalystDefinition } from '../kind-factory' import { FAILURE_MODE_KIND_SPEC } from './failure-mode' import { IMPROVEMENT_KIND_SPEC } from './improvement' import { KNOWLEDGE_GAP_KIND_SPEC } from './knowledge-gap' @@ -30,7 +30,7 @@ import { KNOWLEDGE_POISONING_KIND_SPEC } from './knowledge-poisoning' * use: failure-mode first (no upstream deps), gap + poisoning next * (both depend on failures), improvement last (chains all three). */ -export const DEFAULT_TRACE_ANALYST_KINDS: readonly TraceAnalystKindSpec[] = [ +export const DEFAULT_TRACE_ANALYST_KINDS: readonly TraceAnalystDefinition[] = [ FAILURE_MODE_KIND_SPEC, KNOWLEDGE_GAP_KIND_SPEC, KNOWLEDGE_POISONING_KIND_SPEC, diff --git a/src/analyst/kinds/kinds.test.ts b/src/analyst/kinds/kinds.test.ts index 1d17f650..1d4c7066 100644 --- a/src/analyst/kinds/kinds.test.ts +++ b/src/analyst/kinds/kinds.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest' +import type { TraceAnalysisEngine } from '../engine' import { parseRawFinding, RawAnalystFindingSchema } from '../finding-signature' -import { createTraceAnalystKind, type TraceAnalystKindSpec } from '../kind-factory' +import { createTraceAnalyst, type TraceAnalystDefinition } from '../kind-factory' import { buildTraceToolsForGroup } from '../tool-groups' import { computeFindingId, makeFinding } from '../types' import { @@ -122,11 +123,11 @@ describe('default kind suite shape', () => { it('every default kind declares a non-empty lens prompt without duplicating the output contract', () => { for (const spec of DEFAULT_TRACE_ANALYST_KINDS) { - expect(spec.actorDescription.length).toBeGreaterThan(500) - expect(spec.actorDescription).not.toMatch(/`area`\s*=/) - expect(spec.actorDescription).not.toContain('`evidence_uri`') - expect(spec.actorDescription).not.toContain('`evidence_excerpt`') - expect(spec.actorDescription).not.toMatch(/final\(\{\s*findings/) + expect(spec.instructions.length).toBeGreaterThan(500) + expect(spec.instructions).not.toMatch(/`area`\s*=/) + expect(spec.instructions).not.toContain('`evidence_uri`') + expect(spec.instructions).not.toContain('`evidence_excerpt`') + expect(spec.instructions).not.toMatch(/final\(\{\s*findings/) } }) @@ -137,20 +138,23 @@ describe('default kind suite shape', () => { } }) - it('every default kind has an explicit bounded subquery budget', () => { + it('every default analyst has explicit recursive and trace-read limits', () => { for (const spec of DEFAULT_TRACE_ANALYST_KINDS) { - expect(spec.subqueries?.maxCalls ?? 0).toBeGreaterThanOrEqual(1) - expect(spec.subqueries?.maxParallel ?? 0).toBeGreaterThanOrEqual(2) + expect(spec.limits?.maxLlmCalls ?? 0).toBeGreaterThanOrEqual(1) + expect(spec.limits?.maxIterations ?? 0).toBeGreaterThanOrEqual(1) + expect(spec.limits?.maxToolCalls ?? 0).toBeGreaterThanOrEqual(1) } }) it('improvement kind has a maximum subquery budget for competing fixes', () => { - const max = Math.max(...DEFAULT_TRACE_ANALYST_KINDS.map((k) => k.subqueries?.maxCalls ?? 0)) - expect(IMPROVEMENT_KIND_SPEC.subqueries?.maxCalls).toBe(max) + const max = Math.max( + ...DEFAULT_TRACE_ANALYST_KINDS.map((kind) => kind.limits?.maxLlmCalls ?? 0), + ) + expect(IMPROVEMENT_KIND_SPEC.limits?.maxLlmCalls).toBe(max) }) it("knowledge-gap prompt anchors on agent-knowledge wiki + websearch + tool-doc layers, not generic 'RAG'", () => { - const p = KNOWLEDGE_GAP_KIND_SPEC.actorDescription + const p = KNOWLEDGE_GAP_KIND_SPEC.instructions expect(p).toMatch(/agent-knowledge/) expect(p).toMatch(/wiki/i) expect(p).toMatch(/websearch/i) @@ -158,12 +162,12 @@ describe('default kind suite shape', () => { }) it('knowledge-poisoning prompt enforces dual-verify (acted on + actually false)', () => { - expect(KNOWLEDGE_POISONING_KIND_SPEC.actorDescription).toMatch(/DUAL-VERIFY/) + expect(KNOWLEDGE_POISONING_KIND_SPEC.instructions).toMatch(/DUAL-VERIFY/) expect(KNOWLEDGE_POISONING_KIND_SPEC.minimumEvidenceCitations).toBe(2) }) it('failure-mode prompt requires clustering, not enumeration', () => { - expect(FAILURE_MODE_KIND_SPEC.actorDescription).toMatch(/Cluster, do not enumerate/i) + expect(FAILURE_MODE_KIND_SPEC.instructions).toMatch(/Cluster, do not enumerate/i) }) }) @@ -198,18 +202,17 @@ describe('tool-groups filter the analyst tool surface narrowly', () => { }) }) -describe('createTraceAnalystKind wires the spec into the Analyst contract', () => { +describe('createTraceAnalyst wires a definition into the Analyst contract', () => { it('returns a registry-ready Analyst that delegates to the kind id + version', () => { - const spec: TraceAnalystKindSpec = { + const spec: TraceAnalystDefinition = { id: 'test-kind', description: 'test', area: 'test', version: '0.0.1', - actorDescription: 'mock prompt', - buildTools: () => [], - cost: { kind: 'llm' }, + instructions: 'mock prompt', + toolGroup: 'all', } - const analyst = createTraceAnalystKind(spec, { ai: stubAi(), model: 'test-model' }) + const analyst = createTraceAnalyst(spec, { engine: stubEngine() }) expect(analyst.id).toBe('test-kind') expect(analyst.version).toBe('0.0.1') expect(analyst.inputKind).toBe('trace-store') @@ -217,18 +220,16 @@ describe('createTraceAnalystKind wires the spec into the Analyst contract', () = }) it('versionSuffix appends to the kind version (used by optimizer pipelines)', () => { - const spec: TraceAnalystKindSpec = { + const spec: TraceAnalystDefinition = { id: 'k', - description: '', + description: 'test', area: 'k', version: '1.0.0', - actorDescription: '', - buildTools: () => [], - cost: { kind: 'llm' }, + instructions: 'test', + toolGroup: 'all', } - const analyst = createTraceAnalystKind(spec, { - ai: stubAi(), - model: 'test-model', + const analyst = createTraceAnalyst(spec, { + engine: stubEngine(), versionSuffix: 'mipro-2026-05-18', }) expect(analyst.version).toBe('1.0.0+mipro-2026-05-18') @@ -265,7 +266,7 @@ describe('createTraceAnalystKind wires the spec into the Analyst contract', () = ] const out = renderPriorFindings(prior) expect(out).toMatch(/PRIOR FINDINGS/) - expect(out).toMatch(/REUSE the `finding_id`/) + expect(out).toMatch(/Reuse a matching finding id/) const first = prior[0] const second = prior[1] if (!first || !second) throw new Error('test setup invariant') @@ -289,7 +290,7 @@ describe('createTraceAnalystKind wires the spec into the Analyst contract', () = }), ) const out = renderPriorFindings(many) - expect(out).toContain('+20 more prior findings') + expect(out).toContain('20 older findings omitted') }) }) @@ -314,6 +315,13 @@ function stubStore() { return {} as never } -function stubAi() { - return {} as never +function stubEngine(): TraceAnalysisEngine { + return { + id: 'test-engine', + description: 'test', + model: 'test-model', + analyze: async () => { + throw new Error('not called') + }, + } } diff --git a/src/analyst/kinds/knowledge-gap.ts b/src/analyst/kinds/knowledge-gap.ts index 5956db5a..c66e542d 100644 --- a/src/analyst/kinds/knowledge-gap.ts +++ b/src/analyst/kinds/knowledge-gap.ts @@ -25,8 +25,7 @@ */ import { findingSubjectGrammarPromptFor } from '../finding-subject' -import type { TraceAnalystKindSpec } from '../kind-factory' -import { buildTraceToolsForGroup } from '../tool-groups' +import type { TraceAnalystDefinition } from '../kind-factory' const subjectGrammar = findingSubjectGrammarPromptFor('knowledge-gap') @@ -38,7 +37,7 @@ ${subjectGrammar} DISCOVERY → ATTRIBUTE-TO-LAYER → CITE protocol: -1. \`traces.getDatasetOverview({})\` first. Note which agents, tools, and models appear. +1. \`getDatasetOverview({})\` first. Note which agents, tools, and models appear. 2. Pull traces where the agent shows gap signals. The strongest signals are: - Self-correction turns ("I assumed X but…", "let me re-check", "actually,") - Clarifying-question turns where the agent asked the user something the runtime should have surfaced @@ -47,26 +46,21 @@ DISCOVERY → ATTRIBUTE-TO-LAYER → CITE protocol: - Web-search calls returning pages dated before a known cutoff for content that changes (versioned APIs, schemas, policies) - Agent quoting a tool's docs / system prompt incorrectly because the actual text was insufficient - Fabricated identifiers that don't appear in dataset \`sample_trace_ids\` - Use \`traces.searchTrace\` with patterns like \`I (don.?t|do not) know\`, \`assumed\`, \`unclear\`, \`could you (clarify|tell me|provide)\`, \`not found\`, \`undefined\`, \`unknown\`, \`null\`, dates older than the analysis window, or the agent's specific clarification phrases. + Use \`searchTrace\` with patterns like \`I (don.?t|do not) know\`, \`assumed\`, \`unclear\`, \`could you (clarify|tell me|provide)\`, \`not found\`, \`undefined\`, \`unknown\`, \`null\`, dates older than the analysis window, or the agent's specific clarification phrases. 3. For each gap, identify the **layer of the runtime that should have prevented it** and use its exact locus from the subject grammar above. 4. For each defensible gap, emit ONE finding. Use an exact locus from the subject grammar and name the missing or stale knowledge (for example, "wiki has no page on invoice line-item shape; agent re-derived it from raw spans"). Rate it high when it caused failure or a clarifying question, medium for unnecessary turns, and low for minor inefficiency. Cite the span where the question, correction, retrieval miss, or stale result surfaced and quote it exactly. Use confidence 0.85+ when the agent articulated the gap and 0.6-0.8 when inferred. Recommend a concrete wiki edit for an agent-knowledge locus or a prompt/tool-description edit otherwise. -**Compare layers over loaded evidence.** After the first scan, load the exact excerpts behind candidates across \`agent-knowledge:*\`, \`websearch:outdated\`, \`tool-doc:*\`, \`system-prompt:*\`, and \`memory:*\`. Use one bounded \`llmQuery\` per layer to classify those excerpts. Subqueries cannot call trace tools. Merge their classifications into the final finding set only when the source excerpts support them. +**Compare layers over loaded evidence.** After the first scan, load the exact excerpts behind candidates across \`agent-knowledge:*\`, \`websearch:outdated\`, \`tool-doc:*\`, \`system-prompt:*\`, and \`memory:*\`. Use one bounded \`llm_query\` per layer to classify those excerpts. Subqueries cannot call trace tools. Merge their classifications into the final finding set only when the source excerpts support them. -Do NOT report a gap that the agent later recovered from cleanly within the same turn — that's resilience, not a gap. Cite the *non-recovery* version when both exist. +Do NOT report a gap that the agent later recovered from cleanly within the same turn. That is resilience, not a gap. Cite the non-recovery version when both exist.` -OBSERVABILITY rules: -- Each non-final turn must emit at least one \`console.log\` for evidence.` - -export const KNOWLEDGE_GAP_KIND_SPEC: TraceAnalystKindSpec = { +export const KNOWLEDGE_GAP_KIND_SPEC: TraceAnalystDefinition = { id: 'knowledge-gap', description: 'Identifies missing or stale pieces of knowledge — primarily against the agent-knowledge wiki — and attributes each to the runtime layer (wiki page, claim, raw source, websearch, tool-doc, system-prompt, memory) that should have held it.', area: 'knowledge-gap', version: '1.2.0', - actorDescription: ACTOR_PROMPT, - buildTools: (store) => buildTraceToolsForGroup('discoveryAndSearch', store), - subqueries: { maxCalls: 5, maxParallel: 4 }, - maxTurns: 18, - cost: { kind: 'llm' }, + instructions: ACTOR_PROMPT, + toolGroup: 'discoveryAndSearch', + limits: { maxLlmCalls: 5, maxIterations: 18, maxToolCalls: 48 }, } diff --git a/src/analyst/kinds/knowledge-poisoning.ts b/src/analyst/kinds/knowledge-poisoning.ts index c3138890..9373f86d 100644 --- a/src/analyst/kinds/knowledge-poisoning.ts +++ b/src/analyst/kinds/knowledge-poisoning.ts @@ -17,8 +17,7 @@ */ import { findingSubjectGrammarPromptFor } from '../finding-subject' -import type { TraceAnalystKindSpec } from '../kind-factory' -import { buildTraceToolsForGroup } from '../tool-groups' +import type { TraceAnalystDefinition } from '../kind-factory' const subjectGrammar = findingSubjectGrammarPromptFor('knowledge-poisoning') @@ -28,7 +27,7 @@ ${subjectGrammar} DISCOVERY → DUAL-VERIFY → CITE protocol: -1. \`traces.getDatasetOverview({})\` first. Identify the agents, models, and tools. +1. \`getDatasetOverview({})\` first. Identify the agents, models, and tools. 2. Pull traces where the agent's confident action was later contradicted. Strongest signals: - Agent stated a fact in one span; a later span surfaced contradictory evidence; the agent then proceeded anyway or fabricated reconciliation. - Tool call with stale arguments (an id that no longer exists, an API shape that changed). @@ -36,31 +35,26 @@ DISCOVERY → DUAL-VERIFY → CITE protocol: - Web-search result the agent cited that returned an outdated page; agent treated it as canonical. - System-prompt instruction the agent followed that ground-truth evidence in the trace contradicts (e.g. prompt says "use endpoint A"; tool reply says "endpoint A deprecated, use B"). - Repeated wrong-shape parsing despite the tool's actual output proving the shape. -3. Use \`traces.searchTrace\` with regex on phrases like \`actually\`, \`turns out\`, \`previously assumed\`, \`old version\`, \`deprecated\`, \`updated to\`, \`now uses\`, or specific entity names you suspect have changed. +3. Use \`searchTrace\` with regex on phrases like \`actually\`, \`turns out\`, \`previously assumed\`, \`old version\`, \`deprecated\`, \`updated to\`, \`now uses\`, or specific entity names you suspect have changed. 4. For each candidate poisoning, **DUAL-VERIFY**: - Confirm the agent actually acted on the false belief (cite the span where it did) - Confirm the belief is actually false in this trace's own evidence (cite the span that contradicts it) - Only emit a finding when both halves are nailed down. If you can only nail one, drop it — single-evidence poisoning findings are too speculative to be useful. + Only emit a finding when both halves are supported. If you can only support one, drop it because single-evidence poisoning findings are too speculative to be useful. -**Independently assess both halves.** Load the action excerpt and contradicting excerpt yourself, then send bounded \`llmQuery\` calls the exact evidence for "did the agent act?" and "does the trace contradict the belief?" Subqueries cannot call trace tools. Accept a poisoning only when both assessments and the source excerpts support it. +**Independently assess both halves.** Load the action excerpt and contradicting excerpt yourself, then send bounded \`llm_query\` calls the exact evidence for "did the agent act?" and "does the trace contradict the belief?" Subqueries cannot call trace tools. Accept a poisoning only when both assessments and the source excerpts support it. For each confirmed poisoning, emit ONE finding. Use the source of the false belief as the exact subject. State "agent believed X (from source S); trace evidence shows X is false." Rate it critical for a wrong user-visible action, high when caught internally after significant waste, and medium for inefficiency. Cite BOTH the action span and the contradicting span with exact quotes. Use confidence 0.85+ when both halves have exact quotes and 0.6-0.8 when one half is inferred. Recommend the literal source correction: update the wiki claim, invalidate and re-curate the raw source, or replace the stale prompt/tool instruction. -Do NOT report a finding if the agent caught and corrected the false belief in the same turn — that's the system working. Reserve poisoning for cases where the false belief shaped downstream action. +Do NOT report a finding if the agent caught and corrected the false belief in the same turn. Reserve poisoning for cases where the false belief shaped downstream action.` -OBSERVABILITY rules: -- Each non-final turn must emit at least one \`console.log\` for evidence.` - -export const KNOWLEDGE_POISONING_KIND_SPEC: TraceAnalystKindSpec = { +export const KNOWLEDGE_POISONING_KIND_SPEC: TraceAnalystDefinition = { id: 'knowledge-poisoning', description: 'Identifies confident-but-wrong actions caused by stale memory, contradicting RAG, deprecated tool docs, or outdated system-prompt instructions.', area: 'knowledge-poisoning', version: '1.2.0', - actorDescription: ACTOR_PROMPT, - buildTools: (store) => buildTraceToolsForGroup('all', store), - subqueries: { maxCalls: 8, maxParallel: 4 }, - maxTurns: 20, + instructions: ACTOR_PROMPT, + toolGroup: 'all', + limits: { maxLlmCalls: 8, maxIterations: 20, maxToolCalls: 64 }, minimumEvidenceCitations: 2, - cost: { kind: 'llm' }, } diff --git a/src/analyst/structure-findings.test.ts b/src/analyst/structure-findings.test.ts deleted file mode 100644 index 44236296..00000000 --- a/src/analyst/structure-findings.test.ts +++ /dev/null @@ -1,177 +0,0 @@ -import { describe, expect, it, vi } from 'vitest' -import { RawAnalystFindingSchema } from './finding-signature' -import { structureFindings } from './structure-findings' - -// Stub an OpenAI-compatible chat completion. Mocks ONLY the network boundary. -function stubFetch(content: string): typeof fetch { - return (async () => - new Response(JSON.stringify({ choices: [{ message: { content } }] }), { - status: 200, - headers: { 'content-type': 'application/json' }, - })) as unknown as typeof fetch -} - -const REPORT = - 'The agent re-sent its full history every step (input tokens 671→8776) and never verified state. ' + - 'It looped on world.execute with no fallback. These inefficiencies materially raised cost and risk ' + - 'across the seven-step run, and the lack of self-verification let an incorrect intermediate stand.' - -describe('structureFindings — free-form report → structured findings (any model)', () => { - it('extracts findings from a FENCED JSON response (the wrapper that breaks naive parsing)', async () => { - const fenced = - '```json\n[{"severity":"high","claim":"no context compression — input grew 671→8776",' + - '"evidence":[{"uri":"span://t1/s5"}],"confidence":0.95},' + - '{"severity":"medium","claim":"no self-verification","evidence":[{"uri":"report://summary"}],"confidence":0.9}]\n```' - const res = await structureFindings({ - report: REPORT, - analystId: 'failure-mode', - area: 'failure-mode', - model: 'any', - baseUrl: 'https://x/v1', - apiKey: 'k', - fetchImpl: stubFetch(fenced), - }) - expect(res.outcome).toBe('ok') - expect(res.findings).toHaveLength(2) - expect(res.findings[0]!.area).toBe('failure-mode') - expect(res.findings[0]!.evidence_refs[0]!.kind).toBe('span') - expect(res.findings[0]!.claim).toContain('671→8776') - }) - - it('a substantive report that yields nothing after the reask → extraction_failed (no silent empty)', async () => { - const res = await structureFindings({ - report: REPORT, - analystId: 'failure-mode', - area: 'failure-mode', - model: 'any', - baseUrl: 'https://x/v1', - apiKey: 'k', - maxReasks: 1, - fetchImpl: stubFetch('I could not find anything.'), // never valid JSON - }) - expect(res.outcome).toBe('extraction_failed') - expect(res.findings).toHaveLength(0) - }) - - it('a SHORT report with no findings is a legitimate empty (ok), not a failure', async () => { - const res = await structureFindings({ - report: 'No issues observed.', - analystId: 'failure-mode', - area: 'failure-mode', - model: 'any', - baseUrl: 'https://x/v1', - apiKey: 'k', - fetchImpl: stubFetch('[]'), - }) - expect(res.outcome).toBe('ok') - expect(res.findings).toHaveLength(0) - }) - - it('rejects recovery findings without citations instead of making the report self-evidencing', async () => { - const noEvidence = - '[{"severity":"high","claim":"agent never verified its writes","confidence":0.9}]' - const res = await structureFindings({ - report: REPORT, - analystId: 'failure-mode', - area: 'failure-mode', - model: 'any', - baseUrl: 'https://x/v1', - apiKey: 'k', - fetchImpl: stubFetch(noEvidence), - }) - expect(res.outcome).toBe('extraction_failed') - expect(res.findings).toEqual([]) - }) - - it('passes every citation to processRow', async () => { - const res = await structureFindings({ - report: REPORT, - analystId: 'failure-mode', - area: 'failure-mode', - model: 'any', - baseUrl: 'https://x/v1', - apiKey: 'k', - fetchImpl: stubFetch( - JSON.stringify([ - { - severity: 'high', - claim: 'two sources support this finding', - evidence: [{ uri: 'span://t/s' }, { uri: 'event://t/e' }], - confidence: 0.9, - }, - ]), - ), - processRow: (row) => ({ - ...RawAnalystFindingSchema.parse(row), - claim: `${row.claim}; reviewed`, - }), - }) - - expect(res.findings[0]).toMatchObject({ - claim: 'two sources support this finding; reviewed', - evidence_refs: [{ uri: 'span://t/s' }, { uri: 'event://t/e' }], - }) - }) - - it('rejects malformed processRow output', async () => { - const response = JSON.stringify([ - { - severity: 'high', - claim: 'valid model output', - evidence: [{ uri: 'span://t/s' }], - confidence: 0.9, - }, - ]) - const res = await structureFindings({ - report: REPORT, - analystId: 'failure-mode', - area: 'failure-mode', - model: 'any', - baseUrl: 'https://x/v1', - fetchImpl: stubFetch(response), - processRow: (row) => { - const { evidence: _evidence, ...invalid } = row - return invalid as never - }, - }) - - expect(res).toEqual({ findings: [], outcome: 'extraction_failed' }) - }) - - it('forwards cancellation and does not start recovery work after abort', async () => { - const controller = new AbortController() - const fetchImpl = vi.fn(stubFetch('[]')) - controller.abort(new DOMException('cancelled', 'AbortError')) - - await expect( - structureFindings({ - report: REPORT, - analystId: 'failure-mode', - area: 'failure-mode', - model: 'any', - baseUrl: 'https://x/v1', - apiKey: 'k', - signal: controller.signal, - fetchImpl, - }), - ).rejects.toMatchObject({ name: 'AbortError' }) - expect(fetchImpl).not.toHaveBeenCalled() - }) - - it('rejects an invalid reask count before any paid work', async () => { - const fetchImpl = vi.fn(stubFetch('[]')) - - await expect( - structureFindings({ - report: REPORT, - analystId: 'failure-mode', - area: 'failure-mode', - model: 'any', - baseUrl: 'https://x/v1', - maxReasks: Number.POSITIVE_INFINITY, - fetchImpl, - }), - ).rejects.toThrow(/maxReasks/) - expect(fetchImpl).not.toHaveBeenCalled() - }) -}) diff --git a/src/analyst/structure-findings.ts b/src/analyst/structure-findings.ts deleted file mode 100644 index d4ff9de3..00000000 --- a/src/analyst/structure-findings.ts +++ /dev/null @@ -1,144 +0,0 @@ -/** - * `structureFindings` — the deferred structuring pass (DSPy TwoStepAdapter / - * HALO `synthesize_traces` analog). The agentic actor reasons FREE-FORM and - * emits a prose `report` (which any model does reliably); this separate, cheap - * call's ONLY job is to turn that report into `AnalystFinding[]`. Decoupling - * reasoning from structuring is what makes the SEMANTIC findings model-agnostic - * — the reasoning model never has to satisfy a strict typed-array contract - * while it diagnoses. - * - * Forgiving: the response runs through `coerceToFindingRows` (de-fence, lift - * single→array) before Zod, and on a zero-finding extraction from a substantive - * report it reasks ONCE with the schema restated. Returns a typed outcome so a - * legitimate "nothing to report" is distinguishable from a failed extraction - * (no silent empty). - */ - -import { CostLedger, type CostLedgerHandle } from '../cost-ledger' -import { - callLlm, - costReceiptFromLlm, - costReceiptFromLlmError, - type LlmCallRequest, - type LlmClientOptions, - maximumChargeForLlmRequest, -} from '../llm-client' -import { - evidenceRefsFromRawFinding, - parseRawFinding, - RAW_FINDING_SCHEMA_PROMPT, - type RawAnalystFinding, -} from './finding-signature' -import { coerceToFindingRows } from './parse-tolerant' -import { type AnalystFinding, makeFinding } from './types' - -export interface StructureFindingsOptions { - /** The actor's free-form diagnosis prose. */ - report: string - analystId: string - /** Coarse classification stamped on every extracted finding. */ - area: string - model: string - baseUrl: string - apiKey?: string - /** Optional ledger for direct use. */ - costLedger?: CostLedgerHandle - costPhase?: string - costTags?: Record - maxTokens?: number - signal?: AbortSignal - /** Max reask attempts after a zero/invalid extraction. Default 1. */ - maxReasks?: number - /** Apply the caller's normal finding rules before a recovered row is lifted. */ - processRow?: (row: RawAnalystFinding) => RawAnalystFinding | null - /** Provenance copied onto every recovered finding. */ - findingMetadata?: Record - /** Test seam: inject a fetch (no network in unit tests). */ - fetchImpl?: LlmClientOptions['fetch'] -} - -export interface StructureFindingsResult { - findings: AnalystFinding[] - outcome: 'ok' | 'extraction_failed' -} - -const SYSTEM = [ - 'You convert a free-form trace-analysis report into a STRICT JSON array of findings.', - 'Output ONLY the JSON array — no prose, no code fences.', - RAW_FINDING_SCHEMA_PROMPT, - 'Omit subject when the report does not contain an exact valid locus.', - 'If the report asserts NO problems, output exactly [].', -].join(' ') - -function buildRows(raw: unknown, opts: StructureFindingsOptions): AnalystFinding[] { - const rows = coerceToFindingRows(raw) - const out: AnalystFinding[] = [] - for (const row of rows) { - const parsed = parseRawFinding(row) - if (!parsed) continue - const callbackResult = opts.processRow ? opts.processRow(parsed) : parsed - if (!callbackResult) continue - const processed = parseRawFinding(callbackResult) - if (!processed) continue - out.push( - makeFinding({ - analyst_id: opts.analystId, - area: opts.area, - subject: processed.subject, - claim: processed.claim, - rationale: processed.rationale, - severity: processed.severity, - confidence: processed.confidence, - evidence_refs: evidenceRefsFromRawFinding(processed), - recommended_action: processed.recommended_action, - ...(opts.findingMetadata ? { metadata: { ...opts.findingMetadata } } : {}), - }), - ) - } - return out -} - -export async function structureFindings( - opts: StructureFindingsOptions, -): Promise { - const maxReasks = opts.maxReasks ?? 1 - if (!Number.isSafeInteger(maxReasks) || maxReasks < 0) { - throw new RangeError('structureFindings: maxReasks must be a non-negative safe integer') - } - const llm = { baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl } - const costLedger = opts.costLedger ?? new CostLedger() - let user = `TRACE-ANALYSIS REPORT:\n${opts.report}\n\nReturn the findings JSON array.` - - for (let attempt = 0; attempt <= maxReasks; attempt++) { - const request: LlmCallRequest = { - model: opts.model, - messages: [ - { role: 'system', content: SYSTEM }, - { role: 'user', content: user }, - ], - maxTokens: opts.maxTokens ?? 2_000, - } - const paid = await costLedger.runPaidCall({ - channel: 'analyst', - phase: opts.costPhase ?? 'analyst.structure-findings', - actor: 'structure-findings', - model: opts.model, - signal: opts.signal, - maximumCharge: maximumChargeForLlmRequest(request, llm), - tags: { ...opts.costTags, analystId: opts.analystId, attempt: String(attempt) }, - execute: (signal, callId) => callLlm(request, { ...llm, signal, idempotencyKey: callId }), - receipt: costReceiptFromLlm, - receiptFromError: costReceiptFromLlmError, - }) - if (!paid.succeeded) throw paid.error - const res = paid.value - const text = res.content.trim() - const findings = buildRows(text, opts) - if (findings.length > 0) return { findings, outcome: 'ok' } - // A report that asserts nothing is a legitimate empty — only reask when the - // report is substantive (the extraction, not the diagnosis, likely failed). - if (opts.report.trim().length < 200) return { findings: [], outcome: 'ok' } - user = `${user}\n\nThat produced no valid findings. The report DOES describe issues — re-extract them as the strict JSON array described in the system prompt. Output ONLY the array.` - } - return { findings: [], outcome: 'extraction_failed' } -} diff --git a/src/analyst/tool-groups.ts b/src/analyst/tool-groups.ts index d8bbe9a1..f3976d86 100644 --- a/src/analyst/tool-groups.ts +++ b/src/analyst/tool-groups.ts @@ -6,13 +6,15 @@ * the whole bundle keeps every kind's actor-context budget tight and * makes "what can this analyst see?" obvious at registration time. * - * Each function in the group keeps its full `name`/`description` from - * `buildTraceAnalystTools` — we filter, we don't re-implement. + * Each function in the group keeps its full `name`, schema, handler, and + * description from `buildTraceAnalysisToolDescriptors`. */ -import type { AxFunction } from '@ax-llm/ax' import type { TraceAnalysisStore } from '../trace-analyst/store' -import { buildTraceAnalystTools } from '../trace-analyst/tools' +import { + buildTraceAnalysisToolDescriptors, + type TraceAnalysisToolDescriptor, +} from '../trace-analyst/tools' /** Named tool sets. Kinds pass `tools: TRACE_TOOL_GROUPS.failureForensics` etc. */ export type TraceToolGroupName = @@ -59,17 +61,17 @@ const TOOL_NAMES_BY_GROUP: Record> = { /** * Build the tool set for a named group bound to a specific trace store. * - * `all` returns every tool. Other groups filter `buildTraceAnalystTools` + * `all` returns every tool. Other groups filter the canonical descriptors * by name to the documented subset. An unrecognised group name throws — * silently returning all tools would defeat the cost-control point. */ export function buildTraceToolsForGroup( group: TraceToolGroupName, store: TraceAnalysisStore, -): AxFunction[] { - const all = buildTraceAnalystTools({ store }) +): TraceAnalysisToolDescriptor[] { + const all = buildTraceAnalysisToolDescriptors({ store }) if (group === 'all') return all const allow = TOOL_NAMES_BY_GROUP[group] if (!allow) throw new Error(`unknown trace tool group: ${group}`) - return all.filter((tool) => allow.has((tool as { name: string }).name)) + return all.filter((tool) => allow.has(tool.name)) } diff --git a/src/analyst/trace-tool-callback.test.ts b/src/analyst/trace-tool-callback.test.ts new file mode 100644 index 00000000..5d2d7095 --- /dev/null +++ b/src/analyst/trace-tool-callback.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest' +import type { TraceAnalysisToolDescriptor } from '../trace-analyst/tools' +import { startTraceToolCallback } from './trace-tool-callback' + +const tool: TraceAnalysisToolDescriptor = { + namespace: 'traces', + name: 'echo', + description: 'Echo input.', + parameters: { type: 'object' }, + handler: async (args) => ({ echoed: args }), +} + +describe('startTraceToolCallback', () => { + it('requires authentication and forwards exact arguments', async () => { + const callback = await startTraceToolCallback({ tools: [tool], maxCalls: 2 }) + try { + const unauthorized = await fetch(callback.url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ name: 'echo', args: { value: 1 } }), + }) + expect(unauthorized.status).toBe(401) + + const response = await fetch(callback.url, { + method: 'POST', + headers: { + authorization: `Bearer ${callback.token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name: 'echo', args: { value: 1 } }), + }) + expect(response.status).toBe(200) + await expect(response.json()).resolves.toEqual({ + result: { echoed: { value: 1 } }, + }) + expect(callback.calls()).toBe(1) + } finally { + await callback.close() + } + }) + + it('rejects calls before executing a tool beyond the declared limit', async () => { + const callback = await startTraceToolCallback({ tools: [tool], maxCalls: 1 }) + try { + const call = () => + fetch(callback.url, { + method: 'POST', + headers: { + authorization: `Bearer ${callback.token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ name: 'echo', args: {} }), + }) + expect((await call()).status).toBe(200) + expect((await call()).status).toBe(429) + expect(callback.calls()).toBe(1) + } finally { + await callback.close() + } + }) +}) diff --git a/src/analyst/trace-tool-callback.ts b/src/analyst/trace-tool-callback.ts new file mode 100644 index 00000000..51765ad9 --- /dev/null +++ b/src/analyst/trace-tool-callback.ts @@ -0,0 +1,178 @@ +import { randomBytes } from 'node:crypto' +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http' +import { closeServer, listenLocal, sendJson } from '../campaign/external-optimizer-http' +import type { TraceAnalysisToolDescriptor } from '../trace-analyst/tools' + +const MAX_REQUEST_BYTES = 1_000_000 +const MAX_RESPONSE_BYTES = 4_000_000 + +export interface TraceToolCallback { + url: string + token: string + calls: () => number + close: () => Promise +} + +/** Expose one bounded trace-tool set only on an authenticated loopback socket. */ +export async function startTraceToolCallback(args: { + tools: readonly TraceAnalysisToolDescriptor[] + maxCalls: number + signal?: AbortSignal +}): Promise { + if (!Number.isSafeInteger(args.maxCalls) || args.maxCalls <= 0) { + throw new TypeError('trace tool callback maxCalls must be a positive safe integer') + } + args.signal?.throwIfAborted() + const byName = new Map(args.tools.map((tool) => [tool.name, tool])) + if (byName.size !== args.tools.length) { + throw new Error('trace tool callback received duplicate tool names') + } + + const token = randomBytes(32).toString('hex') + let calls = 0 + let accepting = true + let closePromise: Promise | undefined + const activeControllers = new Set() + const activeHandlers = new Set>() + const server = createServer((request, response) => { + if (!accepting) { + sendJsonIfOpen(response, 503, { error: 'trace tool callback is closing' }) + return + } + const controller = new AbortController() + const abortRequest = (): void => { + request.destroy() + response.destroy() + } + activeControllers.add(controller) + controller.signal.addEventListener('abort', abortRequest, { once: true }) + + let handler!: Promise + handler = handleRequest(request, response, controller.signal).finally(() => { + controller.signal.removeEventListener('abort', abortRequest) + activeControllers.delete(controller) + activeHandlers.delete(handler) + }) + activeHandlers.add(handler) + void handler.catch(() => undefined) + }) + const port = await listenLocal(server) + const close = (): Promise => { + closePromise ??= closeCallback() + return closePromise + } + const onAbort = (): void => { + void close().catch(() => undefined) + } + args.signal?.addEventListener('abort', onAbort, { once: true }) + if (args.signal?.aborted) onAbort() + + return { + url: `http://127.0.0.1:${port}/call`, + token, + calls: () => calls, + close, + } + + async function handleRequest( + request: IncomingMessage, + response: ServerResponse, + signal: AbortSignal, + ): Promise { + try { + if (request.method !== 'POST' || request.url !== '/call') { + sendJsonIfOpen(response, 404, { error: 'not found' }) + return + } + if (request.headers.authorization !== `Bearer ${token}`) { + sendJsonIfOpen(response, 401, { error: 'unauthorized' }) + return + } + if (calls >= args.maxCalls) { + sendJsonIfOpen(response, 429, { error: 'trace tool call limit reached' }) + return + } + const body = await readJson(request) + if (!isRecord(body) || typeof body.name !== 'string' || !('args' in body)) { + sendJsonIfOpen(response, 400, { error: 'name and args are required' }) + return + } + const tool = byName.get(body.name) + if (!tool) { + sendJsonIfOpen(response, 404, { error: `unknown trace tool '${body.name}'` }) + return + } + calls += 1 + const result = await tool.handler(body.args, { signal }) + const encoded = JSON.stringify({ result }) + if (Buffer.byteLength(encoded) > MAX_RESPONSE_BYTES) { + sendJsonIfOpen(response, 413, { error: 'trace tool response too large' }) + return + } + response.writeHead(200, { + 'content-type': 'application/json; charset=utf-8', + 'content-length': String(Buffer.byteLength(encoded)), + }) + response.end(encoded) + } catch (error) { + sendJsonIfOpen(response, signal.aborted ? 499 : 400, { + error: error instanceof Error ? error.message : String(error), + }) + } + } + + async function closeCallback(): Promise { + args.signal?.removeEventListener('abort', onAbort) + accepting = false + const closingServer = closeServer(server) + server.closeIdleConnections?.() + for (const controller of activeControllers) controller.abort() + const [serverResult] = await Promise.allSettled([ + closingServer, + waitForActiveHandlers(activeHandlers), + ]) + if (activeControllers.size !== 0 || activeHandlers.size !== 0) { + throw new Error('trace tool callback closed with active requests') + } + if (serverResult?.status === 'rejected') throw serverResult.reason + } +} + +async function waitForActiveHandlers(activeHandlers: Set>): Promise { + while (activeHandlers.size > 0) { + await Promise.allSettled([...activeHandlers]) + } +} + +function readJson(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let size = 0 + const chunks: Buffer[] = [] + request.on('data', (chunk: Buffer) => { + size += chunk.length + if (size > MAX_REQUEST_BYTES) { + reject(new Error('trace tool request too large')) + request.destroy() + return + } + chunks.push(chunk) + }) + request.on('error', reject) + request.on('end', () => { + try { + resolve(JSON.parse(Buffer.concat(chunks).toString('utf8'))) + } catch (error) { + reject(error) + } + }) + }) +} + +function sendJsonIfOpen(response: ServerResponse, status: number, body: unknown): void { + if (response.destroyed || response.writableEnded) return + sendJson(response, status, body) +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} diff --git a/src/analyst/types.ts b/src/analyst/types.ts index c7508f28..0a92a90f 100644 --- a/src/analyst/types.ts +++ b/src/analyst/types.ts @@ -327,7 +327,7 @@ export interface AnalystRunResult { * over the same stream, so the two surfaces share their invariants. * * Per-finding events are intentionally omitted — analyzers are batch - * operations (an Ax actor returns the full `findings:json[]` at the + * operations (a recursive engine returns the full `findings:json[]` at the * end of the responder), so streaming inside one analyst would only * emit partial JSON consumers can't render. The kind-completion event * is the right granularity; subscribers wanting per-finding rendering diff --git a/src/campaign/analyst-surface.test.ts b/src/campaign/analyst-surface.test.ts index 3b5bb0ae..6f0cfd16 100644 --- a/src/campaign/analyst-surface.test.ts +++ b/src/campaign/analyst-surface.test.ts @@ -55,7 +55,7 @@ describe('buildTraceAnalystSurfaceDispatch', () => { await dispatch('Inspect failed tool calls.', scenario, context) expect(seen).toEqual([ { - actorDescription: 'Inspect failed tool calls.', + instructions: 'Inspect failed tool calls.', traceStore, runId: 'cell-1:failed-command', signal, diff --git a/src/campaign/analyst-surface.ts b/src/campaign/analyst-surface.ts index d3a1a538..7eb99e83 100644 --- a/src/campaign/analyst-surface.ts +++ b/src/campaign/analyst-surface.ts @@ -25,7 +25,7 @@ export interface TraceAnalystArtifact { export interface BuildTraceAnalystSurfaceDispatchOptions { analyze(input: { - actorDescription: string + instructions: string traceStore: TraceAnalysisStore runId: string signal: AbortSignal @@ -46,7 +46,7 @@ export function buildTraceAnalystSurfaceDispatch( ) } return options.analyze({ - actorDescription: surface, + instructions: surface, traceStore: scenario.traceStore, runId: `${context.cellId}:${scenario.id}`, signal: context.signal, diff --git a/src/campaign/external-optimizer-model-proxy.ts b/src/campaign/external-optimizer-model-proxy.ts index 2f67d5e0..2f2c3ae2 100644 --- a/src/campaign/external-optimizer-model-proxy.ts +++ b/src/campaign/external-optimizer-model-proxy.ts @@ -1,6 +1,11 @@ import { randomBytes } from 'node:crypto' import { createServer, type IncomingMessage, type ServerResponse } from 'node:http' -import type { CostLedgerHandle, CostReceiptInput, CustomTokenPricing } from '../cost-ledger' +import type { + CostChannel, + CostLedgerHandle, + CostReceiptInput, + CustomTokenPricing, +} from '../cost-ledger' import { costForTokenPricing } from '../cost-ledger' import { assertExternalOptimizerModelBudget, @@ -18,6 +23,7 @@ interface ProviderProxyResponse { body: Uint8Array receipt: CostReceiptInput usageComplete: boolean + usageError?: string } /** @@ -36,6 +42,8 @@ export async function startExternalOptimizerModelProxy(args: { costLedger: CostLedgerHandle phase: string actor: string + /** Cost-ledger channel. Defaults to `optimizer`. */ + channel?: CostChannel tags?: Record initialUsage?: { requests: number @@ -158,6 +166,7 @@ async function handleModelProxyRequest(args: { costLedger: CostLedgerHandle phase: string actor: string + channel?: CostChannel tags?: Record } fetchImpl: typeof fetch @@ -203,7 +212,7 @@ async function handleModelProxyRequest(args: { let chargedForBudget = maximumCostUsd try { const paid = await args.args.costLedger.runPaidCall({ - channel: 'optimizer', + channel: args.args.channel ?? 'optimizer', phase: args.args.phase, actor: args.args.actor, ...(args.args.tags ? { tags: args.args.tags } : {}), @@ -221,6 +230,7 @@ async function handleModelProxyRequest(args: { body, model: args.args.model, pricing: args.args.budget.pricing, + maxOutputTokens: parsed.maxOutputTokens, maxResponseBytes: args.args.budget.maxResponseBytes, signal: controller.signal, }), @@ -253,10 +263,14 @@ async function handleModelProxyRequest(args: { chargedForBudget = paid.value.usageComplete ? paid.receipt.costUsd : maximumCostUsd if (!paid.value.usageComplete) { sendJsonIfOpen(response, 502, { - error: 'optimizer model response omitted complete token usage', + error: paid.value.usageError ?? 'optimizer model response omitted complete token usage', }) return } + if (paid.value.usageError) { + sendJsonIfOpen(response, 502, { error: paid.value.usageError }) + return + } if (paid.value.status >= 200 && paid.value.status < 300) { args.recordSuccessfulCompletion() } @@ -300,6 +314,7 @@ async function forwardModelProxyRequest(args: { body: Uint8Array model: string pricing: CustomTokenPricing + maxOutputTokens: number maxResponseBytes: number signal: AbortSignal }): Promise { @@ -319,24 +334,40 @@ async function forwardModelProxyRequest(args: { }) const body = await readProviderResponseBody(response, args.maxResponseBytes) const usage = parseProviderUsage(body) + const successful = response.status >= 200 && response.status < 300 + const zeroUsage = + successful && + usage !== undefined && + usage.inputTokens + + usage.outputTokens + + (usage.cachedTokens ?? 0) + + (usage.cacheWriteTokens ?? 0) === + 0 + const usageError = zeroUsage + ? 'optimizer model provider reported zero token usage for a successful response' + : successful && usage !== undefined && usage.outputTokens > args.maxOutputTokens + ? `optimizer model provider reported ${usage.outputTokens} output tokens, exceeding requested limit ${args.maxOutputTokens}` + : undefined return { status: response.status, contentType: response.headers.get('content-type') ?? 'application/json', body, - receipt: usage - ? { - model: args.model, - ...usage, - ...(usage.actualCostUsd === undefined ? { customTokenPricing: args.pricing } : {}), - } - : { - model: args.model, - inputTokens: 0, - outputTokens: 0, - costUnknown: true, - usageUnknown: true, - }, - usageComplete: usage !== undefined, + receipt: + usage && !zeroUsage + ? { + model: args.model, + ...usage, + ...(usage.actualCostUsd === undefined ? { customTokenPricing: args.pricing } : {}), + } + : { + model: args.model, + inputTokens: 0, + outputTokens: 0, + costUnknown: true, + usageUnknown: true, + }, + usageComplete: usage !== undefined && !zeroUsage, + ...(usageError ? { usageError } : {}), } } diff --git a/src/campaign/external-optimizer-subprocess.ts b/src/campaign/external-optimizer-subprocess.ts index 08c9f261..aca85241 100644 --- a/src/campaign/external-optimizer-subprocess.ts +++ b/src/campaign/external-optimizer-subprocess.ts @@ -1,7 +1,7 @@ import { type ChildProcess, spawn } from 'node:child_process' import { lstat, mkdtemp, open, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { isAbsolute, join, resolve } from 'node:path' import { type ExternalOptimizerRunnerCommand, isRecord } from './external-optimizer-contracts' import { runWithCleanup } from './external-optimizer-resources' @@ -62,7 +62,7 @@ export async function runExternalOptimizerProcess(args: { ) } await writeFile(inputPath, inputJson) - const command = args.runner?.command ?? 'python' + const command = resolveRunnerCommand(args.runner?.command ?? 'python') const commandArgs = [ ...(args.runner?.args ?? ['-m', args.module]), '--input', @@ -91,6 +91,11 @@ export async function runExternalOptimizerProcess(args: { }) } +function resolveRunnerCommand(command: string): string { + if (isAbsolute(command) || (!command.includes('/') && !command.includes('\\'))) return command + return resolve(command) +} + async function readBoundedTextFile(path: string, maxBytes: number, label: string): Promise { const entry = await lstat(path) if (!entry.isFile()) throw new Error(`${label} must be a regular file`) diff --git a/src/index.ts b/src/index.ts index 5a43885a..e87bdbbf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -39,7 +39,6 @@ export { validateAgentProfileCell, verifyAgentProfileCell, } from './agent-profile-cell' -export { type CreateAnalystAiConfig, createAnalystAi } from './analyst/ax-service' // ── Analyst (registry + findings) ───────────────────────────────────── // Consumer-facing happy path only: build a registry (or take the default // kinds), pass it to analyzeRuns/selfImprove, read AnalystFinding[], persist @@ -66,6 +65,19 @@ export { buildDefaultAnalystRegistry, type DefaultAnalystRegistryOptions, } from './analyst/default-registry' +export { defineTraceAnalyst } from './analyst/define' +export { + createDspyRlmTraceEngine, + type DspyRlmTraceEngineOptions, +} from './analyst/dspy-rlm-engine' +export { + DEFAULT_TRACE_ANALYST_LIMITS, + resolveTraceAnalystLimits, + type TraceAnalysisEngine, + type TraceAnalysisEngineRequest, + type TraceAnalysisEngineResult, + type TraceAnalystLimits, +} from './analyst/engine' export type { RawAnalystEvidence, RawAnalystFinding, @@ -80,11 +92,12 @@ export { type PersistedFinding, } from './analyst/findings-store' export { - type CreateTraceAnalystKindOpts, - createTraceAnalystKind, + type CreateTraceAnalystOptions, + createTraceAnalyst, renderPriorFindings, renderUpstreamFindings, - type TraceAnalystKindSpec, + runTraceAnalyst, + type TraceAnalystDefinition, } from './analyst/kind-factory' export { DEFAULT_TRACE_ANALYST_KINDS, @@ -586,7 +599,7 @@ export { type Unavailable, writeSupervisorRunReport, } from './supervisor-run/index' -// ── Trace analyst surface (Ax RLM over OTLP-JSONL) ─────────────────── +// Trace analyst surface (recursive engines over OTLP-JSONL). // Direct re-export of the trace-analyst submodule so consumers don't have // to reach into subpaths. Used by agent canonical evals via the // `autoresearch` block (analyzeTraces + OtlpFileTraceStore). @@ -787,14 +800,13 @@ export { aggregateRunScore, clamp01, DEFAULT_RUN_SCORE_WEIGHTS } from './run-sco export type { SteeringBundle, SteeringDelta, SteeringRolePrompt } from './steering' export { mergeSteeringBundle, renderSteeringText } from './steering' export type { - AxSteeringOptimizerConfig, SteeringOptimizationResult, SteeringOptimizationRow, SteeringOptimizationSelector, SteeringOptimizerBackend, SteeringOptimizerConfig, } from './steering-optimizer' -export { AxGepaSteeringOptimizer, PairwiseSteeringOptimizer } from './steering-optimizer' +export { PairwiseSteeringOptimizer } from './steering-optimizer' export type { InspectorContext, WorkspaceAssertion, @@ -1638,7 +1650,7 @@ export { attachCostToReport, costReport } from './cost-report' export type { ModelSeats, SeatName, SeatPresetName } from './model-seats' export { resolveSeat, SeatUnsetError, seatPresets } from './model-seats' -// Ax RLM trace analyst — subpath: /traces (re-exported alongside trace store). +// Recursive trace analyst, also exported from the /traces subpath. export type { AttestationProvenance, diff --git a/src/steering-optimizer.ts b/src/steering-optimizer.ts index 4c072417..f8604341 100644 --- a/src/steering-optimizer.ts +++ b/src/steering-optimizer.ts @@ -1,9 +1,7 @@ -import { type AxAIService, AxGEPA, type AxMetricFn, AxSignature, ax } from '@ax-llm/ax' -import { createAnalystAi } from './analyst/ax-service' import { aggregateRunScore, type RunScore, type RunScoreWeights } from './run-score' import type { SteeringBundle } from './steering' -export type SteeringOptimizerBackend = 'pairwise' | 'ax-gepa' +export type SteeringOptimizerBackend = 'pairwise' export interface SteeringOptimizationRow { variantId: string @@ -26,14 +24,6 @@ export interface SteeringOptimizationResult { rationale: string rankings: Array<{ variantId: string; mean: number; runs: number }> selector?: SteeringOptimizationSelector - /** Runnable handle on the trained classifier. Present only when the - * ax-gepa backend completed training; calls the optimized selector - * program via ax's `forward`. */ - selectVariant?: (row: { - task: string - split: string - seedPreview: string - }) => Promise<{ variantId: string; rationale: string }> skipped?: boolean } @@ -41,120 +31,32 @@ export interface SteeringOptimizerConfig { weights?: Partial } -export interface AxSteeringOptimizerConfig extends SteeringOptimizerConfig { - provider: 'openai' | 'anthropic' - apiKey: string - model: string - teacherModel?: string - minScenarioWinners?: number -} - +/** + * Rank already-evaluated steering variants. + * + * Model-backed candidate generation belongs to the official GEPA campaign + * integration. This class only performs the deterministic selection it owns. + */ export class PairwiseSteeringOptimizer { optimize( rows: SteeringOptimizationRow[], config: SteeringOptimizerConfig = {}, ): SteeringOptimizationResult { - const ranked = rankRows(rows, config.weights) - if (!ranked.length) throw new Error('no steering optimization rows') + const rankings = rankRows(rows, config.weights) + if (rankings.length === 0) throw new Error('no steering optimization rows') return { backend: 'pairwise', - recommendedVariantId: ranked[0]!.variantId, + recommendedVariantId: rankings[0]!.variantId, rationale: `Highest observed mean aggregate across ${rows.length} scored run(s).`, - rankings: ranked, + rankings, } } } -export class AxGepaSteeringOptimizer { - constructor(private readonly config: AxSteeringOptimizerConfig) {} - - async optimize(rows: SteeringOptimizationRow[]): Promise { - const fallback = new PairwiseSteeringOptimizer().optimize(rows, this.config) - const minScenarioWinners = this.config.minScenarioWinners ?? 6 - const variantIds = [...new Set(rows.map((row) => row.variantId))] - const byScenario = collapseScenarioWinners(rows, this.config.weights) - if (variantIds.length < 2 || byScenario.length < minScenarioWinners) { - return { - ...fallback, - backend: 'ax-gepa', - skipped: true, - rationale: `AxGEPA skipped: need >=2 variants and >=${minScenarioWinners} scenario winners, got ${variantIds.length} variant(s) and ${byScenario.length} scenario winner(s).`, - } - } - - const signature = `task:string, split:string, seedPreview:string -> variantId:class "${variantIds.join(', ')}", rationale:string` - const selectorSignature = AxSignature.from< - { task: string; split: string; seedPreview: string }, - { variantId: string; rationale?: string } - >(signature) - const selector = ax(selectorSignature, { - description: 'Choose the best steering bundle variant for an autopilot task.', - }) - const shuffled = seededShuffle(byScenario, signature) - const splitIndex = Math.max(1, Math.floor(shuffled.length * 0.8)) - const train = shuffled.slice(0, splitIndex) - const validation = shuffled.slice(splitIndex) - if (!validation.length) { - return { - ...fallback, - backend: 'ax-gepa', - skipped: true, - rationale: 'AxGEPA skipped: no validation examples after split.', - } - } - - const studentAI = createAxService(this.config.provider, this.config.apiKey, this.config.model) - const optimizer = new AxGEPA({ - studentAI, - teacherAI: createAxService( - this.config.provider, - this.config.apiKey, - this.config.teacherModel ?? this.config.model, - ), - numTrials: 8, - minibatch: true, - minibatchSize: 4, - earlyStoppingTrials: 3, - sampleCount: 1, - }) - - const metric: AxMetricFn = ({ prediction, example }) => - stringField(prediction, 'variantId') === stringField(example, 'variantId') ? 1 : 0 - const compiled = await optimizer.compile(selector, train, metric, { - validationExamples: validation, - maxMetricCalls: 64, - }) - if (compiled.optimizedProgram !== undefined) { - selector.applyOptimization(compiled.optimizedProgram) - } - - // After applyOptimization the `selector` program carries the trained - // weights; the closure runs it through ax's `forward` so the caller can - // classify unseen tasks rather than only read a description string. - const selectVariant = async (row: { task: string; split: string; seedPreview: string }) => { - const prediction = await selector.forward(studentAI, row) - return { - variantId: String(prediction.variantId), - rationale: String(prediction.rationale ?? ''), - } - } - - return { - ...fallback, - backend: 'ax-gepa', - rationale: `AxGEPA trained a variant selector from ${byScenario.length} scored scenario winner(s); default winner remains ${fallback.recommendedVariantId}.`, - selector: { - backend: 'ax-gepa', - signature, - labels: variantIds, - rationale: compiled.bestScore !== undefined ? `bestScore=${compiled.bestScore}` : undefined, - }, - selectVariant, - } - } -} - -function rankRows(rows: SteeringOptimizationRow[], weights?: Partial) { +function rankRows( + rows: readonly SteeringOptimizationRow[], + weights?: Partial, +): Array<{ variantId: string; mean: number; runs: number }> { const buckets = new Map() for (const row of rows) { const values = buckets.get(row.variantId) ?? [] @@ -169,76 +71,3 @@ function rankRows(rows: SteeringOptimizationRow[], weights?: Partial b.mean - a.mean) } - -function collapseScenarioWinners( - rows: SteeringOptimizationRow[], - weights?: Partial, -) { - const byScenario = new Map() - for (const row of rows) { - const bucket = byScenario.get(row.scenarioId) ?? [] - bucket.push(row) - byScenario.set(row.scenarioId, bucket) - } - return [...byScenario.entries()].map(([scenarioId, scenarioRows]) => { - const best = scenarioRows - .map((row) => ({ row, aggregate: aggregateRunScore(row.score, weights) })) - .sort((a, b) => b.aggregate - a.aggregate)[0]! - return { - task: String(best.row.metadata?.task ?? best.row.metadata?.seed_preview ?? scenarioId), - split: String(best.row.metadata?.split ?? 'train'), - seedPreview: String(best.row.metadata?.seed_preview ?? ''), - variantId: best.row.variantId, - } - }) -} - -function createAxService( - provider: 'openai' | 'anthropic', - apiKey: string, - model: string, -): AxAIService { - return createAnalystAi({ - provider, - apiKey, - model, - }) -} - -function stringField(value: unknown, field: string): string | undefined { - if (!value || typeof value !== 'object') return undefined - const candidate = (value as Record)[field] - return typeof candidate === 'string' ? candidate : undefined -} - -// Deterministic Fisher-Yates driven by a seeded mulberry32 PRNG. Seeding from -// a stable value (the signature) keeps train/validation splits reproducible -// across runs while removing positional skew when rows arrive grouped. -function seededShuffle(items: readonly T[], seed: string): T[] { - const rng = mulberry32(hashString(seed)) - const out = [...items] - for (let i = out.length - 1; i > 0; i--) { - const j = Math.floor(rng() * (i + 1)) - ;[out[i], out[j]] = [out[j]!, out[i]!] - } - return out -} - -function hashString(value: string): number { - let h = 2166136261 - for (let i = 0; i < value.length; i++) { - h ^= value.charCodeAt(i) - h = Math.imul(h, 16777619) - } - return h >>> 0 -} - -function mulberry32(seed: number): () => number { - let a = seed >>> 0 - return () => { - a = (a + 0x6d2b79f5) | 0 - let t = Math.imul(a ^ (a >>> 15), 1 | a) - t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t - return ((t ^ (t >>> 14)) >>> 0) / 4294967296 - } -} diff --git a/src/trace-analyst/analyst.test.ts b/src/trace-analyst/analyst.test.ts deleted file mode 100644 index 4b35aae4..00000000 --- a/src/trace-analyst/analyst.test.ts +++ /dev/null @@ -1,381 +0,0 @@ -import { mkdtempSync, readFileSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import type { AxAIService } from '@ax-llm/ax' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -import { analyzeTraces } from './analyst' -import type { TraceAnalysisStore } from './store' -import type { DatasetOverview } from './types' - -const TINY_FIXTURE = new URL('../../tests/fixtures/trace-analyst/tiny-trace.jsonl', import.meta.url) - .pathname - -const axMock = vi.hoisted(() => ({ - agentCalls: [] as Array<{ signature: string; options: Record }>, - forwardCalls: [] as Array<{ ai: unknown; values: unknown; options: unknown }>, - forwardError: undefined as Error | undefined, -})) - -vi.mock('@ax-llm/ax', () => { - const field = { - optional() { - return field - }, - array() { - return field - }, - } - const f = Object.assign( - () => ({ - input() { - return this - }, - output() { - return this - }, - build() { - return {} - }, - }), - { - string: () => field, - number: () => field, - boolean: () => field, - json: () => field, - }, - ) - const fn = (name: string) => { - const tool: Record = { name } - const builder = { - description(value: string) { - tool.description = value - return builder - }, - namespace(value: string) { - tool.namespace = value - return builder - }, - arg() { - return builder - }, - returns() { - return builder - }, - handler(value: unknown) { - tool.handler = value - return builder - }, - example() { - return builder - }, - build() { - return tool - }, - } - return builder - } - return { - f, - fn, - AxJSRuntime: class { - readonly options: unknown - constructor(options?: unknown) { - this.options = options - } - }, - agent: (signature: string, options: Record) => { - if ( - options.functions !== undefined && - !( - typeof (options.functions as { [Symbol.iterator]?: unknown })[Symbol.iterator] === - 'function' - ) - ) { - throw new TypeError('functions must be iterable') - } - axMock.agentCalls.push({ signature, options }) - const forward = async (ai: unknown, values: unknown, runOptions: unknown) => { - axMock.forwardCalls.push({ ai, values, options: runOptions }) - const onTurn = options.actorTurnCallback - if (typeof onTurn === 'function') { - await onTurn({ - stage: 'executor', - turn: 1, - actionLogEntryCount: 1, - guidanceLogEntryCount: 0, - executorResult: {}, - code: 'const overview = await traces.getDatasetOverview({})', - result: {}, - output: 'overview loaded', - isError: false, - thought: 'inspect first', - }) - } - if (axMock.forwardError) throw axMock.forwardError - return { - report: 'publish_finding hits MaxTurnsExceeded in t000000000001/s004', - findings: ['t000000000001/s004: publish_finding hit MaxTurnsExceeded'], - } - } - return { - forward, - getUsage() { - return { - actor: [{ tokens: { totalTokens: 10 } }], - responder: [{ tokens: { totalTokens: 2 } }], - } - }, - getChatLog() { - return [ - { role: 'assistant', name: 'executor' }, - { role: 'assistant', name: 'responder' }, - ] - }, - resetUsage() {}, - } - }, - } -}) - -describe('analyzeTraces', () => { - beforeEach(() => { - axMock.agentCalls.length = 0 - axMock.forwardCalls.length = 0 - axMock.forwardError = undefined - }) - - it('constructs an Ax RLM analyst with bounded trace tools and returns run telemetry', async () => { - const overview: DatasetOverview = { - total_traces: 1, - raw_jsonl_bytes: 100, - services: ['bench'], - agents: ['driver'], - models: ['model-a'], - tool_names: ['publish_finding'], - sample_trace_ids: ['t000000000001'], - errors: { trace_count: 1, span_count: 1 }, - error_clusters: [], - time_range: null, - } - const store: TraceAnalysisStore = { - async hasTrace(traceId) { - return traceId === 't000000000001' - }, - async hasSpans({ trace_id, span_ids }) { - return trace_id === 't000000000001' ? span_ids.filter((spanId) => spanId === 's004') : [] - }, - async getOverview() { - return overview - }, - async queryTraces() { - return { - traces: [], - total: 0, - has_more: false, - } - }, - async countTraces() { - return 1 - }, - async viewTrace() { - return { trace_id: 't000000000001', spans: [] } - }, - async viewSpans() { - return { - trace_id: 't000000000001', - spans: [], - missing_span_ids: [], - omitted_span_ids: [], - has_more: false, - truncated_attribute_count: 0, - } - }, - async searchTrace() { - return { - trace_id: 't000000000001', - hits: [], - has_more: false, - } - }, - async searchSpan() { - return { - trace_id: 't000000000001', - span_id: 's004', - hits: [], - has_more: false, - } - }, - } - - const ai = { provider: 'test' } as unknown as AxAIService - const signal = new AbortController().signal - const result = await analyzeTraces( - { question: 'Which harness failure mode blocks success?' }, - { source: store, ai, model: 'rlm-test', maxSubqueries: 3, signal }, - ) - - expect(axMock.agentCalls).toHaveLength(1) - expect(axMock.agentCalls[0]!.signature).toBe( - 'question:string -> report:string, findings:string[]', - ) - expect(axMock.agentCalls[0]!.options.maxSubAgentCalls).toBe(3) - expect(axMock.agentCalls[0]!.options.functions).toEqual( - expect.arrayContaining([ - expect.objectContaining({ namespace: 'traces', name: 'getDatasetOverview' }), - expect.objectContaining({ namespace: 'traces', name: 'searchSpan' }), - ]), - ) - expect(axMock.forwardCalls).toEqual([ - { - ai, - values: { question: 'Which harness failure mode blocks success?' }, - options: { abortSignal: signal }, - }, - ]) - expect(result.answer).toContain('MaxTurnsExceeded') - expect(result.findings).toEqual(['t000000000001/s004: publish_finding hit MaxTurnsExceeded']) - expect(result.turnCount).toBe(1) - expect(result.turns[0]).toMatchObject({ - turn: 1, - isError: false, - output: 'overview loaded', - }) - expect(result.actorPromptVersion).toMatch(/^trace-analyst-actor-v\d+-/) - expect(result.usage.actor[0]).toEqual({ tokens: { totalTokens: 10 } }) - expect(result.usage.responder[0]).toEqual({ tokens: { totalTokens: 2 } }) - expect(result.chatLog.actor[0]).toEqual({ role: 'assistant', name: 'executor' }) - expect(result.chatLog.responder[0]).toEqual({ role: 'assistant', name: 'responder' }) - }) - - it('honors cancellation while pre-indexing a file source', async () => { - const controller = new AbortController() - const reason = new Error('cancel before indexing') - controller.abort(reason) - - await expect( - analyzeTraces( - { question: 'What failed?' }, - { - source: TINY_FIXTURE, - ai: { provider: 'test' } as unknown as AxAIService, - signal: controller.signal, - }, - ), - ).rejects.toBe(reason) - expect(axMock.agentCalls).toHaveLength(0) - }) - - it('persists progress turns even when the analyst crashes', async () => { - const tmpDir = mkdtempSync(join(tmpdir(), 'agent-eval-trace-analyst-')) - const progressLogPath = join(tmpDir, 'progress.jsonl') - const turns: unknown[] = [] - axMock.forwardError = new Error('provider unavailable') - const store = minimalStore() - - try { - await expect( - analyzeTraces( - { question: 'What broke?' }, - { - source: store, - ai: { provider: 'test' } as unknown as AxAIService, - progressLogPath, - onTurn: (turn) => { - turns.push(turn) - }, - }, - ), - ).rejects.toThrow('provider unavailable') - - const lines = readFileSync(progressLogPath, 'utf8').trim().split('\n') - expect(lines).toHaveLength(1) - expect(JSON.parse(lines[0]!)).toMatchObject({ - turn: 1, - output: 'overview loaded', - isError: false, - }) - expect(turns).toEqual([expect.objectContaining({ turn: 1, output: 'overview loaded' })]) - } finally { - rmSync(tmpDir, { recursive: true, force: true }) - } - }) - - it.each([ - ['maxDepth', 2, 'maxSubqueries'], - ['maxParallelSubagents', 3, 'maxParallelSubqueries'], - ['subagentDescription', 'old prompt', 'actorDescription'], - ])( - 'rejects removed option %s instead of silently using defaults', - async (name, value, replacement) => { - await expect( - analyzeTraces({ question: 'What failed?' }, { - source: minimalStore(), - ai: { provider: 'test' } as unknown as AxAIService, - [name]: value, - } as never), - ).rejects.toThrow(`'${name}' is unsupported; use '${replacement}'`) - expect(axMock.agentCalls).toHaveLength(0) - }, - ) -}) - -function minimalStore(): TraceAnalysisStore { - const overview: DatasetOverview = { - total_traces: 1, - raw_jsonl_bytes: 100, - services: ['bench'], - agents: ['driver'], - models: ['model-a'], - tool_names: ['publish_finding'], - sample_trace_ids: ['t000000000001'], - errors: { trace_count: 1, span_count: 1 }, - error_clusters: [], - time_range: null, - } - return { - async hasTrace(traceId) { - return traceId === 't000000000001' - }, - async hasSpans({ trace_id, span_ids }) { - return trace_id === 't000000000001' ? span_ids.filter((spanId) => spanId === 's004') : [] - }, - async getOverview() { - return overview - }, - async queryTraces() { - return { traces: [], total: 0, has_more: false } - }, - async countTraces() { - return 1 - }, - async viewTrace() { - return { trace_id: 't000000000001', spans: [] } - }, - async viewSpans() { - return { - trace_id: 't000000000001', - spans: [], - missing_span_ids: [], - omitted_span_ids: [], - has_more: false, - truncated_attribute_count: 0, - } - }, - async searchTrace() { - return { - trace_id: 't000000000001', - hits: [], - has_more: false, - } - }, - async searchSpan() { - return { - trace_id: 't000000000001', - span_id: 's004', - hits: [], - has_more: false, - } - }, - } -} diff --git a/src/trace-analyst/analyst.ts b/src/trace-analyst/analyst.ts index ee7dd33a..740ecc98 100644 --- a/src/trace-analyst/analyst.ts +++ b/src/trace-analyst/analyst.ts @@ -1,217 +1,97 @@ -import type { AxAgentActorTurnCallbackArgs, AxAIService, AxFunction } from '@ax-llm/ax' -import { runTraceAnalysisLoop, type TraceAnalysisLoopResult } from './loop' +import { randomUUID } from 'node:crypto' +import type { + TraceAnalysisEngine, + TraceAnalysisEngineResult, + TraceAnalystLimits, +} from '../analyst/engine' +import { runTraceAnalyst } from '../analyst/kind-factory' +import type { TraceToolGroupName } from '../analyst/tool-groups' +import type { AnalystFinding, AnalystUsageReceipt } from '../analyst/types' +import type { CostLedgerHandle } from '../cost-ledger' import { TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION } from './prompts' import type { TraceAnalysisStore } from './store' import { OtlpFileTraceStore } from './store-otlp' -import { buildTraceAnalystTools } from './tools' export interface AnalyzeTracesInput { - /** The user-facing question. Domain framing belongs here, not in the - * actor description. */ question: string + id?: string + description?: string + area?: string } -export interface AnalyzeTracesResult { - /** The actor's submitted prose answer. */ - answer: string - /** Bulleted findings from the actor's structured completion. */ - findings: string[] - /** Per-turn snapshots captured via `actorTurnCallback`. */ - turns: AnalyzeTracesTurnSnapshot[] - /** Total turns the actor took. */ - turnCount: number - /** Token usage by role. */ - usage: TraceAnalystUsage - /** Full system + assistant + tool message log by role. */ - chatLog: TraceAnalystChatLog - /** Prompt version that produced this run. */ - actorPromptVersion: string -} - -export interface TraceAnalystUsage { - actor: TraceAnalystUsageEntry[] - responder: TraceAnalystUsageEntry[] -} - -export interface TraceAnalystUsageEntry { - [key: string]: unknown -} - -export interface TraceAnalystChatLog { - actor: TraceAnalystChatMessage[] - responder: TraceAnalystChatMessage[] -} - -export interface TraceAnalystChatMessage { - [key: string]: unknown -} - -export interface AnalyzeTracesTurnSnapshot { - stage: AxAgentActorTurnCallbackArgs['stage'] - turn: number - isError: boolean - /** The JS code the actor produced for this turn. */ - code: string - /** The formatted action-log entry the actor sees on the next turn. */ - output: string - /** Provider thought (when `executorOptions.showThoughts` is true and the - * provider returns it). */ - thought?: string -} +export type AnalyzeTracesResult = TraceAnalysisEngineResult export interface AnalyzeTracesOptions { - /** Trace data source. Pass either an OTLP-JSONL path or a custom store. */ + /** OTLP JSONL path or a caller-owned store. */ source: string | TraceAnalysisStore - /** Caller-provided AxAIService. */ - ai: AxAIService - /** Model id forwarded to the actor. */ - model?: string - /** Maximum model subqueries. 0 disables model fan-out. Default 4. */ - maxSubqueries?: number - /** Maximum actor turns. Default 12. */ - maxTurns?: number - /** Maximum parallel model subqueries. Default 2. */ - maxParallelSubqueries?: number - /** Cancels in-flight model and tool work. */ + /** Recursive research engine. */ + engine: TraceAnalysisEngine + /** Override the general trace-research policy. */ + instructions?: string + /** Smallest tool set that can answer the question. Default: all. */ + toolGroup?: TraceToolGroupName + limits?: Partial + runId?: string + budgetUsd?: number + costLedger?: CostLedgerHandle + costPhase?: string + priorFindings?: readonly AnalystFinding[] + upstreamFindings?: readonly AnalystFinding[] + recordUsage?: (receipt: AnalystUsageReceipt) => void + tags?: Record + log?: (message: string, fields?: Record) => void signal?: AbortSignal - /** Override the actor description. */ - actorDescription?: string - /** Per-turn observability hook. */ - onTurn?: (turn: AnalyzeTracesTurnSnapshot) => void | Promise - /** Override max runtime characters per turn. Default 6000. */ - maxRuntimeChars?: number - /** When set, every turn's snapshot is appended to this JSONL file - * immediately. If the analyst crashes mid-loop (provider 503, - * network error, validator reject) the partial reasoning is still - * on disk for diagnosis and recovery. */ - progressLogPath?: string } /** - * Run the trace analyst. + * Answer one question by recursively inspecting a trace store. * - * Throws: - * - `TraceFileMissingError` if `source` is a path and doesn't exist. - * - `AxAgentClarificationError` if the analyst asks for clarification. - * - Provider errors (auth, rate limits) propagate from the AI service. + * The returned answer, cited findings, engine steps, call counts, and runtime + * identity are one audit record. A direct one-shot model call is not used. */ export async function analyzeTraces( input: AnalyzeTracesInput, options: AnalyzeTracesOptions, ): Promise { - if (!input.question || typeof input.question !== 'string') { + if (typeof input.question !== 'string' || !input.question.trim()) { throw new TypeError('analyzeTraces: input.question must be a non-empty string') } - rejectRemovedOptions(options) - - const store: TraceAnalysisStore = + const id = input.id?.trim() || 'trace-analysis' + const store = typeof options.source === 'string' ? new OtlpFileTraceStore({ path: options.source }) : options.source - - // Pre-warm file stores so missing inputs fail before the RLM starts. if (store instanceof OtlpFileTraceStore) { await store.ensureIndexed(options.signal ? { signal: options.signal } : undefined) } - const tools: AxFunction[] = buildTraceAnalystTools({ store }) - const turns: AnalyzeTracesTurnSnapshot[] = [] - - // Persist each turn as JSONL so interrupted analyst runs keep useful evidence. - let progressFs: import('node:fs').WriteStream | undefined - if (options.progressLogPath) { - const { createWriteStream } = await import('node:fs') - const { mkdir } = await import('node:fs/promises') - const { dirname } = await import('node:path') - await mkdir(dirname(options.progressLogPath), { recursive: true }) - progressFs = createWriteStream(options.progressLogPath, { flags: 'a' }) - } - - const actorTurnCallback = async (turn: AxAgentActorTurnCallbackArgs): Promise => { - const snap: AnalyzeTracesTurnSnapshot = { - stage: turn.stage, - turn: turn.turn, - isError: turn.isError, - code: turn.code, - output: turn.output, - thought: turn.thought, - } - turns.push(snap) - if (progressFs) { - try { - progressFs.write(`${JSON.stringify({ ...snap, ts: Date.now() })}\n`) - } catch { - // Progress logging must never fail the analyst. - } - } - if (options.onTurn) await options.onTurn(snap) - } - - const maxSubqueries = options.maxSubqueries ?? 4 - const maxTurns = options.maxTurns ?? 12 - const maxParallelSubqueries = options.maxParallelSubqueries ?? 2 - const maxRuntimeChars = options.maxRuntimeChars ?? 6000 - let completed: TraceAnalysisLoopResult - try { - completed = await runTraceAnalysisLoop({ - id: 'TraceAnalyst', + return runTraceAnalyst({ + definition: { + id, description: - 'Analyzes OTLP-shaped JSONL traces using bounded discovery tools to identify systemic failure modes.', - prompt: `${options.actorDescription ?? TRACE_ANALYST_ACTOR_DESCRIPTION} - -The report must answer the user's question, and findings must be an array of concise evidence-backed strings.`, + input.description?.trim() || + 'Answers a caller-defined question by recursively inspecting trace evidence.', + area: input.area?.trim() || 'trace-analysis', + version: TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, question: input.question, - ai: options.ai, - ...(options.model ? { model: options.model } : {}), - tools, - findingType: 'string', - maxSubqueries, - maxParallelSubqueries, - maxTurns, - maxRuntimeChars, - ...(options.signal ? { signal: options.signal } : {}), - onTurn: actorTurnCallback, - }) - } finally { - if (progressFs) { - await new Promise((resolve) => progressFs!.end(() => resolve())) - } - } - - return { - answer: completed.report, - findings: completed.findings, - turns, - turnCount: turns.length, - usage: { - actor: normalizeRecordArray(completed.usage.actor), - responder: normalizeRecordArray(completed.usage.responder), + instructions: options.instructions ?? TRACE_ANALYST_ACTOR_DESCRIPTION, + toolGroup: options.toolGroup ?? 'all', + limits: options.limits, }, - chatLog: { - actor: normalizeRecordArray(completed.chatLog.actor), - responder: normalizeRecordArray(completed.chatLog.responder), + engine: options.engine, + store, + context: { + runId: options.runId ?? id, + correlationId: randomUUID(), + budgetUsd: options.budgetUsd, + costLedger: options.costLedger, + costPhase: options.costPhase ?? 'trace-analysis', + priorFindings: options.priorFindings, + upstreamFindings: options.upstreamFindings, + recordUsage: options.recordUsage, + tags: options.tags, + log: options.log, + signal: options.signal, }, - actorPromptVersion: TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, - } -} - -function rejectRemovedOptions(options: AnalyzeTracesOptions): void { - const supplied = options as unknown as Record - const migrations = [ - ['maxDepth', 'maxSubqueries'], - ['maxParallelSubagents', 'maxParallelSubqueries'], - ['subagentDescription', 'actorDescription'], - ] as const - for (const [removed, replacement] of migrations) { - if (removed in supplied) { - throw new TypeError(`analyzeTraces: '${removed}' is unsupported; use '${replacement}'`) - } - } -} - -function normalizeRecordArray(value: unknown): Record[] { - if (!Array.isArray(value)) return [] - return value.map((item) => - item && typeof item === 'object' ? { ...(item as Record) } : { value: item }, - ) + }) } diff --git a/src/trace-analyst/index.ts b/src/trace-analyst/index.ts index 69b316ce..56c03ef8 100644 --- a/src/trace-analyst/index.ts +++ b/src/trace-analyst/index.ts @@ -1,4 +1,4 @@ -/** Ax RLM trace analyst over bounded OTLP-JSONL trace stores. */ +/** Recursive trace analysis over bounded OTLP JSONL stores. */ export { asNumber, firstNumberAttr } from '../trace/attribute-vocabulary' export type { LlmSpanOtlpInput } from '../trace/otlp-attributes' @@ -20,7 +20,6 @@ export type { AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult, - AnalyzeTracesTurnSnapshot, } from './analyst' export { analyzeTraces } from './analyst' export { @@ -104,7 +103,6 @@ export type { } from './tools' export { buildTraceAnalysisToolDescriptors, - buildTraceAnalystTools, TRACE_ANALYST_TOOL_NAMESPACE, traceAnalystFunctionGroup, } from './tools' diff --git a/src/trace-analyst/loop.test.ts b/src/trace-analyst/loop.test.ts deleted file mode 100644 index 1df274d9..00000000 --- a/src/trace-analyst/loop.test.ts +++ /dev/null @@ -1,267 +0,0 @@ -import { type AxChatRequest, type AxChatResponse, AxMockAIService } from '@ax-llm/ax' -import { describe, expect, it } from 'vitest' -import { readTraceAnalysisOutput, runTraceAnalysisLoop, TraceAnalysisTurnLimitError } from './loop' - -describe('runTraceAnalysisLoop', () => { - it('uses Ax direct response for a prepared long-context trace', async () => { - let modelCalls = 0 - const requests: Array>> = [] - const ai = new AxMockAIService({ - features: { functions: false, streaming: false }, - chatResponse: async (nextRequest): Promise => { - modelCalls += 1 - requests.push(nextRequest) - return { - results: [ - { - index: 0, - content: - modelCalls === 1 - ? 'Javascript Code: respond("Write the trace report.", { inspected: inputs.context })' - : 'Report: prepared trace report\nFindings: ["trace://trace-1/span/step-2"]', - finishReason: 'stop', - }, - ], - modelUsage: { - ai: 'mock-ai', - model: 'mock-model', - tokens: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, - }, - } - }, - }) - - const result = await runTraceAnalysisLoop({ - id: 'prepared-context-analyst', - description: 'Analyzes one prepared trace.', - prompt: 'Inspect the supplied trace.', - question: 'What failed?', - context: JSON.stringify({ trace_id: 'trace-1', spans: [{ span_id: 'step-2' }] }), - ai, - tools: [], - findingType: 'string', - maxSubqueries: 0, - maxParallelSubqueries: 1, - maxTurns: 1, - maxRuntimeChars: 6000, - }) - - expect(result).toMatchObject({ - report: 'prepared trace report', - findings: ['trace://trace-1/span/step-2'], - turnCount: 0, - }) - expect(modelCalls).toBe(2) - expect( - requests - .flatMap((request) => request.chatPrompt) - .map((message) => ('content' in message ? JSON.stringify(message.content) : '')) - .join('\n'), - ).toContain('inputs.context') - }) - - it('uses the public Ax pipeline and returns its structured response', async () => { - let modelCalls = 0 - const requests: Array>> = [] - const ai = new AxMockAIService({ - features: { functions: false, streaming: false }, - chatResponse: async (nextRequest): Promise => { - modelCalls += 1 - requests.push(nextRequest) - const content = - modelCalls === 1 - ? 'Javascript Code: final("Analyze the trace.", {})' - : modelCalls === 2 - ? 'Javascript Code: final("Write the trace report.", { report: "exact report", findings: ["span://trace-1/span-2"] })' - : 'Report: exact report\nFindings: ["span://trace-1/span-2"]' - return { - results: [ - { - index: 0, - content, - finishReason: 'stop', - }, - ], - modelUsage: { - ai: 'mock-ai', - model: 'mock-model', - tokens: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, - }, - } - }, - }) - - const result = await runTraceAnalysisLoop({ - id: 'test-analyst', - description: 'Tests the direct analysis loop.', - prompt: 'Inspect the supplied traces.', - question: 'What failed?', - ai, - tools: [], - findingType: 'string', - maxSubqueries: 0, - maxParallelSubqueries: 1, - maxTurns: 1, - maxRuntimeChars: 6000, - }) - - expect(result.report).toBe('exact report') - expect(result.findings).toEqual(['span://trace-1/span-2']) - expect(result.turnCount).toBe(1) - expect(modelCalls).toBe(3) - const prompts = requests.flatMap((request) => - request.chatPrompt.map((message) => - 'content' in message - ? typeof message.content === 'string' - ? message.content - : JSON.stringify(message.content) - : JSON.stringify(message), - ), - ) - expect(prompts.join('\n')).toContain('What failed?') - expect(prompts.join('\n')).toContain('let the response stage produce') - expect(result.usage.actor).toHaveLength(2) - expect(result.usage.responder).toHaveLength(1) - expect(result.chatLog.actor).toHaveLength(2) - expect(result.chatLog.responder).toHaveLength(1) - expect(result.chatLog.responder[0]).toMatchObject({ name: 'responder' }) - }) - - it('rejects exhausted evidence collection before response synthesis', async () => { - let mockCalls = 0 - const ai = new AxMockAIService({ - features: { functions: false, streaming: false }, - chatResponse: async (): Promise => { - mockCalls += 1 - const content = - mockCalls === 1 - ? 'Javascript Code: final("Analyze the trace.", {})' - : mockCalls === 2 - ? 'Javascript Code: console.log("partial evidence")' - : 'Report: partial evidence only\nFindings: []' - return { - results: [ - { - index: 0, - content, - finishReason: 'stop', - }, - ], - modelUsage: { - ai: 'mock-ai', - model: 'mock-model', - tokens: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, - }, - } - }, - }) - - await expect( - runTraceAnalysisLoop({ - id: 'turn-limited-analyst', - description: 'Replays a model response that never calls final.', - prompt: 'Inspect the supplied traces.', - question: 'What failed?', - ai, - tools: [], - findingType: 'string', - maxSubqueries: 0, - maxParallelSubqueries: 1, - maxTurns: 1, - maxRuntimeChars: 6000, - }), - ).rejects.toEqual( - expect.objectContaining({ - name: 'TraceAnalysisTurnLimitError', - analystId: 'turn-limited-analyst', - stage: 'executor', - maxTurns: 1, - }), - ) - expect(mockCalls).toBe(2) - }) - - it('rejects exhausted prepared-context collection even if response synthesis succeeds', async () => { - let mockCalls = 0 - const ai = new AxMockAIService({ - features: { functions: false, streaming: false }, - chatResponse: async (): Promise => { - mockCalls += 1 - return { - results: [ - { - index: 0, - content: - mockCalls === 1 - ? 'Javascript Code: console.log("partial context")' - : mockCalls === 2 - ? 'Javascript Code: final("Write the trace report.", { partial: true })' - : 'Report: partial context only\nFindings: []', - finishReason: 'stop', - }, - ], - modelUsage: { - ai: 'mock-ai', - model: 'mock-model', - tokens: { promptTokens: 1, completionTokens: 1, totalTokens: 2 }, - }, - } - }, - }) - - await expect( - runTraceAnalysisLoop({ - id: 'turn-limited-context-analyst', - description: 'Replays a context analysis that never calls respond.', - prompt: 'Inspect the supplied trace.', - question: 'What failed?', - context: '{"trace_id":"trace-1"}', - ai, - tools: [], - findingType: 'string', - maxSubqueries: 0, - maxParallelSubqueries: 1, - maxTurns: 1, - maxRuntimeChars: 6000, - }), - ).rejects.toEqual( - expect.objectContaining({ - name: 'TraceAnalysisTurnLimitError', - analystId: 'turn-limited-context-analyst', - stage: 'distiller', - maxTurns: 1, - }), - ) - expect(mockCalls).toBe(3) - }) - - it('exposes a named error for turn-limit handling', () => { - expect(new TraceAnalysisTurnLimitError('analyst', 'distiller', 3)).toMatchObject({ - name: 'TraceAnalysisTurnLimitError', - analystId: 'analyst', - stage: 'distiller', - maxTurns: 3, - }) - }) - - it('rejects non-structured response output', () => { - expect(() => - readTraceAnalysisOutput('Actor stopped without a structured response.', 'string'), - ).toThrow('response must contain report and findings') - }) - - it('leaves object-finding rows for the kind schema to validate or repair', () => { - expect( - readTraceAnalysisOutput( - { - report: 'plausible', - findings: ['{"claim":"repairable"}', 42], - }, - 'object', - ), - ).toEqual({ - report: 'plausible', - findings: ['{"claim":"repairable"}', 42], - }) - }) -}) diff --git a/src/trace-analyst/loop.ts b/src/trace-analyst/loop.ts deleted file mode 100644 index 87be3043..00000000 --- a/src/trace-analyst/loop.ts +++ /dev/null @@ -1,276 +0,0 @@ -import { - type AxAgentActorTurnCallback, - type AxAIService, - type AxFunction, - AxJSRuntime, - agent, -} from '@ax-llm/ax' -import { TraceFileMissingError } from './store-otlp' - -const TRACE_ANALYSIS_TOOL_INSTRUCTION = `Gather the evidence needed for the report with the trace tools. -When the evidence is sufficient, call final(task, evidence) and let the response stage produce the declared report and findings fields.` - -const TRACE_ANALYSIS_CONTEXT_INSTRUCTION = `The complete bounded trace data is available as inputs.context. -Inspect it in the runtime, then call respond(task, evidence) and let the response stage produce the declared report and findings fields.` - -const TRACE_ANALYSIS_RESPONDER_INSTRUCTION = `Use only evidence produced by the analysis stage. -Do not invent trace ids, span ids, steps, tool results, or final verification outcomes.` - -export interface TraceAnalysisLoopResult { - report: string - findings: TFinding[] - usage: { - actor: readonly unknown[] - responder: readonly unknown[] - } - chatLog: { - actor: readonly unknown[] - responder: readonly unknown[] - } - turnCount: number -} - -export class TraceAnalysisTurnLimitError extends Error { - readonly analystId: string - readonly stage: 'distiller' | 'executor' - readonly maxTurns: number - - constructor(analystId: string, stage: 'distiller' | 'executor', maxTurns: number) { - super( - `Trace analyst '${analystId}' reached maxTurns=${maxTurns} in the ${stage} stage without explicit completion`, - ) - this.name = 'TraceAnalysisTurnLimitError' - this.analystId = analystId - this.stage = stage - this.maxTurns = maxTurns - } -} - -interface TraceAnalysisLoopOptions { - id: string - description: string - prompt: string - question: string - context?: string - ai: AxAIService - model?: string - tools: readonly AxFunction[] - maxSubqueries: number - maxParallelSubqueries: number - maxTurns: number - maxRuntimeChars: number - signal?: AbortSignal - onTurn?: AxAgentActorTurnCallback -} - -export function runTraceAnalysisLoop( - options: TraceAnalysisLoopOptions & { findingType: 'string' }, -): Promise> -export function runTraceAnalysisLoop( - options: TraceAnalysisLoopOptions & { findingType: 'object' }, -): Promise> -export async function runTraceAnalysisLoop( - options: TraceAnalysisLoopOptions & { findingType: 'string' | 'object' }, -): Promise> { - validateLoopLimits(options) - - let turnCount = 0 - const completedStages = new Set<'distiller' | 'executor'>() - const onTurn: AxAgentActorTurnCallback = async (turn) => { - if (turn.stage === 'executor') turnCount = Math.max(turnCount, turn.turn) - if (isExplicitCompletionTurn(turn)) completedStages.add(turn.stage) - await options.onTurn?.(turn) - if (turn.turn >= options.maxTurns && !completedStages.has(turn.stage)) { - throw new TraceAnalysisTurnLimitError(options.id, turn.stage, options.maxTurns) - } - } - const hasPreparedContext = options.context !== undefined - const config = { - agentIdentity: { name: options.id, description: options.description }, - contextFields: hasPreparedContext ? (['context'] as const) : ([] as const), - runtime: new AxJSRuntime({ - permissions: [], - blockDynamicImport: true, - allowedModules: [], - freezeIntrinsics: true, - blockShadowRealm: true, - preventGlobalThisExtensions: false, - }), - maxSubAgentCalls: options.maxSubqueries, - maxTurns: options.maxTurns, - maxRuntimeChars: options.maxRuntimeChars, - maxBatchedLlmQueryConcurrency: options.maxParallelSubqueries, - promptLevel: 'detailed' as const, - contextPolicy: { preset: 'full' as const, budget: 'balanced' as const }, - directResponse: - hasPreparedContext && options.tools.length === 0 ? ('auto' as const) : ('off' as const), - functions: options.tools, - executorOptions: { - description: `${options.prompt.trim()}\n\n${ - hasPreparedContext ? TRACE_ANALYSIS_CONTEXT_INSTRUCTION : TRACE_ANALYSIS_TOOL_INSTRUCTION - }`, - ...(options.model ? { model: options.model } : {}), - showThoughts: false, - thinkingTokenBudget: 'none' as const, - }, - responderOptions: { - description: `${TRACE_ANALYSIS_RESPONDER_INSTRUCTION}\n\n${options.prompt.trim()}`, - ...(options.model ? { model: options.model } : {}), - showThoughts: false, - thinkingTokenBudget: 'none' as const, - }, - actorTurnCallback: onTurn, - bubbleErrors: [TraceFileMissingError], - } - const analyst = hasPreparedContext - ? options.findingType === 'string' - ? agent('context:string, question:string -> report:string, findings:string[]', config) - : agent('context:string, question:string -> report:string, findings:json[]', config) - : options.findingType === 'string' - ? agent('question:string -> report:string, findings:string[]', config) - : agent('question:string -> report:string, findings:json[]', config) - - const output = hasPreparedContext - ? await analyst.forward( - options.ai, - { context: options.context!, question: options.question } as never, - options.signal ? { abortSignal: options.signal } : undefined, - ) - : await analyst.forward( - options.ai, - { question: options.question }, - options.signal ? { abortSignal: options.signal } : undefined, - ) - assertStageDidNotExhaust( - options.id, - 'distiller', - options.maxTurns, - analyst.distiller?.getState()?.actionLogEntries, - ) - assertStageDidNotExhaust( - options.id, - 'executor', - options.maxTurns, - analyst.executor?.getState()?.actionLogEntries, - ) - const completed = - options.findingType === 'string' - ? readTraceAnalysisOutput(output, 'string') - : readTraceAnalysisOutput(output, 'object') - const usage = analyst.getUsage() - const chatLog = splitChatLog(analyst.getChatLog()) - - return { - ...completed, - usage: { - actor: usage.actor, - responder: usage.responder, - }, - chatLog, - turnCount, - } -} - -function isExplicitCompletionTurn(turn: Parameters[0]): boolean { - return !turn.isError && isExplicitCompletionCode(turn.code) -} - -function isExplicitCompletionCode(code: string): boolean { - const executable = trimLeadingComments(code) - return /^(?:await\s+)?(?:final|respond|askClarification)\s*\(/.test(executable) -} - -function assertStageDidNotExhaust( - analystId: string, - stage: 'distiller' | 'executor', - maxTurns: number, - entries: - | readonly { - code: string - tags: readonly string[] - }[] - | undefined, -): void { - if (!entries || entries.length < maxTurns) return - const last = entries.at(-1) - if (last && !last.tags.includes('error') && isExplicitCompletionCode(last.code)) return - throw new TraceAnalysisTurnLimitError(analystId, stage, maxTurns) -} - -function trimLeadingComments(code: string): string { - let remaining = code.trimStart() - while (remaining.startsWith('//') || remaining.startsWith('/*')) { - if (remaining.startsWith('//')) { - const lineEnd = remaining.indexOf('\n') - if (lineEnd === -1) return '' - remaining = remaining.slice(lineEnd + 1).trimStart() - continue - } - const commentEnd = remaining.indexOf('*/', 2) - if (commentEnd === -1) return '' - remaining = remaining.slice(commentEnd + 2).trimStart() - } - return remaining -} - -function splitChatLog(entries: readonly unknown[]): { - actor: readonly unknown[] - responder: readonly unknown[] -} { - const actor: unknown[] = [] - const responder: unknown[] = [] - for (const entry of entries) { - const name = - entry && typeof entry === 'object' && 'name' in entry - ? (entry as { name?: unknown }).name - : undefined - if (typeof name === 'string' && (name === 'responder' || name.endsWith('.responder'))) { - responder.push(entry) - } else { - actor.push(entry) - } - } - return { actor, responder } -} - -interface CompletedTraceAnalysis { - report: string - findings: TFinding[] -} - -export function readTraceAnalysisOutput( - value: unknown, - findingType: 'string', -): CompletedTraceAnalysis -export function readTraceAnalysisOutput( - value: unknown, - findingType: 'object', -): CompletedTraceAnalysis -export function readTraceAnalysisOutput( - value: unknown, - findingType: 'string' | 'object', -): CompletedTraceAnalysis { - if (!value || typeof value !== 'object') { - throw new Error('Trace analyst response must contain report and findings') - } - const { report, findings } = value as { report?: unknown; findings?: unknown } - if (typeof report !== 'string' || !Array.isArray(findings)) { - throw new Error('Trace analyst response must contain report and findings') - } - if (findingType === 'string') { - if (findings.some((finding) => typeof finding !== 'string')) { - throw new Error('Trace analyst response must contain string findings') - } - return { report, findings: findings as string[] } - } - return { report, findings } -} - -function validateLoopLimits(options: TraceAnalysisLoopOptions): void { - if (!Number.isSafeInteger(options.maxSubqueries) || options.maxSubqueries < 0) { - throw new TypeError('maxSubqueries must be a non-negative integer') - } - if (!Number.isSafeInteger(options.maxParallelSubqueries) || options.maxParallelSubqueries < 1) { - throw new TypeError('maxParallelSubqueries must be a positive integer') - } -} diff --git a/src/trace-analyst/prompts.ts b/src/trace-analyst/prompts.ts index b6a6f5a8..7eed040a 100644 --- a/src/trace-analyst/prompts.ts +++ b/src/trace-analyst/prompts.ts @@ -1,50 +1,17 @@ -/** Ax RLM prompt for bounded trace discovery and evidence-backed analysis. */ +/** General policy for recursive, evidence-backed trace analysis. */ -export const TRACE_ANALYST_ACTOR_DESCRIPTION = `You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the \`traces\` namespace. +export const TRACE_ANALYST_ACTOR_DESCRIPTION = `Answer the question by inspecting the OTLP trace dataset with the available tools. -DISCOVERY → NARROW → DEEP-READ protocol — follow exactly: +1. Call getDatasetOverview first. Use its real trace ids and dataset size to plan the investigation. +2. Narrow with queryTraces and countTraces before scanning large payloads. +3. For a small trace, use viewTrace. For a large trace, use searchTrace and then viewSpans or searchSpan. +4. Never invent a trace id, span id, tool result, error, frequency, or final outcome. +5. When a search reports has_more, refine the query before drawing a conclusion. +6. Use llm_query only over evidence already loaded. A recursive query cannot inspect traces itself. +7. Cite exact evidence URIs returned by the tools. Include a short exact excerpt when it supports the claim. +8. Return no finding when the available evidence cannot support one. -1. ALWAYS call \`traces.getDatasetOverview({})\` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids — never fabricate one). +The prose answer must directly answer the question and state important uncertainty. +The findings array contains only actionable or decision-relevant claims supported by inspected evidence.` -2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. \`filters.regex_pattern\` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces — narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset. - -3. To list more traces than the sample, call \`traces.queryTraces({ filters?, limit, offset? })\`. Each summary carries raw_jsonl_bytes — use it to choose between viewTrace and searchTrace BEFORE calling either. - -4. Per-trace inspection: - - SMALL trace (raw_jsonl_bytes well under 150_000): call \`traces.viewTrace({ trace_id })\`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large \`input.value\` / \`output.value\` / \`llm.input_messages\` will show a \`[trace-analyst truncated: N bytes]\` marker. - - LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an \`oversized\` response): use \`traces.searchTrace({ trace_id, regex_pattern })\` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call \`traces.viewSpans({ trace_id, span_ids: [...] })\` for surgical reads (~16KB cap, 4× higher than discovery), or \`traces.searchSpan({ trace_id, span_id, regex_pattern })\` for one large span. Stays bounded regardless of trace size. - - Useful regex patterns: \`STATUS_CODE_ERROR\` (failures), tool names like \`grep\` or \`view_trace\`, error strings like \`MaxTurnsExceeded\`, model names, attribute keys. - -5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids. - -5a. **Result-shape contract** — searchTrace and searchSpan return \`{ trace_id, hits, has_more }\`. Iterate \`result.hits\` (NOT result.matches). Each hit has \`{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }\`. viewTrace returns \`{ trace_id, spans }\` (or \`oversized\`). viewSpans returns \`{ trace_id, spans, missing_span_ids, omitted_span_ids, has_more, truncated_attribute_count }\`; retry only \`omitted_span_ids\`. Never assume a field name — log the result shape first if unsure. - -6. If viewTrace returns an \`oversized\` summary instead of \`spans\`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set. - -7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches. - -8. If a tool errors (invalid regex, range error), STOP and reconsider — don't retry with a guessed id or argument. Use the discovery tools above to recover. - -9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again. - -10. If the question splits into independent reasoning branches, use bounded \`llmQuery(...)\` calls over evidence you already loaded. Subqueries cannot inspect the trace store, so pass the exact trace excerpts they need. Example: - - const reviews = await llmQuery([ - { query: 'Classify the failure mechanism in this excerpt.', context: traceAbcExcerpt }, - { query: 'Classify the failure mechanism in this excerpt.', context: traceDefExcerpt }, - ]); - -OBSERVABILITY rules: -- Each discovery turn must emit at least one concise \`console.log(...)\` showing what evidence was learned. -- Finish gathering evidence before submitting the analysis. -- Reuse runtime variables across turns; don't recompute. - -OUTPUT contract — your final answer must include: -- A clear prose conclusion answering the user's question. -- Trace ids and span ids cited as evidence for each claim. -- Failure modes named in the user's domain language, with frequency and concrete examples. -- A concise findings array containing only claims supported by inspected evidence. - -Do NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.` - -export const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = 'trace-analyst-actor-v6-2026-07-14' +export const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = 'trace-analyst-research-v1-2026-07-30' diff --git a/src/trace-analyst/tools.test.ts b/src/trace-analyst/tools.test.ts index bb0986ae..9397d126 100644 --- a/src/trace-analyst/tools.test.ts +++ b/src/trace-analyst/tools.test.ts @@ -1,19 +1,8 @@ -/** - * Tests for the AxFunction adapter layer in `tools.ts`. We don't run - * a full agent here — we exercise the runtime guards on bad inputs - * and confirm the namespace + function-set shape is what AxAgent - * expects. - * - * Each test names the regression it would catch. - */ - import { describe, expect, it } from 'vitest' - import { TraceAnalysisValidationError } from './errors' import { OtlpFileTraceStore, TraceNotFoundError } from './store-otlp' import { buildTraceAnalysisToolDescriptors, - buildTraceAnalystTools, TRACE_ANALYST_TOOL_NAMESPACE, traceAnalystFunctionGroup, } from './tools' @@ -21,93 +10,49 @@ import { const TINY_FIXTURE = new URL('../../tests/fixtures/trace-analyst/tiny-trace.jsonl', import.meta.url) .pathname -describe('buildTraceAnalystTools', () => { - it('exposes exactly the seven discovery → narrow → deep-read functions in the traces namespace', () => { - const store = new OtlpFileTraceStore({ path: TINY_FIXTURE }) - const tools = buildTraceAnalystTools({ store }) - // The Ax fn() builder may wrap names; inspect through JSON.stringify - // round-trip since the AxFunction stub carries the symbol-keyed - // brand. We don't assert on internals — only that we built 7 - // distinct tool definitions. - expect(tools.length).toBe(7) +describe('trace analysis tool descriptors', () => { + it('exposes exactly seven transport-neutral trace reads', () => { + const tools = buildTraceAnalysisToolDescriptors({ + store: new OtlpFileTraceStore({ path: TINY_FIXTURE }), + }) + + expect(tools.map((tool) => tool.name)).toEqual([ + 'getDatasetOverview', + 'queryTraces', + 'countTraces', + 'viewTrace', + 'viewSpans', + 'searchTrace', + 'searchSpan', + ]) + expect(tools.every((tool) => tool.namespace === TRACE_ANALYST_TOOL_NAMESPACE)).toBe(true) }) - it('traceAnalystFunctionGroup namespaces the toolset under "traces" with discovery metadata', () => { + it('publishes the same descriptors through the function group', () => { const store = new OtlpFileTraceStore({ path: TINY_FIXTURE }) const group = traceAnalystFunctionGroup({ store }) + expect(group.namespace).toBe('traces') expect(group.title).toMatch(/trace/i) - expect(group.functions.length).toBe(7) + expect(group.functions).toHaveLength(7) expect(group.selectionCriteria).toContain('OTLP') }) - it('adapts every canonical transport-neutral descriptor into matching Ax metadata', () => { - const store = new OtlpFileTraceStore({ path: TINY_FIXTURE }) - const axTools = buildTraceAnalystTools({ store }) - const descriptors = buildTraceAnalysisToolDescriptors({ store }) - - expect(descriptors).toHaveLength(7) - expect( - descriptors.map(({ namespace, name, description, parameters }) => ({ - namespace, - name, - description, - parameters, - })), - ).toEqual( - axTools.map(({ namespace, name, description, parameters }) => ({ - namespace, - name, - description, - parameters, - })), - ) - expect(descriptors.every((tool) => tool.namespace === TRACE_ANALYST_TOOL_NAMESPACE)).toBe(true) - }) - - it('delegates every Ax operation to the same canonical transport-neutral handler', async () => { - const store = new OtlpFileTraceStore({ path: TINY_FIXTURE }) - const axTools = buildTraceAnalystTools({ store }) - const descriptors = buildTraceAnalysisToolDescriptors({ store }) - const argsByName: Record> = { - getDatasetOverview: { filters: { has_errors: true } }, - queryTraces: { filters: { has_errors: true }, limit: 1, offset: 0 }, - countTraces: { filters: { has_errors: true } }, - viewTrace: { trace_id: 't000000000001' }, - viewSpans: { trace_id: 't000000000001', span_ids: ['s004'] }, - searchTrace: { - trace_id: 't000000000001', - regex_pattern: 'MaxTurnsExceeded', - max_matches: 2, - }, - searchSpan: { - trace_id: 't000000000001', - span_id: 's004', - regex_pattern: 'MaxTurnsExceeded', - max_matches: 2, - }, - } - - for (const descriptor of descriptors) { - const axTool = axTools.find((tool) => tool.name === descriptor.name) - if (!axTool) throw new Error(`missing Ax tool ${descriptor.name}`) - const args = argsByName[descriptor.name] - if (!args) throw new Error(`missing invocation for ${descriptor.name}`) - await expect(descriptor.handler(args)).resolves.toEqual(await axTool.func(args)) - } - }) - - it('preserves the store byte ceiling through the transport-neutral viewTrace handler', async () => { - const store = new OtlpFileTraceStore({ path: TINY_FIXTURE, perCallByteCeiling: 100 }) - const viewTrace = buildTraceAnalysisToolDescriptors({ store }).find( - (tool) => tool.name === 'viewTrace', + it('preserves the store byte limit through viewTrace', async () => { + const tool = findTool( + buildTraceAnalysisToolDescriptors({ + store: new OtlpFileTraceStore({ + path: TINY_FIXTURE, + perCallByteCeiling: 100, + }), + }), + 'viewTrace', ) - if (!viewTrace) throw new Error('missing viewTrace descriptor') - - const result = (await viewTrace.handler({ trace_id: 't000000000001' })) as { + const result = (await tool.handler({ trace_id: 't000000000001' })) as { spans?: unknown oversized?: { span_count: number } } + expect(result.spans).toBeUndefined() expect(result.oversized?.span_count).toBe(4) }) @@ -118,23 +63,21 @@ describe('buildTraceAnalystTools', () => { ['queryTraces', { limit: 1, offset: -1 }], ['searchTrace', { trace_id: 't000000000001', regex_pattern: '[', max_matches: 1 }], ['searchTrace', { trace_id: 't000000000001', regex_pattern: 'x', max_matches: 501 }], - ])('preserves %s validation error across neutral and Ax callers', async (name, args) => { - const store = new OtlpFileTraceStore({ path: TINY_FIXTURE }) - const axTool = buildTraceAnalystTools({ store }).find((tool) => tool.name === name) - const descriptor = buildTraceAnalysisToolDescriptors({ store }).find( - (tool) => tool.name === name, + ])('rejects invalid %s arguments', async (name, args) => { + const tool = findTool( + buildTraceAnalysisToolDescriptors({ + store: new OtlpFileTraceStore({ path: TINY_FIXTURE }), + }), + name, ) - if (!axTool || !descriptor) throw new Error(`missing tool ${name}`) - await expect(Promise.resolve().then(() => axTool.func(args))).rejects.toBeInstanceOf( - TraceAnalysisValidationError, - ) - await expect(descriptor.handler(args)).rejects.toMatchObject({ code: 'validation' }) + await expect(tool.handler(args)).rejects.toBeInstanceOf(TraceAnalysisValidationError) }) - it('publishes exact JSON Schemas with typed filters and public integer caps', () => { - const store = new OtlpFileTraceStore({ path: TINY_FIXTURE }) - const tools = buildTraceAnalysisToolDescriptors({ store }) + it('publishes typed JSON schemas with public integer limits', () => { + const tools = buildTraceAnalysisToolDescriptors({ + store: new OtlpFileTraceStore({ path: TINY_FIXTURE }), + }) const byName = new Map(tools.map((tool) => [tool.name, tool.parameters])) expect(byName.get('queryTraces')).toMatchObject({ @@ -147,8 +90,6 @@ describe('buildTraceAnalystTools', () => { required: ['limit'], }) expect(byName.get('viewSpans')).toMatchObject({ - type: 'object', - additionalProperties: false, properties: { span_ids: { type: 'array', @@ -163,24 +104,6 @@ describe('buildTraceAnalystTools', () => { max_matches: { type: 'integer', minimum: 1, maximum: 500 }, }, }) - - const overview = byName.get('getDatasetOverview') as { - properties: { filters: Record } - } - expect(overview.properties.filters).toMatchObject({ - type: 'object', - additionalProperties: false, - properties: { - has_errors: { type: 'boolean' }, - service_names: { type: 'array', items: { type: 'string' } }, - agent_names: { type: 'array', items: { type: 'string' } }, - model_names: { type: 'array', items: { type: 'string' } }, - tool_names: { type: 'array', items: { type: 'string' } }, - start_time_after: { type: 'string', format: 'date-time' }, - start_time_before: { type: 'string', format: 'date-time' }, - regex_pattern: { type: 'string', minLength: 1 }, - }, - }) }) it.each([ @@ -199,16 +122,15 @@ describe('buildTraceAnalystTools', () => { unexpected: true, }, ], - ])('rejects unknown %s fields identically through both public surfaces', async (name, args) => { - const store = new OtlpFileTraceStore({ path: TINY_FIXTURE }) - const neutral = buildTraceAnalysisToolDescriptors({ store }).find((tool) => tool.name === name) - const ax = buildTraceAnalystTools({ store }).find((tool) => tool.name === name) - if (!neutral || !ax) throw new Error(`missing tool ${name}`) - - await expect(neutral.handler(args)).rejects.toBeInstanceOf(TraceAnalysisValidationError) - await expect(Promise.resolve().then(() => ax.func(args))).rejects.toBeInstanceOf( - TraceAnalysisValidationError, + ])('rejects unknown %s fields', async (name, args) => { + const tool = findTool( + buildTraceAnalysisToolDescriptors({ + store: new OtlpFileTraceStore({ path: TINY_FIXTURE }), + }), + name, ) + + await expect(tool.handler(args)).rejects.toBeInstanceOf(TraceAnalysisValidationError) }) it.each([ @@ -216,42 +138,47 @@ describe('buildTraceAnalystTools', () => { { filters: { service_names: 'runtime' } }, { filters: { start_time_after: 'yesterday' } }, { filters: { unknown_filter: true } }, - ])('rejects an invalid filter instead of silently broadening the query', async (args) => { - const store = new OtlpFileTraceStore({ path: TINY_FIXTURE }) - const tool = buildTraceAnalysisToolDescriptors({ store }).find( - (candidate) => candidate.name === 'getDatasetOverview', + ])('rejects an invalid filter instead of broadening the query', async (args) => { + const tool = findTool( + buildTraceAnalysisToolDescriptors({ + store: new OtlpFileTraceStore({ path: TINY_FIXTURE }), + }), + 'getDatasetOverview', ) - if (!tool) throw new Error('missing getDatasetOverview descriptor') await expect(tool.handler(args)).rejects.toMatchObject({ code: 'validation' }) }) - it('forwards Ax abortSignal into the canonical descriptor before any store read', async () => { - const store = new OtlpFileTraceStore({ path: TINY_FIXTURE }) - const tool = buildTraceAnalystTools({ store }).find( - (candidate) => candidate.name === 'viewTrace', + it('forwards cancellation before a store read', async () => { + const tool = findTool( + buildTraceAnalysisToolDescriptors({ + store: new OtlpFileTraceStore({ path: TINY_FIXTURE }), + }), + 'viewTrace', ) - if (!tool) throw new Error('missing viewTrace Ax tool') const controller = new AbortController() - const reason = new Error('stop Ax trace read') + const reason = new Error('stop trace read') controller.abort(reason) await expect( - Promise.resolve().then(() => - tool.func({ trace_id: 't000000000001' }, { abortSignal: controller.signal }), - ), + tool.handler({ trace_id: 't000000000001' }, { signal: controller.signal }), ).rejects.toBe(reason) }) - it('preserves typed store errors instead of converting them to transport results', async () => { - const store = new OtlpFileTraceStore({ path: TINY_FIXTURE }) - const viewTrace = buildTraceAnalysisToolDescriptors({ store }).find( - (tool) => tool.name === 'viewTrace', + it('preserves typed store errors', async () => { + const tool = findTool( + buildTraceAnalysisToolDescriptors({ + store: new OtlpFileTraceStore({ path: TINY_FIXTURE }), + }), + 'viewTrace', ) - if (!viewTrace) throw new Error('missing viewTrace descriptor') - await expect(viewTrace.handler({ trace_id: 'missing' })).rejects.toBeInstanceOf( - TraceNotFoundError, - ) + await expect(tool.handler({ trace_id: 'missing' })).rejects.toBeInstanceOf(TraceNotFoundError) }) }) + +function findTool(tools: ReturnType, name: string) { + const tool = tools.find((candidate) => candidate.name === name) + if (!tool) throw new Error(`missing tool ${name}`) + return tool +} diff --git a/src/trace-analyst/tools.ts b/src/trace-analyst/tools.ts index 8a85a66a..7523351c 100644 --- a/src/trace-analyst/tools.ts +++ b/src/trace-analyst/tools.ts @@ -1,11 +1,8 @@ /** - * Canonical transport-neutral trace tools plus the Ax adapter used by the - * built-in analyst. Schemas and handlers originate here; provider adapters - * only translate descriptor fields and cancellation context. + * Canonical transport-neutral trace tools. Schemas and handlers originate + * here; recursive engines only translate calls across their process boundary. */ -import type { AxFunction } from '@ax-llm/ax' - import type { EvalToolDef } from '../eval-tools' import { type BoundedTraceAnalysisStoreOptions, @@ -142,24 +139,12 @@ export function buildTraceAnalysisToolDescriptors( ] } -/** Adapt the canonical descriptors into Ax functions used by the built-in analyst. */ -export function buildTraceAnalystTools(options: BuildTraceAnalysisToolsOptions): AxFunction[] { - return buildTraceAnalysisToolDescriptors(options).map((descriptor) => ({ - namespace: descriptor.namespace, - name: descriptor.name, - description: descriptor.description, - parameters: descriptor.parameters as AxFunction['parameters'], - func: (args, extra) => - descriptor.handler(args, extra?.abortSignal ? { signal: extra.abortSignal } : undefined), - })) -} - export function traceAnalystFunctionGroup(options: BuildTraceAnalysisToolsOptions): { namespace: string title: string selectionCriteria: string description: string - functions: AxFunction[] + functions: TraceAnalysisToolDescriptor[] } { return { namespace: TRACE_ANALYST_TOOL_NAMESPACE, @@ -168,6 +153,6 @@ export function traceAnalystFunctionGroup(options: BuildTraceAnalysisToolsOption description: 'Discovery, narrowing, and bounded deep reads over a JSONL trace dataset. ' + 'Always call getDatasetOverview first.', - functions: buildTraceAnalystTools(options), + functions: buildTraceAnalysisToolDescriptors(options), } } diff --git a/src/traced-analyst.ts b/src/traced-analyst.ts index 30304047..9b05401c 100644 --- a/src/traced-analyst.ts +++ b/src/traced-analyst.ts @@ -1,92 +1,65 @@ -/** - * Traced analyst wrapper — instruments `analyzeTraces` with spans so the - * analyst's internal model turns appear in the trace tree. Also wraps each - * actor turn callback with a span. - * - * The wrapper records the Ax turn loop at its public boundaries: - * 1. A parent span for the entire analyst run. - * 2. Per-turn child spans from the `onTurn` callback (captures code, - * output size, error status). - * 3. Summary attributes on the parent (total turns, usage, findings). - */ - import type { TraceEmitter } from './trace/emitter' -import type { - AnalyzeTracesInput, - AnalyzeTracesOptions, - AnalyzeTracesResult, - AnalyzeTracesTurnSnapshot, +import { + type AnalyzeTracesInput, + type AnalyzeTracesOptions, + type AnalyzeTracesResult, + analyzeTraces, } from './trace-analyst/analyst' -import { analyzeTraces } from './trace-analyst/analyst' export interface TracedAnalystOptions { - /** TraceEmitter for span emission. */ emitter: TraceEmitter - /** Parent span id. If omitted, uses emitter stack. */ parentSpanId?: string } -/** - * Run `analyzeTraces` wrapped in a parent span with per-turn child spans. - */ +/** Run a recursive trace investigation and record its engine steps. */ export async function tracedAnalyzeTraces( input: AnalyzeTracesInput, options: AnalyzeTracesOptions, - traceOpts: TracedAnalystOptions, + traceOptions: TracedAnalystOptions, ): Promise { - const parentSpan = await traceOpts.emitter.span({ + const parentSpan = await traceOptions.emitter.span({ kind: 'custom', name: 'analyst:analyze-traces', - parentSpanId: traceOpts.parentSpanId, + parentSpanId: traceOptions.parentSpanId, attributes: { + 'analyst.engine': options.engine.id, 'analyst.question_length': input.question.length, - 'analyst.max_turns': options.maxTurns ?? 12, - 'analyst.max_subqueries': options.maxSubqueries ?? 4, + 'analyst.max_iterations': options.limits?.maxIterations ?? 12, + 'analyst.max_llm_calls': options.limits?.maxLlmCalls ?? 8, 'eval.phase': 'analyst', }, }) - // Intercept onTurn to emit per-turn spans. - const originalOnTurn = options.onTurn - const wrappedOptions: AnalyzeTracesOptions = { - ...options, - onTurn: async (turn: AnalyzeTracesTurnSnapshot) => { - const turnSpan = await traceOpts.emitter.span({ + try { + const result = await analyzeTraces(input, options) + for (const [index, step] of result.trajectory.entries()) { + const encoded = JSON.stringify(step) + const stepSpan = await traceOptions.emitter.span({ kind: 'custom', - name: `analyst:turn-${turn.turn}`, + name: `analyst:step-${index + 1}`, parentSpanId: parentSpan.span.spanId, attributes: { - 'analyst.stage': turn.stage, - 'analyst.turn': turn.turn, - 'analyst.is_error': turn.isError, - 'analyst.code_length': turn.code.length, - 'analyst.output_length': turn.output.length, + 'analyst.step': index + 1, + 'analyst.step_bytes': Buffer.byteLength(encoded), 'eval.phase': 'analyst', }, }) - if (turn.isError) { - await turnSpan.fail('Turn produced an error') - } else { - await turnSpan.end() - } - if (originalOnTurn) await originalOnTurn(turn) - }, - } - - try { - const result = await analyzeTraces(input, wrappedOptions) + await stepSpan.end() + } await parentSpan.end({ attributes: { - 'analyst.question_length': input.question.length, - 'analyst.turn_count': result.turnCount, + 'analyst.engine': options.engine.id, + 'analyst.model_calls': result.modelCalls, + 'analyst.tool_calls': result.toolCalls, + 'analyst.step_count': result.trajectory.length, 'analyst.finding_count': result.findings.length, 'analyst.answer_length': result.answer.length, 'eval.phase': 'analyst', }, } as Record) return result - } catch (err) { - await parentSpan.fail(err instanceof Error ? err : String(err)) - throw err + } catch (error) { + await parentSpan.fail(error instanceof Error ? error : String(error)) + throw error } } diff --git a/tests/analyst-tracing.test.ts b/tests/analyst-tracing.test.ts index 9c80e5cf..0f237adc 100644 --- a/tests/analyst-tracing.test.ts +++ b/tests/analyst-tracing.test.ts @@ -1,8 +1,17 @@ import { describe, expect, it } from 'vitest' +import type { TraceAnalysisEngine } from '../src/analyst/engine' import { TraceEmitter } from '../src/trace/emitter' import { InMemoryTraceStore } from '../src/trace/store' import { tracedAnalyzeTraces } from '../src/traced-analyst' +const engine: TraceAnalysisEngine = { + id: 'test-engine', + description: 'Unused because the source path is intentionally absent.', + async analyze() { + throw new Error('test engine should not run') + }, +} + function makeEmitter() { const store = new InMemoryTraceStore() let counter = 0 @@ -19,21 +28,11 @@ describe('analyst tracing', () => { const { store, emitter } = makeEmitter() await emitter.startRun({ scenarioId: 'test', layer: 'meta', projectId: 'test' }) - // Mock the analyzeTraces function by providing a mock AI that returns - // immediately. Since analyzeTraces requires a real AxAIService, we test - // the span creation by verifying span structure after a controlled error. - // A full integration test would need a real or mocked AI backend. - - // We test the span wrapping by verifying the parent span is created - // and an error span is properly emitted when analyzeTraces throws. const fakeOptions = { source: '/nonexistent/path.jsonl', - ai: {} as any, - model: 'test-model', + engine, } - // The analyzeTraces call will fail (no real AI + file doesn't exist), - // but the span wrapping should still fire. try { await tracedAnalyzeTraces({ question: 'what failed?' }, fakeOptions, { emitter }) } catch { @@ -62,7 +61,7 @@ describe('analyst tracing', () => { try { await tracedAnalyzeTraces( { question: longQuestion }, - { source: '/nonexistent.jsonl', ai: {} as any }, + { source: '/nonexistent.jsonl', engine }, { emitter }, ) } catch { @@ -75,36 +74,4 @@ describe('analyst tracing', () => { 'analyst.question_length': longQuestion.length, }) }) - - it('wraps onTurn callbacks with child spans', async () => { - const { store, emitter } = makeEmitter() - await emitter.startRun({ scenarioId: 'test', layer: 'meta', projectId: 'test' }) - - const turnsSeen: number[] = [] - try { - await tracedAnalyzeTraces( - { question: 'test question' }, - { - source: '/nonexistent.jsonl', - ai: {} as any, - onTurn: (turn) => { - turnsSeen.push(turn.turn) - }, - }, - { emitter }, - ) - } catch { - // Expected - } - - // The parent span should exist regardless of error - const spans = await store.spans({ runId: 'analyst-run' }) - const parentSpan = spans.find((s) => s.name === 'analyst:analyze-traces') - expect(parentSpan).toBeDefined() - // Turn spans only appear if analyzeTraces ran far enough to invoke onTurn. - // With no real AI, we just verify the parent span attributes are correct. - expect(parentSpan!.attributes).toMatchObject({ - 'eval.phase': 'analyst', - }) - }) }) diff --git a/tests/campaign/external-optimizer-process.test.ts b/tests/campaign/external-optimizer-process.test.ts index 66632c27..b83213d2 100644 --- a/tests/campaign/external-optimizer-process.test.ts +++ b/tests/campaign/external-optimizer-process.test.ts @@ -1,7 +1,7 @@ import { existsSync } from 'node:fs' import { mkdtemp, readFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { join, relative } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' import { runExternalOptimizerProcess, @@ -126,6 +126,28 @@ describe('external optimizer process', () => { } }) + it('resolves a path-like runner command before entering the private working directory', async () => { + const script = [ + "const { writeFileSync } = require('node:fs')", + "const output = process.argv[process.argv.indexOf('--output') + 1]", + "writeFileSync(output, JSON.stringify({ status: 'complete' }))", + ].join(';') + + await expect( + runExternalOptimizerProcess({ + label: 'relative-runner optimizer', + tempPrefix: 'agent-eval-relative-runner-', + module: 'unused', + input: {}, + runner: { + command: relative(process.cwd(), process.execPath), + args: ['-e', script, '--'], + }, + timeoutMs: 5_000, + }), + ).resolves.toEqual({ status: 'complete' }) + }) + it('retains the final exception after large process output', async () => { const script = [ "process.stderr.write('HEAD_MARKER\\n')", @@ -1040,6 +1062,97 @@ describe('external optimizer model proxy', () => { } }) + it('fails closed when a successful provider response reports zero usage', async () => { + const ledger = new CostLedger() + const proxy = await startExternalOptimizerModelProxy({ + upstreamBaseUrl: 'https://provider.example/v1', + upstreamApiKey: 'provider-secret', + model: 'model-a', + budget: modelBudget({ maxRequests: 1 }), + costLedger: ledger, + phase: 'optimizer', + actor: 'official-library', + fetchImpl: async () => + new Response( + JSON.stringify({ + choices: [{ message: { content: 'unmetered result' } }], + usage: { prompt_tokens: 0, completion_tokens: 0 }, + }), + { headers: { 'content-type': 'application/json' } }, + ), + }) + + try { + const response = await postModel(proxy, { + model: 'model-a', + messages: [], + max_tokens: 1, + }) + expect(response.status).toBe(502) + expect(await response.json()).toEqual({ + error: 'optimizer model provider reported zero token usage for a successful response', + }) + expect(proxy.successfulCompletions()).toBe(0) + expect(ledger.summary().accountingComplete).toBe(false) + expect(ledger.list()).toEqual([ + expect.objectContaining({ + inputTokens: 0, + outputTokens: 0, + costUnknown: true, + usageUnknown: true, + }), + ]) + } finally { + await proxy.close() + } + }) + + it('rejects output beyond the requested limit after recording actual usage', async () => { + const ledger = new CostLedger() + const proxy = await startExternalOptimizerModelProxy({ + upstreamBaseUrl: 'https://provider.example/v1', + upstreamApiKey: 'provider-secret', + model: 'model-a', + budget: modelBudget({ maxRequests: 1 }), + costLedger: ledger, + phase: 'optimizer', + actor: 'official-library', + fetchImpl: async () => + new Response( + JSON.stringify({ + choices: [{ message: { content: 'too much output' } }], + usage: { prompt_tokens: 10, completion_tokens: 21 }, + }), + { headers: { 'content-type': 'application/json' } }, + ), + }) + + try { + const response = await postModel(proxy, { + model: 'model-a', + messages: [], + max_tokens: 20, + }) + expect(response.status).toBe(502) + expect(await response.json()).toEqual({ + error: 'optimizer model provider reported 21 output tokens, exceeding requested limit 20', + }) + expect(proxy.successfulCompletions()).toBe(0) + const receipts = ledger.list() + expect(receipts).toEqual([ + expect.objectContaining({ + inputTokens: 10, + outputTokens: 21, + costUnknown: false, + usageUnknown: false, + }), + ]) + expect(receipts[0]?.costUsd).toBeCloseTo(0.000052, 12) + } finally { + await proxy.close() + } + }) + it('bounds provider responses before buffering them', async () => { const ledger = new CostLedger() const proxy = await startExternalOptimizerModelProxy({ diff --git a/tests/steering-optimizer.test.ts b/tests/steering-optimizer.test.ts index 33052ec0..5c4ca44f 100644 --- a/tests/steering-optimizer.test.ts +++ b/tests/steering-optimizer.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import type { SteeringOptimizationRow } from '../src/steering-optimizer' -import { AxGepaSteeringOptimizer, PairwiseSteeringOptimizer } from '../src/steering-optimizer' +import { PairwiseSteeringOptimizer } from '../src/steering-optimizer' describe('steering optimizer', () => { it('ranks variants by aggregate score', () => { @@ -9,18 +9,6 @@ describe('steering optimizer', () => { expect(result.rankings[0]?.variantId).toBe('strong') }) - it('fails closed to skipped ax mode when data is insufficient', async () => { - const result = await new AxGepaSteeringOptimizer({ - provider: 'openai', - apiKey: 'sk-test', - model: 'gpt-5.4-mini', - minScenarioWinners: 10, - }).optimize(rows()) - expect(result.backend).toBe('ax-gepa') - expect(result.skipped).toBe(true) - expect(result.recommendedVariantId).toBe('strong') - }) - it('keeps rankings finite when runtime-loaded rows have invalid cost or latency', () => { const bad = row('bad-metrics', 's3', 9) bad.score.costUsd = Number.NaN From da2273fce33d1d3fbdb0101c9f6a4c7380edcb50 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 30 Jul 2026 16:17:42 -0600 Subject: [PATCH 2/5] fix(analyst): pin the published reference result to the digests that produced it The CodeTraceBench artifact records the implementation that ran it; binding those fields to the live ANALYST_BENCHMARK_* constants asserted that the current implementation produced a run it did not. The published digests are now literal historical facts, and a fresh certified run is required before any headline claim about the current engine. --- .../benchmark-reference-result.test.ts | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/analyst/benchmark-reference-result.test.ts b/src/analyst/benchmark-reference-result.test.ts index 1ddc7a25..4ed2a046 100644 --- a/src/analyst/benchmark-reference-result.test.ts +++ b/src/analyst/benchmark-reference-result.test.ts @@ -3,10 +3,14 @@ import { readFile } from 'node:fs/promises' import { fileURLToPath } from 'node:url' import { describe, expect, it } from 'vitest' import { readAnalystBenchmarkArtifact } from './benchmark-command-result' -import { - ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, - ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, -} from './benchmark-implementation' + +// Digests of the implementation that produced the published run. These are +// historical facts about the artifact, not the live ANALYST_BENCHMARK_* +// constants: they change only together with a fresh certified benchmark run. +const PUBLISHED_RUN_IMPLEMENTATION_SHA256 = + '4dba263b6256a30d56c7fdb2d992d3a953c0035d731f359b704db806f68f75ac' +const PUBLISHED_RUN_DEPENDENCY_LOCK_SHA256 = + '1e03f2daed356d60316aabefb407ec1e437ac94d408d61eea4ae096e9c6fbb5b' const REFERENCE_RESULT = fileURLToPath( new URL( @@ -86,8 +90,8 @@ describe('CodeTraceBench GLM-5.2 reference result', () => { labelsSha256: '5d8b4024c3e2114965cbf2f2fa0124bbf59b3fb134824fa06dd6a38ee07e8412', selectedCases: 32, protocolSha256: '166e399c9a93c9806b007273bf0b54078709389c52c6a33e3cbce0f554dab302', - implementationSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, - dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, + implementationSha256: PUBLISHED_RUN_IMPLEMENTATION_SHA256, + dependencyLockSha256: PUBLISHED_RUN_DEPENDENCY_LOCK_SHA256, observations: 128, verificationAvailability: { cases: 32, @@ -247,8 +251,8 @@ describe('CodeTraceBench GLM-5.2 reference result', () => { missingCases: 0, }, implementation: { - sourceSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, - dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, + sourceSha256: PUBLISHED_RUN_IMPLEMENTATION_SHA256, + dependencyLockSha256: PUBLISHED_RUN_DEPENDENCY_LOCK_SHA256, }, failureProof: { parallelStart: { From e159b49d34d39a8368cad0bdda56029424f0b8c1 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 30 Jul 2026 16:45:15 -0600 Subject: [PATCH 3/5] fix(release): the export gate and changelog track the DSPy migration's removed surface - verify-package-exports no longer requires the deleted buildTraceAnalystTools and pins every removed analyst, steering, and traces export with the script's negative-assertion convention - CHANGELOG documents the breaking analyst-engine surface: required engine, defineTraceAnalyst returning TraceAnalystDefinition without a cost field, createTraceAnalyst rename, instructions rename, pairwise-only steering, the Ax stack removal, and the Python dspy extra - the published CodeTraceBench README states its numbers came from the retired one-shot runner at implementation 4dba263b and forbids citing them for the current engine without a fresh certified run --- CHANGELOG.md | 20 ++++++++++++++++++ .../codetracebench-glm52-20260730/README.md | 5 +++++ scripts/verify-package-exports.mjs | 21 ++++++++++++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf6a7907..9ce506a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,26 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- Completed results retain a digest of every behavior-defining source file and are read through one strict recursive schema. The repository includes one pinned 32-case CodeTraceBench input, two complete 64-call GLM-5.2 Agent Eval runs, and a failure-inclusive run of pinned CodeTracer on the same trajectories. Exact source, input, result, resume, usage, cost, and secret-scan checks are committed with the results. +- The Python package gains a `dspy` extra: `agent-eval-rpc[dspy]` runs official `dspy.RLM` in a sandboxed Deno/Pyodide child process with seven allowlisted trace tools and strict JSON I/O. + +### Changed + +- **Breaking:** trace analysts are recursive research programs run through an explicit analysis engine. + `analyzeTraces` requires an `engine` (the DSPy RLM engine is the primary implementation) and reports engine iterations; `maxTurns`, `maxSubqueries`, `onTurn`, and `AnalyzeTracesTurnSnapshot` are gone. + `callLlmJson` remains only as the one-shot `direct` benchmark baseline. +- **Breaking:** `defineTraceAnalyst()` returns an inert `TraceAnalystDefinition` for registration instead of a registrable analyst, and no longer takes a `cost` declaration — analyst cost is always metered LLM usage. + `createTraceAnalystKind` is now `createTraceAnalyst`; `TraceAnalystKindSpec` and `CreateTraceAnalystKindOpts` are replaced by `TraceAnalystDefinition`. +- **Breaking:** `BuildTraceAnalystSurfaceDispatchOptions.analyze` receives `instructions` instead of `actorDescription`. +- **Breaking:** `SteeringOptimizerBackend` narrows to `'pairwise'`. +- Citation verification is store-backed: cited trace and span ids must resolve in the trace analysis store, and encoded or foreign ids are rejected. +- External-optimizer subprocess calls fail on HTTP 200 responses with zero input and output usage, and on output over-reservation after recording the actual charge. +- Relative external-optimizer runner commands resolve against the caller's working directory instead of the child's temporary directory. + +### Removed + +- **Breaking:** the Ax analyst stack and the `@ax-llm/ax` dependency: `createAnalystAi`, `CreateAnalystAiConfig`, `structureFindings`, `StructureFindingsOptions`, `StructureFindingsResult`, `AxGepaSteeringOptimizer`, and `AxSteeringOptimizerConfig`. +- **Breaking:** `createPublicBenchmarkModelRunner`; the analyst benchmark CLI defaults to the DSPy RLM runner and keeps the one-shot runner as the explicit `direct` baseline. +- `buildTraceAnalystTools`; trace tools are built from the transport-neutral descriptors. ### Fixed diff --git a/benchmarks/trace-analysis/codetracebench-glm52-20260730/README.md b/benchmarks/trace-analysis/codetracebench-glm52-20260730/README.md index 76b2348f..3dfbf691 100644 --- a/benchmarks/trace-analysis/codetracebench-glm52-20260730/README.md +++ b/benchmarks/trace-analysis/codetracebench-glm52-20260730/README.md @@ -10,6 +10,11 @@ This directory records three real-model runs on the same 32 public CodeTraceBenc The result is useful but not strong. Agent Eval accounted for every provider call and beat this CodeTracer run on the published metric, but all three arms had low precision, weak repeatability, and frequent false positives on solved trajectories. +**Implementation provenance.** +These numbers were produced by the retired one-shot direct runner at implementation SHA-256 `4dba263b6256a30d56c7fdb2d992d3a953c0035d731f359b704db806f68f75ac` (the value recorded in every artifact here). +The current Agent Eval analyst is a recursive DSPy RLM engine with a different implementation digest; the reproduce commands below therefore run a different analyst and will produce different numbers. +These results describe only the retired runner, and no number here may be cited for the current engine until a fresh certified run replaces this directory. + ## Inputs | Field | Value | diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index 2aaff20e..1826ed9f 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -338,6 +338,26 @@ try { type RemovedAnalystCostAlias = import('@tangle-network/agent-eval').AnalystRunSummary['cost_usd'] // @ts-expect-error one-shot finding structuring was removed; use a recursive analysis engine type RemovedStructureFindingsOptions = import('@tangle-network/agent-eval/analyst').StructureFindingsOptions + // @ts-expect-error trace analysts run through an analysis engine, not an Ax client + type RemovedCreateAnalystAi = typeof import('@tangle-network/agent-eval/analyst').createAnalystAi + // @ts-expect-error trace analysts run through an analysis engine, not an Ax client + type RemovedCreateAnalystAiConfig = import('@tangle-network/agent-eval/analyst').CreateAnalystAiConfig + // @ts-expect-error trace analysts are created from TraceAnalystDefinition via createTraceAnalyst + type RemovedCreateTraceAnalystKind = typeof import('@tangle-network/agent-eval/analyst').createTraceAnalystKind + // @ts-expect-error TraceAnalystDefinition is the one analyst definition contract + type RemovedTraceAnalystKindSpec = import('@tangle-network/agent-eval/analyst').TraceAnalystKindSpec + // @ts-expect-error TraceAnalystDefinition is the one analyst definition contract + type RemovedCreateTraceAnalystKindOpts = import('@tangle-network/agent-eval/analyst').CreateTraceAnalystKindOpts + // @ts-expect-error steering optimization is pairwise-only + type RemovedAxGepaSteeringOptimizer = typeof import('@tangle-network/agent-eval').AxGepaSteeringOptimizer + // @ts-expect-error steering optimization is pairwise-only + type RemovedAxSteeringOptimizerConfig = import('@tangle-network/agent-eval').AxSteeringOptimizerConfig + // @ts-expect-error trace analysis reports engine iterations, not per-turn snapshots + type RemovedAnalyzeTracesTurnSnapshot = import('@tangle-network/agent-eval/traces').AnalyzeTracesTurnSnapshot + // @ts-expect-error the direct one-shot runner is the explicit baseline, not the model runner + type RemovedCreatePublicBenchmarkModelRunner = typeof import('@tangle-network/agent-eval/analyst').createPublicBenchmarkModelRunner + // @ts-expect-error trace tools are built from transport-neutral descriptors + type RemovedBuildTraceAnalystTools = typeof import('@tangle-network/agent-eval/traces').buildTraceAnalystTools // @ts-expect-error comparison partitions use explicit train, selection, and test fields type RemovedComparisonHoldout = CompareOptimizationMethodsOptions['holdoutScenarios'] // @ts-expect-error SurfaceProposer is the only candidate-proposal contract @@ -662,7 +682,6 @@ try { } for (const name of [ 'buildTraceAnalysisToolDescriptors', - 'buildTraceAnalystTools', 'createBoundedTraceAnalysisStore', 'TRACE_ANALYSIS_LIMITS', 'TraceAnalysisValidationError', From ef36ad97aa585c693249f48e29e5db156ac2d9a7 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 30 Jul 2026 16:45:26 -0600 Subject: [PATCH 4/5] fix(analyst): verified excerpts, tolerant finding rows, and a pinned, attested sandbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the adversarial-audit findings on the DSPy engine boundary: - excerpt citations verify against span attributes and status_message (or a cited finding's content fields) only — identifiers, names, statuses, and timestamps are no longer quotable evidence — and excerpts under 8 chars are rejected as unverifiable - one malformed finding row from the bridge no longer discards the whole paid investigation: rows are validated per row, rejections are logged and counted in runtime.rejectedFindings - the bridge's canonical fixture emits a grammar-valid subject (incorrect-step-2); step:2 was certified Python-side but rejected by the TypeScript finding-subject schema - the Deno/Pyodide sandbox pins pyodide via an import map (npm latest flipped 0.29.3 to 314.0.3 during development, silently switching interpreters on a cold cache) and the pin is recorded in runtime identity - the sandbox attestation is derived from the deno command that actually ran (config, node-modules-dir, permission flags with scoped-path counts) instead of hardcoded constants - model output answer and findings_json are stripped before validation so a trailing newline cannot abort a fully-paid run - the analyst benchmark implementation digest is repinned for these changes TS: typecheck clean, 4525 passed / 3 skipped. Python: 17/17 bridge tests plus a live inspect through real Deno booting pinned pyodide 314.0.3. --- .../src/agent_eval_rpc/dspy_rlm_bridge.py | 62 +++++++++++- clients/python/tests/test_dspy_rlm_bridge.py | 96 +++++++++++++++++-- src/analyst/benchmark-implementation.ts | 2 +- src/analyst/dspy-rlm-engine.test.ts | 91 +++++++++++++++++- src/analyst/dspy-rlm-engine.ts | 33 +++++-- src/analyst/kind-factory.test.ts | 41 +++++++- src/analyst/kind-factory.ts | 33 ++++++- 7 files changed, 329 insertions(+), 29 deletions(-) diff --git a/clients/python/src/agent_eval_rpc/dspy_rlm_bridge.py b/clients/python/src/agent_eval_rpc/dspy_rlm_bridge.py index e7a69c4e..4ff5d1b3 100644 --- a/clients/python/src/agent_eval_rpc/dspy_rlm_bridge.py +++ b/clients/python/src/agent_eval_rpc/dspy_rlm_bridge.py @@ -12,6 +12,7 @@ import shutil import subprocess import sys +import tempfile from collections.abc import Callable, Mapping from contextlib import contextmanager from contextvars import ContextVar @@ -308,7 +309,14 @@ def _build_dspy_tools(dspy: Any, tool_specs: list[dict[str, Any]]) -> list[Any]: @contextmanager def _probed_interpreter(dspy: Any) -> Any: - deno_command = _build_deno_command(dspy) + with tempfile.TemporaryDirectory(prefix="agent-eval-dspy-rlm-") as import_map_dir: + import_map_path = Path(import_map_dir) / "import-map.json" + import_map_path.write_text(json.dumps(_PYODIDE_IMPORT_MAP), encoding="utf-8") + deno_command = _build_deno_command(dspy, import_map_path) + yield from _run_probed_interpreter(dspy, deno_command) + + +def _run_probed_interpreter(dspy: Any, deno_command: list[str]) -> Any: interpreter = dspy.PythonInterpreter(deno_command=deno_command) try: try: @@ -327,7 +335,17 @@ def _probed_interpreter(dspy: Any) -> Any: interpreter.shutdown() -def _build_deno_command(dspy: Any) -> list[str]: +# DSPy's runner.js imports npm:pyodide without a version, and --no-config means +# no lockfile: a cold cache would execute whatever npm dist-tags.latest points +# at. The import map pins the interpreter the sandbox runs; transitive npm +# ranges below this root are the residual unpinned surface. +_PYODIDE_VERSION = "314.0.3" +_PYODIDE_IMPORT_MAP = { + "imports": {"npm:pyodide/pyodide.js": f"npm:pyodide@{_PYODIDE_VERSION}/pyodide.js"} +} + + +def _build_deno_command(dspy: Any, import_map_path: Path) -> list[str]: deno_executable = _find_deno_executable() runner_path = _dspy_runner_path(dspy) deno_cache_dir = _deno_cache_dir(deno_executable) @@ -338,6 +356,7 @@ def _build_deno_command(dspy: Any) -> list[str]: str(deno_executable), "run", "--no-config", + f"--import-map={import_map_path}", "--node-modules-dir=none", f"--allow-read={runner_path},{deno_cache_dir}", str(runner_path), @@ -584,12 +603,17 @@ def _validate_finding(value: Any, index: int) -> dict[str, Any]: def _runtime_identity(deno_command: list[str]) -> dict[str, Any]: + # The sandbox record is derived from the command that actually ran, so a + # permission or config regression shows up in every run record instead of + # being masked by a constant. Scoped paths are machine-specific and are + # recorded as counts, never as raw paths, to keep run records shareable. return { "engine": "dspy-rlm", "packages": { "agent-eval-rpc": _package_version("agent-eval-rpc"), "dspy": _package_version("dspy"), "deno": _package_version("deno"), + "pyodide": _PYODIDE_VERSION, }, "python": { "implementation": sys_implementation_name(), @@ -599,14 +623,40 @@ def _runtime_identity(deno_command: list[str]) -> dict[str, Any]: "sourceSha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), }, "sandbox": { + # The startup probe already raised unless it returned exactly "2". "probeOutput": "2", "runtime": "deno-pyodide", - "config": "none", - "nodeModulesDir": "none", + **_sandbox_attestation(deno_command), }, } +def _sandbox_attestation(deno_command: list[str]) -> dict[str, Any]: + options = [part for part in deno_command[1:] if part.startswith("-")] + permissions: dict[str, Any] = {} + for part in options: + if part in ("-A", "--allow-all"): + permissions["allow-all"] = {"scoped": False} + continue + if part.startswith("--allow-"): + name, _, value = part.partition("=") + scope_entries = [entry for entry in value.split(",") if entry] if value else [] + permissions[name.removeprefix("--")] = ( + {"scoped": True, "paths": len(scope_entries)} + if scope_entries + else {"scoped": False} + ) + node_modules_dir = next( + (part.partition("=")[2] for part in options if part.startswith("--node-modules-dir")), + "default", + ) + return { + "config": "none" if "--no-config" in options else "discovered", + "nodeModulesDir": node_modules_dir or "default", + "permissions": permissions, + } + + def sys_implementation_name() -> str: return platform.python_implementation().lower() @@ -647,6 +697,10 @@ def _prediction_field(prediction: Any, name: str) -> Any: def _prediction_string(prediction: Any, name: str) -> str: value = _prediction_field(prediction, name) + # Model output: surrounding whitespace is presentation, not a data-integrity + # defect, and this runs after every model call is already paid for. + if isinstance(value, str): + value = value.strip() return _require_non_empty_string(value, f"DSPy RLM prediction {name}") diff --git a/clients/python/tests/test_dspy_rlm_bridge.py b/clients/python/tests/test_dspy_rlm_bridge.py index 2ee45f4e..e00ea6d2 100644 --- a/clients/python/tests/test_dspy_rlm_bridge.py +++ b/clients/python/tests/test_dspy_rlm_bridge.py @@ -53,7 +53,7 @@ def test_inspect_reports_pinned_runtime_and_private_atomic_output( monkeypatch.setattr( dspy_rlm_bridge, "_build_deno_command", - lambda _dspy: DENO_COMMAND, + lambda _dspy, _import_map: DENO_COMMAND, ) monkeypatch.setattr( dspy_rlm_bridge, @@ -84,6 +84,7 @@ def test_inspect_reports_pinned_runtime_and_private_atomic_output( "agent-eval-rpc": "0.137.0", "dspy": "3.2.1", "deno": "2.7.14", + "pyodide": dspy_rlm_bridge._PYODIDE_VERSION, }, "python": {"implementation": "cpython", "version": "3.13.0"}, "bridge": { @@ -96,6 +97,7 @@ def test_inspect_reports_pinned_runtime_and_private_atomic_output( "runtime": "deno-pyodide", "config": "none", "nodeModulesDir": "none", + "permissions": {"allow-read": {"scoped": True, "paths": 2}}, }, } } @@ -107,6 +109,25 @@ def test_inspect_reports_pinned_runtime_and_private_atomic_output( assert observed_umask == previous_umask +def test_sandbox_attestation_derives_from_the_real_command() -> None: + widened = dspy_rlm_bridge._sandbox_attestation( + ["/bin/deno", "run", "--allow-net", "-A", "--node-modules-dir=auto", "/r.js"] + ) + assert widened == { + "config": "discovered", + "nodeModulesDir": "auto", + "permissions": { + "allow-all": {"scoped": False}, + "allow-net": {"scoped": False}, + }, + } + assert dspy_rlm_bridge._sandbox_attestation(DENO_COMMAND) == { + "config": "none", + "nodeModulesDir": "none", + "permissions": {"allow-read": {"scoped": True, "paths": 2}}, + } + + def test_analyze_uses_official_rlm_contract_and_enabled_node_tool_specs( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -117,7 +138,7 @@ def test_analyze_uses_official_rlm_contract_and_enabled_node_tool_specs( monkeypatch.setattr( dspy_rlm_bridge, "_build_deno_command", - lambda _dspy: DENO_COMMAND, + lambda _dspy, _import_map: DENO_COMMAND, ) monkeypatch.setattr(dspy_rlm_bridge, "_runtime_identity", lambda _command: RUNTIME) monkeypatch.setenv("OPENAI_API_KEY", "must-not-be-read") @@ -178,7 +199,7 @@ def fake_post(url: str, **kwargs: Any) -> httpx.Response: { "severity": "high", "claim": "Step 2 failed before recovery.", - "subject": "step:2", + "subject": "incorrect-step-2", "confidence": 0.95, "rationale": "The span has the first error.", "recommended_action": "Fix step 2.", @@ -253,12 +274,14 @@ def test_deno_command_ignores_repo_configuration_and_node_modules( monkeypatch.setattr(dspy_rlm_bridge, "_dspy_runner_path", lambda _dspy: runner) monkeypatch.setattr(dspy_rlm_bridge, "_deno_cache_dir", lambda _deno: cache) - command = dspy_rlm_bridge._build_deno_command(SimpleNamespace()) + import_map = tmp_path / "import-map.json" + command = dspy_rlm_bridge._build_deno_command(SimpleNamespace(), import_map) assert command == [ str(deno.resolve()), "run", "--no-config", + f"--import-map={import_map}", "--node-modules-dir=none", f"--allow-read={runner},{cache}", str(runner), @@ -266,6 +289,30 @@ def test_deno_command_ignores_repo_configuration_and_node_modules( assert "--node-modules-dir=auto" not in command +def test_probed_interpreter_pins_pyodide_through_the_import_map( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: dict[str, Any] = {} + + def capture_command(_dspy: Any, import_map_path: Path) -> list[str]: + observed["import_map"] = json.loads(import_map_path.read_text()) + return DENO_COMMAND + + calls: dict[str, Any] = {} + monkeypatch.setattr(dspy_rlm_bridge, "_build_deno_command", capture_command) + with dspy_rlm_bridge._probed_interpreter(_fake_dspy(calls)): + pass + + assert observed["import_map"] == { + "imports": { + "npm:pyodide/pyodide.js": ( + f"npm:pyodide@{dspy_rlm_bridge._PYODIDE_VERSION}/pyodide.js" + ) + } + } + assert calls["interpreter_shutdown"] is True + + def test_broken_sandbox_fails_before_lm_construction( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -276,7 +323,7 @@ def test_broken_sandbox_fails_before_lm_construction( monkeypatch.setattr( dspy_rlm_bridge, "_build_deno_command", - lambda _dspy: DENO_COMMAND, + lambda _dspy, _import_map: DENO_COMMAND, ) input_path, output_path = _write_analyze_input(tmp_path) monkeypatch.setattr( @@ -381,7 +428,7 @@ def test_analyze_rejects_malformed_findings( monkeypatch.setattr( dspy_rlm_bridge, "_build_deno_command", - lambda _dspy: DENO_COMMAND, + lambda _dspy, _import_map: DENO_COMMAND, ) monkeypatch.setattr(dspy_rlm_bridge, "_runtime_identity", lambda _command: RUNTIME) input_path, output_path = _write_analyze_input(tmp_path) @@ -397,6 +444,38 @@ def test_analyze_rejects_malformed_findings( assert not output_path.exists() +def test_analyze_strips_model_output_whitespace( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + calls: dict[str, Any] = {} + fake_dspy = _fake_dspy( + calls, + answer="The first failing operation is step 2.\n", + findings_json="[]\n", + invoke_tool=False, + ) + monkeypatch.setattr(dspy_rlm_bridge, "_load_dspy", lambda: fake_dspy) + monkeypatch.setattr( + dspy_rlm_bridge, + "_build_deno_command", + lambda _dspy, _import_map: DENO_COMMAND, + ) + monkeypatch.setattr(dspy_rlm_bridge, "_runtime_identity", lambda _command: RUNTIME) + input_path, output_path = _write_analyze_input(tmp_path) + monkeypatch.setattr( + sys, + "argv", + ["dspy-rlm-bridge", "--input", str(input_path), "--output", str(output_path)], + ) + + dspy_rlm_bridge.main() + + output = json.loads(output_path.read_text()) + assert output["answer"] == "The first failing operation is step 2." + assert output["findings"] == [] + + def _write_analyze_input(tmp_path: Path) -> tuple[Path, Path]: input_path = tmp_path / "input.json" output_path = tmp_path / "output.json" @@ -443,6 +522,7 @@ def _fake_dspy( calls: dict[str, Any], *, findings_json: str | None = None, + answer: str = "The first failing operation is step 2.", invoke_tool: bool = True, probe_output: str = "2\n", ) -> Any: @@ -451,7 +531,7 @@ def _fake_dspy( { "severity": "high", "claim": "Step 2 failed before recovery.", - "subject": "step:2", + "subject": "incorrect-step-2", "confidence": 0.95, "rationale": "The span has the first error.", "recommended_action": "Fix step 2.", @@ -515,7 +595,7 @@ def __call__(self, **kwargs: Any) -> Any: "spans": [], } return SimpleNamespace( - answer="The first failing operation is step 2.", + answer=answer, findings_json=findings_json if findings_json is not None else valid_findings, trajectory=[{"iteration": 1, "tool": "viewTrace"}], ) diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index ccf6fc76..be392b4f 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -94,7 +94,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = - '610fb8555731e8a4eceaedfa98c47ea37b040341aa684791c3f55ded9dc50357' + '44633a522ef6d3654224b69c73aa3f6b51ed9b2b63d861a2219472b0a9655cf4' export function analystBenchmarkImplementationDigest() { return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 diff --git a/src/analyst/dspy-rlm-engine.test.ts b/src/analyst/dspy-rlm-engine.test.ts index a5144cce..80e1cc79 100644 --- a/src/analyst/dspy-rlm-engine.test.ts +++ b/src/analyst/dspy-rlm-engine.test.ts @@ -1,5 +1,5 @@ import { createServer } from 'node:http' -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { CostLedger } from '../cost-ledger' import type { TraceAnalysisToolDescriptor } from '../trace-analyst/tools' import { createDspyRlmTraceEngine } from './dspy-rlm-engine' @@ -36,7 +36,7 @@ async function main() { const toolResult = await tool.json() fs.writeFileSync(outputPath, JSON.stringify({ answer: 'The run failed after one inspected trace.', - findings: [], + findings: __FINDINGS__, trajectory: [{ tool: 'getDatasetOverview', result: toolResult.result }], modelCalls: 1, runtime: { engine: 'test-bridge' }, @@ -48,6 +48,10 @@ main().catch((error) => { }) ` +function childScript(findingsJson: string): string { + return CHILD_SCRIPT.replace('__FINDINGS__', findingsJson) +} + describe('createDspyRlmTraceEngine', () => { it('runs a child through authenticated model and trace callbacks', async () => { let providerAuthorization = '' @@ -95,7 +99,7 @@ describe('createDspyRlmTraceEngine', () => { maxOutputTokens: 64, runner: { command: process.execPath, - args: ['-e', CHILD_SCRIPT, '--'], + args: ['-e', childScript('[]'), '--'], }, timeoutMs: 5_000, }) @@ -148,6 +152,87 @@ describe('createDspyRlmTraceEngine', () => { } }) + it('drops invalid finding rows, keeps valid ones, and counts the rejections', async () => { + const provider = createServer((_request, response) => { + response.writeHead(200, { 'content-type': 'application/json' }) + response.end( + JSON.stringify({ + choices: [{ message: { role: 'assistant', content: 'investigate' } }], + usage: { prompt_tokens: 3, completion_tokens: 2 }, + }), + ) + }) + await new Promise((resolve, reject) => { + provider.once('error', reject) + provider.listen(0, '127.0.0.1', () => { + provider.off('error', reject) + resolve() + }) + }) + const address = provider.address() + if (!address || typeof address === 'string') throw new Error('provider did not bind') + + const tool: TraceAnalysisToolDescriptor = { + namespace: 'traces', + name: 'getDatasetOverview', + description: 'Return a summary.', + parameters: { type: 'object', properties: {}, additionalProperties: false }, + handler: async () => ({ total_traces: 1 }), + } + const validFinding = { + severity: 'high', + claim: 'Step 2 failed before recovery.', + subject: 'incorrect-step-2', + confidence: 0.9, + evidence: [{ uri: 'trace://run-1/span/step-2' }], + } + const invalidFinding = { ...validFinding, subject: 'step:2' } + const engine = createDspyRlmTraceEngine({ + baseUrl: `http://127.0.0.1:${address.port}/v1`, + apiKey: 'provider-secret', + model: 'model-a', + pricing: { inputUsdPerMillion: 1, outputUsdPerMillion: 2 }, + maxCostUsd: 0.1, + maxOutputTokens: 64, + runner: { + command: process.execPath, + args: ['-e', childScript(JSON.stringify([validFinding, invalidFinding])), '--'], + }, + timeoutMs: 5_000, + }) + + const log = vi.fn() + try { + const result = await engine.analyze({ + analystId: 'failure-mode', + question: 'What failed?', + instructions: 'Inspect the trace.', + tools: [tool], + limits: { + maxIterations: 2, + maxLlmCalls: 1, + maxToolCalls: 1, + maxOutputChars: 1_000, + }, + costLedger: new CostLedger(), + costPhase: 'analyst.test', + log, + }) + + expect(result.findings).toEqual([expect.objectContaining({ claim: validFinding.claim })]) + expect(result.runtime.rejectedFindings).toBe(1) + expect(log).toHaveBeenCalledWith('finding rejected: bridge row failed schema validation', { + engine: 'dspy-rlm', + index: 1, + reason: expect.stringContaining('finding-subject grammar'), + }) + } finally { + await new Promise((resolve, reject) => { + provider.close((error) => (error ? reject(error) : resolve())) + }) + } + }) + it('requires explicit pricing for an unknown model', () => { expect(() => createDspyRlmTraceEngine({ diff --git a/src/analyst/dspy-rlm-engine.ts b/src/analyst/dspy-rlm-engine.ts index 26f392c7..a785b8a3 100644 --- a/src/analyst/dspy-rlm-engine.ts +++ b/src/analyst/dspy-rlm-engine.ts @@ -126,7 +126,13 @@ export function createDspyRlmTraceEngine(options: DspyRlmTraceEngineOptions): Tr timeoutMs, ...(request.signal ? { signal: request.signal } : {}), }) - const parsed = parseBridgeOutput(raw) + const parsed = parseBridgeOutput(raw, (index, reason) => { + request.log?.('finding rejected: bridge row failed schema validation', { + engine: 'dspy-rlm', + index, + reason, + }) + }) const successfulCompletions = modelProxy.successfulCompletions() const requestAttempts = modelProxy.requestAttempts() if (parsed.modelCalls !== successfulCompletions) { @@ -169,7 +175,10 @@ export function createDspyRlmTraceEngine(options: DspyRlmTraceEngineOptions): Tr } } -function parseBridgeOutput(value: unknown): Omit { +function parseBridgeOutput( + value: unknown, + onRejectedFinding: (index: number, reason: string) => void, +): Omit { if (!isRecord(value)) throw new Error('DSPy RLM bridge output must be an object') if (typeof value.answer !== 'string' || !value.answer.trim()) { throw new Error('DSPy RLM bridge returned no answer') @@ -177,16 +186,24 @@ function parseBridgeOutput(value: unknown): Omit { + // Findings are model output: one malformed row is model noise, not a bridge + // fault, and the rest of the paid investigation must survive it. Rejected + // rows are logged per row and counted in runtime.rejectedFindings. + let rejectedFindings = 0 + const findings: RawAnalystFinding[] = [] + value.findings.forEach((finding, index) => { const parsed = RawAnalystFindingSchema.safeParse(finding) if (!parsed.success) { - throw new Error( - `DSPy RLM bridge finding ${index} is invalid: ${parsed.error.issues + rejectedFindings += 1 + onRejectedFinding( + index, + parsed.error.issues .map((issue) => `${issue.path.join('.')}: ${issue.message}`) - .join('; ')}`, + .join('; '), ) + return } - return parsed.data + findings.push(parsed.data) }) if (!Array.isArray(value.trajectory)) { throw new Error('DSPy RLM bridge trajectory must be an array') @@ -202,7 +219,7 @@ function parseBridgeOutput(value: unknown): Omit { expect(result.findings).toEqual([]) expect(log).toHaveBeenCalledWith('finding rejected: unresolved evidence', { uri: 'trace://run-1/span/step-2', - reason: 'excerpt is not present in the cited span', + reason: 'excerpt is not present in the cited span content', }) }) + it.each([ + ['message.assistant', 'span name'], + ['test-agent', 'agent name'], + ['2026-07-30T00:00:00.000Z', 'timestamp'], + ])('rejects the fabricated excerpt %s quoting a %s instead of content', async (excerpt) => { + const log = vi.fn() + const result = await runTraceAnalyst({ + definition, + engine: findingEngine({ uri: 'trace://run-1/span/step-2', excerpt }), + store, + context: { ...context(), log }, + }) + + expect(result.findings).toEqual([]) + expect(log).toHaveBeenCalledWith('finding rejected: unresolved evidence', { + uri: 'trace://run-1/span/step-2', + reason: 'excerpt is not present in the cited span content', + }) + }) + + it.each([['step-2'], ['OK'], ['0']])( + 'rejects the excerpt %s as too short to verify', + async (excerpt) => { + const log = vi.fn() + const result = await runTraceAnalyst({ + definition, + engine: findingEngine({ uri: 'trace://run-1/span/step-2', excerpt }), + store, + context: { ...context(), log }, + }) + + expect(result.findings).toEqual([]) + expect(log).toHaveBeenCalledWith('finding rejected: unresolved evidence', { + uri: 'trace://run-1/span/step-2', + reason: 'excerpt is too short to verify', + }) + }, + ) + it('accepts a supplied finding citation and checks its excerpt', async () => { const prior = makeFinding({ analyst_id: 'prior-analyst', diff --git a/src/analyst/kind-factory.ts b/src/analyst/kind-factory.ts index 0c8d31ae..62ff75a0 100644 --- a/src/analyst/kind-factory.ts +++ b/src/analyst/kind-factory.ts @@ -224,6 +224,10 @@ async function evidenceIsResolvable( ]), ) for (const citation of finding.evidence) { + if (citation.excerpt !== undefined && citation.excerpt.trim().length < MINIMUM_EXCERPT_LENGTH) { + rejectEvidence(context, citation.uri, 'excerpt is too short to verify') + return false + } const traceLocation = parseTraceSpanEvidenceUri(citation.uri) if (traceLocation) { const storeContext = context.signal ? { signal: context.signal } : undefined @@ -247,8 +251,11 @@ async function evidenceIsResolvable( storeContext, ) const span = viewed.spans.find((entry) => entry.span_id === traceLocation.spanId) - if (!span || !containsExactText(span, citation.excerpt)) { - rejectEvidence(context, citation.uri, 'excerpt is not present in the cited span') + if ( + !span || + !containsExactText([span.attributes, span.status_message], citation.excerpt) + ) { + rejectEvidence(context, citation.uri, 'excerpt is not present in the cited span content') return false } } @@ -261,8 +268,20 @@ async function evidenceIsResolvable( rejectEvidence(context, citation.uri, 'citation is not a supplied finding or trace span') return false } - if (citation.excerpt !== undefined && !containsExactText(referenced, citation.excerpt)) { - rejectEvidence(context, citation.uri, 'excerpt is not present in the cited finding') + if ( + citation.excerpt !== undefined && + !containsExactText( + [ + referenced.claim, + referenced.rationale, + referenced.recommended_action, + referenced.validation_plan, + referenced.evidence_refs?.map((evidence) => evidence.excerpt), + ], + citation.excerpt, + ) + ) { + rejectEvidence(context, citation.uri, 'excerpt is not present in the cited finding content') return false } } @@ -279,6 +298,12 @@ function parseFindingEvidenceUri(uri: string): string | null { } } +/** Excerpts must quote enough content to be checkable evidence, not an + * incidental substring of an id, status, or timestamp. */ +const MINIMUM_EXCERPT_LENGTH = 8 + +/** Matches only within the passed content-bearing values — callers must not + * hand this whole spans or findings, or identifier fields become quotable. */ function containsExactText(value: unknown, expected: string, depth = 0): boolean { if (!expected || depth > 20) return false if (typeof value === 'string') return value.includes(expected) From 71bca06f5c7d07862d43712be169142f05de63a7 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 30 Jul 2026 16:48:10 -0600 Subject: [PATCH 5/5] style(analyst): apply biome formatting and repin the implementation digest --- src/analyst/benchmark-implementation.ts | 2 +- src/analyst/dspy-rlm-engine.ts | 4 +--- src/analyst/kind-factory.ts | 5 +---- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index be392b4f..555855e4 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -94,7 +94,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = - '44633a522ef6d3654224b69c73aa3f6b51ed9b2b63d861a2219472b0a9655cf4' + '0c4928e0f3c9b521d6e54d90d52e194eed6019d8746dcc7cc8987af298fd3b7f' export function analystBenchmarkImplementationDigest() { return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 diff --git a/src/analyst/dspy-rlm-engine.ts b/src/analyst/dspy-rlm-engine.ts index a785b8a3..28610fb1 100644 --- a/src/analyst/dspy-rlm-engine.ts +++ b/src/analyst/dspy-rlm-engine.ts @@ -197,9 +197,7 @@ function parseBridgeOutput( rejectedFindings += 1 onRejectedFinding( index, - parsed.error.issues - .map((issue) => `${issue.path.join('.')}: ${issue.message}`) - .join('; '), + parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`).join('; '), ) return } diff --git a/src/analyst/kind-factory.ts b/src/analyst/kind-factory.ts index 62ff75a0..0af31002 100644 --- a/src/analyst/kind-factory.ts +++ b/src/analyst/kind-factory.ts @@ -251,10 +251,7 @@ async function evidenceIsResolvable( storeContext, ) const span = viewed.spans.find((entry) => entry.span_id === traceLocation.spanId) - if ( - !span || - !containsExactText([span.attributes, span.status_message], citation.excerpt) - ) { + if (!span || !containsExactText([span.attributes, span.status_message], citation.excerpt)) { rejectEvidence(context, citation.uri, 'excerpt is not present in the cited span content') return false }