diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/driver.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/driver.py index 8d5657ece..e81421ca8 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/driver.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/driver.py @@ -40,7 +40,9 @@ import contextlib import importlib import inspect -from collections.abc import Awaitable, Callable +import os +import tempfile +from collections.abc import Awaitable, Callable, Iterator from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -73,6 +75,92 @@ #: agent-timeout code -- kept distinct from per-command crashes. AGENT_TIMEOUT_REASON_CODE = "harbor_agent_timeout_error" +#: Maximum characters of captured construction output appended to an error. +CAPTURED_OUTPUT_LIMIT = 600 + + +@contextlib.contextmanager +def _capture_process_output() -> Iterator[Callable[[], str]]: + """Capture fd 1/2 for the duration of the block, then replay them. + + A miner constructor typically shells out (``pip install ...``). The child + writes to the *inherited* file descriptors, so Python-level redirection + never sees it and ``CalledProcessError`` keeps only argv + exit status — + the actual reason ("Directory is not installable", a resolver conflict, a + missing wheel) is lost from the trial record. Capturing at the fd level is + the only way to keep it. + + Each stream is replayed to its original descriptor on exit so the runner's + own logs stay byte-complete; the returned callable yields the captured text + and is only meaningful after the block has exited. + """ + + captured: dict[str, str] = {"text": ""} + + def read() -> str: + return captured["text"] + + saved: dict[int, int] = {} + buffers: dict[int, Any] = {} + try: + for fd in (1, 2): + handle = tempfile.TemporaryFile() + saved[fd] = os.dup(fd) + os.dup2(handle.fileno(), fd) + buffers[fd] = handle + except OSError: + # Never let observability break execution: restore and run uncaptured. + for fd, original in saved.items(): + with contextlib.suppress(OSError): + os.dup2(original, fd) + os.close(original) + for handle in buffers.values(): + with contextlib.suppress(OSError): + handle.close() + yield read + return + + try: + yield read + finally: + chunks: list[str] = [] + for fd in (1, 2): + data = b"" + handle = buffers.get(fd) + if handle is not None: + try: + handle.flush() + handle.seek(0) + data = handle.read() + except OSError: + data = b"" + original = saved.get(fd) + if original is not None: + with contextlib.suppress(OSError): + os.dup2(original, fd) + os.close(original) + if data: + with contextlib.suppress(OSError): + os.write(fd, data) + chunks.append(data.decode("utf-8", "replace")) + if handle is not None: + with contextlib.suppress(OSError): + handle.close() + captured["text"] = "".join(chunks) + + +def _captured_detail(text: str) -> str: + """Render captured output as a single-line suffix for an error message.""" + + lines = [line.strip() for line in text.splitlines() if line.strip()] + if not lines: + return "" + joined = " / ".join(lines) + if len(joined) > CAPTURED_OUTPUT_LIMIT: + joined = joined[-CAPTURED_OUTPUT_LIMIT:] + joined = f"…{joined}" + return f" | output: {joined}" + class AgentLoadError(Exception): """The agent class could not be located / loaded (typed, fail-fast). @@ -246,24 +334,41 @@ async def drive( session: TmuxSession | None = None try: # 1. Load + construct the agent. Any failure here is the - # submission's fault -> submission_code_failed. - try: - agent_cls = self._agent_class or load_agent_class(self._import_path) - init_kwargs = dict(self._extra_init_kwargs) - if resolved_agent_env: - init_kwargs.setdefault("extra_env", dict(resolved_agent_env)) - agent = agent_cls( - logs_dir=Path(logs_dir) if logs_dir is not None else None, - model_name=model_name, - **init_kwargs, + # submission's fault -> submission_code_failed. Construction is + # wrapped in an fd-level capture because miner constructors shell + # out, and the child's diagnostics would otherwise never reach + # the trial record. + load_error: AgentLoadError | None = None + construction_error: Exception | None = None + with _capture_process_output() as captured_output: + try: + agent_cls = self._agent_class or load_agent_class(self._import_path) + init_kwargs = dict(self._extra_init_kwargs) + if resolved_agent_env: + init_kwargs.setdefault("extra_env", dict(resolved_agent_env)) + agent = agent_cls( + logs_dir=Path(logs_dir) if logs_dir is not None else None, + model_name=model_name, + **init_kwargs, + ) + except AgentLoadError as exc: + load_error = exc + except Exception as exc: + construction_error = exc + if load_error is not None: + return AgentRunResult( + status="failed", + reason_code=load_error.reason_code, + error=f"{load_error}{_captured_detail(captured_output())}", ) - except AgentLoadError as exc: - return AgentRunResult(status="failed", reason_code=exc.reason_code, error=str(exc)) - except Exception as exc: + if construction_error is not None: return AgentRunResult( status="failed", reason_code=AGENT_LOAD_FAILED_REASON_CODE, - error=f"agent construction failed: {exc}", + error=( + f"agent construction failed: {construction_error}" + f"{_captured_detail(captured_output())}" + ), ) agent_info = self._safe_agent_info(agent) diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/orchestrator.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/orchestrator.py index b3cf06e7a..c9d7c7f94 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/orchestrator.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/orchestrator.py @@ -43,7 +43,10 @@ from pathlib import Path from typing import Any -from agent_challenge.evaluation.own_runner.driver import AgentDriver +from agent_challenge.evaluation.own_runner.driver import ( + AGENT_LOAD_FAILED_REASON_CODE, + AgentDriver, +) from agent_challenge.evaluation.own_runner.reason_codes import ( REASON_CODES, is_known_reason_code, @@ -105,6 +108,17 @@ #: finalizes with its sibling trials intact. TRIAL_CRASH_REASON_CODE = "harbor_trial_failed" +#: How many construction failures confirm a broken submission package. +#: +#: Agent construction (unpack + install + import + instantiate) is +#: task-independent: a ZIP missing its manifest, or importing a module it never +#: shipped, breaks identically on every task. Re-running all 30 tasks to +#: relearn that one fact burns hours of wall clock and real LLM budget, so the +#: job stops once this many trials have failed that way. The threshold is >1 so +#: a single transient fault (a flaky package index during ``pip install``) +#: cannot abort an otherwise healthy job. +CONSTRUCTION_FAILURE_ABORT_THRESHOLD = 2 + # Fail fast at import if the taxonomy ever drops a code we emit. assert TRIAL_TIMEOUT_REASON_CODE in REASON_CODES assert TRIAL_CRASH_REASON_CODE in REASON_CODES @@ -443,9 +457,11 @@ async def run(self, tasks: Sequence[TaskSpec]) -> JobResult: state_lock = asyncio.Lock() in_flight = 0 peak = 0 + construction_failures = 0 + short_circuit = False async def execute(trial_id: TrialId) -> TrialOutcome: - nonlocal in_flight, peak + nonlocal in_flight, peak, construction_failures, short_circuit # Resume: a persisted result means this trial is already done -- load # it WITHOUT acquiring the semaphore (it never re-runs, never counts # toward in-flight, never double-counts). @@ -453,11 +469,24 @@ async def execute(trial_id: TrialId) -> TrialOutcome: if persisted is not None: return persisted + task = task_lookup[trial_id.task_name] + + # Fail-fast: once enough trials have proven the submission's agent + # cannot be constructed, the remaining trials would burn budget to + # reproduce the same packaging error. Resolve them immediately as + # explicit, self-describing failures instead of running them. + async with state_lock: + aborted = short_circuit + if aborted: + outcome = self._short_circuited_outcome(trial_id, task) + self._persist_trial(trial_id, outcome) + await self._notify_trial_listener(trial_id, outcome) + return outcome + async with semaphore: async with state_lock: in_flight += 1 peak = max(peak, in_flight) - task = task_lookup[trial_id.task_name] try: # Backstop: bound the whole trial (prepare + drive + verify + # teardown) so one stalled sub-step can never wedge @@ -481,6 +510,13 @@ async def execute(trial_id: TrialId) -> TrialOutcome: finally: async with state_lock: in_flight -= 1 + # Count construction failures so a broken package trips the + # fail-fast gate above for every trial not yet started. + if outcome.reason_code == AGENT_LOAD_FAILED_REASON_CODE: + async with state_lock: + construction_failures += 1 + if construction_failures >= CONSTRUCTION_FAILURE_ABORT_THRESHOLD: + short_circuit = True # Persist immediately so a later crash cannot lose a finished # trial (and a resume skips it). self._persist_trial(trial_id, outcome) @@ -541,6 +577,34 @@ def _crashed_outcome( error_text=f"trial crashed: {type(exc).__name__}: {exc}", ) + def _short_circuited_outcome(self, trial_id: TrialId, task: TaskSpec) -> TrialOutcome: + """Failed outcome for a trial never run because the package is broken. + + Mirrors :meth:`_timed_out_outcome` (``status="failed"``, ``errored=True``, + ``rewards=None``) so the job still aggregates every planned trial and the + totals stay honest -- the task was planned, scored 0, and says exactly + why it never executed. + """ + + return TrialOutcome( + task_name=trial_id.task_name, + trial_name=trial_id.trial_name, + status="failed", + rewards=None, + reason_code=AGENT_LOAD_FAILED_REASON_CODE, + errored=True, + agent_name=self._config.agent_name, + model_name=self._config.model_name, + source=task.source, + error_text=( + "short-circuit: agent construction failed on " + f"{CONSTRUCTION_FAILURE_ABORT_THRESHOLD} earlier trials, so this " + "trial was not run. Agent construction is task-independent -- fix " + "the submission package (installable project + importable modules) " + "and resubmit." + ), + ) + # -- lock / persistence ------------------------------------------------ def _check_or_write_lock(self) -> None: diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/result_schema.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/result_schema.py index fde74d5e1..640d60032 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/result_schema.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/result_schema.py @@ -38,8 +38,8 @@ import json import sys -from collections.abc import Mapping -from typing import IO, Any +from collections.abc import Mapping, Sequence +from typing import IO, Any, Protocol # The prefix the runner scans for (runner.py:1499). Must be byte-identical. RESULT_LINE_PREFIX = "BASE_BENCHMARK_RESULT=" @@ -196,6 +196,204 @@ def _as_int(value: Any) -> Any: return value +# --------------------------------------------------------------------------- # +# Trial failure diagnostics (additive observability on the result line) # +# --------------------------------------------------------------------------- # + + +class _TrialDiagnosticSource(Protocol): + """Minimal trial-outcome surface consumed by :func:`build_trial_diagnostics`.""" + + task_name: str + trial_name: str + status: str + errored: bool + reason_code: str | None + error_text: str | None + + +class _TextRedactor(Protocol): + """Narrow redaction seam (satisfied by :class:`~.redaction.LogRedactor`).""" + + def redact(self, text: str | None) -> str | None: ... + + +#: Marker ``reason_code`` on the final entry when failed trials exceed ``limit``. +DIAGNOSTICS_TRUNCATED_REASON = "diagnostics_truncated" + +#: Explicit suffix appended when an individual ``error_text`` is length-capped. +ERROR_TEXT_TRUNCATION_MARKER = "\u2026[truncated]" + + +def _truncate_error_text(text: str | None, max_error_chars: int) -> str | None: + """Cap ``text`` at ``max_error_chars`` and append an explicit ellipsis marker.""" + + if text is None: + return None + if max_error_chars < 0: + raise ValueError("max_error_chars must be >= 0") + if len(text) <= max_error_chars: + return text + return text[:max_error_chars] + ERROR_TEXT_TRUNCATION_MARKER + + +def build_trial_diagnostics( + outcomes: Sequence[Any], + *, + limit: int = 20, + max_error_chars: int = 2000, + redactor: _TextRedactor | None = None, +) -> list[dict[str, Any]]: + """Project failed/errored trial outcomes into additive diagnostics dicts. + + Production Terminal-Bench crashes currently surface only the opaque + ``harbor_trial_failed`` reason code. This helper lifts the per-trial + ``error_text`` (and identity fields) into a compact list that can ride on + the ``BASE_BENCHMARK_RESULT`` line under the additive ``trial_diagnostics`` + key without touching the five core contract fields. + + Inclusion rule + -------------- + A trial is included iff ``errored is True`` OR ``status == "failed"``. + Successful trials are omitted so a fully green job stays free of noise. + + Ordering + cap + -------------- + Input order is preserved (deterministic). At most ``limit`` real entries are + kept. When more failed trials exist, a final marker entry is appended with + ``reason_code="diagnostics_truncated"`` and an ``error_text`` that records how + many additional failures were omitted (so the list length is ``limit + 1`` + when truncated). + + Redaction + --------- + ``error_text`` may carry gateway tokens or miner env values. When a + ``redactor`` is supplied (typically :class:`LogRedactor`), every emitted + ``error_text`` is passed through ``redactor.redact`` before truncation. + Callers MUST supply an active redactor whenever secrets may still be present. + + Parameters + ---------- + outcomes : + Trial outcomes (duck-typed; :class:`TrialOutcome` satisfies the protocol). + limit : + Max real (non-marker) diagnostic entries. Must be >= 0. + max_error_chars : + Per-entry cap on ``error_text`` length before the ellipsis marker. + redactor : + Optional text redactor (``LogRedactor`` or any object with ``redact``). + + Returns + ------- + list[dict[str, Any]] + Each entry has keys ``task_name``, ``trial_name``, ``status``, + ``errored``, ``reason_code``, ``error_text``. Empty when no + failed/errored trials. + """ + + if limit < 0: + raise ValueError("limit must be >= 0") + + selected: list[Any] = [ + outcome + for outcome in outcomes + if bool(getattr(outcome, "errored", False)) + or getattr(outcome, "status", None) == "failed" + ] + if not selected: + return [] + + omitted = max(0, len(selected) - limit) + kept = selected[:limit] + entries: list[dict[str, Any]] = [] + for outcome in kept: + raw_error = getattr(outcome, "error_text", None) + if redactor is not None: + raw_error = redactor.redact(raw_error) + entries.append( + { + "task_name": outcome.task_name, + "trial_name": outcome.trial_name, + "status": outcome.status, + "errored": bool(outcome.errored), + "reason_code": outcome.reason_code, + "error_text": _truncate_error_text(raw_error, max_error_chars), + } + ) + + if omitted: + entries.append( + { + "task_name": "__truncated__", + "trial_name": "__truncated__", + "status": "failed", + "errored": True, + "reason_code": DIAGNOSTICS_TRUNCATED_REASON, + "error_text": f"+{omitted} more failed trial(s) omitted", + } + ) + return entries + + +def collapse_error_text_to_single_line(text: str | None) -> str: + """Collapse newlines/CRs in ``text`` so a stderr diagnostic stays one line.""" + + if not text: + return "" + return " ".join(text.replace("\r", "\n").splitlines()) + + +def format_trial_diagnostic_stderr_line(entry: Mapping[str, Any]) -> str: + """Format one flat ``key=value`` stderr diagnostic for a failed trial. + + Shape mirrors ``agent_challenge_reason_code=...`` breadcrumbs elsewhere:: + + agent_challenge_trial_diagnostic task= trial= reason= error= + + Newlines inside ``error`` are collapsed so the whole diagnostic is one line. + """ + + task = entry.get("task_name") or "" + trial = entry.get("trial_name") or "" + reason = entry.get("reason_code") + reason_s = "" if reason is None else str(reason) + raw_error = entry.get("error_text") + error = collapse_error_text_to_single_line( + raw_error if isinstance(raw_error, str) else None + ) + return ( + f"agent_challenge_trial_diagnostic task={task} trial={trial} " + f"reason={reason_s} error={error}" + ) + + +def emit_trial_diagnostic_stderr_lines( + entries: Sequence[Mapping[str, Any]], + *, + stream: IO[str] | None = None, +) -> list[str]: + """Print one stderr diagnostic per real (non-truncation-marker) entry. + + Truncation marker entries (``reason_code == diagnostics_truncated``) are + skipped so operators only see concrete per-trial failures on stderr. Returns + the lines that were written. + """ + + target = stream if stream is not None else sys.stderr + written: list[str] = [] + for entry in entries: + if entry.get("reason_code") == DIAGNOSTICS_TRUNCATED_REASON: + continue + line = format_trial_diagnostic_stderr_line(entry) + target.write(line + "\n") + written.append(line) + try: + target.flush() + except Exception: # noqa: BLE001 - best-effort flush only + pass + return written + + # --------------------------------------------------------------------------- # # Harbor outcome derivation (reproduces runner.py:1410-1437 byte-for-byte) # # --------------------------------------------------------------------------- # @@ -267,13 +465,19 @@ def emit_benchmark_result_line(payload: Mapping[str, Any], *, stream: IO[str] | __all__ = [ "BENCHMARK_RESULT_SCHEMA", + "DIAGNOSTICS_TRUNCATED_REASON", + "ERROR_TEXT_TRUNCATION_MARKER", "REQUIRED_FIELDS", "RESULT_LINE_PREFIX", "STATUS_VALUES", "ResultSchemaError", "build_benchmark_result", + "build_trial_diagnostics", + "collapse_error_text_to_single_line", "derive_benchmark_result_from_stats", "emit_benchmark_result_line", + "emit_trial_diagnostic_stderr_lines", "format_benchmark_result_line", + "format_trial_diagnostic_stderr_line", "validate_benchmark_result", ] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner_backend.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner_backend.py index c315c38d5..fbea7595c 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner_backend.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner_backend.py @@ -100,7 +100,9 @@ ) from agent_challenge.evaluation.own_runner.result_schema import ( build_benchmark_result, + build_trial_diagnostics, emit_benchmark_result_line, + emit_trial_diagnostic_stderr_lines, ) from agent_challenge.evaluation.own_runner.taskdefs import ( DATASET_ID, @@ -1214,6 +1216,32 @@ def _derive_manifest_sha256(*, agent_hash: str, task_ids: Sequence[str], compose return hashlib.sha256(descriptor.encode()).hexdigest() + +def _emit_time_log_redactor() -> LogRedactor: + """Build a defense-in-depth redactor from process-env secret values. + + Trial outcomes are already redacted before they reach :class:`JobResult`, but + emit-time diagnostics re-apply redaction so a secret that only appears in the + process environment (or slipped past the trial wrapper) never rides out on + the ``BASE_BENCHMARK_RESULT`` line or the stderr diagnostic breadcrumbs. + """ + + miner_values: list[str] = [] + for key in ( + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "HF_API_TOKEN", + ): + value = os.environ.get(key) + if value: + miner_values.append(value) + return LogRedactor( + gateway_token=os.environ.get(GATEWAY_TOKEN_ENV), + miner_env_values=miner_values, + ) + + def _emit_job_result( result: JobResult, task_ids: Sequence[str], @@ -1239,6 +1267,18 @@ def _emit_job_result( task_id: list(scores) for task_id, scores in collect_trial_scores(result.trial_outcomes).items() } + # Additive per-trial failure diagnostics so production crashes are not + # opaque ``harbor_trial_failed`` shells. Omitted entirely when every + # trial succeeded so a green job's result line stays byte-identical. + # ``getattr`` keeps lightweight test doubles (SimpleNamespace without + # trial_outcomes) working the same as a real JobResult with []. + diagnostics = build_trial_diagnostics( + getattr(result, "trial_outcomes", None) or (), + redactor=_emit_time_log_redactor(), + ) + if diagnostics: + payload["trial_diagnostics"] = diagnostics + emit_trial_diagnostic_stderr_lines(diagnostics) # Temporary NO_PHALA host mode: mark the legacy line unattested and # attach guest_artifact_proof when the agent ZIP provenance is known. # Attested path below is untouched when NO_PHALA is off. diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py index c8c8cc52b..5f35a5791 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py @@ -1601,7 +1601,7 @@ def _terminal_bench_broker_limits() -> DockerLimits: network=os.environ.get("CHALLENGE_DOCKER_BROKER_NETWORK", "default"), read_only=True, user=settings.docker_user, - tmpfs=("/tmp:rw,nosuid,size=2g",), + tmpfs=("/tmp:rw,nosuid,exec,size=2g",), ulimits=("nofile=1024:1024",), cap_drop=("ALL",), security_opt=("no-new-privileges",), diff --git a/packages/challenges/agent-challenge/tests/test_container_config.py b/packages/challenges/agent-challenge/tests/test_container_config.py index 92c1f7116..08abda6cc 100644 --- a/packages/challenges/agent-challenge/tests/test_container_config.py +++ b/packages/challenges/agent-challenge/tests/test_container_config.py @@ -240,3 +240,22 @@ def _job() -> EvaluationJob: status="queued", selected_tasks_json="[]", ) + + +def test_terminal_bench_broker_tmpfs_allows_exec() -> None: + """Miner agents pip-install into /tmp because the rootfs is read-only. + + Docker's ``--tmpfs`` defaults to ``noexec``, so native wheels (tiktoken, + pydantic-core, numpy) fail to load with "failed to map segment from shared + object" and every such submission scores 0 for an infrastructure reason. + The analyzer container keeps ``noexec`` on purpose; this runner cannot. + """ + from agent_challenge.evaluation.runner import _terminal_bench_broker_limits + + limits = _terminal_bench_broker_limits() + tmp_mounts = [m for m in limits.tmpfs if m.startswith("/tmp:")] + assert tmp_mounts, "the runner must expose a writable /tmp tmpfs" + options = tmp_mounts[0].split(":", 1)[1].split(",") + assert "exec" in options, f"/tmp must be executable, got {tmp_mounts[0]}" + assert "noexec" not in options, f"/tmp must not be noexec, got {tmp_mounts[0]}" + assert "nosuid" in options, "nosuid hardening must be preserved" diff --git a/packages/challenges/agent-challenge/tests/test_own_runner_driver.py b/packages/challenges/agent-challenge/tests/test_own_runner_driver.py index 86fde78bf..8db4e0349 100644 --- a/packages/challenges/agent-challenge/tests/test_own_runner_driver.py +++ b/packages/challenges/agent-challenge/tests/test_own_runner_driver.py @@ -731,3 +731,38 @@ async def test_drive_end_to_end_against_real_container() -> None: text=True, ) assert inspect.returncode != 0, "driver must remove the container on exit" + + +class FdWritingFailingAgent: + """Agent whose constructor fails after a subprocess wrote to fd 2. + + Mirrors production: the miner's ``__init__`` shells out to pip, pip prints + the real reason on fd 2, and ``CalledProcessError.__str__`` keeps only the + argv and exit status. The reason must not be lost. + """ + + def __init__(self, logs_dir=None, model_name=None, **kwargs) -> None: + os.write(2, b"ERROR: Directory '/workspace/agent' is not installable.\n") + raise subprocess.CalledProcessError( + 1, ["python", "-m", "pip", "install", "/workspace/agent"] + ) + + @staticmethod + def name() -> str: + return "fd-writing-failing-agent" + + +async def test_construction_failure_reports_the_captured_fd_output() -> None: + """The operator must see WHY construction failed, not just which argv.""" + driver = AgentDriver(agent_class=FdWritingFailingAgent) + result = await driver.drive( + environment=_tmux_present_env(), + instruction="do the thing", + start_session=False, + ) + assert result.status == "failed" + assert result.reason_code == "harbor_submission_code_failed" + assert "agent construction failed" in result.error + assert "is not installable" in result.error, ( + f"captured fd output must reach error_text, got: {result.error!r}" + ) diff --git a/packages/challenges/agent-challenge/tests/test_own_runner_orchestrator.py b/packages/challenges/agent-challenge/tests/test_own_runner_orchestrator.py index 3c88b5271..dfe93b538 100644 --- a/packages/challenges/agent-challenge/tests/test_own_runner_orchestrator.py +++ b/packages/challenges/agent-challenge/tests/test_own_runner_orchestrator.py @@ -963,3 +963,49 @@ async def _preparer(trial_id: TrialId, task: TaskSpec) -> PreparedTrial: assert result.resolved == 2 assert result.total == 2 assert result.pass_at_k == {"agent__adhoc": {2: 1.0}} + + +# =========================================================================== +# Fail-fast: a submission whose agent cannot even be constructed +# =========================================================================== + + +async def test_construction_failures_short_circuit_the_remaining_trials( + tmp_path: Path, +) -> None: + """A package that cannot construct fails identically on every task. + + Agent construction is task-independent: a ZIP missing its manifest or its + imported modules breaks the same way for all 30 tasks. Running them all + burns hours of LLM budget to relearn one fact, so the job must stop after a + small confirmation threshold and mark the rest explicitly. + """ + attempted: list[str] = [] + + async def _run(trial_id: TrialId, task: TaskSpec) -> TrialOutcome: + attempted.append(trial_id.trial_name) + return TrialOutcome( + task_name=trial_id.task_name, + trial_name=trial_id.trial_name, + status="failed", + rewards=None, + reason_code="harbor_submission_code_failed", + errored=True, + error_text="agent construction failed: no pyproject.toml", + ) + + tasks = [TaskSpec(f"task-{i}") for i in range(10)] + orch = TrialJobOrchestrator( + config=JobConfig(n_attempts=1, n_concurrent=1), + job_dir=tmp_path / "job", + trial_runner=_run, + ) + result = await orch.run(tasks) + + assert len(attempted) <= 2, f"must stop early, ran {len(attempted)} trials" + # Every task still accounted for, and the skipped ones say why. + assert result.n_total_trials == 10 + assert result.score == 0.0 + skipped = [o for o in result.trial_outcomes if "short-circuit" in (o.error_text or "")] + assert len(skipped) == 10 - len(attempted) + assert all(o.reason_code == "harbor_submission_code_failed" for o in skipped) diff --git a/packages/challenges/agent-challenge/tests/test_own_runner_trial_diagnostics.py b/packages/challenges/agent-challenge/tests/test_own_runner_trial_diagnostics.py new file mode 100644 index 000000000..c9bd8a2e7 --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_own_runner_trial_diagnostics.py @@ -0,0 +1,293 @@ +"""Trial failure diagnostics on the BASE_BENCHMARK_RESULT line + stderr. + +Production Terminal-Bench failures currently collapse to the opaque +``harbor_trial_failed`` reason code with no persisted detail. These tests pin +the additive ``trial_diagnostics`` payload field and the matching one-line +stderr breadcrumbs so the real per-trial crash cause is visible without +breaking the five-field harbor contract. +""" + +from __future__ import annotations + +import json + +import pytest + +from agent_challenge.evaluation.own_runner.orchestrator import JobResult, TrialOutcome +from agent_challenge.evaluation.own_runner.redaction import ( + REDACTED_GATEWAY_TOKEN, + LogRedactor, +) +from agent_challenge.evaluation.own_runner.result_schema import ( + RESULT_LINE_PREFIX, + build_benchmark_result, + build_trial_diagnostics, + format_benchmark_result_line, + validate_benchmark_result, +) +from agent_challenge.evaluation.own_runner_backend import _emit_job_result + + +def _failed( + task: str, + attempt: int = 0, + *, + reason: str | None = "harbor_trial_failed", + error: str | None = "trial crashed: RuntimeError: boom", + errored: bool = True, + status: str = "failed", +) -> TrialOutcome: + return TrialOutcome( + task_name=task, + trial_name=f"{task}__attempt-{attempt}", + status=status, + rewards=None, + reason_code=reason, + errored=errored, + error_text=error, + ) + + +def _ok(task: str = "ok-task") -> TrialOutcome: + return TrialOutcome( + task_name=task, + trial_name=f"{task}__attempt-0", + status="completed", + rewards={"reward": 1.0}, + errored=False, + ) + + +def _job( + outcomes: list[TrialOutcome], + *, + status: str = "failed", + reason: str | None = None, + score: float = 0.0, + resolved: int | None = None, +) -> JobResult: + total = len(outcomes) + errored = sum(1 for o in outcomes if o.errored) + resolved_n = resolved if resolved is not None else 0 + return JobResult( + status=status, + score=score, + resolved=resolved_n, + total=total, + reason_code=reason, + pass_at_k={}, + n_total_trials=total, + n_completed_trials=total - errored, + n_errored_trials=errored, + trial_outcomes=outcomes, + benchmark_result=build_benchmark_result( + status=status, + score=score, + resolved=resolved_n, + total=total, + reason_code=reason, + ), + ) + + +# --------------------------------------------------------------------------- # +# S1 — build_trial_diagnostics filters + shape +# --------------------------------------------------------------------------- # + + +def test_build_includes_only_failed_or_errored_preserving_order() -> None: + outcomes = [ + _ok("a"), + _failed("b", error="err-b"), + _ok("c"), + _failed("d", error="err-d", errored=False, status="failed"), # failed, not errored + TrialOutcome( # errored but status completed (defensive) + task_name="e", + trial_name="e__attempt-0", + status="completed", + errored=True, + reason_code="harbor_trial_failed", + error_text="err-e", + ), + ] + diags = build_trial_diagnostics(outcomes) + assert [d["task_name"] for d in diags] == ["b", "d", "e"] + for entry in diags: + assert set(entry) >= { + "task_name", + "trial_name", + "status", + "errored", + "reason_code", + "error_text", + } + assert diags[0]["error_text"] == "err-b" + assert diags[0]["reason_code"] == "harbor_trial_failed" + assert diags[0]["errored"] is True + assert diags[1]["errored"] is False + assert diags[1]["status"] == "failed" + + +def test_build_returns_empty_for_all_successful() -> None: + assert build_trial_diagnostics([_ok("x"), _ok("y")]) == [] + + +def test_build_truncates_error_text_with_ellipsis_marker() -> None: + long = "X" * 5000 + diags = build_trial_diagnostics([_failed("t", error=long)], max_error_chars=100) + assert len(diags) == 1 + text = diags[0]["error_text"] + assert text is not None + assert len(text) < 5000 + assert text.startswith("X" * 100) + assert text.endswith("…[truncated]") + assert "X" * 101 not in text.replace("…[truncated]", "") + + +def test_build_caps_at_limit_and_appends_truncation_marker() -> None: + outcomes = [_failed(f"t{i}", error=f"e{i}") for i in range(5)] + diags = build_trial_diagnostics(outcomes, limit=3) + # 3 real entries + 1 marker documenting the remainder. + assert len(diags) == 4 + assert [d["task_name"] for d in diags[:3]] == ["t0", "t1", "t2"] + marker = diags[-1] + assert marker["reason_code"] == "diagnostics_truncated" + assert marker["errored"] is True + assert marker["status"] == "failed" + assert "2" in (marker["error_text"] or "") # omitted count + + +# --------------------------------------------------------------------------- # +# S3 — redaction is mandatory on error_text +# --------------------------------------------------------------------------- # + + +def test_build_redacts_secret_in_error_text() -> None: + secret = "scoped-gw-token-9f2a-SECRET" + redactor = LogRedactor(gateway_token=secret) + diags = build_trial_diagnostics( + [_failed("leak", error=f"trial crashed: RuntimeError: used {secret}")], + redactor=redactor, + ) + assert len(diags) == 1 + assert secret not in (diags[0]["error_text"] or "") + assert REDACTED_GATEWAY_TOKEN in (diags[0]["error_text"] or "") + + +def test_emitted_payload_redacts_secret_in_trial_diagnostics( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """End-to-end: secret in outcome.error_text must not appear on the result line.""" + + monkeypatch.delenv("CHALLENGE_PHALA_ATTESTATION_ENABLED", raising=False) + monkeypatch.delenv("PHALA_ATTESTATION_ENABLED", raising=False) + secret = "or-live-secret-token-abc123" + # Emit-time redactor sources secrets from process env (defense in depth). + monkeypatch.setenv("OPENROUTER_API_KEY", secret) + monkeypatch.setenv("BASE_GATEWAY_TOKEN", secret) + + result = _job( + [_failed("crash-task", error=f"trial crashed: ValueError: bearer {secret}")] + ) + rc = _emit_job_result(result, ["crash-task"]) + assert rc == 0 + captured = capsys.readouterr() + out_line = [ln for ln in captured.out.splitlines() if ln.startswith(RESULT_LINE_PREFIX)][-1] + assert secret not in out_line + payload = json.loads(out_line[len(RESULT_LINE_PREFIX) :]) + assert "trial_diagnostics" in payload + assert secret not in json.dumps(payload["trial_diagnostics"]) + # stderr diagnostic also redacted + assert secret not in captured.err + assert any("agent_challenge_trial_diagnostic" in ln for ln in captured.err.splitlines()) + + +# --------------------------------------------------------------------------- # +# S4 — schema accepts additive trial_diagnostics +# --------------------------------------------------------------------------- # + + +def test_payload_with_trial_diagnostics_validates() -> None: + payload = build_benchmark_result( + status="failed", score=0.0, resolved=0, total=1, reason_code="harbor_trial_failed" + ) + payload["trial_diagnostics"] = build_trial_diagnostics( + [_failed("t", error="trial crashed: RuntimeError: boom")] + ) + validate_benchmark_result(payload) # must not raise + line = format_benchmark_result_line(payload) + assert "trial_diagnostics" in line + parsed = json.loads(line[len(RESULT_LINE_PREFIX) :]) + assert parsed["status"] == "failed" + assert parsed["reason_code"] == "harbor_trial_failed" + assert isinstance(parsed["trial_diagnostics"], list) + assert parsed["trial_diagnostics"][0]["task_name"] == "t" + + +# --------------------------------------------------------------------------- # +# S5 — _emit_job_result legacy wiring +# --------------------------------------------------------------------------- # + + +def test_emit_adds_trial_diagnostics_only_when_nonempty( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.delenv("CHALLENGE_PHALA_ATTESTATION_ENABLED", raising=False) + monkeypatch.delenv("PHALA_ATTESTATION_ENABLED", raising=False) + + # Successful job: no trial_diagnostics key (byte-compatible additive contract). + ok = _job([_ok("hello-world")], status="completed", reason=None, score=1.0, resolved=1) + assert _emit_job_result(ok, ["hello-world"]) == 0 + out_ok = capsys.readouterr().out + line_ok = [ln for ln in out_ok.splitlines() if ln.startswith(RESULT_LINE_PREFIX)][-1] + payload_ok = json.loads(line_ok[len(RESULT_LINE_PREFIX) :]) + assert "trial_diagnostics" not in payload_ok + + # Failed job: key present with the crashed trial. + failed = _job( + [_failed("boom", error="trial crashed: OSError: no space left on device")], + status="failed", + reason="harbor_trial_failed", + ) + assert _emit_job_result(failed, ["boom"]) == 0 + captured = capsys.readouterr() + line_fail = [ln for ln in captured.out.splitlines() if ln.startswith(RESULT_LINE_PREFIX)][-1] + payload_fail = json.loads(line_fail[len(RESULT_LINE_PREFIX) :]) + assert "trial_diagnostics" in payload_fail + assert payload_fail["trial_diagnostics"][0]["task_name"] == "boom" + assert "no space left" in payload_fail["trial_diagnostics"][0]["error_text"] + + # stderr one-liner, single line, newlines collapsed. + diag_lines = [ + ln for ln in captured.err.splitlines() if ln.startswith("agent_challenge_trial_diagnostic") + ] + assert len(diag_lines) == 1 + diag = diag_lines[0] + assert "task=boom" in diag + assert "trial=boom__attempt-0" in diag + assert "reason=harbor_trial_failed" in diag + assert "error=" in diag + assert "\n" not in diag + + +def test_emit_stderr_collapses_newlines_in_error( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.delenv("CHALLENGE_PHALA_ATTESTATION_ENABLED", raising=False) + monkeypatch.delenv("PHALA_ATTESTATION_ENABLED", raising=False) + result = _job( + [_failed("nl", error="line1\nline2\r\nline3")], + status="failed", + reason="harbor_trial_failed", + ) + assert _emit_job_result(result, ["nl"]) == 0 + err = capsys.readouterr().err + prefix = "agent_challenge_trial_diagnostic" + diag_lines = [ln for ln in err.splitlines() if ln.startswith(prefix)] + assert len(diag_lines) == 1 + assert "line1" in diag_lines[0] and "line2" in diag_lines[0] and "line3" in diag_lines[0] + # The diagnostic itself is exactly one physical line. + assert diag_lines[0].count("\n") == 0