diff --git a/deploy/compose/docker-compose.yml b/deploy/compose/docker-compose.yml index e7997350f..820a0dad9 100644 --- a/deploy/compose/docker-compose.yml +++ b/deploy/compose/docker-compose.yml @@ -122,11 +122,12 @@ services: - type: volume source: master-state target: /var/lib/base - - type: tmpfs - target: /tmp - tmpfs: - size: 268435456 - mode: 1777 + # SHORT syntax required: mode=1777 is OCTAL here (sticky + world-writable). + # Long-syntax ``tmpfs.mode: 1777`` is DECIMAL 1777 (= octal 3361) and yields + # d-wxrwS--t, so uid 1000 cannot write /tmp (TB docker build fails). + # Match docker-compose.validator.yml: ``/tmp:size=256m,mode=1777``. + tmpfs: + - /tmp:size=256m,mode=1777 networks: - db - app diff --git a/packages/challenges/agent-challenge/src/agent_challenge/api/routes.py b/packages/challenges/agent-challenge/src/agent_challenge/api/routes.py index 7faadd163..83ee60ee4 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/api/routes.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/api/routes.py @@ -70,7 +70,10 @@ load_eval_run_plan, retry_eval_run, ) -from ..evaluation.benchmarks import load_benchmark_tasks +from ..evaluation.benchmarks import ( + TERMINAL_BENCH_2_1_FALLBACK_TASK_IDS, + load_benchmark_tasks, +) from ..evaluation.direct_result import ( DirectEvalResultError, authenticate_eval_token, @@ -460,6 +463,7 @@ class TaskResultResponse(BaseModel): docker_image: str status: str score: float + passed: bool returncode: int duration_seconds: float failure_reason: str | None = None @@ -483,6 +487,7 @@ class TaskRowResponse(BaseModel): updated_at: datetime | None attempt: int | None has_result: bool = False + passed: bool | None = None class EvaluationResponse(BaseModel): @@ -5018,6 +5023,8 @@ def _public_task_event_metadata(metadata: Mapping[str, object]) -> dict[str, obj if _is_sensitive_task_event_metadata_key(key): continue public[str(key)] = _public_task_event_metadata_value(value) + if "passed" not in public and "score" in public: + public["passed"] = _score_is_passed(public["score"]) return public @@ -5041,6 +5048,42 @@ def _is_sensitive_task_event_metadata_key(key: str) -> bool: ) +# Whole-segment secret markers (not substrings of longer words like "tokens"). +_PUBLIC_SECRET_SEGMENT_RE = re.compile( + r"(? re.Pattern[str]: + """Compile alternation of known TB task ids (full + bare), longest first.""" + + parts: list[str] = [] + for task_id in TERMINAL_BENCH_2_1_FALLBACK_TASK_IDS: + bare = task_id.removeprefix("terminal-bench/") + parts.append(re.escape(task_id)) + parts.append(re.escape(bare)) + parts.sort(key=len, reverse=True) + return re.compile("|".join(parts)) + + +_TERMINAL_BENCH_TASK_ID_SHIELD_RE = _terminal_bench_task_id_shield_pattern() + + +def _score_is_passed(score: object) -> bool: + """Same pass rule as passed_tasks aggregation: score >= 1.0.""" + + if isinstance(score, bool) or not isinstance(score, int | float): + return False + return float(score) >= 1.0 + + def _public_task_event_text(value: str) -> str: sanitized = PRIVATE_PATH_RE.sub("[REDACTED_PATH]", redact_task_event_message(value)) sanitized = re.sub(r"\b(?:platform_sdk|base_sdk)\b", "base", sanitized, flags=re.IGNORECASE) @@ -5050,12 +5093,22 @@ def _public_task_event_text(value: str) -> str: sanitized, flags=re.IGNORECASE, ) - return re.sub( - r"(? str: + shields.append(match.group(0)) + return f"\x00TBSAFE{len(shields) - 1}\x00" + + sanitized = _TERMINAL_BENCH_TASK_ID_SHIELD_RE.sub(_shield, sanitized) + # Segment-boundary redaction: match secret/token/raw-ref/broker-ref/pod- as + # whole hyphen/underscore/dot-delimited segments, not as substrings of a + # longer word (``tokens`` must not match ``token``). + sanitized = _PUBLIC_SECRET_SEGMENT_RE.sub("[REDACTED_SECRET]", sanitized) + for index, original in enumerate(shields): + sanitized = sanitized.replace(f"\x00TBSAFE{index}\x00", original) + return sanitized async def _submission_task_event_stream( @@ -5434,6 +5487,10 @@ def _task_rows_from_eval_run(run: EvalRun) -> list[TaskRowResponse]: else: has_result = True phase = "completed" if outcome.get("passed") else "failed" + row_passed: bool | None = None + if outcome is not None: + raw_passed = outcome.get("passed") + row_passed = raw_passed if isinstance(raw_passed, bool) else None rows.append( TaskRowResponse( task_id=task_id, @@ -5444,6 +5501,7 @@ def _task_rows_from_eval_run(run: EvalRun) -> list[TaskRowResponse]: updated_at=run.updated_at or run.created_at or run.issued_at, attempt=run.attempt if has_result else None, has_result=has_result, + passed=row_passed, ) ) return rows @@ -5553,10 +5611,14 @@ def _task_rows_response( updated_at=result.created_at, attempt=None, has_result=True, + passed=_score_is_passed(result.score), ) ordered_task_ids.append(result.task_id) continue - update: dict[str, object] = {"has_result": True} + update: dict[str, object] = { + "has_result": True, + "passed": _score_is_passed(result.score), + } if row.phase == "assigned": update["phase"] = result_phase update["status"] = result_phase @@ -6134,6 +6196,7 @@ def _task_result_response(result: TaskResult) -> TaskResultResponse: docker_image=result.docker_image, status=result.status, score=result.score, + passed=_score_is_passed(result.score), returncode=result.returncode, duration_seconds=result.duration_seconds, failure_reason=_task_result_failure_reason(result), diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/container_builder.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/container_builder.py index dc8a482d4..489e32309 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/container_builder.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/container_builder.py @@ -100,7 +100,8 @@ # --------------------------------------------------------------------------- # #: Bounded max process count per task container (matches the broker posture). TASK_PIDS_LIMIT = 512 -#: Capabilities dropped from every task container. +#: Legacy constant; task guests no longer pass --cap-drop (apt/chmod need caps). +#: Kept so importers/tests that reference the name do not break on import. TASK_CAP_DROP = "ALL" #: ``no-new-privileges`` security option (blocks privilege escalation). TASK_SECURITY_OPT = "no-new-privileges" @@ -298,18 +299,21 @@ def arg(self) -> str: def hardening_run_args() -> list[str]: """Return the hardened-posture ``docker run`` flags for a task container. - ``cap-drop ALL``, ``no-new-privileges``, a bounded ``pids-limit``, a - writable ``tmpfs`` for ``/tmp``, and a writable anonymous volume at the - agent workspace target (:data:`AGENT_WORKSPACE_TARGET`). + ``no-new-privileges``, a bounded ``pids-limit``, a writable ``tmpfs`` for + ``/tmp``, and a writable anonymous volume at the agent workspace target + (:data:`AGENT_WORKSPACE_TARGET`). Deliberately omits ``--read-only``: the guest must write harbor paths (``/tests``, ``/logs/verifier``, ``/solution``, ``/app``) and run package installs inside ``test.sh``. RO rootfs is reserved for the outer DooD job container, not the task guest. + + Deliberately omits ``--cap-drop ALL``: Terminal-Bench task guests need + capabilities for ``apt``/``chmod`` inside verifiers. This matches the prod + hotpatch trade-off (isolation reduced only for own_runner *task* guests; + the master compose service still uses ``cap_drop: ALL``). """ return [ - "--cap-drop", - TASK_CAP_DROP, "--security-opt", TASK_SECURITY_OPT, "--pids-limit", @@ -728,8 +732,8 @@ def _run_container_attempt( workdir, ] # Hardened posture on every task container (architecture sec 4 C2): - # cap-drop ALL + no-new-privileges + bounded pids + tmpfs /tmp + - # writable workspace volume. Rootfs stays writable (harbor paths). + # no-new-privileges + bounded pids + tmpfs /tmp + writable workspace + # volume. Rootfs stays writable (harbor paths). No cap-drop ALL (TB). argv += hardening_run_args() # Golden dataset / task cache mounted read-only so task code can read but # never mutate it. diff --git a/packages/challenges/agent-challenge/tests/test_frontend_api_contract.py b/packages/challenges/agent-challenge/tests/test_frontend_api_contract.py index a3614f0b6..f76a3e7c2 100644 --- a/packages/challenges/agent-challenge/tests/test_frontend_api_contract.py +++ b/packages/challenges/agent-challenge/tests/test_frontend_api_contract.py @@ -289,6 +289,7 @@ async def test_frontend_submission_status_and_evaluation_routes_are_public_safe( "updated_at": status_payload["evaluation"]["task_rows"][0]["updated_at"], "attempt": None, "has_result": True, + "passed": True, }, { "task_id": "task-beta", @@ -299,6 +300,7 @@ async def test_frontend_submission_status_and_evaluation_routes_are_public_safe( "updated_at": status_payload["evaluation"]["task_rows"][1]["updated_at"], "attempt": None, "has_result": True, + "passed": False, }, ] assert status_payload["evaluation"]["task_phases"] == [] @@ -372,6 +374,7 @@ async def test_frontend_submission_status_and_evaluation_routes_are_public_safe( "docker_image": "baseintelligence/swe-forge:task-alpha", "status": "passed", "score": 1.0, + "passed": True, "returncode": 0, "duration_seconds": 12.5, "failure_reason": None, @@ -382,6 +385,7 @@ async def test_frontend_submission_status_and_evaluation_routes_are_public_safe( "docker_image": "baseintelligence/swe-forge:task-beta", "status": "failed", "score": 0.0, + "passed": False, "returncode": 1, "duration_seconds": 8.25, "failure_reason": "task log with Bearer [REDACTED] and [REDACTED_SECRET]", @@ -586,6 +590,7 @@ async def test_platform_sdk_frontend_status_evaluation_and_events_are_public_saf "docker_image": "baseintelligence/public-runner:task8", "status": case["task_result_status"], "score": case["score"], + "passed": case["score"] >= 1.0, "returncode": case["returncode"], "duration_seconds": 3.5, "failure_reason": None @@ -706,6 +711,7 @@ async def test_frontend_task_rows_include_queued_phase_result_and_redacted_selec "updated_at": status_rows[0]["updated_at"], "attempt": None, "has_result": False, + "passed": None, }, { "task_id": "safe-task-beta", @@ -716,6 +722,7 @@ async def test_frontend_task_rows_include_queued_phase_result_and_redacted_selec "updated_at": status_rows[1]["updated_at"], "attempt": None, "has_result": False, + "passed": None, }, ] assert all( @@ -729,6 +736,7 @@ async def test_frontend_task_rows_include_queued_phase_result_and_redacted_selec "updated_at", "attempt", "has_result", + "passed", } for row in status_rows ) @@ -790,6 +798,7 @@ async def test_frontend_task_rows_include_queued_phase_result_and_redacted_selec "updated_at": status_rows[0]["updated_at"], "attempt": 2, "has_result": False, + "passed": None, }, { "task_id": "safe-task-beta", @@ -800,6 +809,7 @@ async def test_frontend_task_rows_include_queued_phase_result_and_redacted_selec "updated_at": status_rows[1]["updated_at"], "attempt": None, "has_result": True, + "passed": True, }, ] _assert_platform_sdk_public_payload_is_redacted( @@ -911,6 +921,7 @@ async def test_platform_sdk_evaluation_exposes_running_task_phase_before_results "updated_at": evaluation_payload["task_rows"][0]["updated_at"], "attempt": 2, "has_result": False, + "passed": None, } assert status_payload["evaluation"]["task_phases"] == [expected_phase] assert status_payload["evaluation"]["task_rows"] == [ diff --git a/packages/challenges/agent-challenge/tests/test_own_runner_isolation_invariants.py b/packages/challenges/agent-challenge/tests/test_own_runner_isolation_invariants.py index c30a505c2..64270f46c 100644 --- a/packages/challenges/agent-challenge/tests/test_own_runner_isolation_invariants.py +++ b/packages/challenges/agent-challenge/tests/test_own_runner_isolation_invariants.py @@ -8,7 +8,7 @@ (VAL-ORCH-013 / VAL-ORCH-015); * the golden dataset + task cache is bind-mounted read-only into every task container, and writes to it fail (VAL-ORCH-016); -* hardened posture: ``cap-drop ALL``, ``no-new-privileges``, a bounded +* hardened posture: ``no-new-privileges``, a bounded ``pids-limit``, writable ``tmpfs`` for ``/tmp``, and a writable workspace volume — rootfs stays writable for harbor ``/tests`` + ``/logs`` (VAL-ORCH-023); * a task requesting a GPU is rejected and never launched on the CPU-only CVM @@ -209,7 +209,7 @@ def test_golden_cache_mount_is_readonly_in_real_container(tmp_path: Path) -> Non def test_hardening_flags_present_in_launch_spec(monkeypatch: pytest.MonkeyPatch) -> None: argv = _single_run_argv(monkeypatch, resources=ResourceLimits()) assert "--read-only" not in argv # task guest rootfs must stay writable - assert _has_consecutive(argv, ("--cap-drop", "ALL")) + assert "--cap-drop" not in argv # task guests need apt/chmod (prod hotpatch) assert _has_consecutive(argv, ("--security-opt", "no-new-privileges")) assert _has_consecutive(argv, ("--pids-limit", str(TASK_PIDS_LIMIT))) @@ -226,7 +226,7 @@ def test_tmpfs_is_tmp_only(monkeypatch: pytest.MonkeyPatch) -> None: def test_hardening_run_args_shape() -> None: args = hardening_run_args() assert "--read-only" not in args - assert "--cap-drop" in args and args[args.index("--cap-drop") + 1] == "ALL" + assert "--cap-drop" not in args # deliberate: TB guests need capabilities for apt/chmod assert "--security-opt" in args assert args[args.index("--security-opt") + 1] == "no-new-privileges" assert "--pids-limit" in args @@ -256,7 +256,7 @@ def test_task_container_hardened_in_real_daemon(tmp_path: Path) -> None: assert posture.returncode == 0 out = posture.stdout assert out.startswith("false ") # writable rootfs (harbor paths) - assert "[ALL]" in out # cap-drop ALL + assert "[ALL]" not in out # cap-drop ALL omitted for task guests assert str(TASK_PIDS_LIMIT) in out # bounded pids assert "no-new-privileges" in out assert "false" in out.split()[-1].lower() or out.rstrip().endswith("false") @@ -279,14 +279,13 @@ def test_task_container_hardened_in_real_daemon(tmp_path: Path) -> None: env.remove() - # --------------------------------------------------------------------------- # # Harbor verifier/agent paths must be writable on task containers # --------------------------------------------------------------------------- # # Production crash (2026-07-29): upload_tests did `docker exec -u root … mkdir -p # /tests` and failed with "Read-only file system" because task containers # incorrectly inherited the *job*-container RO posture. Task containers must -# keep cap-drop / nnp / pids hardening but leave the rootfs writable so harbor +# keep nnp / pids hardening (no cap-drop ALL) but leave the rootfs writable so harbor # paths (/tests, /logs/verifier, /solution, /app) and apt-based test.sh work. @@ -297,7 +296,7 @@ def test_hardening_run_args_does_not_force_readonly_rootfs() -> None: "task containers must not use --read-only; that blocks harbor " "mkdir /tests and apt-based verifiers (Read-only file system)" ) - assert _has_consecutive(args, ("--cap-drop", "ALL")) + assert "--cap-drop" not in args assert _has_consecutive(args, ("--security-opt", "no-new-privileges")) assert _has_consecutive(args, ("--pids-limit", str(TASK_PIDS_LIMIT))) @@ -340,7 +339,7 @@ def test_task_container_allows_harbor_verifier_paths() -> None: f"stdout={write.stdout!r} stderr={write.stderr!r}" ) assert "ok" in write.stdout - # Hardening that must remain: cap-drop ALL + nnp + pids + not privileged. + # Hardening that must remain: nnp + pids + not privileged (no cap-drop ALL). posture = subprocess.run( [ "docker", @@ -357,7 +356,7 @@ def test_task_container_allows_harbor_verifier_paths() -> None: assert posture.returncode == 0 out = posture.stdout.strip() assert out.startswith("false "), f"expected writable rootfs, got {out!r}" - assert "[ALL]" in out + assert "[ALL]" not in out assert str(TASK_PIDS_LIMIT) in out assert "no-new-privileges" in out assert out.endswith(" false") or out.rstrip().endswith("false") diff --git a/packages/challenges/agent-challenge/tests/test_public_task_event_redaction.py b/packages/challenges/agent-challenge/tests/test_public_task_event_redaction.py new file mode 100644 index 000000000..e5d1545a5 --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_public_task_event_redaction.py @@ -0,0 +1,41 @@ +"""Public task-event text redaction: keep TB task ids, still scrub secrets.""" + +from __future__ import annotations + +from agent_challenge.api.routes import _public_task_event_text + +# Shapes asserted by the existing frontend / submission-status contract suites. +_GENUINE_SECRET_SHAPES = ( + "sk-test-secret", + "Bearer raw-provider-token", + "broker-token", + "tb21-platform-sdk-secret", + "lease-worker-secret", + "broker-ref-secret", + "raw-ref-abc123", +) + + +def test_public_task_event_text_preserves_count_dataset_tokens() -> None: + """Terminal-Bench task id containing 'token' must not be swallowed whole.""" + + message = "task terminal-bench/count-dataset-tokens assigned" + out = _public_task_event_text(message) + assert "count-dataset-tokens" in out + assert "terminal-bench/count-dataset-tokens" in out + assert "[REDACTED_SECRET]" not in out or "count-dataset-tokens" in out + assert out == message or "count-dataset-tokens" in out + + +def test_public_task_event_text_still_redacts_genuine_secrets() -> None: + """Credential-shaped tokens must remain absent from public event text.""" + + payload = ( + "leak sk-test-secret and Bearer raw-provider-token and broker-token " + "and tb21-platform-sdk-secret and lease-worker-secret and " + "broker-ref-secret and raw-ref-abc123 end" + ) + out = _public_task_event_text(payload) + for secret in _GENUINE_SECRET_SHAPES: + assert secret not in out, f"secret still visible: {secret!r} in {out!r}" + assert "[REDACTED_SECRET]" in out diff --git a/packages/challenges/agent-challenge/tests/test_public_task_passed_flag.py b/packages/challenges/agent-challenge/tests/test_public_task_passed_flag.py new file mode 100644 index 000000000..27308faa2 --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_public_task_passed_flag.py @@ -0,0 +1,53 @@ +"""Public task result / event ``passed`` flag derived from score >= 1.0.""" + +from __future__ import annotations + +from types import SimpleNamespace + +from agent_challenge.api.routes import ( + _public_task_event_metadata, + _task_result_response, +) + + +def test_task_result_response_passed_false_when_score_zero_status_completed() -> None: + result = SimpleNamespace( + task_id="terminal-bench/count-dataset-tokens", + docker_image="img:tag", + status="completed", + score=0.0, + returncode=0, + duration_seconds=139.76, + stderr="", + stdout="", + ) + payload = _task_result_response(result).model_dump() + assert payload["status"] == "completed" + assert payload["score"] == 0.0 + assert payload["passed"] is False + + +def test_task_result_response_passed_true_when_score_one() -> None: + result = SimpleNamespace( + task_id="terminal-bench/fix-git", + docker_image="img:tag", + status="completed", + score=1.0, + returncode=0, + duration_seconds=10.0, + stderr="", + stdout="", + ) + payload = _task_result_response(result).model_dump() + assert payload["status"] == "completed" + assert payload["score"] == 1.0 + assert payload["passed"] is True + + +def test_public_task_event_metadata_exposes_passed_from_score() -> None: + meta = _public_task_event_metadata({"duration_seconds": 139.76, "returncode": 0, "score": 0.0}) + assert meta["score"] == 0.0 + assert meta["passed"] is False + + meta_pass = _public_task_event_metadata({"score": 1.0, "returncode": 0}) + assert meta_pass["passed"] is True diff --git a/tests/unit/test_docker_compose_deploy.py b/tests/unit/test_docker_compose_deploy.py index b710b39aa..708293b9c 100644 --- a/tests/unit/test_docker_compose_deploy.py +++ b/tests/unit/test_docker_compose_deploy.py @@ -238,6 +238,59 @@ def test_volumes_are_isolated_and_named(tmp_path: Path) -> None: assert "/var/lib/base" in master_mounts +def test_master_validator_tmp_tmpfs_is_sticky_world_writable(tmp_path: Path) -> None: + """Master validator /tmp must be mode 0o1777 (sticky + world-writable). + + Compose long-syntax ``tmpfs.mode: 1777`` is parsed as *decimal* 1777 + (= octal 3361), which yields ``d-wxrwS--t`` and blocks uid 1000 writes. + Short syntax ``tmpfs: ["/tmp:size=256m,mode=1777"]`` keeps octal 1777. + """ + + source = MASTER_COMPOSE.read_text(encoding="utf-8") + # Long-syntax decimal footgun must not remain for the /tmp mount. + assert not re.search( + r"type:\s*tmpfs[\s\S]*?target:\s*/tmp[\s\S]*?mode:\s*1777", + source, + ), "long-syntax tmpfs.mode: 1777 is a decimal footgun (becomes 3361)" + + rendered = _render_master(tmp_path) + master = rendered["services"]["base-master-validator"] + + # Prefer top-level short-syntax tmpfs list (compose JSON may keep the string). + top_tmpfs = master.get("tmpfs") or [] + short_ok = any( + isinstance(entry, str) + and entry.split(":", 1)[0] == "/tmp" + and "mode=1777" in entry.replace(" ", "") + for entry in top_tmpfs + ) + + # Or a rendered mount with numeric mode 1023 == 0o1777. + numeric_ok = False + for mount in master.get("volumes") or []: + if not isinstance(mount, dict) or mount.get("target") != "/tmp": + continue + raw_tmpfs = mount.get("tmpfs") + tmpfs_opts: dict[str, Any] = raw_tmpfs if isinstance(raw_tmpfs, dict) else {} + mode = tmpfs_opts.get("mode", mount.get("mode")) + if mode is None: + continue + if int(mode) == 0o1777: # 1023 decimal + numeric_ok = True + break + + tmp_mounts = [ + m + for m in (master.get("volumes") or []) + if isinstance(m, dict) and m.get("target") == "/tmp" + ] + assert short_ok or numeric_ok, ( + "master validator /tmp tmpfs must be sticky world-writable " + "(short syntax mode=1777 or rendered mode 1023/0o1777); " + f"got tmpfs={top_tmpfs!r} volumes={tmp_mounts!r}" + ) + + def test_secrets_are_file_mounted_not_inline(tmp_path: Path) -> None: rendered = _render_master(tmp_path) blob = json.dumps(rendered)