fix(tb): scoring tmpfs, redaction, passed flag, cap-drop

deploy/compose/docker-compose.yml

@@ -122,11 +122,12 @@ services:
       - type: volume
         source: master-state
         target: /var/lib/base
-      - type: tmpfs
-        target: /tmp
-        tmpfs:
-          size: 268435456
-          mode: 1777
+    # SHORT syntax required: mode=1777 is OCTAL here (sticky + world-writable).
+    # Long-syntax ``tmpfs.mode: 1777`` is DECIMAL 1777 (= octal 3361) and yields
+    # d-wxrwS--t, so uid 1000 cannot write /tmp (TB docker build fails).
+    # Match docker-compose.validator.yml: ``/tmp:size=256m,mode=1777``.
+    tmpfs:
+      - /tmp:size=256m,mode=1777
     networks:
       - db
       - app

packages/challenges/agent-challenge/src/agent_challenge/api/routes.py

@@ -70,7 +70,10 @@
     load_eval_run_plan,
     retry_eval_run,
 )
-from ..evaluation.benchmarks import load_benchmark_tasks
+from ..evaluation.benchmarks import (
+    TERMINAL_BENCH_2_1_FALLBACK_TASK_IDS,
+    load_benchmark_tasks,
+)
 from ..evaluation.direct_result import (
     DirectEvalResultError,
     authenticate_eval_token,
@@ -460,6 +463,7 @@ class TaskResultResponse(BaseModel):
     docker_image: str
     status: str
     score: float
+    passed: bool
     returncode: int
     duration_seconds: float
     failure_reason: str | None = None
@@ -483,6 +487,7 @@ class TaskRowResponse(BaseModel):
     updated_at: datetime | None
     attempt: int | None
     has_result: bool = False
+    passed: bool | None = None
 
 
 class EvaluationResponse(BaseModel):
@@ -5018,6 +5023,8 @@ def _public_task_event_metadata(metadata: Mapping[str, object]) -> dict[str, obj
         if _is_sensitive_task_event_metadata_key(key):
             continue
         public[str(key)] = _public_task_event_metadata_value(value)
+    if "passed" not in public and "score" in public:
+        public["passed"] = _score_is_passed(public["score"])
     return public
 
 
@@ -5041,6 +5048,42 @@ def _is_sensitive_task_event_metadata_key(key: str) -> bool:
     )
 
 
+# Whole-segment secret markers (not substrings of longer words like "tokens").
+_PUBLIC_SECRET_SEGMENT_RE = re.compile(
+    r"(?<![A-Za-z0-9_.-])"
+    r"(?:"
+    r"(?:[A-Za-z0-9_.]+[-_.])*(?:secret|raw-ref|broker-ref|pod-)[A-Za-z0-9_.-]*"
+    r"|"
+    r"(?:[A-Za-z0-9_.]+[-_.])*token(?:[-_.][A-Za-z0-9_.]+)*"
+    r")"
+    r"(?![A-Za-z0-9_.-])",
+    flags=re.IGNORECASE,
+)
+
+
+def _terminal_bench_task_id_shield_pattern() -> re.Pattern[str]:
+    """Compile alternation of known TB task ids (full + bare), longest first."""
+
+    parts: list[str] = []
+    for task_id in TERMINAL_BENCH_2_1_FALLBACK_TASK_IDS:
+        bare = task_id.removeprefix("terminal-bench/")
+        parts.append(re.escape(task_id))
+        parts.append(re.escape(bare))
+    parts.sort(key=len, reverse=True)
+    return re.compile("|".join(parts))
+
+
+_TERMINAL_BENCH_TASK_ID_SHIELD_RE = _terminal_bench_task_id_shield_pattern()
+
+
+def _score_is_passed(score: object) -> bool:
+    """Same pass rule as passed_tasks aggregation: score >= 1.0."""
+
+    if isinstance(score, bool) or not isinstance(score, int | float):
+        return False
+    return float(score) >= 1.0
+
+
 def _public_task_event_text(value: str) -> str:
     sanitized = PRIVATE_PATH_RE.sub("[REDACTED_PATH]", redact_task_event_message(value))
     sanitized = re.sub(r"\b(?:platform_sdk|base_sdk)\b", "base", sanitized, flags=re.IGNORECASE)
@@ -5050,12 +5093,22 @@ def _public_task_event_text(value: str) -> str:
         sanitized,
         flags=re.IGNORECASE,
     )
-    return re.sub(
-        r"(?<![A-Za-z0-9_.-])[A-Za-z0-9_.-]*(?:secret|token|raw-ref|broker-ref|pod-)[A-Za-z0-9_.-]*(?![A-Za-z0-9_.-])",
-        "[REDACTED_SECRET]",
-        sanitized,
-        flags=re.IGNORECASE,
-    )
+    # Shield known Terminal-Bench task identifiers so a future regex change cannot
+    # swallow legitimate ids that happen to contain "token" (e.g. count-dataset-tokens).
+    shields: list[str] = []
+
+    def _shield(match: re.Match[str]) -> str:
+        shields.append(match.group(0))
+        return f"\x00TBSAFE{len(shields) - 1}\x00"
+
+    sanitized = _TERMINAL_BENCH_TASK_ID_SHIELD_RE.sub(_shield, sanitized)
+    # Segment-boundary redaction: match secret/token/raw-ref/broker-ref/pod- as
+    # whole hyphen/underscore/dot-delimited segments, not as substrings of a
+    # longer word (``tokens`` must not match ``token``).
+    sanitized = _PUBLIC_SECRET_SEGMENT_RE.sub("[REDACTED_SECRET]", sanitized)
+    for index, original in enumerate(shields):
+        sanitized = sanitized.replace(f"\x00TBSAFE{index}\x00", original)
+    return sanitized
 
 
 async def _submission_task_event_stream(
@@ -5434,6 +5487,10 @@ def _task_rows_from_eval_run(run: EvalRun) -> list[TaskRowResponse]:
         else:
             has_result = True
             phase = "completed" if outcome.get("passed") else "failed"
+        row_passed: bool | None = None
+        if outcome is not None:
+            raw_passed = outcome.get("passed")
+            row_passed = raw_passed if isinstance(raw_passed, bool) else None
         rows.append(
             TaskRowResponse(
                 task_id=task_id,
@@ -5444,6 +5501,7 @@ def _task_rows_from_eval_run(run: EvalRun) -> list[TaskRowResponse]:
                 updated_at=run.updated_at or run.created_at or run.issued_at,
                 attempt=run.attempt if has_result else None,
                 has_result=has_result,
+                passed=row_passed,
             )
         )
     return rows
@@ -5553,10 +5611,14 @@ def _task_rows_response(
                 updated_at=result.created_at,
                 attempt=None,
                 has_result=True,
+                passed=_score_is_passed(result.score),
             )
             ordered_task_ids.append(result.task_id)
             continue
-        update: dict[str, object] = {"has_result": True}
+        update: dict[str, object] = {
+            "has_result": True,
+            "passed": _score_is_passed(result.score),
+        }
         if row.phase == "assigned":
             update["phase"] = result_phase
             update["status"] = result_phase
@@ -6134,6 +6196,7 @@ def _task_result_response(result: TaskResult) -> TaskResultResponse:
         docker_image=result.docker_image,
         status=result.status,
         score=result.score,
+        passed=_score_is_passed(result.score),
         returncode=result.returncode,
         duration_seconds=result.duration_seconds,
         failure_reason=_task_result_failure_reason(result),

packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/container_builder.py

@@ -100,7 +100,8 @@
 # --------------------------------------------------------------------------- #
 #: Bounded max process count per task container (matches the broker posture).
 TASK_PIDS_LIMIT = 512
-#: Capabilities dropped from every task container.
+#: Legacy constant; task guests no longer pass --cap-drop (apt/chmod need caps).
+#: Kept so importers/tests that reference the name do not break on import.
 TASK_CAP_DROP = "ALL"
 #: ``no-new-privileges`` security option (blocks privilege escalation).
 TASK_SECURITY_OPT = "no-new-privileges"
@@ -298,18 +299,21 @@ def arg(self) -> str:
 def hardening_run_args() -> list[str]:
     """Return the hardened-posture ``docker run`` flags for a task container.
 
-    ``cap-drop ALL``, ``no-new-privileges``, a bounded ``pids-limit``, a
-    writable ``tmpfs`` for ``/tmp``, and a writable anonymous volume at the
-    agent workspace target (:data:`AGENT_WORKSPACE_TARGET`).
+    ``no-new-privileges``, a bounded ``pids-limit``, a writable ``tmpfs`` for
+    ``/tmp``, and a writable anonymous volume at the agent workspace target
+    (:data:`AGENT_WORKSPACE_TARGET`).
 
     Deliberately omits ``--read-only``: the guest must write harbor paths
     (``/tests``, ``/logs/verifier``, ``/solution``, ``/app``) and run package
     installs inside ``test.sh``. RO rootfs is reserved for the outer DooD job
     container, not the task guest.
+
+    Deliberately omits ``--cap-drop ALL``: Terminal-Bench task guests need
+    capabilities for ``apt``/``chmod`` inside verifiers. This matches the prod
+    hotpatch trade-off (isolation reduced only for own_runner *task* guests;
+    the master compose service still uses ``cap_drop: ALL``).
     """
     return [
-        "--cap-drop",
-        TASK_CAP_DROP,
         "--security-opt",
         TASK_SECURITY_OPT,
         "--pids-limit",
@@ -728,8 +732,8 @@ def _run_container_attempt(
             workdir,
         ]
         # Hardened posture on every task container (architecture sec 4 C2):
-        # cap-drop ALL + no-new-privileges + bounded pids + tmpfs /tmp +
-        # writable workspace volume. Rootfs stays writable (harbor paths).
+        # no-new-privileges + bounded pids + tmpfs /tmp + writable workspace
+        # volume. Rootfs stays writable (harbor paths). No cap-drop ALL (TB).
         argv += hardening_run_args()
         # Golden dataset / task cache mounted read-only so task code can read but
         # never mutate it.

packages/challenges/agent-challenge/tests/test_frontend_api_contract.py

@@ -289,6 +289,7 @@ async def test_frontend_submission_status_and_evaluation_routes_are_public_safe(
             "updated_at": status_payload["evaluation"]["task_rows"][0]["updated_at"],
             "attempt": None,
             "has_result": True,
+            "passed": True,
         },
         {
             "task_id": "task-beta",
@@ -299,6 +300,7 @@ async def test_frontend_submission_status_and_evaluation_routes_are_public_safe(
             "updated_at": status_payload["evaluation"]["task_rows"][1]["updated_at"],
             "attempt": None,
             "has_result": True,
+            "passed": False,
         },
     ]
     assert status_payload["evaluation"]["task_phases"] == []
@@ -372,6 +374,7 @@ async def test_frontend_submission_status_and_evaluation_routes_are_public_safe(
             "docker_image": "baseintelligence/swe-forge:task-alpha",
             "status": "passed",
             "score": 1.0,
+            "passed": True,
             "returncode": 0,
             "duration_seconds": 12.5,
             "failure_reason": None,
@@ -382,6 +385,7 @@ async def test_frontend_submission_status_and_evaluation_routes_are_public_safe(
             "docker_image": "baseintelligence/swe-forge:task-beta",
             "status": "failed",
             "score": 0.0,
+            "passed": False,
             "returncode": 1,
             "duration_seconds": 8.25,
             "failure_reason": "task log with Bearer [REDACTED] and [REDACTED_SECRET]",
@@ -586,6 +590,7 @@ async def test_platform_sdk_frontend_status_evaluation_and_events_are_public_saf
                 "docker_image": "baseintelligence/public-runner:task8",
                 "status": case["task_result_status"],
                 "score": case["score"],
+                "passed": case["score"] >= 1.0,
                 "returncode": case["returncode"],
                 "duration_seconds": 3.5,
                 "failure_reason": None
@@ -706,6 +711,7 @@ async def test_frontend_task_rows_include_queued_phase_result_and_redacted_selec
             "updated_at": status_rows[0]["updated_at"],
             "attempt": None,
             "has_result": False,
+            "passed": None,
         },
         {
             "task_id": "safe-task-beta",
@@ -716,6 +722,7 @@ async def test_frontend_task_rows_include_queued_phase_result_and_redacted_selec
             "updated_at": status_rows[1]["updated_at"],
             "attempt": None,
             "has_result": False,
+            "passed": None,
         },
     ]
     assert all(
@@ -729,6 +736,7 @@ async def test_frontend_task_rows_include_queued_phase_result_and_redacted_selec
             "updated_at",
             "attempt",
             "has_result",
+            "passed",
         }
         for row in status_rows
     )
@@ -790,6 +798,7 @@ async def test_frontend_task_rows_include_queued_phase_result_and_redacted_selec
             "updated_at": status_rows[0]["updated_at"],
             "attempt": 2,
             "has_result": False,
+            "passed": None,
         },
         {
             "task_id": "safe-task-beta",
@@ -800,6 +809,7 @@ async def test_frontend_task_rows_include_queued_phase_result_and_redacted_selec
             "updated_at": status_rows[1]["updated_at"],
             "attempt": None,
             "has_result": True,
+            "passed": True,
         },
     ]
     _assert_platform_sdk_public_payload_is_redacted(
@@ -911,6 +921,7 @@ async def test_platform_sdk_evaluation_exposes_running_task_phase_before_results
         "updated_at": evaluation_payload["task_rows"][0]["updated_at"],
         "attempt": 2,
         "has_result": False,
+        "passed": None,
     }
     assert status_payload["evaluation"]["task_phases"] == [expected_phase]
     assert status_payload["evaluation"]["task_rows"] == [

packages/challenges/agent-challenge/tests/test_own_runner_isolation_invariants.py

@@ -8,7 +8,7 @@
   (VAL-ORCH-013 / VAL-ORCH-015);
 * the golden dataset + task cache is bind-mounted read-only into every task
   container, and writes to it fail (VAL-ORCH-016);
-* hardened posture: ``cap-drop ALL``, ``no-new-privileges``, a bounded
+* hardened posture: ``no-new-privileges``, a bounded
   ``pids-limit``, writable ``tmpfs`` for ``/tmp``, and a writable workspace
   volume — rootfs stays writable for harbor ``/tests`` + ``/logs`` (VAL-ORCH-023);
 * a task requesting a GPU is rejected and never launched on the CPU-only CVM
@@ -209,7 +209,7 @@ def test_golden_cache_mount_is_readonly_in_real_container(tmp_path: Path) -> Non
 def test_hardening_flags_present_in_launch_spec(monkeypatch: pytest.MonkeyPatch) -> None:
     argv = _single_run_argv(monkeypatch, resources=ResourceLimits())
     assert "--read-only" not in argv  # task guest rootfs must stay writable
-    assert _has_consecutive(argv, ("--cap-drop", "ALL"))
+    assert "--cap-drop" not in argv  # task guests need apt/chmod (prod hotpatch)
     assert _has_consecutive(argv, ("--security-opt", "no-new-privileges"))
     assert _has_consecutive(argv, ("--pids-limit", str(TASK_PIDS_LIMIT)))
 
@@ -226,7 +226,7 @@ def test_tmpfs_is_tmp_only(monkeypatch: pytest.MonkeyPatch) -> None:
 def test_hardening_run_args_shape() -> None:
     args = hardening_run_args()
     assert "--read-only" not in args
-    assert "--cap-drop" in args and args[args.index("--cap-drop") + 1] == "ALL"
+    assert "--cap-drop" not in args  # deliberate: TB guests need capabilities for apt/chmod
     assert "--security-opt" in args
     assert args[args.index("--security-opt") + 1] == "no-new-privileges"
     assert "--pids-limit" in args
@@ -256,7 +256,7 @@ def test_task_container_hardened_in_real_daemon(tmp_path: Path) -> None:
         assert posture.returncode == 0
         out = posture.stdout
         assert out.startswith("false ")  # writable rootfs (harbor paths)
-        assert "[ALL]" in out  # cap-drop ALL
+        assert "[ALL]" not in out  # cap-drop ALL omitted for task guests
         assert str(TASK_PIDS_LIMIT) in out  # bounded pids
         assert "no-new-privileges" in out
         assert "false" in out.split()[-1].lower() or out.rstrip().endswith("false")
@@ -279,14 +279,13 @@ def test_task_container_hardened_in_real_daemon(tmp_path: Path) -> None:
         env.remove()
 
 
-
 # --------------------------------------------------------------------------- #
 # Harbor verifier/agent paths must be writable on task containers
 # --------------------------------------------------------------------------- #
 # Production crash (2026-07-29): upload_tests did `docker exec -u root … mkdir -p
 # /tests` and failed with "Read-only file system" because task containers
 # incorrectly inherited the *job*-container RO posture. Task containers must
-# keep cap-drop / nnp / pids hardening but leave the rootfs writable so harbor
+# keep nnp / pids hardening (no cap-drop ALL) but leave the rootfs writable so harbor
 # paths (/tests, /logs/verifier, /solution, /app) and apt-based test.sh work.
 
 
@@ -297,7 +296,7 @@ def test_hardening_run_args_does_not_force_readonly_rootfs() -> None:
         "task containers must not use --read-only; that blocks harbor "
         "mkdir /tests and apt-based verifiers (Read-only file system)"
     )
-    assert _has_consecutive(args, ("--cap-drop", "ALL"))
+    assert "--cap-drop" not in args
     assert _has_consecutive(args, ("--security-opt", "no-new-privileges"))
     assert _has_consecutive(args, ("--pids-limit", str(TASK_PIDS_LIMIT)))
 
@@ -340,7 +339,7 @@ def test_task_container_allows_harbor_verifier_paths() -> None:
             f"stdout={write.stdout!r} stderr={write.stderr!r}"
         )
         assert "ok" in write.stdout
-        # Hardening that must remain: cap-drop ALL + nnp + pids + not privileged.
+        # Hardening that must remain: nnp + pids + not privileged (no cap-drop ALL).
         posture = subprocess.run(
             [
                 "docker",
@@ -357,7 +356,7 @@ def test_task_container_allows_harbor_verifier_paths() -> None:
         assert posture.returncode == 0
         out = posture.stdout.strip()
         assert out.startswith("false "), f"expected writable rootfs, got {out!r}"
-        assert "[ALL]" in out
+        assert "[ALL]" not in out
         assert str(TASK_PIDS_LIMIT) in out
         assert "no-new-privileges" in out
         assert out.endswith(" false") or out.rstrip().endswith("false")