From f5a008b88b3352f180d13a55d24085ebceb87f04 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:04:37 +0000 Subject: [PATCH] fix(agent-challenge): supply LLM_MODEL to the evaluated agent Every Terminal-Bench evaluation scored 0 and every result.json carried "model_name": null. The packaged agent fails closed without a concrete model id: agent run failed: A concrete model id is required: set LLM_MODEL (the measured review harness supplies it under .rules). Three independent layers dropped the model id, so no submission could ever score: 1. the runner never set LLM_MODEL -- the platform only holds it as CHALLENGE_LLM_MODEL (settings.llm_model), which the agent never reads; 2. sanitize_miner_env_for_job stripped a miner-supplied LLM_MODEL because the name is not token/key shaped; 3. AGENT_ENV_ALLOWLIST admitted only LLM_COST_LIMIT and OPENROUTER_API_KEY. The trial died in ~5s per task, before any LLM call, which is why the whole leaderboard sat at zero while OPENROUTER_API_KEY was reaching the agent correctly. _terminal_bench_env now publishes settings.llm_model as LLM_MODEL, ahead of the sanitized miner merge so a miner may override it; the model name joins both the miner product allowlist and the agent sandbox allowlist. A miner can therefore bring their own key and their own model. When an operator blanks llm_model nothing is injected, so the agent still fails loudly rather than running an unmeasured default. Updated pins keep their intent: gateway/URL/host/proxy names stay excluded, secret scrubbing still asserted, and the ex-gateway test now asserts LLM_MODEL comes from settings and that no gateway base_url or token leaks into the job env. Verified: 7 new tests RED -> GREEN; full agent-challenge suite shows zero regressions (26 pre-existing failures on main, 26 with this change, identical sets; 2286 -> 2293 passing). --- .../evaluation/own_runner/isolation.py | 3 + .../src/agent_challenge/evaluation/runner.py | 8 ++ .../agent_challenge/submissions/miner_env.py | 4 + .../tests/test_deepseek_via_gateway.py | 9 ++- .../tests/test_llm_model_reaches_agent.py | 78 +++++++++++++++++++ .../tests/test_miner_env_lock.py | 9 ++- .../tests/test_own_runner_isolation.py | 7 +- .../tests/test_worker_security.py | 4 + 8 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 packages/challenges/agent-challenge/tests/test_llm_model_reaches_agent.py diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/isolation.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/isolation.py index 7f53f099c..5fa7c16f3 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/isolation.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/isolation.py @@ -66,6 +66,9 @@ AGENT_ENV_ALLOWLIST: frozenset[str] = frozenset( { "LLM_COST_LIMIT", + # VAL-LLM-MODEL: the packaged agent fails closed without a concrete + # model id ("set LLM_MODEL"). Not secret-shaped, never a URL/host. + "LLM_MODEL", "OPENROUTER_API_KEY", } ) diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py index 1749e68ee..c8c8cc52b 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py @@ -1631,6 +1631,14 @@ def _terminal_bench_env( "BASE_BENCHMARK_DATASET": settings.terminal_bench_dataset, **TERMINAL_BENCH_WRITABLE_ENV, } + # VAL-LLM-MODEL: the packaged agent fails closed without a concrete model id + # ("A concrete model id is required: set LLM_MODEL"). The platform holds it + # as CHALLENGE_LLM_MODEL, which the agent never reads, so publish it under + # the name the agent expects. A miner-supplied LLM_MODEL overrides this in + # the sanitized merge below. Left unset when the operator blanks it, so the + # agent still fails loudly rather than running an unmeasured default. + if settings.llm_model: + env["LLM_MODEL"] = settings.llm_model for name in settings.harbor_forward_env_vars: value = os.environ.get(name) if value and name not in TERMINAL_BENCH_CONTROL_ENV_KEYS: diff --git a/packages/challenges/agent-challenge/src/agent_challenge/submissions/miner_env.py b/packages/challenges/agent-challenge/src/agent_challenge/submissions/miner_env.py index fe925cac9..36b610160 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/submissions/miner_env.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/submissions/miner_env.py @@ -46,6 +46,10 @@ { "OPENROUTER_API_KEY", "LLM_COST_LIMIT", + # VAL-LLM-MODEL: a miner may bring their own key *and* their own model + # id. Not a URL/host/proxy name, and URL-shaped values stay rejected by + # ``looks_like_url_value``, so this opens no endpoint-injection hole. + "LLM_MODEL", "EVAL_RUN_TOKEN", "REVIEW_SESSION_TOKEN", } diff --git a/packages/challenges/agent-challenge/tests/test_deepseek_via_gateway.py b/packages/challenges/agent-challenge/tests/test_deepseek_via_gateway.py index a35361b45..6889b12c3 100644 --- a/packages/challenges/agent-challenge/tests/test_deepseek_via_gateway.py +++ b/packages/challenges/agent-challenge/tests/test_deepseek_via_gateway.py @@ -173,7 +173,14 @@ async def test_deepseek_no_longer_routes_through_master_gateway( assert "BASE_LLM_GATEWAY_URL" not in env assert "BASE_GATEWAY_TOKEN" not in env assert "DEEPSEEK_API_KEY" not in env - assert "LLM_MODEL" not in env + # VAL-LLM-MODEL: LLM_MODEL is no longer gateway-derived. It is supplied + # from measured platform settings because the packaged agent fails closed + # without a concrete model id, so its presence is NOT gateway injection. + # Assert it carries the settings value and that nothing from the residual + # gateway (base_url / token) leaks into the job env. + assert env.get("LLM_MODEL") == "x-ai/grok-4.5" + assert "master-gateway.test" not in str(env) + assert "scoped-assignment-token" not in str(env) serialized = json.dumps(env, sort_keys=True) assert "api.deepseek.com" not in serialized diff --git a/packages/challenges/agent-challenge/tests/test_llm_model_reaches_agent.py b/packages/challenges/agent-challenge/tests/test_llm_model_reaches_agent.py new file mode 100644 index 000000000..8a218583f --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_llm_model_reaches_agent.py @@ -0,0 +1,78 @@ +"""The LLM model id must reach the agent (VAL-LLM-MODEL). + +The packaged agent fails closed when ``LLM_MODEL`` is unset:: + + agent run failed: A concrete model id is required: set LLM_MODEL + (the measured review harness supplies it under .rules). + +Three independent layers dropped it, so *every* evaluation scored 0 and every +``result.json`` carried ``"model_name": null``: + +1. the runner never set ``LLM_MODEL`` -- the platform only holds + ``CHALLENGE_LLM_MODEL`` (settings ``llm_model``), which the agent never reads; +2. :func:`sanitize_miner_env_for_job` stripped a miner-supplied ``LLM_MODEL`` + because the name is not token/key shaped; +3. :data:`AGENT_ENV_ALLOWLIST` admitted only ``LLM_COST_LIMIT`` and + ``OPENROUTER_API_KEY``. + +A miner may bring their own key *and* their own model; when they supply neither, +the measured platform model is used. +""" + +from __future__ import annotations + +from agent_challenge.core.config import settings +from agent_challenge.evaluation.own_runner.isolation import ( + AGENT_ENV_ALLOWLIST, + filter_agent_env, +) +from agent_challenge.evaluation.runner import _terminal_bench_env +from agent_challenge.submissions.miner_env import ( + sanitize_miner_env_for_job, + validate_miner_env, +) + + +def test_agent_env_allowlist_admits_llm_model() -> None: + """Layer 3: the agent sandbox must be allowed to see the model id.""" + assert "LLM_MODEL" in AGENT_ENV_ALLOWLIST + + +def test_filter_agent_env_keeps_llm_model() -> None: + """Layer 3: the final filter must not strip the model id.""" + kept = filter_agent_env({"LLM_MODEL": "x-ai/grok-4.5", "OPENROUTER_API_KEY": "sk-or-x"}) + assert kept["LLM_MODEL"] == "x-ai/grok-4.5" + assert kept["OPENROUTER_API_KEY"] == "sk-or-x" + + +def test_runner_injects_platform_llm_model() -> None: + """Layer 1: with no miner env, the measured platform model is supplied.""" + env = _terminal_bench_env() + assert env["LLM_MODEL"] == settings.llm_model + assert env["LLM_MODEL"], "platform model id must not be empty" + + +def test_miner_may_supply_own_llm_model() -> None: + """Layer 2: a miner bringing their own key may also choose their model.""" + assert sanitize_miner_env_for_job({"LLM_MODEL": "openai/gpt-5"}) == { + "LLM_MODEL": "openai/gpt-5" + } + assert validate_miner_env({"LLM_MODEL": "openai/gpt-5"}) == {"LLM_MODEL": "openai/gpt-5"} + + +def test_miner_llm_model_overrides_platform_default() -> None: + """A miner-supplied model wins over the platform default.""" + env = _terminal_bench_env({"LLM_MODEL": "openai/gpt-5"}) + assert env["LLM_MODEL"] == "openai/gpt-5" + + +def test_llm_model_value_must_not_be_a_url() -> None: + """Admitting the name must not open an endpoint-injection hole.""" + assert sanitize_miner_env_for_job({"LLM_MODEL": "http://evil.example/v1"}) == {} + + +def test_llm_model_is_not_treated_as_a_secret() -> None: + """The model id is not secret-shaped, so it must not be flagged as one.""" + from agent_challenge.evaluation.own_runner.isolation import disallowed_secret_keys + + assert "LLM_MODEL" not in disallowed_secret_keys({"LLM_MODEL": "x-ai/grok-4.5"}) diff --git a/packages/challenges/agent-challenge/tests/test_miner_env_lock.py b/packages/challenges/agent-challenge/tests/test_miner_env_lock.py index 4ef4cf3b6..dc6e8499c 100644 --- a/packages/challenges/agent-challenge/tests/test_miner_env_lock.py +++ b/packages/challenges/agent-challenge/tests/test_miner_env_lock.py @@ -163,8 +163,13 @@ def test_terminal_bench_env_ignores_miner_base_log_stream_override() -> None: # --------------------------------------------------------------------------- # # VAL-ACLOCK-005 / 006 — agent sandbox allowlist + chokepoint # --------------------------------------------------------------------------- # -def test_agent_env_allowlist_exactly_or_key_and_cost_limit() -> None: - assert AGENT_ENV_ALLOWLIST == frozenset({"OPENROUTER_API_KEY", "LLM_COST_LIMIT"}) +def test_agent_env_allowlist_exactly_key_cost_limit_and_model() -> None: + # VAL-LLM-MODEL: LLM_MODEL joins the sandbox allowlist because the packaged + # agent fails closed without a concrete model id. The exact pin is + # deliberate: nothing else enters this set without an explicit decision. + assert AGENT_ENV_ALLOWLIST == frozenset( + {"OPENROUTER_API_KEY", "LLM_COST_LIMIT", "LLM_MODEL"} + ) def test_filter_agent_env_strips_url_proxy_and_extra_secrets() -> None: diff --git a/packages/challenges/agent-challenge/tests/test_own_runner_isolation.py b/packages/challenges/agent-challenge/tests/test_own_runner_isolation.py index 6bbd95681..bd0898d94 100644 --- a/packages/challenges/agent-challenge/tests/test_own_runner_isolation.py +++ b/packages/challenges/agent-challenge/tests/test_own_runner_isolation.py @@ -108,7 +108,12 @@ def test_allowlist_excludes_base_gateway_vars() -> None: assert "BASE_LLM_GATEWAY_URL" not in AGENT_ENV_ALLOWLIST assert "BASE_GATEWAY_TOKEN" not in AGENT_ENV_ALLOWLIST assert "LLM_COST_LIMIT" in AGENT_ENV_ALLOWLIST - assert AGENT_ENV_ALLOWLIST == frozenset({"LLM_COST_LIMIT", "OPENROUTER_API_KEY"}) + # VAL-LLM-MODEL: the packaged agent fails closed without a concrete model + # id, so LLM_MODEL is admitted alongside the key and cost limit. The + # gateway/URL/host/proxy exclusions above remain the invariant. + assert AGENT_ENV_ALLOWLIST == frozenset( + {"LLM_COST_LIMIT", "LLM_MODEL", "OPENROUTER_API_KEY"} + ) def test_harness_control_keys_are_not_secrets() -> None: diff --git a/packages/challenges/agent-challenge/tests/test_worker_security.py b/packages/challenges/agent-challenge/tests/test_worker_security.py index 107dcfa2e..9134ce996 100644 --- a/packages/challenges/agent-challenge/tests/test_worker_security.py +++ b/packages/challenges/agent-challenge/tests/test_worker_security.py @@ -195,10 +195,14 @@ async def test_worker_broker_path_scrubs_token_and_signature_metadata_and_keeps_ "hello-world", ] benchmark_spec = executor.specs[1] + # VAL-LLM-MODEL: the measured model id now ships in the job env (the packaged + # agent fails closed without a concrete LLM_MODEL). It is a plain model name, + # not a secret -- the assert_no_untrusted_secret guards below still apply. assert benchmark_spec.env == { "BASE_AGENT_PATH": "/workspace/agent", "BASE_BENCHMARK_DATASET": "terminal-bench/terminal-bench-2-1", "HOME": "/tmp", + "LLM_MODEL": "x-ai/grok-4.5", "XDG_CACHE_HOME": "/tmp/.cache", } assert_no_untrusted_secret(