From eceef43bafd895381e642d03425e1acff967b6c0 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:43:55 +0000 Subject: [PATCH 1/3] fix(proxy): allow miner env/launch routes under attested allowlist Prod enables agent_challenge_attested_routes_enabled, which blocked GET|PUT /env, POST /env/confirm-empty, and POST /launch with local 404. Treat env/launch as signed routes flag-independently via the existing _is_agent_challenge_env_route SSOT so the enabled-mode allowlist admits them and miner X-Hotkey/X-Signature/X-Nonce/X-Timestamp headers are preserved (otherwise signed PUT residual 401 after allowlisting). --- src/base/master/app_proxy.py | 18 +-- .../test_agent_challenge_attested_proxy.py | 126 +++++++++++++++++- 2 files changed, 131 insertions(+), 13 deletions(-) diff --git a/src/base/master/app_proxy.py b/src/base/master/app_proxy.py index bbcd36e6..7fbb9637 100644 --- a/src/base/master/app_proxy.py +++ b/src/base/master/app_proxy.py @@ -425,11 +425,11 @@ def _is_agent_challenge_signed_route( ``X-Timestamp``) must survive the generic ``/challenges/{slug}`` passthrough so the challenge can verify them. - Full attested mode uses the exact review/eval allowlist (plus signed - ``POST /submissions``). Legacy mode keeps env/launch signed surfaces, and - also preserves the same exact review/eval signed row so dual-flag AC - prepare/deploy cannot residual as HTTP 401 when the Base allowlist flag - is not yet flipped. + Exact review/eval signed rows, public ``POST /submissions``, and miner + env/launch surfaces always need signature headers — independent of the + attested-routes allowlist flag. Env/launch must stay signed under full + attested mode so miners can attach OPENROUTER_API_KEY (or confirm-empty) + without residual HTTP 401 after the allowlist admits the path. """ if slug != "agent-challenge": @@ -440,10 +440,10 @@ def _is_agent_challenge_signed_route( if _is_agent_challenge_exact_review_eval_signed_route(method, path): return True - if not attested_routes_enabled: - return _is_agent_challenge_env_route(slug, method, path) - - return False + # Miner env/launch: same flag-independence. Feeds enabled-mode allowlist + # via signed_route(..., attested=True) and preserve_miner_signature_headers. + _ = attested_routes_enabled + return _is_agent_challenge_env_route(slug, method, path) def _is_agent_challenge_enabled_mode_allowed_route( diff --git a/tests/unit/test_agent_challenge_attested_proxy.py b/tests/unit/test_agent_challenge_attested_proxy.py index 6f658f71..e649a4fa 100644 --- a/tests/unit/test_agent_challenge_attested_proxy.py +++ b/tests/unit/test_agent_challenge_attested_proxy.py @@ -492,10 +492,14 @@ async def handler(request: httpx.Request) -> httpx.Response: ("POST", "/submissions/sub-1/eval/status"), ("POST", "/submissions/sub-1/eval/result"), ("POST", "/submissions/sub-1/eval/key-release"), - ("GET", "/submissions/sub-1/env"), - ("PUT", "/submissions/sub-1/env"), - ("POST", "/submissions/sub-1/env/confirm-empty"), - ("POST", "/submissions/sub-1/launch"), + # Miner env/launch shapes are allowlisted + signature-preserved (FIX G). + # Wrong methods on those shapes stay denied. + ("DELETE", "/submissions/sub-1/env"), + ("POST", "/submissions/sub-1/env"), + ("PUT", "/submissions/sub-1/env/confirm-empty"), + ("GET", "/submissions/sub-1/env/confirm-empty"), + ("GET", "/submissions/sub-1/launch"), + ("PUT", "/submissions/sub-1/launch"), # review/v1 guest capability table is allowlisted + Authorization preserved # (see test_review_capability_routes_preserve_authorization_bearer). Neighbor # aliases and unconstrained assignment paths remain blocked below. @@ -828,6 +832,120 @@ async def handler(request: httpx.Request) -> httpx.Response: assert "authorization" not in headers +@pytest.mark.parametrize("attested_routes_enabled", [True, False]) +@pytest.mark.parametrize( + ("method", "path", "upstream_path"), + ( + ( + "GET", + "/challenges/agent-challenge/submissions/13/env", + "/submissions/13/env", + ), + ( + "PUT", + "/challenges/agent-challenge/submissions/13/env", + "/submissions/13/env", + ), + ( + "POST", + "/challenges/agent-challenge/submissions/13/env/confirm-empty", + "/submissions/13/env/confirm-empty", + ), + ( + "POST", + "/challenges/agent-challenge/submissions/13/launch", + "/submissions/13/launch", + ), + ), +) +def test_miner_env_launch_routes_allowed_and_preserve_signature_headers( + attested_routes_enabled: bool, + method: str, + path: str, + upstream_path: str, +) -> None: + """FIX G: env/launch must work under attested allowlist AND keep miner sig headers. + + Prod sets agent_challenge_attested_routes_enabled=true. Without these shapes in + the enabled-mode allowlist the public edge returns local 404 Proxy path not found. + Even after allowlisting, signed PUT/POST would residual 401 if X-Hotkey/X-Signature + were stripped. Both flag states must forward + preserve the four miner headers. + """ + + captured: dict[str, Any] = {} + + async def handler(request: httpx.Request) -> httpx.Response: + captured["method"] = request.method + captured["path"] = request.url.path + captured["headers"] = request.headers + captured["body"] = await request.aread() + return httpx.Response( + 200, + content=b'{"ok":true,"env":"forwarded"}', + headers={"content-type": "application/json"}, + ) + + client = _proxy_client(handler, attested_routes_enabled=attested_routes_enabled) + request_body = b'{"OPENROUTER_API_KEY":"sk-test"}' if method == "PUT" else b"" + response = client.request( + method, + path, + content=request_body, + headers={ + "Content-Type": "application/json", + "X-Hotkey": "5D7D4EGayNMinerHotkeyExampleForTestOnly", + "X-Signature": "0x" + ("ef" * 32), + "X-Nonce": "fresh-nonce-miner-env", + "X-Timestamp": "1700000013", + "Authorization": "Bearer should-not-forward", + "X-Base-Verified-Hotkey": "forged", + "X-Allowlist-Digest": "caller-allowlist", + }, + ) + + assert response.status_code == 200 + assert response.json() == {"ok": True, "env": "forwarded"} + assert captured["method"] == method + assert captured["path"] == upstream_path + if method == "PUT": + assert captured["body"] == request_body + headers: httpx.Headers = captured["headers"] + assert headers["x-hotkey"] == "5D7D4EGayNMinerHotkeyExampleForTestOnly" + assert headers["x-signature"].startswith("0x") + assert headers["x-nonce"] == "fresh-nonce-miner-env" + assert headers["x-timestamp"] == "1700000013" + assert "authorization" not in headers + if attested_routes_enabled: + assert "x-base-verified-hotkey" not in headers + assert "x-allowlist-digest" not in headers + + +@pytest.mark.parametrize( + ("method", "path"), + ( + ("GET", "submissions/13/env"), + ("PUT", "submissions/13/env"), + ("POST", "submissions/13/env/confirm-empty"), + ("POST", "submissions/13/launch"), + ), +) +def test_enabled_mode_allowlist_accepts_miner_env_launch_shapes( + method: str, + path: str, +) -> None: + assert _is_agent_challenge_enabled_mode_allowed_route( + "agent-challenge", + method, + path, + ) + assert not _is_blocked_agent_challenge_proxy_path( + "agent-challenge", + method, + path, + attested_routes_enabled=True, + ) + + def test_forged_trust_headers_do_not_elevate_private_routes() -> None: """VAL-ACAT-047/048: forged trust headers never open private aliases.""" From 450072f87ea391b95525e6ec208e60b5c3d22686 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:43:55 +0000 Subject: [PATCH 2/3] fix(agent-challenge): start submissions unconfirmed for miner env gate _persist_submission hardcoded env_confirmed_empty=True, so analysis allow always auto-enqueued credential-less evaluation and waiting_miner_env was dead. New rows start unconfirmed; miners must PUT /env or POST /env/confirm-empty before evaluation. Legacy backfill paths untouched. --- .../src/agent_challenge/api/routes.py | 4 +-- .../tests/test_analyzer_lifecycle.py | 35 ++++++++++--------- .../tests/test_lifecycle_e2e.py | 10 ++++++ 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/packages/challenges/agent-challenge/src/agent_challenge/api/routes.py b/packages/challenges/agent-challenge/src/agent_challenge/api/routes.py index 83ee60ee..4db18066 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/api/routes.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/api/routes.py @@ -6399,8 +6399,8 @@ async def _persist_submission( artifact_path=artifact.artifact_path, raw_status="received", effective_status="received", - env_confirmed_empty=True, - env_confirmed_empty_at=datetime.now(UTC), + env_confirmed_empty=False, + env_confirmed_empty_at=None, signature=signature, signature_nonce=signature_nonce, signature_timestamp=signature_timestamp, diff --git a/packages/challenges/agent-challenge/tests/test_analyzer_lifecycle.py b/packages/challenges/agent-challenge/tests/test_analyzer_lifecycle.py index 41490cc9..3e7111f5 100644 --- a/packages/challenges/agent-challenge/tests/test_analyzer_lifecycle.py +++ b/packages/challenges/agent-challenge/tests/test_analyzer_lifecycle.py @@ -387,21 +387,27 @@ async def test_worker_allow_with_confirmed_empty_env_enqueues_one_evaluation_and assert job_count == 1 -async def test_create_submission_marks_env_confirmed_empty_and_auto_enqueues_on_allow( +async def test_create_submission_starts_unconfirmed_and_parks_on_allow_without_env( client, database_session, monkeypatch, signed_submission_override, tmp_path, ): + """FIX E: new submissions must not pre-confirm empty env. + + Hardcoding env_confirmed_empty=True at create made waiting_miner_env + unreachable — analysis allow always auto-enqueued credential-less eval. + Miners attach OPENROUTER_API_KEY via PUT /env or call confirm-empty first. + """ configure_master(monkeypatch, tmp_path) await submit_agent(client, {"agent.py": "def solve(value):\n return value + 1\n"}) async with database_session() as session: submission = await session.scalar(select(AgentSubmission)) assert submission is not None - assert submission.env_confirmed_empty is True - assert submission.env_confirmed_empty_at is not None + assert submission.env_confirmed_empty is False + assert submission.env_confirmed_empty_at is None summary = await run_next_analysis( session, lease_owner="analysis-worker", @@ -411,16 +417,16 @@ async def test_create_submission_marks_env_confirmed_empty_and_auto_enqueues_on_ assert summary is not None assert summary.verdict == "allow" - assert summary.evaluation_job_id is not None + assert summary.evaluation_job_id is None async with database_session() as session: submission = await session.scalar(select(AgentSubmission)) job_count = await session.scalar(select(func.count(EvaluationJob.id))) assert submission is not None - assert submission.raw_status == "tb_queued" - assert submission.env_locked_at is not None - assert submission.latest_evaluation_job_id is not None - assert job_count == 1 + assert submission.raw_status == "waiting_miner_env" + assert submission.env_locked_at is None + assert submission.latest_evaluation_job_id is None + assert job_count == 0 async def test_analysis_commits_before_llm_call_to_release_connection( @@ -744,18 +750,15 @@ async def test_llm_standby_requeues_when_gateway_token_becomes_available( ) assert submission is not None - assert submission.raw_status == "tb_completed" + assert submission.raw_status == "waiting_miner_env" assert analysis_count == 2 - assert events[-9:] == [ + assert events[-6:] == [ "llm_standby", "analysis_queued", "ast_running", "llm_running", "analysis_allowed", "waiting_miner_env", - "tb_queued", - "tb_running", - "tb_completed", ] @@ -1027,15 +1030,15 @@ async def test_gate_clean_submission_allows_and_records_ast_and_rules( assert summary is not None assert summary.verdict == "allow" - assert summary.evaluation_job_id is not None + assert summary.evaluation_job_id is None async with database_session() as session: submission = await session.scalar(select(AgentSubmission)) job_count = await session.scalar(select(func.count(EvaluationJob.id))) analysis_run = await session.scalar(select(AnalysisRun)) assert submission is not None - assert submission.raw_status == "tb_queued" - assert job_count == 1 + assert submission.raw_status == "waiting_miner_env" + assert job_count == 0 report = json.loads(analysis_run.report_json) assert report["ast"]["verdict"] == "clean" assert report["ast"]["verdict_reason"] diff --git a/packages/challenges/agent-challenge/tests/test_lifecycle_e2e.py b/packages/challenges/agent-challenge/tests/test_lifecycle_e2e.py index cb8c88fb..0a1bf4d7 100644 --- a/packages/challenges/agent-challenge/tests/test_lifecycle_e2e.py +++ b/packages/challenges/agent-challenge/tests/test_lifecycle_e2e.py @@ -189,6 +189,11 @@ async def test_signed_allow_lifecycle_recovers_terminal_bench_and_scores_weight( reviewer = StaticReviewer("allow") async with database_session() as session: + submission = await session.get(AgentSubmission, submission_id) + assert submission is not None + # E2E full TB path: miner already confirmed empty env (FIX E default is unconfirmed). + submission.env_confirmed_empty = True + await session.flush() summary = await run_analysis_for_submission( session, submission_id, @@ -590,6 +595,11 @@ async def _submit_and_analyze(client, database_session, *, reviewer: StaticRevie assert response.status_code == 201 submission_id = response.json()["submission_id"] async with database_session() as session: + submission = await session.get(AgentSubmission, submission_id) + assert submission is not None + # E2E full TB path: miner already confirmed empty env (FIX E default is unconfirmed). + submission.env_confirmed_empty = True + await session.flush() await run_analysis_for_submission( session, submission_id, From 3ecafbb146a899f032365eb430f20fd9c0ab9a69 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:43:55 +0000 Subject: [PATCH 3/3] fix(agent-challenge): fail-closed evaluation enqueue status gate _validate_evaluation_enqueue_status previously fell through for unknown statuses (including accidental enqueue from non-ready states). Explicitly allow in-flight TB statuses, waiting_miner_env when confirmed, legacy analysis_allowed, and terminal re-eval labels; raise ValueError otherwise. --- .../src/agent_challenge/evaluation/runner.py | 21 ++- .../agent-challenge/tests/test_evaluation.py | 134 ++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py index 0f38df6a..367e15c2 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py @@ -240,10 +240,29 @@ def _validate_evaluation_enqueue_status( if confirmed_miner_env: return raise ValueError("submission is waiting for miner environment confirmation") - if submission.raw_status in {"queued", "tb_queued", "tb_running", "tb_failed_retryable"}: + if submission.raw_status in { + "queued", + "tb_queued", + "tb_running", + "tb_failed_retryable", + # Operator / owner re-eval of terminal submissions. + # create_evaluation_job maps internal terminals → tb_queued and + # public terminals → queued. Must be explicit — never fail-open. + "tb_completed", + "tb_failed_final", + "completed", + "valid", + "invalid", + "suspicious", + "error", + "overridden_valid", + "overridden_invalid", + "evaluating", + }: return if submission.raw_status == "analysis_allowed": raise ValueError("submission is waiting for miner environment confirmation") + raise ValueError(f"submission status {submission.raw_status!r} cannot enqueue evaluation") async def create_evaluation_job( diff --git a/packages/challenges/agent-challenge/tests/test_evaluation.py b/packages/challenges/agent-challenge/tests/test_evaluation.py index fe1cba95..e617c33d 100644 --- a/packages/challenges/agent-challenge/tests/test_evaluation.py +++ b/packages/challenges/agent-challenge/tests/test_evaluation.py @@ -25,6 +25,7 @@ benchmark_tasks_from_json, benchmark_tasks_to_json, ) +from agent_challenge.evaluation.runner import _validate_evaluation_enqueue_status from agent_challenge.evaluation.worker import run_worker_once from agent_challenge.models import ( AgentSubmission, @@ -232,6 +233,8 @@ async def test_run_evaluation_job_scores_all_tasks(database_session, monkeypatch name="agent-a", agent_hash="abc123", artifact_uri=str(agent_dir), + raw_status="queued", + effective_status="queued", ) session.add(submission) await session.flush() @@ -316,6 +319,8 @@ async def test_create_evaluation_job_selects_at_most_twenty_tasks( name="agent-max-twenty", agent_hash="max-twenty-selection", artifact_uri=str(agent_dir), + raw_status="queued", + effective_status="queued", ) session.add(submission) await session.flush() @@ -368,6 +373,113 @@ async def test_create_terminal_bench_evaluation_job_selects_at_most_twenty_tasks assert {task.benchmark for task in selected_tasks} == {"terminal_bench"} +def _enqueue_status_submission( + raw_status: str, + *, + env_confirmed_empty: bool = False, + env_locked_at: datetime | None = None, + env_compatibility_reason: str | None = None, +) -> AgentSubmission: + return AgentSubmission( + miner_hotkey="hotkey-enqueue-gate", + name="agent-enqueue-gate", + agent_hash=f"enqueue-{raw_status}", + artifact_uri="/tmp/agent-enqueue-gate", + raw_status=raw_status, + effective_status=raw_status, + env_confirmed_empty=env_confirmed_empty, + env_locked_at=env_locked_at, + env_compatibility_reason=env_compatibility_reason, + ) + + +@pytest.mark.parametrize( + "raw_status", + ( + "queued", + "tb_queued", + "tb_running", + "tb_failed_retryable", + "tb_completed", + "tb_failed_final", + "completed", + "valid", + "invalid", + "suspicious", + "error", + "overridden_valid", + "overridden_invalid", + "evaluating", + ), +) +def test_validate_evaluation_enqueue_status_allows_known_ready_statuses( + raw_status: str, +) -> None: + """FIX F: allowlist includes in-flight TB statuses and terminal re-eval.""" + _validate_evaluation_enqueue_status( + _enqueue_status_submission(raw_status), + confirmed_miner_env=False, + ) + + +def test_validate_evaluation_enqueue_status_allows_waiting_when_confirmed() -> None: + _validate_evaluation_enqueue_status( + _enqueue_status_submission("waiting_miner_env"), + confirmed_miner_env=True, + ) + + +def test_validate_evaluation_enqueue_status_allows_legacy_analysis_allowed() -> None: + locked_at = datetime.now(UTC) + _validate_evaluation_enqueue_status( + _enqueue_status_submission( + "analysis_allowed", + env_confirmed_empty=True, + env_locked_at=locked_at, + env_compatibility_reason="pre_env_gate_analysis_allowed", + ), + confirmed_miner_env=False, + ) + + +@pytest.mark.parametrize( + "raw_status", + ( + "received", + "analysis_queued", + "analysis_rejected", + "cancelled", + "admin_paused", + "review_running", + ), +) +def test_validate_evaluation_enqueue_status_rejects_invalid_statuses( + raw_status: str, +) -> None: + """FIX F: statuses outside the allowlist must raise, not fail-open.""" + with pytest.raises(ValueError, match="cannot enqueue evaluation"): + _validate_evaluation_enqueue_status( + _enqueue_status_submission(raw_status), + confirmed_miner_env=False, + ) + + +def test_validate_evaluation_enqueue_status_rejects_unconfirmed_waiting() -> None: + with pytest.raises(ValueError, match="waiting for miner environment"): + _validate_evaluation_enqueue_status( + _enqueue_status_submission("waiting_miner_env"), + confirmed_miner_env=False, + ) + + +def test_validate_evaluation_enqueue_status_rejects_non_legacy_analysis_allowed() -> None: + with pytest.raises(ValueError, match="waiting for miner environment"): + _validate_evaluation_enqueue_status( + _enqueue_status_submission("analysis_allowed"), + confirmed_miner_env=False, + ) + + @pytest.mark.parametrize("raw_status", ["tb_completed", "tb_failed_final"]) async def test_create_evaluation_job_revalidates_internal_terminal_submission( database_session, @@ -502,6 +614,8 @@ async def test_run_evaluation_job_records_failed_task_events( name="agent-a", agent_hash="failed-task-events", artifact_uri=str(agent_dir), + raw_status="queued", + effective_status="queued", ) session.add(submission) await session.flush() @@ -561,6 +675,8 @@ async def test_run_evaluation_job_records_terminal_event_after_log_cap( name="agent-a", agent_hash="log-cap-terminal-event", artifact_uri=str(agent_dir), + raw_status="queued", + effective_status="queued", ) session.add(submission) await session.flush() @@ -615,6 +731,8 @@ async def test_run_evaluation_job_persists_failure(database_session, monkeypatch name="agent-a", agent_hash="def456", artifact_uri=str(agent_dir), + raw_status="queued", + effective_status="queued", ) session.add(submission) await session.flush() @@ -657,6 +775,8 @@ def analyzer(_workspace, *, reviewer=None): name="agent-a", agent_hash="containerfail", artifact_uri=str(agent_dir), + raw_status="queued", + effective_status="queued", ) session.add(submission) await session.flush() @@ -707,6 +827,8 @@ async def test_run_evaluation_job_runs_terminal_bench_task(database_session, mon name="agent-a", agent_hash="ghi789", artifact_uri=str(agent_dir), + raw_status="queued", + effective_status="queued", ) session.add(submission) await session.flush() @@ -811,6 +933,8 @@ def run(self, spec, timeout_seconds: int): name="agent-lockprobe-swe", agent_hash="lockprobe-swe", artifact_uri=str(agent_dir), + raw_status="queued", + effective_status="queued", ) session.add(submission) await session.flush() @@ -913,6 +1037,8 @@ def run(self, spec, timeout_seconds: int): name="agent-attempt-commit", agent_hash="attempt-commit-hash", artifact_uri=str(agent_dir), + raw_status="queued", + effective_status="queued", ) session.add(submission) await session.flush() @@ -959,6 +1085,8 @@ async def test_run_evaluation_job_skips_already_persisted_task_result( name="agent-idempotent", agent_hash="idempotent-hash", artifact_uri=str(agent_dir), + raw_status="queued", + effective_status="queued", ) session.add(submission) await session.flush() @@ -2053,6 +2181,8 @@ async def test_base_sdk_retry_requeues_then_final_fails_at_worker_cap( name="platform-sdk-retry-agent", agent_hash="platform-sdk-retry-hash", artifact_uri=str(agent_dir), + raw_status="queued", + effective_status="queued", ) session.add(submission) await session.flush() @@ -2142,6 +2272,8 @@ def analyzer(_workspace, *, reviewer=None): name="agent-a", agent_hash="reviewer123", artifact_uri=str(agent_dir), + raw_status="queued", + effective_status="queued", ) session.add(submission) await session.flush() @@ -2188,6 +2320,8 @@ async def test_terminal_bench_mounts_extracted_zip_workspace( name="agent-a", agent_hash="zip789", artifact_uri=str(agent_zip), + raw_status="queued", + effective_status="queued", artifact_path=str(agent_zip), ) session.add(submission)