fix(proxy+agent-challenge): unblock miner env routes and engage env gate

packages/challenges/agent-challenge/src/agent_challenge/api/routes.py

@@ -6399,8 +6399,8 @@ async def _persist_submission(
         artifact_path=artifact.artifact_path,
         raw_status="received",
         effective_status="received",
-        env_confirmed_empty=True,
-        env_confirmed_empty_at=datetime.now(UTC),
+        env_confirmed_empty=False,
+        env_confirmed_empty_at=None,
         signature=signature,
         signature_nonce=signature_nonce,
         signature_timestamp=signature_timestamp,

packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py

@@ -240,10 +240,29 @@ def _validate_evaluation_enqueue_status(
         if confirmed_miner_env:
             return
         raise ValueError("submission is waiting for miner environment confirmation")
-    if submission.raw_status in {"queued", "tb_queued", "tb_running", "tb_failed_retryable"}:
+    if submission.raw_status in {
+        "queued",
+        "tb_queued",
+        "tb_running",
+        "tb_failed_retryable",
+        # Operator / owner re-eval of terminal submissions.
+        # create_evaluation_job maps internal terminals → tb_queued and
+        # public terminals → queued. Must be explicit — never fail-open.
+        "tb_completed",
+        "tb_failed_final",
+        "completed",
+        "valid",
+        "invalid",
+        "suspicious",
+        "error",
+        "overridden_valid",
+        "overridden_invalid",
+        "evaluating",
+    }:
         return
     if submission.raw_status == "analysis_allowed":
         raise ValueError("submission is waiting for miner environment confirmation")
+    raise ValueError(f"submission status {submission.raw_status!r} cannot enqueue evaluation")
 
 
 async def create_evaluation_job(

packages/challenges/agent-challenge/tests/test_analyzer_lifecycle.py

@@ -387,21 +387,27 @@ async def test_worker_allow_with_confirmed_empty_env_enqueues_one_evaluation_and
     assert job_count == 1
 
 
-async def test_create_submission_marks_env_confirmed_empty_and_auto_enqueues_on_allow(
+async def test_create_submission_starts_unconfirmed_and_parks_on_allow_without_env(
     client,
     database_session,
     monkeypatch,
     signed_submission_override,
     tmp_path,
 ):
+    """FIX E: new submissions must not pre-confirm empty env.
+
+    Hardcoding env_confirmed_empty=True at create made waiting_miner_env
+    unreachable — analysis allow always auto-enqueued credential-less eval.
+    Miners attach OPENROUTER_API_KEY via PUT /env or call confirm-empty first.
+    """
     configure_master(monkeypatch, tmp_path)
     await submit_agent(client, {"agent.py": "def solve(value):\n    return value + 1\n"})
 
     async with database_session() as session:
         submission = await session.scalar(select(AgentSubmission))
         assert submission is not None
-        assert submission.env_confirmed_empty is True
-        assert submission.env_confirmed_empty_at is not None
+        assert submission.env_confirmed_empty is False
+        assert submission.env_confirmed_empty_at is None
         summary = await run_next_analysis(
             session,
             lease_owner="analysis-worker",
@@ -411,16 +417,16 @@ async def test_create_submission_marks_env_confirmed_empty_and_auto_enqueues_on_
 
     assert summary is not None
     assert summary.verdict == "allow"
-    assert summary.evaluation_job_id is not None
+    assert summary.evaluation_job_id is None
     async with database_session() as session:
         submission = await session.scalar(select(AgentSubmission))
         job_count = await session.scalar(select(func.count(EvaluationJob.id)))
 
     assert submission is not None
-    assert submission.raw_status == "tb_queued"
-    assert submission.env_locked_at is not None
-    assert submission.latest_evaluation_job_id is not None
-    assert job_count == 1
+    assert submission.raw_status == "waiting_miner_env"
+    assert submission.env_locked_at is None
+    assert submission.latest_evaluation_job_id is None
+    assert job_count == 0
 
 
 async def test_analysis_commits_before_llm_call_to_release_connection(
@@ -744,18 +750,15 @@ async def test_llm_standby_requeues_when_gateway_token_becomes_available(
         )
 
     assert submission is not None
-    assert submission.raw_status == "tb_completed"
+    assert submission.raw_status == "waiting_miner_env"
     assert analysis_count == 2
-    assert events[-9:] == [
+    assert events[-6:] == [
         "llm_standby",
         "analysis_queued",
         "ast_running",
         "llm_running",
         "analysis_allowed",
         "waiting_miner_env",
-        "tb_queued",
-        "tb_running",
-        "tb_completed",
     ]
 
 
@@ -1027,15 +1030,15 @@ async def test_gate_clean_submission_allows_and_records_ast_and_rules(
 
     assert summary is not None
     assert summary.verdict == "allow"
-    assert summary.evaluation_job_id is not None
+    assert summary.evaluation_job_id is None
     async with database_session() as session:
         submission = await session.scalar(select(AgentSubmission))
         job_count = await session.scalar(select(func.count(EvaluationJob.id)))
         analysis_run = await session.scalar(select(AnalysisRun))
 
     assert submission is not None
-    assert submission.raw_status == "tb_queued"
-    assert job_count == 1
+    assert submission.raw_status == "waiting_miner_env"
+    assert job_count == 0
     report = json.loads(analysis_run.report_json)
     assert report["ast"]["verdict"] == "clean"
     assert report["ast"]["verdict_reason"]

packages/challenges/agent-challenge/tests/test_evaluation.py

@@ -25,6 +25,7 @@
     benchmark_tasks_from_json,
     benchmark_tasks_to_json,
 )
+from agent_challenge.evaluation.runner import _validate_evaluation_enqueue_status
 from agent_challenge.evaluation.worker import run_worker_once
 from agent_challenge.models import (
     AgentSubmission,
@@ -232,6 +233,8 @@ async def test_run_evaluation_job_scores_all_tasks(database_session, monkeypatch
             name="agent-a",
             agent_hash="abc123",
             artifact_uri=str(agent_dir),
+            raw_status="queued",
+            effective_status="queued",
         )
         session.add(submission)
         await session.flush()
@@ -316,6 +319,8 @@ async def test_create_evaluation_job_selects_at_most_twenty_tasks(
             name="agent-max-twenty",
             agent_hash="max-twenty-selection",
             artifact_uri=str(agent_dir),
+            raw_status="queued",
+            effective_status="queued",
         )
         session.add(submission)
         await session.flush()
@@ -368,6 +373,113 @@ async def test_create_terminal_bench_evaluation_job_selects_at_most_twenty_tasks
     assert {task.benchmark for task in selected_tasks} == {"terminal_bench"}
 
 
+def _enqueue_status_submission(
+    raw_status: str,
+    *,
+    env_confirmed_empty: bool = False,
+    env_locked_at: datetime | None = None,
+    env_compatibility_reason: str | None = None,
+) -> AgentSubmission:
+    return AgentSubmission(
+        miner_hotkey="hotkey-enqueue-gate",
+        name="agent-enqueue-gate",
+        agent_hash=f"enqueue-{raw_status}",
+        artifact_uri="/tmp/agent-enqueue-gate",
+        raw_status=raw_status,
+        effective_status=raw_status,
+        env_confirmed_empty=env_confirmed_empty,
+        env_locked_at=env_locked_at,
+        env_compatibility_reason=env_compatibility_reason,
+    )
+
+
+@pytest.mark.parametrize(
+    "raw_status",
+    (
+        "queued",
+        "tb_queued",
+        "tb_running",
+        "tb_failed_retryable",
+        "tb_completed",
+        "tb_failed_final",
+        "completed",
+        "valid",
+        "invalid",
+        "suspicious",
+        "error",
+        "overridden_valid",
+        "overridden_invalid",
+        "evaluating",
+    ),
+)
+def test_validate_evaluation_enqueue_status_allows_known_ready_statuses(
+    raw_status: str,
+) -> None:
+    """FIX F: allowlist includes in-flight TB statuses and terminal re-eval."""
+    _validate_evaluation_enqueue_status(
+        _enqueue_status_submission(raw_status),
+        confirmed_miner_env=False,
+    )
+
+
+def test_validate_evaluation_enqueue_status_allows_waiting_when_confirmed() -> None:
+    _validate_evaluation_enqueue_status(
+        _enqueue_status_submission("waiting_miner_env"),
+        confirmed_miner_env=True,
+    )
+
+
+def test_validate_evaluation_enqueue_status_allows_legacy_analysis_allowed() -> None:
+    locked_at = datetime.now(UTC)
+    _validate_evaluation_enqueue_status(
+        _enqueue_status_submission(
+            "analysis_allowed",
+            env_confirmed_empty=True,
+            env_locked_at=locked_at,
+            env_compatibility_reason="pre_env_gate_analysis_allowed",
+        ),
+        confirmed_miner_env=False,
+    )
+
+
+@pytest.mark.parametrize(
+    "raw_status",
+    (
+        "received",
+        "analysis_queued",
+        "analysis_rejected",
+        "cancelled",
+        "admin_paused",
+        "review_running",
+    ),
+)
+def test_validate_evaluation_enqueue_status_rejects_invalid_statuses(
+    raw_status: str,
+) -> None:
+    """FIX F: statuses outside the allowlist must raise, not fail-open."""
+    with pytest.raises(ValueError, match="cannot enqueue evaluation"):
+        _validate_evaluation_enqueue_status(
+            _enqueue_status_submission(raw_status),
+            confirmed_miner_env=False,
+        )
+
+
+def test_validate_evaluation_enqueue_status_rejects_unconfirmed_waiting() -> None:
+    with pytest.raises(ValueError, match="waiting for miner environment"):
+        _validate_evaluation_enqueue_status(
+            _enqueue_status_submission("waiting_miner_env"),
+            confirmed_miner_env=False,
+        )
+
+
+def test_validate_evaluation_enqueue_status_rejects_non_legacy_analysis_allowed() -> None:
+    with pytest.raises(ValueError, match="waiting for miner environment"):
+        _validate_evaluation_enqueue_status(
+            _enqueue_status_submission("analysis_allowed"),
+            confirmed_miner_env=False,
+        )
+
+
 @pytest.mark.parametrize("raw_status", ["tb_completed", "tb_failed_final"])
 async def test_create_evaluation_job_revalidates_internal_terminal_submission(
     database_session,
@@ -502,6 +614,8 @@ async def test_run_evaluation_job_records_failed_task_events(
             name="agent-a",
             agent_hash="failed-task-events",
             artifact_uri=str(agent_dir),
+            raw_status="queued",
+            effective_status="queued",
         )
         session.add(submission)
         await session.flush()
@@ -561,6 +675,8 @@ async def test_run_evaluation_job_records_terminal_event_after_log_cap(
             name="agent-a",
             agent_hash="log-cap-terminal-event",
             artifact_uri=str(agent_dir),
+            raw_status="queued",
+            effective_status="queued",
         )
         session.add(submission)
         await session.flush()
@@ -615,6 +731,8 @@ async def test_run_evaluation_job_persists_failure(database_session, monkeypatch
             name="agent-a",
             agent_hash="def456",
             artifact_uri=str(agent_dir),
+            raw_status="queued",
+            effective_status="queued",
         )
         session.add(submission)
         await session.flush()
@@ -657,6 +775,8 @@ def analyzer(_workspace, *, reviewer=None):
             name="agent-a",
             agent_hash="containerfail",
             artifact_uri=str(agent_dir),
+            raw_status="queued",
+            effective_status="queued",
         )
         session.add(submission)
         await session.flush()
@@ -707,6 +827,8 @@ async def test_run_evaluation_job_runs_terminal_bench_task(database_session, mon
             name="agent-a",
             agent_hash="ghi789",
             artifact_uri=str(agent_dir),
+            raw_status="queued",
+            effective_status="queued",
         )
         session.add(submission)
         await session.flush()
@@ -811,6 +933,8 @@ def run(self, spec, timeout_seconds: int):
             name="agent-lockprobe-swe",
             agent_hash="lockprobe-swe",
             artifact_uri=str(agent_dir),
+            raw_status="queued",
+            effective_status="queued",
         )
         session.add(submission)
         await session.flush()
@@ -913,6 +1037,8 @@ def run(self, spec, timeout_seconds: int):
             name="agent-attempt-commit",
             agent_hash="attempt-commit-hash",
             artifact_uri=str(agent_dir),
+            raw_status="queued",
+            effective_status="queued",
         )
         session.add(submission)
         await session.flush()
@@ -959,6 +1085,8 @@ async def test_run_evaluation_job_skips_already_persisted_task_result(
             name="agent-idempotent",
             agent_hash="idempotent-hash",
             artifact_uri=str(agent_dir),
+            raw_status="queued",
+            effective_status="queued",
         )
         session.add(submission)
         await session.flush()
@@ -2053,6 +2181,8 @@ async def test_base_sdk_retry_requeues_then_final_fails_at_worker_cap(
             name="platform-sdk-retry-agent",
             agent_hash="platform-sdk-retry-hash",
             artifact_uri=str(agent_dir),
+            raw_status="queued",
+            effective_status="queued",
         )
         session.add(submission)
         await session.flush()
@@ -2142,6 +2272,8 @@ def analyzer(_workspace, *, reviewer=None):
             name="agent-a",
             agent_hash="reviewer123",
             artifact_uri=str(agent_dir),
+            raw_status="queued",
+            effective_status="queued",
         )
         session.add(submission)
         await session.flush()
@@ -2188,6 +2320,8 @@ async def test_terminal_bench_mounts_extracted_zip_workspace(
             name="agent-a",
             agent_hash="zip789",
             artifact_uri=str(agent_zip),
+            raw_status="queued",
+            effective_status="queued",
             artifact_path=str(agent_zip),
         )
         session.add(submission)

packages/challenges/agent-challenge/tests/test_lifecycle_e2e.py

@@ -189,6 +189,11 @@ async def test_signed_allow_lifecycle_recovers_terminal_bench_and_scores_weight(
 
     reviewer = StaticReviewer("allow")
     async with database_session() as session:
+        submission = await session.get(AgentSubmission, submission_id)
+        assert submission is not None
+        # E2E full TB path: miner already confirmed empty env (FIX E default is unconfirmed).
+        submission.env_confirmed_empty = True
+        await session.flush()
         summary = await run_analysis_for_submission(
             session,
             submission_id,
@@ -590,6 +595,11 @@ async def _submit_and_analyze(client, database_session, *, reviewer: StaticRevie
     assert response.status_code == 201
     submission_id = response.json()["submission_id"]
     async with database_session() as session:
+        submission = await session.get(AgentSubmission, submission_id)
+        assert submission is not None
+        # E2E full TB path: miner already confirmed empty env (FIX E default is unconfirmed).
+        submission.env_confirmed_empty = True
+        await session.flush()
         await run_analysis_for_submission(
             session,
             submission_id,