diff --git a/AGENTS.md b/AGENTS.md index f5c038a3f..3c2902682 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -112,8 +112,9 @@ PostgreSQL (control-plane durability) env keys. 5. **AC review callback** hard-pinned to `https://chain.joinbase.ai/challenges/agent-challenge`. -6. **AGATE eval gate:** package LLM rules **residual** + `package_tree_sha` proof - **before** TEE auth; otherwise no eval / attestation. +6. **AGATE eval gate (host-trust):** package LLM rules **residual** (host kinds + admitted) + `package_tree_sha` proof **before** eval under host-trust; + never claim TEE / independent attestation. Phala dual flags are removed. 7. **Agent models:** no closed model catalog; **ban personal finetunes**. 8. **Tbench 2.1 tasks:** baked image content + digest; no miner-supplied task URL. 9. Do not wipe production Postgres / `KEY_FILE` / wallets unless a feature diff --git a/packages/challenges/agent-challenge/README.md b/packages/challenges/agent-challenge/README.md index 497ccf7c1..1e18d8f21 100644 --- a/packages/challenges/agent-challenge/README.md +++ b/packages/challenges/agent-challenge/README.md @@ -42,7 +42,12 @@ Self-deploy CLI accuracy fixtures remain under `docs/miner/self-deploy.md` and ## Agent-driven eval gate (product pin) -Scored path order: package + measured LLM rules residual → `package_tree_sha` -proof → fresh TEE authorization → **ONLY THEN** eval prepare / KR / score. -Host-static analyzer alone is not enough (**no eval prepare** without residual + -tree SHA). **no closed catalog** of models; **personal finetunes** banned. +**Host-trust only (T40):** Phala TEE dual flags and CVM attestation are removed. +Never claim TEE, tamper-proof execution, or independent hardware verification. + +Scored path order: package + host LLM rules residual → `package_tree_sha` +proof → host-trust eval prepare / score. Residual + tree SHA still required +before eval under host-trust. **no closed catalog** of models; **personal +finetunes** banned. + +See [docs/host-trust.md](docs/host-trust.md) and [docs/no-phala-mode.md](docs/no-phala-mode.md). diff --git a/packages/challenges/agent-challenge/config.example.yaml b/packages/challenges/agent-challenge/config.example.yaml index eba44d7b9..8dc540fd6 100644 --- a/packages/challenges/agent-challenge/config.example.yaml +++ b/packages/challenges/agent-challenge/config.example.yaml @@ -48,17 +48,12 @@ challenge: # prebuilt runner image's native Docker environment inside a privileged Docker-in-Docker # runner launched as a BASE broker job, without Daytona credentials. terminal_bench_execution_backend: own_runner - # Phala TEE attested-result emission (architecture sec 8). Opt-in and OFF by - # default: with the flag off the canonical image runs the legacy own_runner - # path byte-identically (R=1, epsilon=0 harbor scoring, no dstack access). This - # is the validator-config view of the in-image gate; its env var - # CHALLENGE_PHALA_ATTESTATION_ENABLED is the same switch the image reads. - phala_attestation_enabled: false - # Attested review is a separate explicit switch. Production full-attested - # mode enables this together with phala_attestation_enabled; mixed/single- - # sided settings fail closed at startup. Default OFF keeps legacy status and - # gateway-backed review byte-identical (no review field / no review session). - attested_review_enabled: false + # REMOVED product path (T40): Phala TEE dual flags. Both MUST stay false. + # ChallengeSettings rejects any true value. Product path is unattested / + # host-trust only (CHALLENGE_NO_PHALA / CHALLENGE_UNATTESTED_EXECUTION). + # Do not re-enable; there is no TEE product path. + phala_attestation_enabled: false # REMOVED (T40) — must stay false; host-trust only + attested_review_enabled: false # REMOVED (T40) — must stay false; host-trust only # Normative review TTLs, response caps, mutation rates, and resource limits. # Declarations mirror validation-contract resource limits; defaults are the # documented hard maxima. Routes/lifecycle code must read these keys. @@ -119,13 +114,10 @@ challenge: keep_good_tasks_policy: "off" keep_good_tasks_drop_lowest: 0 keep_good_tasks_threshold: 0.0 - # Low-rate replay-audit sampler (architecture sec 4 C6 / sec 8, defense-in-depth). - # Tier-driven replay fractions over the ATTESTED submission population: a VERIFIED - # Phala-tdx attestation is high-trust and audited at the low "attested" rate; an - # unverifiable/failed attestation is low-trust and audited at the higher - # "unverified" rate (higher trust => strictly lower rate). A rate of 0 disables - # auditing for that tier. The sampler only runs when phala_attestation_enabled is - # on, so flag-off scoring/weights stay byte-identical to legacy. Both rates [0,1]. + # Low-rate replay-audit sampler (defense-in-depth). Under host-trust (T40) the + # sampler stays inert because phala_attestation_enabled is permanently false. + # Rates retained for config compatibility only — not a TEE product path. + # Both rates [0,1]. replay_audit_attested_rate: 0.02 replay_audit_unverified_rate: 0.10 # Deterministic sampler seed: the same seed reproduces the identical sampled diff --git a/packages/challenges/agent-challenge/docker/review/phala_pre_launch.sh b/packages/challenges/agent-challenge/docker/review/phala_pre_launch.sh old mode 100644 new mode 100755 index 16a2430a9..1e9bdced3 --- a/packages/challenges/agent-challenge/docker/review/phala_pre_launch.sh +++ b/packages/challenges/agent-challenge/docker/review/phala_pre_launch.sh @@ -1,320 +1,8 @@ #!/bin/bash -echo "----------------------------------------------" -echo "Running Phala Cloud Pre-Launch Script v0.0.15" -echo "----------------------------------------------" -set -e - -# Function: notify host - -notify_host() { - if command -v dstack-util >/dev/null 2>&1; then - dstack-util notify-host -e "$1" -d "$2" - else - tdxctl notify-host -e "$1" -d "$2" - fi -} - -notify_host_hoot_info() { - notify_host "boot.progress" "$1" -} - -notify_host_hoot_error() { - notify_host "boot.error" "$1" -} - -# Function: Perform Docker cleanup -perform_cleanup() { - echo "Pruning unused images" - docker image prune -af - echo "Pruning unused volumes" - docker volume prune -f - notify_host_hoot_info "docker cleanup completed" -} - -# Function: Check Docker login status without exposing credentials -check_docker_login() { - local registry="$1" - - # When registry is specified, check auth entry for that registry in Docker config - if [[ -n "$registry" ]]; then - local docker_config_path="${DOCKER_CONFIG:-$HOME/.docker}/config.json" - if [[ -f "$docker_config_path" ]] && grep -q "$registry" "$docker_config_path"; then - return 0 - else - return 1 - fi - fi - - # Fallback check when no explicit registry is provided - if docker info 2>/dev/null | grep -q "Username"; then - return 0 - else - return 1 - fi -} - -# Main logic starts here -echo "Starting login process..." - -# Check if Docker credentials exist -if [[ -n "$DSTACK_DOCKER_USERNAME" && -n "$DSTACK_DOCKER_PASSWORD" ]]; then - echo "Docker credentials found" - DOCKER_REGISTRY_TARGET="${DSTACK_DOCKER_REGISTRY:-ghcr.io}" - echo "Target Docker registry: $DOCKER_REGISTRY_TARGET" - - # Check if already logged in - if check_docker_login "$DSTACK_DOCKER_REGISTRY"; then - echo "Already logged in to Docker registry: $DOCKER_REGISTRY_TARGET" - else - echo "Logging in to Docker registry: $DOCKER_REGISTRY_TARGET" - # Login without exposing password in process list - if [[ -n "$DSTACK_DOCKER_REGISTRY" ]]; then - echo "$DSTACK_DOCKER_PASSWORD" | docker login -u "$DSTACK_DOCKER_USERNAME" --password-stdin "$DSTACK_DOCKER_REGISTRY" - else - echo "$DSTACK_DOCKER_PASSWORD" | docker login -u "$DSTACK_DOCKER_USERNAME" --password-stdin - fi - - if [ $? -eq 0 ]; then - echo "Docker login successful: $DOCKER_REGISTRY_TARGET" - else - echo "Docker login failed: $DOCKER_REGISTRY_TARGET" - notify_host_hoot_error "docker login failed" - exit 1 - fi - fi -# Check if AWS ECR credentials exist -elif [[ -n "$DSTACK_AWS_ACCESS_KEY_ID" && -n "$DSTACK_AWS_SECRET_ACCESS_KEY" && -n "$DSTACK_AWS_REGION" && -n "$DSTACK_AWS_ECR_REGISTRY" ]]; then - echo "AWS ECR credentials found" - - # Check if AWS CLI is installed - if [ ! -f "./aws/dist/aws" ]; then - notify_host_hoot_info "awscli not installed, installing..." - echo "AWS CLI not installed, installing..." - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.24.14.zip" -o "awscliv2.zip" - echo "6ff031a26df7daebbfa3ccddc9af1450 awscliv2.zip" | md5sum -c - if [ $? -ne 0 ]; then - echo "MD5 checksum failed" - notify_host_hoot_error "awscli install failed" - exit 1 - fi - unzip awscliv2.zip &> /dev/null - else - echo "AWS CLI is already installed: ./aws/dist/aws" - fi - - # Set AWS credentials as environment variables - export AWS_ACCESS_KEY_ID="$DSTACK_AWS_ACCESS_KEY_ID" - export AWS_SECRET_ACCESS_KEY="$DSTACK_AWS_SECRET_ACCESS_KEY" - export AWS_DEFAULT_REGION="$DSTACK_AWS_REGION" - - # Set session token if provided (for temporary credentials) - if [[ -n "$DSTACK_AWS_SESSION_TOKEN" ]]; then - echo "AWS session token found, using temporary credentials" - export AWS_SESSION_TOKEN="$DSTACK_AWS_SESSION_TOKEN" - fi - - # Test AWS credentials before attempting ECR login - echo "Testing AWS credentials..." - if ! ./aws/dist/aws sts get-caller-identity &> /dev/null; then - echo "AWS credentials test failed" - # For session token credentials, this might be expected if they're expired - # Log warning but don't fail startup - if [[ -n "$DSTACK_AWS_SESSION_TOKEN" ]]; then - echo "Warning: AWS temporary credentials may have expired, continuing startup" - notify_host_hoot_info "AWS temporary credentials may have expired" - else - echo "AWS credentials test failed" - notify_host_hoot_error "Invalid AWS credentials" - exit 1 - fi - else - echo "Logging in to AWS ECR..." - ./aws/dist/aws ecr get-login-password --region $DSTACK_AWS_REGION | docker login --username AWS --password-stdin "$DSTACK_AWS_ECR_REGISTRY" - if [ $? -eq 0 ]; then - echo "AWS ECR login successful" - notify_host_hoot_info "AWS ECR login successful" - else - echo "AWS ECR login failed" - # For session token credentials, don't fail startup if login fails - if [[ -n "$DSTACK_AWS_SESSION_TOKEN" ]]; then - echo "Warning: AWS ECR login failed with temporary credentials, continuing startup" - notify_host_hoot_info "AWS ECR login failed with temporary credentials" - else - notify_host_hoot_error "AWS ECR login failed" - exit 1 - fi - fi - fi -fi - -perform_cleanup - -# -# GHCR image pull access verification (pure HTTP, no docker daemon) -# -if [[ "$DOCKER_REGISTRY_TARGET" == "ghcr.io" && -n "$DSTACK_DOCKER_USERNAME" && -n "$DSTACK_DOCKER_PASSWORD" ]]; then - COMPOSE_IMAGES=$(grep 'image:' /dstack/docker-compose.yaml 2>/dev/null | awk '{print $2}' | tr -d '"'"'" || true) - for img in $COMPOSE_IMAGES; do - [[ "$img" != ghcr.io/* ]] && continue - repo="${img#ghcr.io/}"; repo="${repo%%:*}" - tag="${img##*:}"; [[ "$tag" == "$img" || "$tag" == "$repo" ]] && tag="latest" - echo "Verifying GHCR pull access: $img" - token=$(curl -sf -u "$DSTACK_DOCKER_USERNAME:$DSTACK_DOCKER_PASSWORD" "https://ghcr.io/token?service=ghcr.io&scope=repository:${repo}:pull" | jq -r '.token // empty' || true) - if [[ -z "$token" ]]; then - echo "ERROR: GHCR token exchange failed for $img" - notify_host_hoot_error "GHCR token exchange failed: $img" - exit 1 - fi - http_code=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $token" -H "Accept: application/vnd.oci.image.index.v1+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json" "https://ghcr.io/v2/${repo}/manifests/${tag}") - if [[ "$http_code" != "200" ]]; then - echo "ERROR: GHCR pull access denied for $img (HTTP $http_code)" - notify_host_hoot_error "GHCR pull access denied: $img (HTTP $http_code)" - exit 1 - fi - echo "GHCR pull access OK: $img" - done -fi - -# -# Pull latest images from docker-compose.yaml so existing CVMs pick up new tags. -# Pull is fail-soft: short-lived tokens may expire between login and pull, but -# falling back to already-cached images is preferable to blocking boot. -# -echo "Images before pull:" -docker images --format '{{.Repository}}:{{.Tag}} {{.ID}} ({{.CreatedSince}})' - -echo "Pulling latest images from /dstack/docker-compose.yaml..." -if docker compose -f /dstack/docker-compose.yaml pull; then - echo "docker compose pull completed" - notify_host_hoot_info "docker compose pull completed" -else - echo "WARNING: docker compose pull failed; continuing with existing images" - notify_host_hoot_info "docker compose pull failed; using existing images" -fi - -echo "Images after pull:" -docker images --format '{{.Repository}}:{{.Tag}} {{.ID}} ({{.CreatedSince}})' - -# -# Set root password. -# -echo "Setting root password.." - -# Check if password files are writable -PASSWD_WRITABLE=true -if [ ! -w /etc/passwd ]; then - echo "Warning: /etc/passwd is read-only" - PASSWD_WRITABLE=false -fi -if [ ! -w /etc/shadow ]; then - echo "Warning: /etc/shadow is read-only" - PASSWD_WRITABLE=false -fi - -if [ "$PASSWD_WRITABLE" = "false" ]; then - echo "Skipping password setup due to read-only file system" -else - # Check if chpasswd is available - if command -v chpasswd >/dev/null 2>&1; then - echo "Using chpasswd method" - - if [ -n "$DSTACK_ROOT_PASSWORD" ]; then - echo "Setting root password from user.." - echo "root:$DSTACK_ROOT_PASSWORD" | chpasswd - unset DSTACK_ROOT_PASSWORD - echo "Root password set/updated from DSTACK_ROOT_PASSWORD" - elif [ -z "$(grep '^root:' /etc/shadow 2>/dev/null | cut -d: -f2)" ]; then - echo "Setting random root password.." - DSTACK_ROOT_PASSWORD=$( - LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | dd bs=1 count=32 2>/dev/null - ) - echo "root:$DSTACK_ROOT_PASSWORD" | chpasswd - unset DSTACK_ROOT_PASSWORD - echo "Root password set (random auto-init)" - else - echo "Root password already set; no changes." - fi - else - echo "Using passwd method" - - if [ -n "$DSTACK_ROOT_PASSWORD" ]; then - echo "Setting root password from user.." - echo "$DSTACK_ROOT_PASSWORD" | passwd --stdin root 2>/dev/null || printf '%s -%s -' "$DSTACK_ROOT_PASSWORD" "$DSTACK_ROOT_PASSWORD" | passwd root - unset DSTACK_ROOT_PASSWORD - echo "Root password set/updated from DSTACK_ROOT_PASSWORD" - elif [ -z "$(grep '^root:' /etc/shadow 2>/dev/null | cut -d: -f2)" ]; then - echo "Setting random root password.." - DSTACK_ROOT_PASSWORD=$( - LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | dd bs=1 count=32 2>/dev/null - ) - echo "$DSTACK_ROOT_PASSWORD" | passwd --stdin root 2>/dev/null || printf '%s -%s -' "$DSTACK_ROOT_PASSWORD" "$DSTACK_ROOT_PASSWORD" | passwd root - unset DSTACK_ROOT_PASSWORD - echo "Root password set (random auto-init)" - else - echo "Root password already set; no changes." - fi - fi -fi - -# -# Set SSH authorized keys -# -if mkdir -p /home/root/.ssh 2>/dev/null; then - if [[ -n "$DSTACK_ROOT_PUBLIC_KEY" ]]; then - echo "$DSTACK_ROOT_PUBLIC_KEY" > /home/root/.ssh/authorized_keys - unset $DSTACK_ROOT_PUBLIC_KEY - echo "Root public key set" - fi - if [[ -n "$DSTACK_AUTHORIZED_KEYS" ]]; then - echo "$DSTACK_AUTHORIZED_KEYS" > /home/root/.ssh/authorized_keys - unset $DSTACK_AUTHORIZED_KEYS - echo "Root authorized_keys set" - fi - - if [[ -f /dstack/user_config ]] && jq empty /dstack/user_config 2>/dev/null; then - if [[ $(jq 'has("ssh_authorized_keys")' /dstack/user_config 2>/dev/null) == "true" ]]; then - jq -j '.ssh_authorized_keys' /dstack/user_config >> /home/root/.ssh/authorized_keys - # Remove duplicates if there are multiple keys - if [[ $(cat /home/root/.ssh/authorized_keys | wc -l) -gt 1 ]]; then - sort -u /home/root/.ssh/authorized_keys > /home/root/.ssh/authorized_keys.tmp - mv /home/root/.ssh/authorized_keys.tmp /home/root/.ssh/authorized_keys - fi - echo "Set root authorized_keys from user preferences, total" $(cat /home/root/.ssh/authorized_keys | wc -l) "keys" - fi - fi -else - echo "Warning: Cannot create /home/root/.ssh directory (read-only file system?)" - echo "Skipping SSH key setup" -fi - -if [[ -S /var/run/dstack.sock ]]; then - export DSTACK_APP_ID=$(curl -s --unix-socket /var/run/dstack.sock http://dstack/Info | jq -j .app_id) -elif [[ -S /var/run/tappd.sock ]]; then - export DSTACK_APP_ID=$(curl -s --unix-socket /var/run/tappd.sock http://dstack/prpc/Tappd.Info | jq -j .app_id) -fi -# Check if DSTACK_GATEWAY_DOMAIN is not set, try to get it from user_config or app-compose.json -# Priority: user_config > app-compose.json -if [[ -z "$DSTACK_GATEWAY_DOMAIN" ]]; then - # First try to get from /dstack/user_config if it exists and is valid JSON - if [[ -f /dstack/user_config ]] && jq empty /dstack/user_config 2>/dev/null; then - if [[ $(jq 'has("default_gateway_domain")' /dstack/user_config 2>/dev/null) == "true" ]]; then - export DSTACK_GATEWAY_DOMAIN=$(jq -j '.default_gateway_domain' /dstack/user_config) - fi - fi - - # If still not set, try to get from app-compose.json - if [[ -z "$DSTACK_GATEWAY_DOMAIN" ]] && [[ $(jq 'has("default_gateway_domain")' app-compose.json) == "true" ]]; then - export DSTACK_GATEWAY_DOMAIN=$(jq -j '.default_gateway_domain' app-compose.json) - fi -fi -if [[ -n "$DSTACK_GATEWAY_DOMAIN" ]]; then - export DSTACK_APP_DOMAIN=$DSTACK_APP_ID"."$DSTACK_GATEWAY_DOMAIN -fi - -echo "----------------------------------------------" -echo "Script execution completed" -echo "----------------------------------------------" +# Phala Cloud Pre-Launch Script +# T40 host-trust stub: kept only so residual review compose generators and +# offline compose_hash helpers still resolve a measured helper string. +# Product path does not launch Phala CVMs. +set -euo pipefail +echo "phala_pre_launch stub (T40 host-trust only) — not a TEE product path" >&2 +exit 0 diff --git a/packages/challenges/agent-challenge/docs/host-trust.md b/packages/challenges/agent-challenge/docs/host-trust.md new file mode 100644 index 000000000..64ee3b80b --- /dev/null +++ b/packages/challenges/agent-challenge/docs/host-trust.md @@ -0,0 +1,10 @@ +# Host-trust execution (product path) + +After Phala TEE removal (T40), Agent Challenge production scoring is **host-trust only**: + +- Enable via `CHALLENGE_UNATTESTED_EXECUTION` / `CHALLENGE_NO_PHALA` / `NO_PHALA` +- Results are marked `attested=false` / `attestation_status=unattested` +- Integrity still uses `package_tree_sha` + AGATE residual (host residual kinds) +- **No** TDX quotes, **no** Phala CVM, **no** independent hardware attestation + +Do not describe this mode as TEE, tamper-proof, or independently verified. diff --git a/packages/challenges/agent-challenge/docs/miner/self-deploy.md b/packages/challenges/agent-challenge/docs/miner/self-deploy.md index 6304f2277..83a26e015 100644 --- a/packages/challenges/agent-challenge/docs/miner/self-deploy.md +++ b/packages/challenges/agent-challenge/docs/miner/self-deploy.md @@ -1,3 +1,5 @@ +> **T40/T41 product path:** Phala TEE dual flags and CVM attestation are **removed**. Host-trust / unattested only. Do not treat historical Phala CVM / TEE sections below as a shipping product path. + > OpenAPI: https://chain.joinbase.ai/challenges/agent-challenge/openapi.json · Day-1: root docs/miner/getting-started.md # Miner self-deploy (How-to advanced) diff --git a/packages/challenges/agent-challenge/docs/no-phala-mode.md b/packages/challenges/agent-challenge/docs/no-phala-mode.md index 31d438c13..0778beedf 100644 --- a/packages/challenges/agent-challenge/docs/no-phala-mode.md +++ b/packages/challenges/agent-challenge/docs/no-phala-mode.md @@ -1,3 +1,7 @@ +> **T40 product path:** Phala TEE dual flags and CVM attestation are **removed**. +> This document describes the **host-trust unattested** product mode. +> Never claim TEE, tamper-proof execution, or independent verification. + # NO_PHALA mode (temporary host-local unattested execution) **Status:** temporary operator escape hatch while Phala CVMs are disabled. diff --git a/packages/challenges/agent-challenge/docs/validator/self-deploy.md b/packages/challenges/agent-challenge/docs/validator/self-deploy.md index bbda4e9b8..fd04a6000 100644 --- a/packages/challenges/agent-challenge/docs/validator/self-deploy.md +++ b/packages/challenges/agent-challenge/docs/validator/self-deploy.md @@ -1,3 +1,5 @@ +> **T40/T41 product path:** Phala TEE dual flags and CVM attestation are **removed**. Host-trust / unattested only. Do not treat historical Phala CVM / TEE sections below as a shipping product path. + > OpenAPI: https://chain.joinbase.ai/challenges/agent-challenge/openapi.json · Day-1: root docs/miner/getting-started.md # Validator / operator self-deploy surfaces diff --git a/packages/challenges/agent-challenge/src/agent_challenge/canonical/attested_result.py b/packages/challenges/agent-challenge/src/agent_challenge/canonical/attested_result.py index f692b9dc5..d3e7e4459 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/canonical/attested_result.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/canonical/attested_result.py @@ -1,1170 +1,67 @@ -"""In-image attested-result emission for the canonical Phala eval image (M1). - -After the ``own_runner`` pipeline produces its per-task results, the canonical -image (running inside a Phala Intel TDX CVM) calls dstack ``get_quote(report_data)`` -and emits an *attested-result envelope* on the BASE ``ExecutionProof`` Phala tier -(architecture.md sec 6). The envelope rides along the SAME single, parseable -``BASE_BENCHMARK_RESULT=`` line the legacy path already emits (additive-only, so -the host-side parser is unaffected), extended with: - -* ``execution_proof`` -- an ``ExecutionProof`` (``tier == "phala-tdx"``) whose - ``attestation`` carries ``{tdx_quote, event_log, report_data, measurement, - vm_config}`` (a :class:`base.schemas.worker.PhalaAttestation`); and -* ``attestation_binding`` -- the architecture-sec-6 ``report_data`` preimage in - the clear (``agent_hash``, sorted ``task_ids``, per-task ``scores`` + - ``scores_digest``, ``validator_nonce``, ``canonical_measurement``) so a - validator can recompute ``report_data`` and check it against the quote. - -Trust & fail-closed invariants: - -* ``report_data`` is derived by :mod:`agent_challenge.canonical.report_data` - (byte-identical to base's single-source helper) and never exceeds the 64-byte - TDX field -- the 32-byte sec-6 digest is what is handed to ``get_quote``. -* If a genuine quote cannot be produced (dstack socket unavailable, - ``get_quote`` raises/times out, or returns an empty/malformed quote) the image - **fails closed**: :func:`emit_attested_or_failclosed` emits an explicit - ``failed`` result with a reason code and NO attestation envelope. It never - fabricates a ``tdx_quote``/``report_data`` and never emits a passing result as - if it were attested (VAL-IMG-034). - -base's ``ExecutionProof``/``PhalaAttestation`` models are not importable inside -the lean canonical image, so the envelope is built as plain dicts and validated -by self-contained conformance checks that mirror base's required fields/types. -The exact envelope shape is pinned to base's real models in -``base/tests/unit/test_worker_proof_phala.py`` (cross-repo conformance guard). -""" +"""Attested-result emission removed (T40). Host-trust only.""" from __future__ import annotations -import json -import re -import sys -from collections.abc import Iterable, Mapping -from dataclasses import dataclass -from typing import IO, Any, Protocol, runtime_checkable +from typing import Any -from agent_challenge.canonical import report_data as rd -from agent_challenge.canonical.measurement import ( - CANONICAL_MEASUREMENT_FIELDS, - CanonicalMeasurement, -) -from agent_challenge.evaluation.own_runner.result_schema import ( - RESULT_LINE_PREFIX, - build_benchmark_result, - emit_benchmark_result_line, - validate_benchmark_result, -) - -#: Phala Intel TDX tier value for ``ExecutionProof.tier``. MUST equal base's -#: ``PHALA_TDX_TIER`` so the emitted envelope is recognized by base's verifier. -PHALA_TDX_TIER = "phala-tdx" - -#: ``ExecutionProof.version`` the image emits (mirrors base ``EXECUTION_PROOF_VERSION``). -EXECUTION_PROOF_VERSION = 1 - -#: Additive key on the ``BASE_BENCHMARK_RESULT=`` payload carrying the envelope. +# Stable result-line key retained so host-trust tests can assert absence. EXECUTION_PROOF_RESULT_KEY = "execution_proof" -#: Additive key carrying the sec-6 ``report_data`` preimage (verifier-checkable). -ATTESTATION_BINDING_RESULT_KEY = "attestation_binding" -#: Reason code emitted on the fail-closed path when a genuine quote cannot be -#: produced (see :mod:`agent_challenge.evaluation.own_runner.reason_codes`). +# Legacy fail-closed reason code (wire/DB compat); not a TEE claim. PHALA_ATTESTATION_FAILED_REASON = "phala_attestation_failed" -#: Full measurement register set carried in the attestation (runtime ``rtmr3`` -#: included; the static, allowlist-pinnable subset is -#: :data:`CANONICAL_MEASUREMENT_FIELDS`). -MEASUREMENT_FIELDS: tuple[str, ...] = ( - "mrtd", - "rtmr0", - "rtmr1", - "rtmr2", - "rtmr3", - "compose_hash", - "os_image_hash", -) -_HEX64_RE = re.compile(r"^[0-9a-f]{64}$") -_HEX96_RE = re.compile(r"^[0-9a-f]{96}$") -_HEX128_RE = re.compile(r"^[0-9a-f]{128}$") - -#: Required attestation-payload fields (mirrors base ``PhalaAttestation``). -ATTESTATION_REQUIRED_FIELDS: tuple[str, ...] = ( - "tdx_quote", - "event_log", - "report_data", - "measurement", - "vm_config", -) - -#: Required ExecutionProof fields (mirrors base ``ExecutionProof``). -EXECUTION_PROOF_REQUIRED_FIELDS: tuple[str, ...] = ( - "version", - "tier", - "manifest_sha256", - "worker_signature", - "attestation", -) - -#: Max width of the TDX ``report_data`` field handed to ``get_quote``. -MAX_REPORT_DATA_BYTES = rd.PHALA_REPORT_DATA_BYTES - - -class EnvelopeSchemaError(ValueError): - """Raised when an attestation envelope violates the Phala-tier schema.""" - class AttestationEmissionError(RuntimeError): - """Raised when a genuine quote cannot be produced (drives fail-closed).""" - - -@runtime_checkable -class QuoteProvider(Protocol): - """A source of TDX quotes (dstack ``DstackClient`` in production).""" - - def get_quote(self, report_data: bytes) -> Any: # pragma: no cover - protocol - ... - - -@dataclass(frozen=True) -class QuoteResult: - """A validated quote: non-empty hex ``quote`` + parsed ``event_log``/``vm_config``.""" - - quote: str - event_log: list[dict[str, Any]] - vm_config: dict[str, Any] - - -# --------------------------------------------------------------------------- # -# dstack quote provider (lazy import so the module loads without a live socket) -# --------------------------------------------------------------------------- # -#: Live dstack get_quote frequently exceeds the SDK default of 3s. Bound key- -#: release and score quote acquisition so the acquire path cannot hang -#: indefinitely before the raw TCP dial on 8701. -DSTACK_QUOTE_TIMEOUT_SECONDS = 90.0 + """TEE attested emission is unavailable after Phala removal.""" class DstackQuoteProvider: - """Adapts the dstack SDK ``DstackClient`` to :class:`QuoteProvider`. - - ``dstack_sdk`` is imported lazily on first use so this module (and its - conformance/parse tests) import cleanly without the SDK's runtime socket. - The client connects to ``/var/run/dstack.sock`` inside the CVM by default. - Quote RPCs are bounded by :data:`DSTACK_QUOTE_TIMEOUT_SECONDS` so a stuck - guest socket cannot leave the CVM silent at eval_prepared for 30 minutes. - """ - - def __init__( - self, - endpoint: str | None = None, - *, - timeout_seconds: float = DSTACK_QUOTE_TIMEOUT_SECONDS, - ) -> None: - self._endpoint = endpoint - self._timeout_seconds = float(timeout_seconds) - self._client: Any | None = None - - def _get_client(self) -> Any: - if self._client is None: - from dstack_sdk import DstackClient - - # Prefer the timeout-aware constructor when present; fall back for - # older SDK stubs used in offline unit tests. - try: - self._client = ( - DstackClient(self._endpoint, timeout=self._timeout_seconds) - if self._endpoint - else DstackClient(timeout=self._timeout_seconds) - ) - except TypeError: - self._client = DstackClient(self._endpoint) if self._endpoint else DstackClient() - return self._client - - def get_quote(self, report_data: bytes) -> Any: - from agent_challenge.canonical.wallclock import WallclockTimeout, call_with_wallclock - - client = self._get_client() - get_quote = getattr(client, "get_quote", None) - if not callable(get_quote): - raise AttestationEmissionError("dstack client lacks get_quote") - - # Daemon-thread wallclock: never re-join a hung dstack get_quote after - # the deadline (ThreadPoolExecutor.__exit__ shutdown(wait=True) would). - try: - return call_with_wallclock( - lambda: get_quote(report_data), - timeout_seconds=self._timeout_seconds, - label="get_quote", - ) - except WallclockTimeout as exc: - raise AttestationEmissionError( - f"dstack get_quote exceeded {self._timeout_seconds:.0f}s wallclock" - ) from exc - except Exception as exc: # noqa: BLE001 - fail closed on any RPC error - text = str(exc).lower() - if "timeout" in text or "timed out" in text: - raise AttestationEmissionError(f"dstack get_quote timed out: {exc}") from exc - raise - - -# --------------------------------------------------------------------------- # -# Quote acquisition (fail-closed) -# --------------------------------------------------------------------------- # -def _coerce_event_log(raw: Any) -> list[dict[str, Any]]: - """Legacy shallow coerce (JSON string → list of dicts). Prefer KR normalize. - - Kept for call-sites that only need JSON parsing. Score emission goes through - :func:`obtain_quote`, which reuses the key-release normalizers so live - dstack GetQuote residuals (empty IMR3 digests, ``0x`` casing) pass RTMR3 - self-check and ``validate_eval_phala_attestation``. - """ - - if raw is None: - return [] - if isinstance(raw, str): - try: - raw = json.loads(raw) if raw.strip() else [] - except json.JSONDecodeError as exc: - raise AttestationEmissionError(f"quote event_log is not valid JSON: {exc}") from exc - if not isinstance(raw, list): - raise AttestationEmissionError("quote event_log is not a list of events") - return [dict(event) for event in raw] - - -def _coerce_vm_config(raw: Any) -> dict[str, Any]: - """Parse a dstack ``vm_config`` (JSON string, dict, or empty) to a plain dict. + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise AttestationEmissionError("DstackQuoteProvider removed with Phala TEE (T40)") - Does not project onto the schema-v2 key set; callers that emit Eval - attestation must run :func:`_project_eval_vm_config` so wire validate sees - exactly ``{vcpu, memory_mb, os_image_hash}``. - """ + def get_quote(self, *args: Any, **kwargs: Any) -> Any: + raise AttestationEmissionError("DstackQuoteProvider removed with Phala TEE (T40)") - if raw is None or raw == "": - return {} - if isinstance(raw, str): - try: - raw = json.loads(raw) if raw.strip() else {} - except json.JSONDecodeError as exc: - raise AttestationEmissionError(f"quote vm_config is not valid JSON: {exc}") from exc - if not isinstance(raw, Mapping): - raise AttestationEmissionError("quote vm_config is not an object") - return dict(raw) - -def _project_eval_vm_config( - raw: Any, - *, - os_image_hash: str | None = None, -) -> dict[str, Any]: - """Map dstack/host vm_config onto the sealed schema-v2 Eval key set. - - Port of review_runtime ``_normalize_vm_config`` for the score emit path: - - * ``cpu_count`` → ``vcpu``; ``memory_size`` (bytes) → ``memory_mb`` via floor // 1MiB - * extras dropped so ``set(result) == {vcpu, memory_mb, os_image_hash}`` - * ints coerced; ``os_image_hash`` prefers measurement-derived when provided - * fail closed when vcpu/cpu_count or memory_mb/memory_size cannot be derived - (score domain does not invent silent defaults like review's 1 / 2048) - """ - - if raw is None or raw == "": - raw = {} - if isinstance(raw, str): - text = raw.strip() - if not text: - raw = {} - else: - try: - raw = json.loads(text) - except json.JSONDecodeError as exc: - raise AttestationEmissionError(f"quote vm_config is not valid JSON: {exc}") from exc - if not isinstance(raw, Mapping): - raise AttestationEmissionError("quote vm_config is not an object") - - # dstack historically exposed cpu_count/memory_size (bytes); schema-v2 is - # exactly {vcpu, memory_mb, os_image_hash}. - if "vcpu" in raw: - vcpu = raw["vcpu"] - elif "cpu_count" in raw: - vcpu = raw["cpu_count"] - else: - raise AttestationEmissionError("quote vm_config missing vcpu (or dstack cpu_count)") - - if "memory_mb" in raw: - memory_mb = raw["memory_mb"] - elif "memory_size" in raw: - try: - memory_bytes = int(raw["memory_size"]) - except (TypeError, ValueError) as exc: - raise AttestationEmissionError("quote memory_size is invalid") from exc - if memory_bytes <= 0: - raise AttestationEmissionError("quote memory_size must be positive") - memory_mb = max(1, memory_bytes // (1024 * 1024)) - else: - raise AttestationEmissionError("quote vm_config missing memory_mb (or dstack memory_size)") - - try: - vcpu_i = int(vcpu) - memory_mb_i = int(memory_mb) - except (TypeError, ValueError) as exc: - raise AttestationEmissionError("quote vm_config vcpu/memory are invalid") from exc - if vcpu_i < 1 or memory_mb_i < 1: - raise AttestationEmissionError("quote vm_config vcpu/memory must be positive") - - image = raw.get("os_image_hash", os_image_hash) - if image is not None and os_image_hash is not None and image != os_image_hash: - # Prefer the measurement-derived image hash; dstack may omit the nested - # field or populate an untrusted host surface. - image = os_image_hash - if image is None and os_image_hash is not None: - image = os_image_hash - - return { - "vcpu": vcpu_i, - "memory_mb": memory_mb_i, - "os_image_hash": image, - } - - -def _score_normalize_quote_hex(quote: Any) -> str: - """Normalize score-path quote hex via KR ``_normalize_quote_hex`` (0x/case).""" - - from agent_challenge.keyrelease.client import ( - KeyReleaseProtocolError, - _normalize_quote_hex, - ) - - try: - return _normalize_quote_hex(quote) - except KeyReleaseProtocolError as exc: - raise AttestationEmissionError(f"get_quote quote_hex cannot be normalized: {exc}") from exc - - -def _visible_ascii_event_id(raw: Any) -> str: - """Map a raw dstack ``event`` field onto 0..128 visible ASCII (may be empty). - - Strips bytes outside U+0021..U+007E (``!``..``~``). Non-strings become empty - so the caller can decide between derivation vs fail-closed. - """ - - if not isinstance(raw, str): - return "" - cleaned = "".join(ch for ch in raw if "!" <= ch <= "~") - if len(cleaned) > 128: - cleaned = cleaned[:128] - return cleaned - - -def _derive_event_id_from_type(event_type: Any) -> str: - """Stable 1–128 visible fallback when the wire-required name is empty. - - Preferred when the entry is not RTMR3-runtime-bound (empty name would not - alter ``runtime_event_digest`` / RTMR3). Uses ``event-type-`` token form - when ``event_type`` is a non-negative int; otherwise the generic ``event``. - """ - - if ( - isinstance(event_type, int) - and not isinstance(event_type, bool) - and 0 <= event_type <= 0xFFFFFFFF - ): - candidate = f"event-type-{event_type}" - if 1 <= len(candidate) <= 128 and all("!" <= ch <= "~" for ch in candidate): - return candidate - return "event" - - -def _project_eval_event_log(raw: Any) -> list[dict[str, Any]]: - """Project GetQuote event_log so every entry.event is wire-legal 1–128. - - Applied *after* KR coerce + empty-IMR3 fill. Live residual after score - vm_config project (image@sha256:ffbb60a9): emit failed with - ``event_log[].event must be a 1-128 character visible ASCII id`` because - early IMR0–2 boot entries (and occasionally incomplete shapes) arrive with - empty / missing / null / control-containing ``event`` names. - - Policy (fail closed for RTMR3 integrity): - * non-empty string → strip control/non-visible bytes; keep if still 1–128 - * empty after normalize AND entry is IMR3 + dstack runtime type → fail closed - (event name is bound into ``runtime_event_digest``; inventing breaks RTMR3) - * empty after normalize AND not RTMR3-runtime-bound → derive from event_type - tokens (``event-type-N``) or ``event`` fallback - * when an entry already has the closed 5-key shape, re-emit that shape with - the projected event id; incomplete offline fixtures keep their keys and - only get event projected when coercible (legacy compatibility) - """ - - from agent_challenge.keyrelease.client import ( - KeyReleaseProtocolError, - _normalize_framed_event_log, - ) - from agent_challenge.keyrelease.quote import ( - APP_IMR, - DSTACK_RUNTIME_EVENT_TYPE, +def emit_attested_eval_result_from_plan(*args: Any, **kwargs: Any) -> Any: + raise AttestationEmissionError( + "emit_attested_eval_result_from_plan removed with Phala TEE (T40); " + "use host-trust unattested path (mark_result_unattested)" ) - if raw is None or raw == "": - return [] - # Allow callers to pass already-coerced lists without re-JSON parsing. - try: - if isinstance(raw, list) and raw and all(isinstance(item, Mapping) for item in raw): - entries = [dict(item) for item in raw] - else: - entries = _normalize_framed_event_log(raw, enforce_schema=False) - except KeyReleaseProtocolError as exc: - raise AttestationEmissionError(f"get_quote event_log cannot be projected: {exc}") from exc - except (ValueError, TypeError) as exc: - raise AttestationEmissionError(f"get_quote event_log cannot be projected: {exc}") from exc - - projected: list[dict[str, Any]] = [] - for index, raw_entry in enumerate(entries): - if not isinstance(raw_entry, Mapping): - raise AttestationEmissionError( - f"event_log[{index}] is not an object; refusing score emission" - ) - entry = dict(raw_entry) - imr = entry.get("imr") - event_type = entry.get("event_type") - event_raw = entry.get("event", "") - event_id = _visible_ascii_event_id(event_raw) - - closed_keys = {"imr", "event_type", "digest", "event", "event_payload"} - closed_shape = closed_keys <= set(entry) or set(entry) == closed_keys - has_int_imr = isinstance(imr, int) and not isinstance(imr, bool) - has_int_type = isinstance(event_type, int) and not isinstance(event_type, bool) - - if not event_id: - rtmr3_bound = ( - has_int_imr - and has_int_type - and imr == APP_IMR - and event_type == DSTACK_RUNTIME_EVENT_TYPE - ) - if rtmr3_bound: - raise AttestationEmissionError( - f"event_log[{index}] IMR3 runtime event is empty after project " - "(RTMR3-bound; refuse invented event id)" - ) - # Live residual: empty/missing/null/control IMR0–2 names. Invent a - # wire-legal id when the entry has event_type (or had an event key) - # so close-to-wire shapes pass schema-v2. Incomplete offline fakes - # that never had event/event_type keep their partial keys. - if closed_shape or "event" in entry or has_int_type: - event_id = _derive_event_id_from_type(event_type) - - if event_id: - if not (1 <= len(event_id) <= 128): - raise AttestationEmissionError( - f"event_log[{index}].event cannot be projected to a 1-128 visible ASCII id" - ) - if any(not ("!" <= ch <= "~") for ch in event_id): - raise AttestationEmissionError( - f"event_log[{index}].event is not visible ASCII after project" - ) - - if has_int_imr and has_int_type: - # Emit closed 5-key for live / KR-normalized residual. - digest = entry.get("digest", "") - if not isinstance(digest, str): - digest = "" - payload = entry.get("event_payload", "") - if not isinstance(payload, str): - payload = "" - if not event_id: - raise AttestationEmissionError( - f"event_log[{index}].event cannot be projected to a 1-128 visible ASCII id" - ) - projected.append( - { - "imr": imr, - "event_type": event_type, - "digest": digest, - "event": event_id, - "event_payload": payload, - } - ) - continue - - # Incomplete offline fixtures: keep original keys, only set event when - # we have a projected id (else leave as-is for legacy assert equality). - out = dict(entry) - if event_id: - out["event"] = event_id - projected.append(out) - return projected +def emit_attested_result(*args: Any, **kwargs: Any) -> Any: + raise AttestationEmissionError("emit_attested_result removed with Phala TEE (T40)") -def _score_normalize_event_log(raw: Any) -> list[dict[str, Any]]: - """Normalize score-path event_log via KR helpers + event-id project. - Live residual order: - 1. KR coerce (0x/case, closed keys, base64→hex) + empty IMR3 fill - 2. project every ``event`` onto 1–128 visible ASCII for schema-v2 wire +def build_phala_attestation(*args: Any, **kwargs: Any) -> Any: + raise AttestationEmissionError("build_phala_attestation removed with Phala TEE (T40)") - KR alone leaves early IMR0–2 dstack names empty/missing; emit then failed with - ``event_log[].event must be a 1-128 character visible ASCII id``. - """ - from agent_challenge.keyrelease.client import ( - KeyReleaseProtocolError, - _normalize_framed_event_log, +def emit_failclosed_result(*, total: int = 0, **kwargs: Any) -> None: + """Print a failed BASE_BENCHMARK_RESULT line (Phala emit path removed).""" + from agent_challenge.evaluation.own_runner.result_schema import ( + build_benchmark_result, + emit_benchmark_result_line, ) - try: - if raw is None: - return [] - filled = _normalize_framed_event_log(raw, enforce_schema=False) - return _project_eval_event_log(filled) - except AttestationEmissionError: - raise - except KeyReleaseProtocolError as exc: - raise AttestationEmissionError(f"get_quote event_log cannot be normalized: {exc}") from exc - except (ValueError, TypeError) as exc: - raise AttestationEmissionError(f"get_quote event_log cannot be normalized: {exc}") from exc - - -def obtain_quote(provider: QuoteProvider, report_data_digest: bytes) -> QuoteResult: - """Call ``provider.get_quote`` and return a validated :class:`QuoteResult`. - - Fail-closed: raises :class:`AttestationEmissionError` on ANY failure -- the - provider raising/timing out, or returning an empty/malformed quote -- so the - caller never mistakes a missing quote for a genuine attestation. The digest - handed to ``get_quote`` is guarded to never exceed the 64-byte TDX field. - - **Score-domain normalize:** reuses key-release GetQuote normalizers - (quote_hex lower/0x strip, event_log coerce + empty IMR3 - ``runtime_event_digest`` fill, closed-key projection) then projects every - ``event_log[].event`` onto schema-v2's 1–128 visible ASCII id so emit's - RTMR3 self-check and ``validate_eval_phala_attestation`` accept the same - live dstack shapes that KR framed grant already accepted (plus empty-name - early boot entries, which wire rejects after KR alone). - """ - - if not isinstance(report_data_digest, (bytes, bytearray)): - raise AttestationEmissionError("report_data handed to get_quote must be bytes") - if len(report_data_digest) > MAX_REPORT_DATA_BYTES: - raise AttestationEmissionError( - f"report_data is {len(report_data_digest)} bytes (> {MAX_REPORT_DATA_BYTES}); " - "refusing to hand an oversized value to get_quote" - ) - - try: - response = provider.get_quote(bytes(report_data_digest)) - except AttestationEmissionError: - raise - except Exception as exc: # noqa: BLE001 - fail closed on any provider failure - raise AttestationEmissionError(f"get_quote failed: {exc}") from exc - - quote_raw = getattr(response, "quote", None) - if quote_raw is None and isinstance(response, Mapping): - quote_raw = response.get("quote") - if not isinstance(quote_raw, str) or not quote_raw.strip(): - raise AttestationEmissionError("get_quote returned an empty or malformed quote") - quote = _score_normalize_quote_hex(quote_raw) - - if isinstance(response, Mapping): - event_raw = response.get("event_log") - vm_raw = response.get("vm_config") - else: - event_raw = getattr(response, "event_log", None) - vm_raw = getattr(response, "vm_config", None) - event_log = _score_normalize_event_log(event_raw) - # Project dstack-shaped vm_config (cpu_count/memory_size/extras) onto the - # exact schema-v2 set {vcpu, memory_mb, os_image_hash}. Empty raw stays - # {} so callers that fill via explicit env override still work; non-empty - # surfaces must be fully projectable or fail closed. - parsed_vm = _coerce_vm_config(vm_raw) - if not parsed_vm: - vm_config: dict[str, Any] = {} - else: - image_hint = parsed_vm.get("os_image_hash") - image_hint_str = image_hint if isinstance(image_hint, str) else None - vm_config = _project_eval_vm_config(parsed_vm, os_image_hash=image_hint_str) - return QuoteResult(quote=quote, event_log=event_log, vm_config=vm_config) - - -# --------------------------------------------------------------------------- # -# Envelope construction -# --------------------------------------------------------------------------- # -def build_measurement( - canonical_measurement: CanonicalMeasurement | Mapping[str, Any], *, rtmr3: str -) -> dict[str, str]: - """Full measurement register set for the attestation (static subset + ``rtmr3``).""" - - if isinstance(canonical_measurement, CanonicalMeasurement): - source: Mapping[str, Any] = canonical_measurement.as_dict() - elif isinstance(canonical_measurement, Mapping): - source = canonical_measurement - else: - raise TypeError( - "canonical_measurement must be a CanonicalMeasurement or mapping, " - f"not {type(canonical_measurement).__name__}" - ) - measurement = {field: str(source[field]) for field in CANONICAL_MEASUREMENT_FIELDS} - measurement["rtmr3"] = str(rtmr3) - return measurement - - -def build_phala_attestation( - *, - tdx_quote: str, - event_log: Iterable[Mapping[str, Any]], - report_data_hex: str, - measurement: Mapping[str, Any], - vm_config: Mapping[str, Any] | None = None, -) -> dict[str, Any]: - """Build (and conformance-check) a ``PhalaAttestation``-shaped payload dict.""" - - attestation: dict[str, Any] = { - "tdx_quote": tdx_quote, - "event_log": [dict(event) for event in event_log], - "report_data": report_data_hex, - "measurement": dict(measurement), - "vm_config": dict(vm_config) if vm_config else {}, - } - validate_phala_attestation(attestation) - return attestation - - -def placeholder_worker_signature() -> dict[str, str]: - """A schema-valid, explicitly-empty tier-0 worker signature. - - The Phala tier's trust anchor is the hardware quote; the sr25519 - ``worker_signature`` layer is (re)bound by the validator-side base adapter - (milestone M4). Until a worker signer is wired into the image the emitter - uses this explicit placeholder rather than fabricating a signature. - """ - - return {"worker_pubkey": "", "sig": ""} - - -def build_execution_proof_envelope( - *, - manifest_sha256: str, - attestation: Mapping[str, Any], - worker_signature: Mapping[str, str] | None = None, - tier: str = PHALA_TDX_TIER, - version: int = EXECUTION_PROOF_VERSION, - image_digest: str | None = None, - provider: Mapping[str, Any] | None = None, -) -> dict[str, Any]: - """Build (and conformance-check) an ``ExecutionProof``-shaped envelope dict.""" - - envelope: dict[str, Any] = { - "version": version, - "tier": tier, - "manifest_sha256": manifest_sha256, - "worker_signature": dict(worker_signature) - if worker_signature is not None - else placeholder_worker_signature(), - "attestation": dict(attestation), - } - if image_digest is not None: - envelope["image_digest"] = image_digest - if provider is not None: - envelope["provider"] = dict(provider) - validate_execution_proof_envelope(envelope) - return envelope - - -def build_attestation_binding( - *, - agent_hash: str, - task_ids: Iterable[str], - scores: Mapping[str, Any], - scores_digest: str, - validator_nonce: str, - canonical_measurement: CanonicalMeasurement | Mapping[str, Any], -) -> dict[str, Any]: - """The sec-6 ``report_data`` preimage in the clear (verifier-recomputable).""" - - if isinstance(canonical_measurement, CanonicalMeasurement): - measurement_source: Mapping[str, Any] = canonical_measurement.as_dict() - else: - measurement_source = canonical_measurement - return { - "agent_hash": agent_hash, - "task_ids": sorted(task_ids), - "scores": dict(scores), - "scores_digest": scores_digest, - "validator_nonce": validator_nonce, - "canonical_measurement": { - field: str(measurement_source[field]) for field in CANONICAL_MEASUREMENT_FIELDS - }, - } - - -# --------------------------------------------------------------------------- # -# Conformance validation (mirrors base's required fields / types) -# --------------------------------------------------------------------------- # -def validate_phala_attestation(payload: Any) -> None: - """Validate ``payload`` conforms to the base ``PhalaAttestation`` schema.""" - - if not isinstance(payload, Mapping): - raise EnvelopeSchemaError(f"attestation must be an object, got {type(payload).__name__}") - for field in ATTESTATION_REQUIRED_FIELDS: - if field not in payload: - raise EnvelopeSchemaError(f"attestation missing required field {field!r}") - quote = payload["tdx_quote"] - if ( - not isinstance(quote, str) - or not quote - or len(quote) % 2 - or len(quote) > 2 * 64 * 1024 - or quote != quote.lower() - or any(character not in "0123456789abcdef" for character in quote) - ): - raise EnvelopeSchemaError("attestation.tdx_quote must be lowercase bounded hex") - report_data = payload["report_data"] - if not isinstance(report_data, str) or _HEX128_RE.fullmatch(report_data) is None: - raise EnvelopeSchemaError("attestation.report_data must be 64-byte lowercase hex") - if not isinstance(payload["event_log"], list): - raise EnvelopeSchemaError("attestation.event_log must be a list") - if not all(isinstance(event, Mapping) for event in payload["event_log"]): - raise EnvelopeSchemaError("attestation.event_log entries must be objects") - if not isinstance(payload["vm_config"], Mapping): - raise EnvelopeSchemaError("attestation.vm_config must be an object") - measurement = payload["measurement"] - if not isinstance(measurement, Mapping): - raise EnvelopeSchemaError("attestation.measurement must be an object") - for field in MEASUREMENT_FIELDS: - if field not in measurement: - raise EnvelopeSchemaError(f"attestation.measurement missing register {field!r}") - width = _HEX96_RE if field.startswith(("mrtd", "rtmr")) else _HEX64_RE - if not isinstance(measurement[field], str) or width.fullmatch(measurement[field]) is None: - raise EnvelopeSchemaError( - f"attestation.measurement.{field} must have its exact lowercase hex width" - ) - - -def validate_execution_proof_envelope(payload: Any) -> None: - """Validate ``payload`` conforms to the base ``ExecutionProof`` Phala tier.""" - - if not isinstance(payload, Mapping): - raise EnvelopeSchemaError( - f"execution_proof must be an object, got {type(payload).__name__}" - ) - for field in EXECUTION_PROOF_REQUIRED_FIELDS: - if field not in payload: - raise EnvelopeSchemaError(f"execution_proof missing required field {field!r}") - if not isinstance(payload["version"], int) or isinstance(payload["version"], bool): - raise EnvelopeSchemaError("execution_proof.version must be an integer") - if not isinstance(payload["tier"], (int, str)) or isinstance(payload["tier"], bool): - raise EnvelopeSchemaError("execution_proof.tier must be an int or string") - if not isinstance(payload["manifest_sha256"], str) or not payload["manifest_sha256"]: - raise EnvelopeSchemaError("execution_proof.manifest_sha256 must be a non-empty string") - signature = payload["worker_signature"] - if not isinstance(signature, Mapping): - raise EnvelopeSchemaError("execution_proof.worker_signature must be an object") - for field in ("worker_pubkey", "sig"): - if field not in signature: - raise EnvelopeSchemaError(f"execution_proof.worker_signature missing {field!r}") - if not isinstance(signature[field], str): - raise EnvelopeSchemaError(f"execution_proof.worker_signature.{field} must be a string") - validate_phala_attestation(payload["attestation"]) - - -# --------------------------------------------------------------------------- # -# Extended result assembly + emission -# --------------------------------------------------------------------------- # -def build_attested_benchmark_result( - *, - benchmark_result: Mapping[str, Any], - execution_proof: Mapping[str, Any], - attestation_binding: Mapping[str, Any], -) -> dict[str, Any]: - """Extend a five-field benchmark result with the (additive) attestation blocks. - - The legacy five-field result contract is preserved byte-for-byte; the - envelope + binding ride along as additive keys. The result is re-validated - against the benchmark-result schema so the extended line stays parseable. - """ - - validate_benchmark_result(benchmark_result) - validate_execution_proof_envelope(execution_proof) - extended = dict(benchmark_result) - extended[EXECUTION_PROOF_RESULT_KEY] = dict(execution_proof) - extended[ATTESTATION_BINDING_RESULT_KEY] = dict(attestation_binding) - validate_benchmark_result(extended) - return extended - - -def emit_attested_benchmark_result( - *, - benchmark_result: Mapping[str, Any], - canonical_measurement: CanonicalMeasurement | Mapping[str, Any], - rtmr3: str, - agent_hash: str, - task_ids: Iterable[str], - scores: Mapping[str, Any], - quote_provider: QuoteProvider, - manifest_sha256: str, - validator_nonce: str | None = None, - eval_run_id: str | None = None, - submission_id: str | None = None, - score_nonce: str | None = None, - score_record: Mapping[str, Any] | None = None, - image_digest: str | None = None, - worker_signature: Mapping[str, str] | None = None, - vm_config: Mapping[str, Any] | None = None, - unit_id: str = "", - stream: IO[str] | None = None, -) -> str: - """Emit an attested ``BASE_BENCHMARK_RESULT=`` line for a completed run. - - Legacy callers receive the original additive benchmark-result envelope. - Supplying every schema-v2 Eval argument emits the exact Eval result request - v1 instead. Both paths fail closed if a genuine quote cannot be produced. - """ - - v2_values = (eval_run_id, submission_id, score_nonce, score_record, image_digest) - if any(value is not None for value in v2_values): - if validator_nonce is not None or not all(value is not None for value in v2_values): - raise AttestationEmissionError( - "schema-v2 emission requires eval run, submission, score nonce, " - "score record, image digest, and no validator_nonce" - ) - return _emit_schema_v2_eval_result( - canonical_measurement=canonical_measurement, - rtmr3=rtmr3, - agent_hash=agent_hash, - task_ids=task_ids, - quote_provider=quote_provider, - manifest_sha256=manifest_sha256, - eval_run_id=str(eval_run_id), - submission_id=str(submission_id), - score_nonce=str(score_nonce), - score_record=score_record, - image_digest=str(image_digest), - vm_config=vm_config, - stream=stream, - ) - - if validator_nonce is None: - raise AttestationEmissionError( - "validator_nonce is required for legacy attestation emission" - ) - digest = rd.report_data( - canonical_measurement=canonical_measurement, - agent_hash=agent_hash, - task_ids=task_ids, - scores_digest=rd.scores_digest(scores), - validator_nonce=validator_nonce, - ) - report_data_field = rd.to_report_data_field(digest) - - quote = obtain_quote(quote_provider, digest) - - measurement = build_measurement(canonical_measurement, rtmr3=rtmr3) - attestation = build_phala_attestation( - tdx_quote=quote.quote, - event_log=quote.event_log, - report_data_hex=report_data_field, - measurement=measurement, - vm_config=vm_config if vm_config is not None else quote.vm_config, - ) - envelope = build_execution_proof_envelope( - manifest_sha256=manifest_sha256, - attestation=attestation, - worker_signature=worker_signature, - ) - binding = build_attestation_binding( - agent_hash=agent_hash, - task_ids=task_ids, - scores=scores, - scores_digest=rd.scores_digest(scores), - validator_nonce=validator_nonce, - canonical_measurement=canonical_measurement, - ) - extended = build_attested_benchmark_result( - benchmark_result=benchmark_result, - execution_proof=envelope, - attestation_binding=binding, - ) - return emit_benchmark_result_line(extended, stream=stream) - - -def _emit_schema_v2_eval_result( - *, - canonical_measurement: CanonicalMeasurement | Mapping[str, Any], - rtmr3: str, - agent_hash: str, - task_ids: Iterable[str], - quote_provider: QuoteProvider, - manifest_sha256: str, - eval_run_id: str, - submission_id: str, - score_nonce: str, - score_record: Mapping[str, Any], - image_digest: str, - vm_config: Mapping[str, Any] | None, - stream: IO[str] | None, -) -> str: - """Emit the schema-closed Eval result request v1 on the sole result line.""" - - from agent_challenge.canonical import eval_wire as ew - - try: - score_digest = ew.score_record_digest(score_record) - binding = ew.build_score_binding( - canonical_measurement=canonical_measurement, - agent_hash=agent_hash, - eval_run_id=eval_run_id, - score_nonce=score_nonce, - scores_digest=score_digest, - task_ids=list(task_ids), - ) - report_data_hex = ew.score_report_data_hex(binding) - quote = obtain_quote(quote_provider, bytes.fromhex(report_data_hex)) - # A production TDX v4 quote carries the authoritative RTMR3 register. - # Recompute it from the ordered event log and reject divergence. Short - # synthetic quote fixtures used by offline legacy tests retain their - # supplied runtime value because they have no parseable TD report. - try: - from agent_challenge.keyrelease.quote import parse_tdx_quote_v4, replay_rtmr3 - - parsed_rtmr3 = parse_tdx_quote_v4(quote.quote).rtmr3 - replayed_rtmr3 = replay_rtmr3(quote.event_log).rtmr3 - if parsed_rtmr3 != replayed_rtmr3: - raise AttestationEmissionError("quote RTMR3 does not match its event log") - rtmr3 = parsed_rtmr3 - except AttestationEmissionError: - raise - except Exception: - # Offline compatibility fixtures may use opaque quote bytes. A - # genuine production TDX v4 quote is parseable here and therefore - # always takes the authoritative quote/event-log path above. - pass - measurement = build_measurement(canonical_measurement, rtmr3=rtmr3) - # Project env override OR quote.vm_config onto exact - # {vcpu, memory_mb, os_image_hash} before wire validate. Live residual: - # raw dstack extras (cpu_count/memory_size/qemu_*) hit "invalid fields". - measurement_image = measurement.get("os_image_hash") - measurement_image_str = measurement_image if isinstance(measurement_image, str) else None - raw_for_project: Any = vm_config if vm_config is not None else quote.vm_config - projected_vm = _project_eval_vm_config( - raw_for_project, - os_image_hash=measurement_image_str, - ) - attestation = ew.validate_eval_phala_attestation( - { - "tdx_quote": quote.quote, - "event_log": quote.event_log, - "report_data": report_data_hex, - "measurement": measurement, - "vm_config": projected_vm, - } - ) - execution_proof = ew.validate_eval_execution_proof( - { - "version": EXECUTION_PROOF_VERSION, - "tier": PHALA_TDX_TIER, - "manifest_sha256": manifest_sha256, - "image_digest": image_digest, - "provider": None, - "worker_signature": placeholder_worker_signature(), - "attestation": attestation, - } - ) - request = ew.validate_eval_result_request( - { - "schema_version": 1, - "eval_run_id": eval_run_id, - "submission_id": submission_id, - "agent_hash": agent_hash, - "score_record": score_record, - "scores_digest": score_digest, - "execution_proof": execution_proof, - } - ) - except (ew.EvalWireError, ValueError, TypeError) as exc: - raise AttestationEmissionError(f"schema-v2 Eval emission is invalid: {exc}") from exc - - # Host process_direct_eval_result requires raw POST body bytes == - # eval_wire.canonical_json_v1(validated). Default json.dumps separators - # insert spaces and arrive as result_noncanonical; emit compact sorted - # form that is byte-identical to the host apply path. - body = ew.canonical_json_v1(request).decode("utf-8") - line = RESULT_LINE_PREFIX + body - target = stream if stream is not None else sys.stdout - target.write(line + "\n") - return line - - -def emit_attested_eval_result_from_plan( - *, - eval_plan: Mapping[str, Any], - score_record: Mapping[str, Any], - rtmr3: str, - quote_provider: QuoteProvider, - manifest_sha256: str, - vm_config: Mapping[str, Any] | None = None, - stream: IO[str] | None = None, -) -> str: - """Emit a strict Eval result using only immutable plan-derived bindings.""" - - from agent_challenge.canonical import eval_wire as ew - - try: - plan = ew.validate_eval_plan(eval_plan) - task_ids = [task["task_id"] for task in plan["selected_tasks"]] - validated_record = ew.validate_canonical_score_record( - score_record, - scoring_policy=plan["scoring_policy"], - expected_eval_run_id=plan["eval_run_id"], - expected_task_ids=task_ids, - expected_k=plan["k"], - ) - measurement = { - "mrtd": plan["eval_app"]["measurement"]["mrtd"], - "rtmr0": plan["eval_app"]["measurement"]["rtmr0"], - "rtmr1": plan["eval_app"]["measurement"]["rtmr1"], - "rtmr2": plan["eval_app"]["measurement"]["rtmr2"], - "compose_hash": plan["eval_app"]["compose_hash"], - "os_image_hash": plan["eval_app"]["measurement"]["os_image_hash"], - } - except ew.EvalWireError as exc: - raise AttestationEmissionError( - f"invalid immutable Eval plan or score record: {exc}" - ) from exc - return emit_attested_benchmark_result( - benchmark_result=build_benchmark_result( - status="completed", - score=ew.decode_score_f64be(validated_record["final"]["job_score_f64be"]), - resolved=validated_record["final"]["passed_tasks"], - total=validated_record["final"]["total_tasks"], - reason_code=None, - ), - canonical_measurement=measurement, - rtmr3=rtmr3, - agent_hash=plan["agent_hash"], - task_ids=task_ids, - scores={}, - quote_provider=quote_provider, - manifest_sha256=manifest_sha256, - eval_run_id=plan["eval_run_id"], - submission_id=plan["submission_id"], - score_nonce=plan["score_nonce"], - score_record=validated_record, - image_digest=plan["eval_app"]["image_ref"], - vm_config=vm_config, - stream=stream, - ) - - -def emit_failclosed_result( - *, - total: int, - reason_code: str = PHALA_ATTESTATION_FAILED_REASON, - stream: IO[str] | None = None, -) -> str: - """Emit a ``failed`` result with NO attestation (the fail-closed line). - - Never carries an ``execution_proof``/``attestation_binding`` and never a - passing score, so a missing/failed quote can never be mistaken downstream for - a genuine attested result. - """ - - failed = build_benchmark_result( + payload = build_benchmark_result( status="failed", score=0.0, resolved=0, total=int(total), - reason_code=reason_code, + reason_code=PHALA_ATTESTATION_FAILED_REASON, ) - return emit_benchmark_result_line(failed, stream=stream) - - -def emit_attested_or_failclosed( - *, - benchmark_result: Mapping[str, Any], - canonical_measurement: CanonicalMeasurement | Mapping[str, Any], - rtmr3: str, - agent_hash: str, - task_ids: Iterable[str], - scores: Mapping[str, Any], - quote_provider: QuoteProvider, - manifest_sha256: str, - validator_nonce: str | None = None, - eval_run_id: str | None = None, - submission_id: str | None = None, - score_nonce: str | None = None, - score_record: Mapping[str, Any] | None = None, - image_digest: str | None = None, - worker_signature: Mapping[str, str] | None = None, - vm_config: Mapping[str, Any] | None = None, - unit_id: str = "", - stream: IO[str] | None = None, -) -> tuple[str, bool]: - """Emit the attested line, or a fail-closed line if no genuine quote exists. - - Returns ``(emitted_line, attested)``. ``attested`` is ``True`` only when a - genuine quote was obtained and the attested envelope was emitted; on any - :class:`AttestationEmissionError` it is ``False`` and the emitted line is an - explicit ``failed`` result with no fabricated attestation (VAL-IMG-034). - """ - - try: - line = emit_attested_benchmark_result( - benchmark_result=benchmark_result, - canonical_measurement=canonical_measurement, - rtmr3=rtmr3, - agent_hash=agent_hash, - task_ids=task_ids, - scores=scores, - validator_nonce=validator_nonce, - quote_provider=quote_provider, - manifest_sha256=manifest_sha256, - eval_run_id=eval_run_id, - submission_id=submission_id, - score_nonce=score_nonce, - score_record=score_record, - image_digest=image_digest, - worker_signature=worker_signature, - vm_config=vm_config, - unit_id=unit_id, - stream=stream, - ) - return line, True - except AttestationEmissionError: - total = _result_total(benchmark_result, task_ids) - line = emit_failclosed_result(total=total, stream=stream) - return line, False - - -def _result_total(benchmark_result: Mapping[str, Any], task_ids: Iterable[str]) -> int: - """Best-effort task total for the fail-closed line (result ``total`` else count).""" - - total = benchmark_result.get("total") if isinstance(benchmark_result, Mapping) else None - if isinstance(total, int) and not isinstance(total, bool): - return total - return len(list(task_ids)) + emit_benchmark_result_line(payload) __all__ = [ - "ATTESTATION_BINDING_RESULT_KEY", - "ATTESTATION_REQUIRED_FIELDS", "AttestationEmissionError", "DstackQuoteProvider", - "EXECUTION_PROOF_REQUIRED_FIELDS", "EXECUTION_PROOF_RESULT_KEY", - "EXECUTION_PROOF_VERSION", - "EnvelopeSchemaError", - "MEASUREMENT_FIELDS", "PHALA_ATTESTATION_FAILED_REASON", - "PHALA_TDX_TIER", - "QuoteProvider", - "QuoteResult", - "build_attestation_binding", - "build_attested_benchmark_result", - "build_execution_proof_envelope", - "build_measurement", "build_phala_attestation", - "emit_attested_benchmark_result", "emit_attested_eval_result_from_plan", - "emit_attested_or_failclosed", + "emit_attested_result", "emit_failclosed_result", - "obtain_quote", - "placeholder_worker_signature", - "validate_execution_proof_envelope", - "validate_phala_attestation", ] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/canonical/compose.py b/packages/challenges/agent-challenge/src/agent_challenge/canonical/compose.py deleted file mode 100644 index 3ce80019b..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/canonical/compose.py +++ /dev/null @@ -1,555 +0,0 @@ -"""Generate the Phala ``app-compose`` the miner deploys (architecture §4 C2). - -The miner self-deploys a Phala TDX CPU CVM running the canonical eval image. The -CVM is described by an ``app-compose.json`` document that embeds a docker-compose -file plus dstack deployment flags. This module generates that document such that: - -* **Only the orchestrator service is declared** — the ~89 Terminal-Bench task - images are NOT static compose services. They are pinned by digest via the - golden manifest (mounted read-only) and launched dynamically as siblings on - the guest Docker socket (DooD) at runtime (VAL-ORCH-032). -* **No secrets are embedded** — the compose carries no gateway token, no - miner-env values, no provider ``*_API_KEY``, and no Phala API key. Secrets are - supplied at deploy time via dstack ``encrypted_env`` for the names listed in - ``allowed_envs`` (VAL-ORCH-033). -* **Generation is deterministic** — the same inputs always produce byte-identical - output, so the SHA-256 compose-hash is stable and matches the value dstack - measures into RTMR3 on deploy (VAL-ORCH-034). - -**Critical byte-for-byte contract (library/measurement-tooling.md):** the bytes -actually written to ``app-compose.json`` and deployed MUST equal -:func:`agent_challenge.canonical.measurement.normalize_app_compose` of the -generated document verbatim. :func:`render_app_compose` is the ONLY serializer a -deployer should use — never a separate ``json.dumps``/pretty-print — otherwise -the live compose-hash will not equal the offline -:func:`agent_challenge.canonical.measurement.compose_hash` and the pinned -allowlist match (M6 verification) fails. - -Import-light (stdlib + the measurement helper) so it loads in the lean image. -""" - -from __future__ import annotations - -import json -import re -from collections.abc import Mapping, Sequence -from pathlib import Path -from typing import Any - -from agent_challenge.canonical.key_release_endpoint import ( - DEFAULT_KEY_RELEASE_RA_TLS_PORT, - is_validator_key_release_authority, - parse_key_release_authority, - parse_raw_key_release_bake, -) -from agent_challenge.canonical.live_registry import LIVE_REGISTRY_ENV -from agent_challenge.canonical.measurement import compose_hash, normalize_app_compose -from agent_challenge.evaluation.own_runner.dood import ( - DOCKER_SOCKET_PATH, - DSTACK_SOCKET_PATH, -) -from agent_challenge.keyrelease.client import ( - KEY_RELEASE_TLS_CA_ENV, - KEY_RELEASE_TLS_CERT_ENV, - KEY_RELEASE_TLS_KEY_ENV, - KEY_RELEASE_URL_ENV, -) - -#: Production raw RA-TLS endpoint env names (mirrored from the validator listener -#: without importing the server module, which must stay out of the lean image). -RA_TLS_HOST_ENV = "KEY_RELEASE_RA_TLS_HOST" -RA_TLS_PORT_ENV = "KEY_RELEASE_RA_TLS_PORT" - -#: Default in-CVM paths for raw RA-TLS client mTLS material. -DEFAULT_KEY_RELEASE_TLS_CERT_PATH = "/run/secrets/ra_tls/client.crt" -DEFAULT_KEY_RELEASE_TLS_KEY_PATH = "/run/secrets/ra_tls/client.key" -DEFAULT_KEY_RELEASE_TLS_CA_PATH = "/run/secrets/ra_tls/ca.crt" - -#: dstack app-compose runner + manifest version for a docker-compose app. -APP_COMPOSE_MANIFEST_VERSION = 2 -APP_COMPOSE_RUNNER = "docker-compose" - -#: Default canonical app name. -DEFAULT_APP_NAME = "agent-challenge-canonical" - -#: Repository root (``src/agent_challenge/canonical/compose.py`` → repo). -REPO_ROOT = Path(__file__).resolve().parents[3] -#: Phala Cloud injects this fixed boot helper into every app-compose document. -#: The measured compose_hash includes it, so the local generator must emit the -#: same bytes or ``POST /cvms/provision`` identity checks fail closed. Shared -#: with the review compose path (same vendor helper file). -PHALA_PRE_LAUNCH_SCRIPT_PATH = REPO_ROOT / "docker" / "review" / "phala_pre_launch.sh" -#: Default Phala Cloud ``features`` factor measured into compose_hash. -PHALA_DEFAULT_FEATURES: tuple[str, ...] = ("kms", "tproxy-net") -#: Top-level keys of the provision-compatible (Phala-envelope) eval app-compose. -#: Review uses the same envelope factors with a disjoint service inventory. -PHALA_APP_COMPOSE_ENVELOPE_KEYS: frozenset[str] = frozenset( - { - "manifest_version", - "name", - "runner", - "docker_compose_file", - "kms_enabled", - "gateway_enabled", - "tproxy_enabled", - "local_key_provider_enabled", - "public_logs", - "public_sysinfo", - "public_tcbinfo", - "no_instance_id", - "secure_time", - "storage_fs", - "features", - "allowed_envs", - "pre_launch_script", - } -) - -#: In-CVM paths for the orchestrator job dir, task cache, and golden manifest. -DEFAULT_JOB_DIR = "/opt/agent-challenge/job" -DEFAULT_CACHE_ROOT = "/opt/agent-challenge/task-cache" -DEFAULT_GOLDEN_DIR = "/opt/agent-challenge/golden" -DEFAULT_DIGEST_MANIFEST = "/opt/agent-challenge/golden/dataset-digest.json" - -# Guest sockets bind-mounted into the orchestrator (DooD + attestation) are -# single-sourced from :mod:`agent_challenge.evaluation.own_runner.dood` (the DooD -# launch-policy reference) so the compose mounts and the socket-exposure guard can -# never diverge: DOCKER_SOCKET_PATH / DSTACK_SOCKET_PATH are imported above. - -#: Orchestrator service name in the generated compose. -ORCHESTRATOR_SERVICE = "orchestrator" - -#: Env var NAMES injected at deploy via dstack ``encrypted_env`` (values NEVER in -#: the compose bytes). These are the per-run Phala binding inputs only. -#: VAL-ACAT-013: Base LLM gateway names (``BASE_GATEWAY_TOKEN``, -#: ``BASE_LLM_GATEWAY_URL``, …) are intentionally **absent**. Production key-release -#: identity is bound into static compose env as ``KEY_RELEASE_RA_TLS_HOST`` / -#: ``KEY_RELEASE_RA_TLS_PORT`` plus the required client mTLS path names. -#: VAL-ACLOCK-009: free ``CHALLENGE_PHALA_KEY_RELEASE_URL`` remains listed only so -#: existing measure-time pin hashes stay byte-stable when the HTTPS placeholder is -#: baked as static compose env. It is **not** a miner trust root: plan wire rejects -#: free HTTP KR hosts, and :func:`encrypt_eval_secrets` refuses free URL values that -#: are not the validator RA-TLS authority matching the signed plan. -DEFAULT_ALLOWED_ENVS: tuple[str, ...] = ( - "CHALLENGE_PHALA_AGENT_HASH", - "CHALLENGE_PHALA_ATTESTATION_ENABLED", - "CHALLENGE_PHALA_CANONICAL_MEASUREMENT", - "CHALLENGE_PHALA_EVAL_PLAN", - "CHALLENGE_PHALA_KEY_RELEASE_URL", - "CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", - "CHALLENGE_PHALA_RA_TLS_SERVER_CA_FILE", - "CHALLENGE_PHALA_RTMR3", - "CHALLENGE_PHALA_VALIDATOR_NONCE", - # Residual ORCH public_logs probes (VAL-ORCH-009/010/014/022): opt-in only, - # secret-free marker emission for non-dev (no SSH) live residual scrape. - "CHALLENGE_RESIDUAL_ORCH_PROBES", - KEY_RELEASE_TLS_CERT_ENV, - KEY_RELEASE_TLS_KEY_ENV, - KEY_RELEASE_TLS_CA_ENV, - "LLM_COST_LIMIT", - "EVAL_RUN_TOKEN", - # Mid-run progress reporter (observability only; never score-bearing). - "EVAL_PROGRESS_BASE_URL", - "EVAL_RUN_ID", - "EVAL_SUBMISSION_ID", - # Local CVM signing secret name only — value never leaves the guest / never hits master APIs. - "RUNNER_HOTKEY_MNEMONIC", - # Measured OpenRouter (eval agent inside measured CVM only when product allows). - # Never Base gateway; keys stay miner/session encrypted_env on attested guests. - "OPENROUTER_API_KEY", -) - - -#: Env names that may appear in encrypted_env / runner config but are NOT part of -#: the measured compose_hash pin (``04011776…``). Progress/telemetry is optional -#: observability injected at deploy; baking the names into allowed_envs would -#: rotate every historical pin. ``encrypt_eval_secrets`` still accepts these via -#: ``DEFAULT_ALLOWED_ENVS``; compose generation for eval pins uses -#: :data:`MEASURED_ALLOWED_ENVS`. -PROGRESS_OPTIONAL_ENVS: tuple[str, ...] = ( - "EVAL_PROGRESS_BASE_URL", - "EVAL_RUN_ID", - "EVAL_SUBMISSION_ID", - "RUNNER_HOTKEY_MNEMONIC", -) - -#: Pin-stable allowlist used when rendering measured app-compose bytes. -MEASURED_ALLOWED_ENVS: tuple[str, ...] = tuple( - name for name in DEFAULT_ALLOWED_ENVS if name not in set(PROGRESS_OPTIONAL_ENVS) -) - -_DIGEST_PIN_RE = re.compile(r"@sha256:[0-9a-f]{64}$") -_DIGEST_REF_RE = re.compile(r"^(?:sha256:)?[0-9a-f]{64}$") - - -class ComposeGenerationError(ValueError): - """A compose could not be generated deterministically / safely.""" - - -def _parse_raw_key_release_endpoint(endpoint: str) -> tuple[str, int] | None: - """Return ``(host, port)`` for production raw RA-TLS bake into static compose env. - - Accepts ``host:8701``, ``ratls://host:port``, ``tls://host:port``, or - ``tcp://host:port``. HTTP(S) URLs and non-raw authorities return ``None`` so - the measure-time HTTPS pin placeholder can still land as a non-baked static - compose factor for offline hash matching (compose pin only — not plan trust). - """ - - return parse_raw_key_release_bake(endpoint) - - -def assert_digest_pinned(image_ref: str, *, what: str = "image") -> str: - """Require ``image_ref`` to be pinned by an immutable ``@sha256:`` digest. - - A floating tag (``:latest``) would break reproducibility of the compose-hash - and the canonical measurement, so it is rejected fail-closed. - """ - - if not isinstance(image_ref, str) or not _DIGEST_PIN_RE.search(image_ref): - raise ComposeGenerationError( - f"{what} must be digest-pinned (repo@sha256:<64hex>), got {image_ref!r}" - ) - return image_ref - - -def golden_task_image_digests(manifest: Mapping[str, Any]) -> dict[str, str]: - """Return ``task_id -> sha256:`` for the golden manifest's task images. - - Terminal-Bench task images are pinned by digest here (never a mutable tag) and - are launched dynamically via DooD at runtime, so they never appear as static - compose services (VAL-ORCH-032). - """ - - tasks = manifest.get("tasks") - if not isinstance(tasks, Mapping): - raise ComposeGenerationError("golden manifest has no 'tasks' mapping") - pins: dict[str, str] = {} - for task_id, entry in tasks.items(): - if not isinstance(entry, Mapping): - raise ComposeGenerationError(f"golden manifest task {task_id!r} is not a mapping") - ref = entry.get("harbor_registry_ref") or entry.get("content_digest_sha256") - if not isinstance(ref, str) or not _DIGEST_REF_RE.match(ref): - raise ComposeGenerationError( - f"golden manifest task {task_id!r} is not digest-pinned: {ref!r}" - ) - pins[str(task_id)] = ref if ref.startswith("sha256:") else f"sha256:{ref}" - return pins - - -def load_golden_manifest(path: Path | str) -> dict[str, Any]: - """Load a golden ``dataset-digest.json`` manifest from disk.""" - - data = json.loads(Path(path).read_text(encoding="utf-8")) - if not isinstance(data, dict): - raise ComposeGenerationError("golden manifest is not a JSON object") - return data - - -# --------------------------------------------------------------------------- # -# Deterministic YAML emitter (stdlib only; sorted keys; JSON-quoted scalars) -# --------------------------------------------------------------------------- # -def _yaml_scalar(value: Any) -> str: - """Render a scalar as a valid, unambiguous YAML token. - - Strings are emitted as JSON double-quoted literals (a valid YAML flow scalar), - which sidesteps YAML's quoting rules for values like ``"8700:8700"`` or an - ``@sha256:`` image ref. ``bool``/``int``/``None`` map to YAML ``true``/``false``, - the integer literal, and ``null``. - """ - - if value is None: - return "null" - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, int): - return str(value) - if isinstance(value, str): - return json.dumps(value, ensure_ascii=False) - raise ComposeGenerationError(f"unsupported compose scalar type: {type(value).__name__}") - - -def _emit_yaml(value: Any, indent: int = 0) -> list[str]: - """Emit ``value`` as deterministic block YAML (mapping keys sorted). - - Supports nested mappings and lists of scalars — the shape a docker-compose - document needs. Lists of mappings are intentionally unsupported (not needed) - so the emitter stays small and correct. - """ - - pad = " " * indent - lines: list[str] = [] - if isinstance(value, Mapping): - for key in sorted(value, key=str): - child = value[key] - key_token = f"{pad}{json.dumps(str(key), ensure_ascii=False)}:" - if isinstance(child, Mapping): - if child: - lines.append(key_token) - lines.extend(_emit_yaml(child, indent + 1)) - else: - lines.append(f"{key_token} {{}}") - elif isinstance(child, (list, tuple)): - if child: - lines.append(key_token) - lines.extend(_emit_yaml(list(child), indent + 1)) - else: - lines.append(f"{key_token} []") - else: - lines.append(f"{key_token} {_yaml_scalar(child)}") - elif isinstance(value, (list, tuple)): - for item in value: - if isinstance(item, (Mapping, list, tuple)): - raise ComposeGenerationError("nested list items are not supported in compose YAML") - lines.append(f"{pad}- {_yaml_scalar(item)}") - else: # pragma: no cover - top-level is always a mapping - raise ComposeGenerationError("compose YAML root must be a mapping") - return lines - - -def _render_docker_compose_yaml(services: Mapping[str, Any]) -> str: - """Render the docker-compose ``services`` block as deterministic block YAML.""" - - document = {"services": services} - return "\n".join(_emit_yaml(document)) + "\n" - - -def phala_pre_launch_script() -> str: - """Return the fixed Phala Cloud pre-launch helper measured into compose_hash. - - Phala stores the helper without a trailing newline in app-compose JSON. The - offline generator must produce the same text as ``POST /cvms/provision`` - rewrites for reverse-matched identity (review and eval). - """ - - if not PHALA_PRE_LAUNCH_SCRIPT_PATH.is_file(): - raise ComposeGenerationError( - f"Phala pre-launch script is missing from the checkout ({PHALA_PRE_LAUNCH_SCRIPT_PATH})" - ) - text = PHALA_PRE_LAUNCH_SCRIPT_PATH.read_text(encoding="utf-8") - if not text.startswith("#!/bin/bash") or "Phala Cloud Pre-Launch Script" not in text: - raise ComposeGenerationError("Phala pre-launch script is not the expected vendor helper") - return text.rstrip("\n") - - -# --------------------------------------------------------------------------- # -# Compose generation -# --------------------------------------------------------------------------- # -def build_orchestrator_service( - *, - orchestrator_image: str, - command: Sequence[str], - static_env: Mapping[str, str], - passthrough_env: Sequence[str], - golden_dir: str, - cache_root: str, -) -> dict[str, Any]: - """Build the single orchestrator compose service (no per-task services). - - Mounts the guest Docker + dstack sockets (DooD + attestation) and the golden - manifest + task cache read-only; it is NOT privileged and starts no inner - dockerd. ``static_env`` are non-secret ``NAME=value`` config entries; - ``passthrough_env`` are secret/per-run NAMES injected at deploy via - ``encrypted_env`` (name-only, so no value is ever written here). - """ - - environment = sorted( - [f"{name}={value}" for name, value in static_env.items()] - + [str(name) for name in passthrough_env if name not in static_env] - ) - # Guest Docker + dstack sockets only. Golden and task-cache material live in - # the measured canonical image at golden_dir/cache_root; bind-mounting empty - # guest host paths over those directories would hide the image assets and break - # live DooD/key-release evaluation. - volumes = sorted( - [ - f"{DOCKER_SOCKET_PATH}:{DOCKER_SOCKET_PATH}", - f"{DSTACK_SOCKET_PATH}:{DSTACK_SOCKET_PATH}", - ] - ) - return { - "image": assert_digest_pinned(orchestrator_image, what="orchestrator image"), - "restart": "no", - "command": list(command), - "environment": environment, - "volumes": volumes, - } - - -def generate_app_compose( - *, - orchestrator_image: str, - name: str = DEFAULT_APP_NAME, - command: Sequence[str] | None = None, - allowed_envs: Sequence[str] = MEASURED_ALLOWED_ENVS, - key_release_url: str | None = None, - attestation_enabled: bool = True, - job_dir: str = DEFAULT_JOB_DIR, - cache_root: str = DEFAULT_CACHE_ROOT, - golden_dir: str = DEFAULT_GOLDEN_DIR, - digest_manifest_path: str = DEFAULT_DIGEST_MANIFEST, - live_registry_manifest_path: str | None = None, - kms_enabled: bool = True, - public_logs: bool = True, - public_sysinfo: bool = True, -) -> dict[str, Any]: - """Generate the deterministic Phala ``app-compose`` document (architecture §4 C2). - - The document declares only the orchestrator service; task images are pinned by - digest via the golden manifest and launched dynamically. No secret VALUE is - ever embedded — the gateway token and per-run binding inputs are injected at - deploy via ``encrypted_env`` for the ``allowed_envs`` NAMES. Serialize the - result with :func:`render_app_compose` (never a separate ``json.dumps``) so the - deployed bytes hash to :func:`compose_hash` of this document. - """ - - if command is None: - command = ( - "run", - "--job-dir", - job_dir, - "--cache-root", - cache_root, - "--digest-manifest", - digest_manifest_path, - ) - - # Non-secret static configuration (never a credential): DooD target + the - # in-CVM cache/manifest paths the orchestrator reads. - static_env = { - "DOCKER_HOST": f"unix://{DOCKER_SOCKET_PATH}", - "CHALLENGE_OWN_RUNNER_CACHE_ROOT": cache_root, - "CHALLENGE_OWN_RUNNER_DIGEST_MANIFEST": digest_manifest_path, - } - # Production raw RA-TLS endpoint is host + port 8701 (no HTTP URL). Bake the - # parsed operator endpoint and fixed client credential path names into the - # measured compose so the in-CVM client has no HTTP fallback path. Legacy - # callers can still pass a non-raw URL for offline tests; that path keeps - # the older KEY_RELEASE_URL long enough for flag-off compatibility only. - if key_release_url and str(key_release_url).strip(): - endpoint = str(key_release_url).strip() - host_port = _parse_raw_key_release_endpoint(endpoint) - if host_port is not None: - host, port = host_port - static_env[RA_TLS_HOST_ENV] = host - static_env[RA_TLS_PORT_ENV] = str(port) - static_env[KEY_RELEASE_TLS_CERT_ENV] = DEFAULT_KEY_RELEASE_TLS_CERT_PATH - static_env[KEY_RELEASE_TLS_KEY_ENV] = DEFAULT_KEY_RELEASE_TLS_KEY_PATH - static_env[KEY_RELEASE_TLS_CA_ENV] = DEFAULT_KEY_RELEASE_TLS_CA_PATH - else: - static_env[KEY_RELEASE_URL_ENV] = endpoint - - # Optional live-subset task-image resolution: point the in-CVM DooD builder at - # the live-registry side manifest (mounted read-only in the golden dir). Only - # added when a path is supplied, so the DEFAULT compose bytes / compose-hash - # are byte-identical (offline / flag-off resolution unchanged). - if live_registry_manifest_path and str(live_registry_manifest_path).strip(): - static_env[LIVE_REGISTRY_ENV] = str(live_registry_manifest_path).strip() - - service = build_orchestrator_service( - orchestrator_image=orchestrator_image, - command=command, - static_env=static_env, - passthrough_env=sorted(set(allowed_envs)), - golden_dir=golden_dir, - cache_root=cache_root, - ) - docker_compose_file = _render_docker_compose_yaml({ORCHESTRATOR_SERVICE: service}) - - # Phala Cloud rewrites missing envelope factors into the measured AppCompose - # (pre_launch_script, features, tproxy/public_tcbinfo/storage_fs, secure_time). - # Emit them offline so local compose_hash equals POST /cvms/provision's hash - # (parity with the review reverse-match path). Eval never imports review.compose - # so docker socket / golden mounts remain eval-only while sharing the vendor - # helper and default features list. - compose: dict[str, Any] = { - "manifest_version": APP_COMPOSE_MANIFEST_VERSION, - "name": name, - "runner": APP_COMPOSE_RUNNER, - "docker_compose_file": docker_compose_file, - "kms_enabled": kms_enabled, - "gateway_enabled": False, - # Phala still emits the deprecated alias alongside gateway_enabled; both - # appear in the measured compose_hash returned by provision. - "tproxy_enabled": True, - "local_key_provider_enabled": False, - "public_logs": public_logs, - "public_sysinfo": public_sysinfo, - "public_tcbinfo": True, - "no_instance_id": False, - "secure_time": False, - "storage_fs": "zfs", - "features": list(PHALA_DEFAULT_FEATURES), - "allowed_envs": sorted(set(allowed_envs)), - "pre_launch_script": phala_pre_launch_script(), - } - return compose - - -def render_app_compose(compose: Mapping[str, Any]) -> str: - """The exact ``app-compose.json`` text to deploy (== normalize_app_compose). - - This is the ONLY serializer a deployer may use for the app-compose file: it is - byte-for-byte :func:`normalize_app_compose`, so the deployed file hashes to - :func:`compose_hash` of ``compose`` and therefore to the live CVM - ``compose_hash`` / RTMR3 ``compose-hash`` event. - """ - - return normalize_app_compose(compose) - - -def render_app_compose_bytes(compose: Mapping[str, Any]) -> bytes: - """The exact ``app-compose.json`` bytes to deploy (UTF-8 of :func:`render_app_compose`).""" - - return render_app_compose(compose).encode("utf-8") - - -def write_app_compose(path: Path | str, compose: Mapping[str, Any]) -> str: - """Write the deployable ``app-compose.json`` bytes to ``path`` and return them.""" - - text = render_app_compose(compose) - Path(path).write_text(text, encoding="utf-8") - return text - - -def app_compose_hash(compose: Mapping[str, Any]) -> str: - """SHA-256 (hex) of the deployable app-compose bytes (== measurement.compose_hash).""" - - return compose_hash(compose) - - -__all__ = [ - "APP_COMPOSE_MANIFEST_VERSION", - "APP_COMPOSE_RUNNER", - "DEFAULT_ALLOWED_ENVS", - "MEASURED_ALLOWED_ENVS", - "PROGRESS_OPTIONAL_ENVS", - "DEFAULT_APP_NAME", - "DEFAULT_CACHE_ROOT", - "DEFAULT_DIGEST_MANIFEST", - "DEFAULT_GOLDEN_DIR", - "DEFAULT_JOB_DIR", - "DEFAULT_KEY_RELEASE_RA_TLS_PORT", - "DEFAULT_KEY_RELEASE_TLS_CA_PATH", - "DEFAULT_KEY_RELEASE_TLS_CERT_PATH", - "DEFAULT_KEY_RELEASE_TLS_KEY_PATH", - "ORCHESTRATOR_SERVICE", - "PHALA_APP_COMPOSE_ENVELOPE_KEYS", - "PHALA_DEFAULT_FEATURES", - "PHALA_PRE_LAUNCH_SCRIPT_PATH", - "RA_TLS_HOST_ENV", - "RA_TLS_PORT_ENV", - "REPO_ROOT", - "ComposeGenerationError", - "app_compose_hash", - "assert_digest_pinned", - "build_orchestrator_service", - "generate_app_compose", - "golden_task_image_digests", - "is_validator_key_release_authority", - "load_golden_manifest", - "parse_key_release_authority", - "phala_pre_launch_script", - "render_app_compose", - "render_app_compose_bytes", - "write_app_compose", -] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/canonical/entrypoint.py b/packages/challenges/agent-challenge/src/agent_challenge/canonical/entrypoint.py index 40b166e39..843db277d 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/canonical/entrypoint.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/canonical/entrypoint.py @@ -1,646 +1,14 @@ -"""Canonical eval image entrypoint. - -Stable command the canonical image runs. ``--help`` and the default invocation -touch only the standard library so the image entrypoint is always invokable for -a dry check; the own_runner evaluation pipeline is imported lazily so an actual -``run`` delegates to the unchanged :mod:`agent_challenge.evaluation.own_runner_backend`. - -Before a real ``run``, the production RA-TLS path materializes a dstack-issued -client certificate under ``/run/secrets/ra_tls`` so the key-release client can -present end-to-end attested mTLS credentials to the validator raw listener. The -validator *server* CA is never fabricated from the guest chain: it must be -supplied by the deploy (``CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM`` or a pre-written -``CHALLENGE_PHALA_RA_TLS_CA_FILE``). Measured compose ships a single leading -``run`` that is normalized to the backend subcommand before key acquisition. - -When a production raw RA-TLS endpoint is configured (static -``KEY_RELEASE_RA_TLS_HOST``/``PORT``, or signed plan -``CHALLENGE_PHALA_EVAL_PLAN.key_release_endpoint`` on a measure-time HTTP(S) -placeholder pin), the bootstrap path always emits flushed, secret-free stage -markers and fails closed within a hard wallclock: missing server CA, -``GetTlsKey`` hang/failure, or an unreachable raw listener produce -``stage=fail reason=...`` and a non-zero exit instead of a silent multi-minute -CVM billed at eval_prepared with zero TCP dials. Never invents PEM roots. - -After ``material_ready``, the entrypoint also exports the **public-only** -client fullchain (leaf + intermediates) via -:mod:`agent_challenge.canonical.public_client_fullchain` so operators can -harvest the chain for host ``KEY_RELEASE_RA_TLS_CA_FILE`` / client-trust install -when non-dev Phala disables SSH/SCP. Export surfaces (``phala logs`` + well-known -``/var/log/agent-challenge/client-fullchain.pem``) never contain private keys; -private keys remain only on the mTLS dial paths under ``/run/secrets/ra_tls``. -""" +"""Canonical CVM entrypoint removed with Phala (T40).""" from __future__ import annotations -import argparse -import os -import socket -import ssl -import sys -from collections.abc import Sequence -from pathlib import Path - -from agent_challenge.canonical.public_client_fullchain import ( - PublicFullchainExportError, - export_public_client_fullchain, -) -from agent_challenge.canonical.wallclock import WallclockTimeout, call_with_wallclock - -PROG = "agent-challenge-canonical" - -#: Production paths baked by :mod:`agent_challenge.canonical.compose`. -DEFAULT_RA_TLS_DIR = Path("/run/secrets/ra_tls") -DEFAULT_RA_TLS_CERT = DEFAULT_RA_TLS_DIR / "client.crt" -DEFAULT_RA_TLS_KEY = DEFAULT_RA_TLS_DIR / "client.key" -DEFAULT_RA_TLS_CA = DEFAULT_RA_TLS_DIR / "ca.crt" - -#: Env carrying the PEM of the CA that signed the validator raw RA-TLS listener -#: certificate (host-side ``KEY_RELEASE_RA_TLS_CERT_FILE``). Distinct from the -#: dstack KMS CA that issues the guest client certificate. -SERVER_CA_PEM_ENV = "CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM" -SERVER_CA_FILE_ENV = "CHALLENGE_PHALA_RA_TLS_SERVER_CA_FILE" - -#: Hard wallclock for guest ``GetTlsKey`` (seconds). Live dstack can exceed the -#: SDK default; past this the guest exits with stage=fail rather than billing. -GET_TLS_KEY_WALLCLOCK_SECONDS = 90.0 -#: TCP dial budget after materials are ready so the host log shows a dial (or -#: a fast closed failure) before framed RA-TLS begins. -TCP_CONNECT_TIMEOUT_SECONDS = 15.0 -#: Marker prefix shared by host log scrapers (secret-free). -RA_TLS_BOOTSTRAP_MARKER = "ra_tls_bootstrap" - - -def build_parser() -> argparse.ArgumentParser: - """Help/check-only parser. ``run`` is intentionally not registered here. - - Measured compose ships ``command: [run, --job-dir, ...]``. argparse's - ``REMAINDER`` still refuses unknown ``--`` options on a subparser, so the - production ``run`` path is handled manually in :func:`main`. - """ - - parser = argparse.ArgumentParser( - prog=PROG, - description="Canonical Agent Challenge evaluation entrypoint (wraps own_runner).", - ) - subparsers = parser.add_subparsers(dest="command") - subparsers.add_parser( - "check", - help="verify the own_runner eval pipeline is importable inside the image and exit", - ) - return parser - - -_OWN_RUNNER_MODULES = ( - "orchestrator.py", - "container_builder.py", - "result_schema.py", - "taskdefs.py", - "reward.py", - "verifier_runner.py", -) - - -def _run_check() -> int: - # Verify the own_runner eval modules are present at the expected locations - # without importing the heavy evaluation package (which pulls the API/chain - # stack via ``evaluation.__init__``), so the dry check works in the lean - # canonical image too. - import agent_challenge - - evaluation = Path(agent_challenge.__file__).resolve().parent / "evaluation" - own_runner = evaluation / "own_runner" - missing = [name for name in _OWN_RUNNER_MODULES if not (own_runner / name).is_file()] - if not (evaluation / "own_runner_backend.py").is_file(): - missing.append("own_runner_backend.py") - if missing: - raise RuntimeError(f"own_runner modules missing from image: {', '.join(missing)}") - print("canonical eval entrypoint OK: own_runner modules present") - return 0 - - -def _normalize_backend_argv(args: list[str]) -> list[str]: - """Normalize measured compose argv to the own_runner_backend ``run`` form. - - The Phala docker-compose command is ``["run", "--job-dir", ...]`` because - the image entrypoint already owns a top-level ``run`` subcommand. The - remainder may therefore be either: - - * ``["run", "--task", ..., "--job-dir", ...]`` (legacy double-run), or - * ``["--job-dir", ...]`` / ``["--task", ...]`` (compose shape). - - In the latter case, prepend the backend ``run`` token so argparse sees the - required subcommand. Never invent ``--task`` values here: the backend pulls - the immutable Eval plan's selected tasks when CLI tasks are omitted on the - Phala path. - """ - - tokens = list(args) - # argparse.REMAINDER keeps a leading "--"; strip a pure separator. - if tokens and tokens[0] == "--": - tokens = tokens[1:] - if not tokens: - return ["run"] - if tokens[0] == "run": - return tokens - return ["run", *tokens] - - -def _emit_bootstrap_marker(stage: str, **fields: str | int | bool) -> None: - """Print a flushed, secret-free bootstrap progress marker for host log scrapers. - - Never includes PEMs, keys, token values, or other secret material. Field - values are already-sanitized present/missing flags or non-secret host/port. - """ - - parts = [f"{RA_TLS_BOOTSTRAP_MARKER} stage={stage}"] - for key, value in fields.items(): - if isinstance(value, bool): - text = "present" if value else "missing" - else: - text = str(value).replace("\n", " ").replace("\r", " ").strip() - # Belt-and-suspenders: never echo obviously secret-looking blobs. - if "BEGIN " in text.upper() or len(text) > 256: - text = "redacted" - parts.append(f"{key}={text}") - print(" ".join(parts), flush=True) - try: - sys.stdout.flush() - sys.stderr.flush() - except Exception: # noqa: BLE001 - best-effort flush only - pass - - -def normalize_server_ca_pem(raw: str) -> str: - """Return an OpenSSL-loadable multi-line PEM, or raise ValueError. - - Live residual: Phala ``encrypted_env`` (or shelling) can collapse a multi-line - certificate into a single line with *literal* ``\\n`` / ``\\r\\n`` escape - sequences. Weak ``BEGIN CERTIFICATE`` substring checks still accept that - shape, but OpenSSL reports ``NO_CERTIFICATE_OR_CRL_FOUND`` when building the - trust store via ``create_default_context(cafile=...)``. - - Contract: - * empty / whitespace-only → ``empty_server_ca`` - * missing PEM markers after unescape → ``malformed_server_ca`` - * PEM markers present but unloadable by OpenSSL → ``malformed_server_ca`` - * success → trailing-newline PEM that ``SSLContext.load_verify_locations - (cadata=...)`` accepts - """ - - if raw is None: - raise ValueError("empty_server_ca: server CA PEM is empty") - text = str(raw).strip() - if not text: - raise ValueError("empty_server_ca: server CA PEM is empty") - - # Unescape only when the payload looks collapsed (one logical line) and still - # carries PEM markers via literal backslash-n sequences. Do not rewrite - # already multi-line PEMs that happen to include backslash characters. - if "BEGIN CERTIFICATE" in text and "\n" not in text and "\\n" in text: - text = text.replace("\\r\\n", "\n").replace("\\n", "\n") - elif "BEGIN CERTIFICATE" in text and "\\n" in text and text.count("\n") < 2: - # Semi-collapsed: a few real newlines but body still escaped. - text = text.replace("\\r\\n", "\n").replace("\\n", "\n") - - text = text.replace("\r\n", "\n").replace("\r", "\n").strip() - if not text: - raise ValueError("empty_server_ca: server CA PEM is empty after normalize") - if "BEGIN CERTIFICATE" not in text or "END CERTIFICATE" not in text: - raise ValueError("malformed_server_ca: PEM markers missing after normalize") - if not text.endswith("\n"): - text = text + "\n" - - # Hard gate: OpenSSL must accept the CA bytes *before* we write ca.crt or - # hand the path to create_default_context (prevents opaque pre-frame SSLError). - try: - probe = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - probe.load_verify_locations(cadata=text) - except ssl.SSLError as exc: - raise ValueError(f"malformed_server_ca: OpenSSL rejected server CA PEM ({exc})") from exc - return text - - -def preload_server_ca_pem(pem: str) -> str: - """Normalize + OpenSSL-preload a server CA PEM; alias of :func:`normalize_server_ca_pem`.""" - - return normalize_server_ca_pem(pem) - - -def _server_ca_presence() -> str: - """Return ``present`` or ``missing`` without reading secret PEM into logs. - - Marker only: a bare ``BEGIN CERTIFICATE`` substring counts as present so the - start marker is informative before full OpenSSL preload. Unloadable PEMs are - rejected later with a durable ``malformed_server_ca`` reason. - """ - - pem = (os.environ.get(SERVER_CA_PEM_ENV) or "").strip() - if pem and "BEGIN CERTIFICATE" in pem: - return "present" - for env_name in (SERVER_CA_FILE_ENV, "CHALLENGE_PHALA_RA_TLS_CA_FILE"): - raw = (os.environ.get(env_name) or "").strip() - if not raw: - continue - path = Path(raw) - if path.is_file(): - try: - text = path.read_text(encoding="utf-8") - except OSError: - continue - if "BEGIN CERTIFICATE" in text: - return "present" - return "missing" - - -def _resolve_server_ca_pem() -> str: - """Return the validator raw-listener CA PEM (OpenSSL-loadable), or fail closed.""" - - pem = (os.environ.get(SERVER_CA_PEM_ENV) or "").strip() - if pem and "BEGIN CERTIFICATE" in pem: - try: - return normalize_server_ca_pem(pem) - except ValueError as exc: - raise RuntimeError(str(exc)) from exc - ca_file = (os.environ.get(SERVER_CA_FILE_ENV) or "").strip() - if ca_file: - path = Path(ca_file) - if path.is_file(): - text = path.read_text(encoding="utf-8") - if "BEGIN CERTIFICATE" in text: - try: - return normalize_server_ca_pem(text) - except ValueError as exc: - raise RuntimeError(str(exc)) from exc - # A pre-provisioned CA file path counts only when it already holds real PEM - # (deploy mounts the validator server CA). Empty placeholder paths fail closed. - configured = (os.environ.get("CHALLENGE_PHALA_RA_TLS_CA_FILE") or "").strip() - if configured: - path = Path(configured) - if path.is_file(): - text = path.read_text(encoding="utf-8") - if "BEGIN CERTIFICATE" in text: - try: - return normalize_server_ca_pem(text) - except ValueError as exc: - raise RuntimeError(str(exc)) from exc - raise RuntimeError( - "missing_server_ca: raw RA-TLS path requires the validator server CA " - f"({SERVER_CA_PEM_ENV} or {SERVER_CA_FILE_ENV} or a non-empty " - "CHALLENGE_PHALA_RA_TLS_CA_FILE); refusing to trust the guest dstack chain" - ) - - -def _call_get_tls_key_with_wallclock(client: object) -> object: - """Invoke ``GetTlsKey`` with a hard wallclock; raise on hang/failure. - - Uses a daemon-thread wallclock (not ThreadPoolExecutor) so a hung dstack - RPC never re-joins on the timeout path. Fail-closed stage=fail markers are - still emitted by the provision caller. - """ - - get_tls_key = getattr(client, "get_tls_key", None) - if not callable(get_tls_key): - raise RuntimeError("gettlskey_unavailable: dstack client lacks get_tls_key") - - def _invoke() -> object: - return get_tls_key( - subject="agent-challenge-key-release", - usage_ra_tls=True, - usage_server_auth=False, - usage_client_auth=True, - ) - - try: - return call_with_wallclock( - _invoke, - timeout_seconds=GET_TLS_KEY_WALLCLOCK_SECONDS, - label="GetTlsKey", - ) - except WallclockTimeout as exc: - raise RuntimeError( - f"gettlskey_timeout: GetTlsKey exceeded {GET_TLS_KEY_WALLCLOCK_SECONDS:.0f}s" - ) from exc - except Exception as exc: # noqa: BLE001 - any dstack failure fails closed - text = str(exc).lower() - if "timeout" in text or "timed out" in text: - raise RuntimeError(f"gettlskey_timeout: {exc}") from exc - raise RuntimeError(f"gettlskey_failed: {exc}") from exc - - -def _probe_raw_tcp(host: str, port: int) -> None: - """Force a TCP dial so the host log sees connect activity, or fail closed.""" - - try: - with socket.create_connection((host, port), timeout=TCP_CONNECT_TIMEOUT_SECONDS): - pass - except OSError as exc: - _emit_bootstrap_marker( - "tcp_connect_fail", - host=host, - port=port, - server_ca=_server_ca_presence(), - reason="tcp_connect_fail", - ) - raise RuntimeError( - f"tcp_connect_fail: cannot dial raw RA-TLS listener {host}:{port}: {exc}" - ) from exc - _emit_bootstrap_marker( - "tcp_connect_ok", - host=host, - port=port, - server_ca="present", - ) - - -def _parse_raw_host_port(endpoint: str) -> tuple[str, int] | None: - """Parse a production raw RA-TLS authority (host:8701 / ratls://host:port). - - Mirrors :func:`agent_challenge.canonical.compose._parse_raw_key_release_endpoint` - without importing compose (lean-image boundary). Returns ``None`` for HTTP(S) - and non-raw authorities so the measure-time HTTPS placeholder does not invent - a raw bake from the placeholder URL. - """ - - value = (endpoint or "").strip() - if not value: - return None - scheme = "" - authority = value - if "://" in value: - scheme, authority = value.split("://", 1) - scheme = scheme.lower() - if scheme in {"http", "https"}: - return None - if scheme not in {"ratls", "tls", "tcp"}: - return None - if "/" in authority: - authority = authority.split("/", 1)[0] - if ":" not in authority: - return None - host, port_text = authority.rsplit(":", 1) - try: - port = int(port_text) - except ValueError: - return None - host = host.strip().strip("[]") - if not host or not 1 <= port <= 65535: - return None - # Bare host:port without scheme is production only on the RA-TLS listener port. - if not scheme and port != 8701: - return None - return host, port - - -def _resolve_raw_ra_tls_host_port() -> tuple[str, int] | None: - """Return production raw RA-TLS ``(host, port)`` for GetTlsKey provisioning. - - Sources, in order (never invent PEMs / listener endpoints): - - 1. Static compose env ``KEY_RELEASE_RA_TLS_HOST`` + ``PORT`` (raw bake path). - 2. Signed plan ``CHALLENGE_PHALA_EVAL_PLAN.key_release_endpoint`` when the - measure-time pin used an HTTPS placeholder that left host/port unset - (Path B residual: guest still dials raw ``host:8701`` from the plan, so - client mTLS material must exist before that dial). - 3. Legacy ``CHALLENGE_PHALA_KEY_RELEASE_URL`` when it is a raw authority. - - Returns ``None`` for flag-off / non-raw legacy HTTP paths so provision is - skipped entirely outside production raw key-release. - """ - - host = (os.environ.get("KEY_RELEASE_RA_TLS_HOST") or "").strip() - port_text = (os.environ.get("KEY_RELEASE_RA_TLS_PORT") or "").strip() - if host and port_text: - try: - port = int(port_text) - except ValueError as exc: - raise RuntimeError( - f"invalid_port: KEY_RELEASE_RA_TLS_PORT is not an integer: {port_text!r}" - ) from exc - if not 1 <= port <= 65535: - raise RuntimeError(f"invalid_port: KEY_RELEASE_RA_TLS_PORT out of range: {port}") - return host, port - - # Plan-first fallback: measure-time HTTPS pin leaves static HOST/PORT empty. - raw_plan = (os.environ.get("CHALLENGE_PHALA_EVAL_PLAN") or "").strip() - if raw_plan: - try: - import json - - plan = json.loads(raw_plan) - except (json.JSONDecodeError, TypeError, ValueError): - plan = None - if isinstance(plan, dict): - endpoint = str(plan.get("key_release_endpoint") or "").strip() - parsed = _parse_raw_host_port(endpoint) if endpoint else None - if parsed is not None: - return parsed - - url = (os.environ.get("CHALLENGE_PHALA_KEY_RELEASE_URL") or "").strip() - if url: - return _parse_raw_host_port(url) - return None - - -def _provision_ra_tls_client_material() -> None: - """Issue a dstack RA-TLS client cert when the production raw path is configured. - - The measured compose pins ``CHALLENGE_PHALA_RA_TLS_{CERT,KEY,CA}_FILE`` at - ``/run/secrets/ra_tls/*`` when host/port were baked raw. When the pin used a - measure-time HTTPS placeholder, those env names may be absent; resolve the - raw production endpoint from the signed plan (never invent PEMs) and still - materialize default paths under ``/run/secrets/ra_tls``. When cert/key files - are not already present and the guest dstack socket is available, request a - client-auth + RA-TLS certificate with ``GetTlsKey`` and write the chain + - key in place. The CA file is *always* the validator server-trust CA (never - the dstack guest intermediate). Fail closed (raise) for the raw RA-TLS - production path; legacy HTTP key-release skips this entirely. - - Emits flushed, secret-free stage markers (start / material_ready / - tcp_connect_ok|fail) and always dials the raw listener after materials are - ready so a silent multi-minute hang at eval_prepared is impossible when - a raw host+port is configured (compose env or signed plan endpoint). - """ - - host_port = _resolve_raw_ra_tls_host_port() - if host_port is None: - return # not the production raw path - - host, port = host_port - # Export host/port so own_runner KR dial + residual scrapers see the raw path - # even when measure-time compose left them unset (HTTP placeholder pin). - os.environ["KEY_RELEASE_RA_TLS_HOST"] = host - os.environ["KEY_RELEASE_RA_TLS_PORT"] = str(port) - - server_ca_state = _server_ca_presence() - _emit_bootstrap_marker( - "start", - host=host, - port=port, - server_ca=server_ca_state, - ) - - cert_path = Path( - (os.environ.get("CHALLENGE_PHALA_RA_TLS_CERT_FILE") or "").strip() or DEFAULT_RA_TLS_CERT - ) - key_path = Path( - (os.environ.get("CHALLENGE_PHALA_RA_TLS_KEY_FILE") or "").strip() or DEFAULT_RA_TLS_KEY - ) - ca_path = Path( - (os.environ.get("CHALLENGE_PHALA_RA_TLS_CA_FILE") or "").strip() or DEFAULT_RA_TLS_CA - ) - - server_ca_pem = _resolve_server_ca_pem() - need_client = not (cert_path.is_file() and key_path.is_file()) - if need_client: - from dstack_sdk import DstackClient - - # SDK timeout is an advisory; the wallclock wrapper is the hard deadline. - client = DstackClient(timeout=int(GET_TLS_KEY_WALLCLOCK_SECONDS)) - response = _call_get_tls_key_with_wallclock(client) - key_pem = getattr(response, "key", None) or "" - chain = list(getattr(response, "certificate_chain", None) or []) - if not isinstance(key_pem, str) or not key_pem.strip(): - raise RuntimeError("gettlskey_failed: dstack GetTlsKey returned no client private key") - if not chain or not all(isinstance(item, str) and item.strip() for item in chain): - raise RuntimeError( - "gettlskey_failed: dstack GetTlsKey returned no client certificate chain" - ) - - cert_path.parent.mkdir(parents=True, exist_ok=True) - # Leaf first, then intermediates for a complete client chain file. - cert_path.write_text("".join(chain), encoding="utf-8") - key_path.write_text(key_pem if key_pem.endswith("\n") else key_pem + "\n", encoding="utf-8") - os.chmod(key_path, 0o600) - - if not cert_path.is_file() or not key_path.is_file(): - raise RuntimeError("client_material_incomplete: raw RA-TLS client cert/key files missing") - - # Always materialize the *server* trust CA at the configured CA path so the - # key-release client verifies the validator listener (not a guest issuer). - ca_path.parent.mkdir(parents=True, exist_ok=True) - ca_path.write_text(server_ca_pem, encoding="utf-8") - - cert_text = cert_path.read_text(encoding="utf-8") - key_text = key_path.read_text(encoding="utf-8") - if not cert_text.strip() or not key_text.strip(): - raise RuntimeError("client_material_empty: raw RA-TLS client cert/key must be non-empty") - - os.environ["CHALLENGE_PHALA_RA_TLS_CERT_FILE"] = str(cert_path) - os.environ["CHALLENGE_PHALA_RA_TLS_KEY_FILE"] = str(key_path) - os.environ["CHALLENGE_PHALA_RA_TLS_CA_FILE"] = str(ca_path) - - # Observability: materialize the leaf SPKI digest so the acquire path never - # falls back to sha256(b"") when env SPKI/PUBKEY are unset (live residual). - if not (os.environ.get("CHALLENGE_PHALA_RA_TLS_SPKI_SHA256") or "").strip(): - try: - import hashlib - - from cryptography import x509 - from cryptography.hazmat.primitives import serialization - - certificate = x509.load_pem_x509_certificate(cert_path.read_bytes()) - spki = certificate.public_key().public_bytes( - serialization.Encoding.DER, - serialization.PublicFormat.SubjectPublicKeyInfo, - ) - os.environ["CHALLENGE_PHALA_RA_TLS_SPKI_SHA256"] = hashlib.sha256(spki).hexdigest() - except (OSError, ValueError): - # Non-fatal: acquire-time resolution still derives from the cert file. - pass - - _emit_bootstrap_marker( - "material_ready", - host=host, - port=port, - server_ca="present", - client_cert="present", - client_key="present", - ) - - # Public-only fullchain export for operator client-trust install. Private - # key never leaves key_path; fail closed when public chain cannot be exported - # (missing/empty/shredded) so residual harvest does not silently skip. - try: - export = export_public_client_fullchain(cert_path=cert_path) - except PublicFullchainExportError as export_exc: - raise RuntimeError(str(export_exc)) from export_exc - _emit_bootstrap_marker( - "public_fullchain_exported", - host=host, - port=port, - cert_count=export.cert_count, - pem_len=export.pem_len, - path_written="yes" if export.path_written else "no", - ) - - # Host must see a TCP hit (even before framed release) or the guest exits. - _probe_raw_tcp(host, port) - - -def _run_eval(args: list[str]) -> int: - try: - _provision_ra_tls_client_material() - except Exception as exc: # noqa: BLE001 - every provision error is durable fail-closed - host = (os.environ.get("KEY_RELEASE_RA_TLS_HOST") or "").strip() or "unset" - port = (os.environ.get("KEY_RELEASE_RA_TLS_PORT") or "").strip() or "unset" - message = str(exc) - reason = message.split(":", 1)[0].strip() if message else "provision_failed" - if not reason or " " in reason: - reason = "provision_failed" - # Collapse verbose RuntimeError prefixes to stable, secret-free reasons. - for known in ( - "malformed_server_ca", - "empty_server_ca", - "missing_server_ca", - "gettlskey_timeout", - "gettlskey_failed", - "gettlskey_unavailable", - "tcp_connect_fail", - "invalid_port", - "client_material_incomplete", - "client_material_empty", - "public_chain_missing", - "private_key_in_export", - ): - if known in message: - reason = known - break - _emit_bootstrap_marker( - "fail", - host=host, - port=port, - server_ca=_server_ca_presence(), - reason=reason, - ) - print(f"canonical eval RA-TLS bootstrap failed: {reason}", file=sys.stderr, flush=True) - return 1 - - from agent_challenge.evaluation.own_runner_backend import main as backend_main - - return backend_main(_normalize_backend_argv(args)) - -def main(argv: Sequence[str] | None = None) -> int: - tokens = list(argv) if argv is not None else None - if tokens is None: - tokens = list(sys.argv[1:]) +class CanonicalEntrypointRemoved(RuntimeError): + pass - # Production path: first token is ``run`` and everything after (including bare - # ``--job-dir`` flags from measured compose) is backend argv. Do not route - # this through argparse subparsers: REMAINDER cannot capture leading options. - if tokens and tokens[0] == "run": - return _run_eval(tokens[1:]) - parser = build_parser() - if not tokens: - return _run_check() - namespace = parser.parse_args(tokens) - if namespace.command in (None, "check"): - return _run_check() - parser.error(f"unknown command: {namespace.command}") # pragma: no cover - return 2 # pragma: no cover +def main(*args, **kwargs): # noqa: ANN001, ANN002 + raise CanonicalEntrypointRemoved("canonical entrypoint removed with Phala TEE (T40)") -if __name__ == "__main__": # pragma: no cover - thin CLI shim - raise SystemExit(main()) +__all__ = ["CanonicalEntrypointRemoved", "main"] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/canonical/live_registry.py b/packages/challenges/agent-challenge/src/agent_challenge/canonical/live_registry.py index e04250f89..d21535c2f 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/canonical/live_registry.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/canonical/live_registry.py @@ -104,9 +104,7 @@ def assert_live_registry_ref(ref: Any, *, what: str = "registry ref") -> str: f"{what} must not use the retired mathiiss namespace (D6), got {pinned!r}" ) if lowered.startswith("docker.io/") or lowered.startswith("index.docker.io/"): - raise LiveRegistryError( - f"{what} must not use docker.io (D6 — GHCR only), got {pinned!r}" - ) + raise LiveRegistryError(f"{what} must not use docker.io (D6 — GHCR only), got {pinned!r}") if not lowered.startswith(LIVE_REGISTRY_HOST_PREFIX): raise LiveRegistryError( f"{what} must be a GHCR digest-pinned ref " diff --git a/packages/challenges/agent-challenge/src/agent_challenge/canonical/public_client_fullchain.py b/packages/challenges/agent-challenge/src/agent_challenge/canonical/public_client_fullchain.py deleted file mode 100644 index 33cf4acf1..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/canonical/public_client_fullchain.py +++ /dev/null @@ -1,324 +0,0 @@ -"""Public-only client fullchain export for operator client-trust install. - -After guest ``GetTlsKey`` materializes ``/run/secrets/ra_tls/client.crt`` -(leaf + intermediates), non-dev Phala blocks SSH/SCP and secret-free bootstrap -markers omit PEM, so operators cannot harvest the public chain needed to set -``KEY_RELEASE_RA_TLS_CA_FILE`` / host client-trust distinct from the server CA. - -This module exposes **public-only** material via two surfaces (no invent PEMs): - -1. Flushed log lines labeled ``ra_tls_public_fullchain`` that host scrapers already - reach with ``phala logs`` (intentionally contain ``BEGIN CERTIFICATE`` blocks). -2. A well-known non-secret path under ``/var/log/agent-challenge/`` for any host - inspect tooling that can read guest files without full SSH. - -Hard rules: -* NEVER includes private key material (``BEGIN PRIVATE KEY`` / PKCS8 / etc.). -* Private key stays only on the mTLS paths used for the raw dial. -* Fail closed when the public chain is missing or empty. -* Never invents roots or issuer PEMs. -""" - -from __future__ import annotations - -import hashlib -import re -import sys -from dataclasses import dataclass -from pathlib import Path - -#: Host log / serial scrapers look for this prefix (intentional public PEM). -PUBLIC_FULLCHAIN_MARKER = "ra_tls_public_fullchain" - -#: Default path (public-only PEM). Not under /run/secrets and never holds keys. -DEFAULT_PUBLIC_EXPORT_PATH = Path("/var/log/agent-challenge/client-fullchain.pem") - -#: Override well-known export path (tests / residual ops). -PUBLIC_EXPORT_PATH_ENV = "CHALLENGE_PHALA_PUBLIC_CLIENT_FULLCHAIN_PATH" - -_CERT_BLOCK_RE = re.compile( - r"-----BEGIN CERTIFICATE-----\r?\n" - r"[A-Za-z0-9+/=\r\n]+" - r"-----END CERTIFICATE-----", - re.MULTILINE, -) - -_PRIVATE_KEY_RE = re.compile( - r"-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE KEY-----", - re.IGNORECASE, -) - -_PRIVATE_HINTS = ( - "BEGIN PRIVATE KEY", - "BEGIN RSA PRIVATE KEY", - "BEGIN EC PRIVATE KEY", - "BEGIN ENCRYPTED PRIVATE KEY", - "BEGIN OPENSSH PRIVATE KEY", -) - - -@dataclass(frozen=True) -class PublicFullchainExport: - """Result of a successful public-only fullchain export (safe to log).""" - - pem: str - cert_count: int - pem_len: int - sha256_hex: str - leaf_spki_sha256: str | None - export_path: str | None - path_written: bool - - -class PublicFullchainExportError(RuntimeError): - """Fail-closed error for missing/invalid public fullchain export.""" - - -def assert_public_only(surface: str, *, label: str = "export_surface") -> None: - """Raise when any private-key material appears in an export surface. - - The check is intentionally wide (header substrings + regex) so unit tests - and production log emission share one fail-closed gate. - """ - - text = surface or "" - upper = text.upper() - for hint in _PRIVATE_HINTS: - if hint in upper: - raise PublicFullchainExportError( - f"private_key_in_export: {label} contains private key material" - ) - if _PRIVATE_KEY_RE.search(text): - raise PublicFullchainExportError( - f"private_key_in_export: {label} matched private key PEM header" - ) - - -def extract_public_fullchain_pem(cert_pem: str) -> str: - """Return leaf+intermediates CERTIFICATE blocks only (no invent, no keys). - - Raises: - PublicFullchainExportError: when chain is empty/missing or result is not - public-only. - """ - - if cert_pem is None or not str(cert_pem).strip(): - raise PublicFullchainExportError("public_chain_missing: client certificate chain is empty") - - raw = str(cert_pem) - # Fail closed if operator accidentally passed a combined cert+key blob. - assert_public_only(raw, label="input_cert_pem") - - blocks = _CERT_BLOCK_RE.findall(raw) - if not blocks: - raise PublicFullchainExportError( - "public_chain_missing: no BEGIN CERTIFICATE blocks in client chain" - ) - - pieces: list[str] = [] - for block in blocks: - body = block.strip() - if not body.endswith("\n"): - body = body + "\n" - pieces.append(body) - pem = "".join(pieces) - if not pem.endswith("\n"): - pem = pem + "\n" - - assert_public_only(pem, label="extracted_fullchain") - if "BEGIN CERTIFICATE" not in pem or "END CERTIFICATE" not in pem: - raise PublicFullchainExportError( - "public_chain_missing: extracted fullchain lacks PEM markers" - ) - return pem - - -def _leaf_spki_sha256(pem: str) -> str | None: - """Best-effort leaf SPKI digest; Non-fatal when cryptography is unavailable.""" - - try: - from cryptography import x509 - from cryptography.hazmat.primitives import serialization - except ImportError: # pragma: no cover - cryptography is a product dep - return None - try: - # load_pem_x509_certificate accepts the first cert in a multi-PEM string - # only when given a single block; take first block explicitly. - first = _CERT_BLOCK_RE.search(pem) - if first is None: - return None - cert = x509.load_pem_x509_certificate(first.group(0).encode("utf-8")) - spki = cert.public_key().public_bytes( - serialization.Encoding.DER, - serialization.PublicFormat.SubjectPublicKeyInfo, - ) - return hashlib.sha256(spki).hexdigest() - except (ValueError, TypeError): - return None - - -def resolve_public_export_path(explicit: Path | str | None = None) -> Path: - """Resolve well-known public export path (env override wins over default).""" - - if explicit is not None: - return Path(explicit) - import os - - raw = (os.environ.get(PUBLIC_EXPORT_PATH_ENV) or "").strip() - if raw: - return Path(raw) - return DEFAULT_PUBLIC_EXPORT_PATH - - -def write_public_fullchain_file(pem: str, path: Path) -> bool: - """Write public-only PEM to ``path``. Returns True on success, False on I/O fail. - - Never writes private keys. Caller must pass already-validated public PEM. - """ - - assert_public_only(pem, label="write_path_input") - try: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(pem, encoding="utf-8") - # Public material only; world-readable so host inspect tools can scrape. - try: - path.chmod(0o644) - except OSError: - pass - return path.is_file() and path.stat().st_size > 0 - except OSError: - return False - - -def emit_public_fullchain_log( - pem: str, - *, - cert_count: int, - pem_len: int, - sha256_hex: str, - leaf_spki_sha256: str | None, - export_path: str | None, - path_written: bool, -) -> str: - """Emit flushed public-fullchain log surface; return combined text for tests. - - Stage markers carry only redacted lengths/digests; PEM body is intentional - public CERTIFICATE blocks. Private keys are rejected before emit. - """ - - assert_public_only(pem, label="log_emit_pem") - meta_parts = [ - f"{PUBLIC_FULLCHAIN_MARKER} stage=export_ready", - f"cert_count={cert_count}", - f"pem_len={pem_len}", - f"sha256={sha256_hex}", - ] - if leaf_spki_sha256: - meta_parts.append(f"leaf_spki_sha256={leaf_spki_sha256}") - if export_path: - meta_parts.append(f"path={export_path}") - meta_parts.append(f"path_written={'yes' if path_written else 'no'}") - meta_line = " ".join(meta_parts) - - end_line = f"{PUBLIC_FULLCHAIN_MARKER} stage=export_end sha256={sha256_hex}" - body = pem if pem.endswith("\n") else pem + "\n" - combined = f"{meta_line}\n{body}{end_line}\n" - assert_public_only(combined, label="log_emit_combined") - - print(meta_line, flush=True) - # Multi-line PEM intentionally printed so phala logs harvestors can scrape. - sys.stdout.write(body) - sys.stdout.flush() - print(end_line, flush=True) - try: - sys.stdout.flush() - sys.stderr.flush() - except Exception: # noqa: BLE001 - best-effort flush only - pass - return combined - - -def export_public_client_fullchain( - *, - cert_path: Path | str, - export_path: Path | str | None = None, - emit_log: bool = True, - write_file: bool = True, -) -> PublicFullchainExport: - """Read guest client.crt and export public-only fullchain (fail-closed). - - Args: - cert_path: Path to the leaf+intermediates client certificate file. - export_path: Optional override for the well-known public path. - emit_log: When True, print the public PEM on the log surface. - write_file: When True, attempt the well-known path write. - - Returns: - :class:`PublicFullchainExport` with digests and path status. - - Raises: - PublicFullchainExportError: missing chain, private key present, I/O for - cert read fails, or export surface validation fails. - """ - - path = Path(cert_path) - if not path.is_file(): - raise PublicFullchainExportError( - f"public_chain_missing: client certificate file not found: {path}" - ) - try: - raw = path.read_text(encoding="utf-8") - except OSError as exc: - raise PublicFullchainExportError( - f"public_chain_missing: cannot read client certificate: {exc}" - ) from exc - - pem = extract_public_fullchain_pem(raw) - cert_count = len(_CERT_BLOCK_RE.findall(pem)) - pem_len = len(pem.encode("utf-8")) - sha256_hex = hashlib.sha256(pem.encode("utf-8")).hexdigest() - leaf_spki = _leaf_spki_sha256(pem) - - target = resolve_public_export_path(export_path) - path_written = False - if write_file: - path_written = write_public_fullchain_file(pem, target) - - export = PublicFullchainExport( - pem=pem, - cert_count=cert_count, - pem_len=pem_len, - sha256_hex=sha256_hex, - leaf_spki_sha256=leaf_spki, - export_path=str(target), - path_written=path_written, - ) - - if emit_log: - combined = emit_public_fullchain_log( - pem, - cert_count=cert_count, - pem_len=pem_len, - sha256_hex=sha256_hex, - leaf_spki_sha256=leaf_spki, - export_path=str(target), - path_written=path_written, - ) - assert_public_only(combined, label="post_emit_surface") - # Dedicated safeguard: private key material must never reach the surface. - if "PRIVATE KEY" in combined.upper(): - raise PublicFullchainExportError( - "private_key_in_export: log surface leaked private key material" - ) - - return export - - -def export_surface_contains_private_key(surface: str) -> bool: - """Predicate helper for tests: True when surface has private key material.""" - - try: - assert_public_only(surface, label="surface_probe") - except PublicFullchainExportError: - return True - return "PRIVATE KEY" in (surface or "").upper() diff --git a/packages/challenges/agent-challenge/src/agent_challenge/canonical/secrets_scan.py b/packages/challenges/agent-challenge/src/agent_challenge/canonical/secrets_scan.py deleted file mode 100644 index 84b7dc0f9..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/canonical/secrets_scan.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Secret scanning for the canonical eval image and build artifacts. - -Detects credential-shaped values and golden-test plaintext markers so the -canonical image (and any generated artifact) can be proven free of baked-in -secrets. Matches are reported as ``(member, pattern)`` pairs and never carry the -matched secret value, so scan results are always safe to log. -""" - -from __future__ import annotations - -import re -import tarfile -from dataclasses import dataclass -from pathlib import Path - -# The golden-plaintext marker is assembled from fragments so this scanner's own -# source (which ships inside the image) never self-matches; only real golden -# files carry the contiguous marker. -_GOLDEN_MARKER = "harbor-independence/" + "oracle-golden" - -# High-confidence patterns: each is specific enough not to fire on a stock -# python base image, yet catches the secret classes VAL-IMG-005 forbids. -SECRET_PATTERNS: dict[str, re.Pattern[str]] = { - "phala_api_key": re.compile(r"phak_[A-Za-z0-9]{16,}"), - "openai_key": re.compile(r"sk-(?:proj-)?[A-Za-z0-9]{32,}"), - "anthropic_key": re.compile(r"sk-ant-[A-Za-z0-9_-]{20,}"), - "aws_access_key": re.compile(r"AKIA[0-9A-Z]{16}"), - # A REAL PEM private key: the BEGIN header followed by a newline, optional - # legacy encryption headers (Proc-Type/DEK-Info, as in "traditional" OpenSSL - # encrypted keys), a base64 body, and the matching END footer. Requiring the - # body (not a bare header) avoids false positives on library source/binaries - # that embed the header string as a constant (e.g. cryptography's ssh.py - # ``_SK_START``); allowing the header block keeps legacy encrypted keys caught. - "pem_private_key": re.compile( - r"-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE KEY-----\r?\n" - r"(?:(?:Proc-Type|DEK-Info):[^\r\n]*\r?\n)*" - r"(?:\r?\n)?" - r"[A-Za-z0-9+/=\r\n]+" - r"-----END (?:[A-Z0-9 ]+ )?PRIVATE KEY-----" - ), - "golden_oracle_plaintext": re.compile(re.escape(_GOLDEN_MARKER)), -} - -# Skip members larger than this when scanning (secrets are small; huge binary -# blobs only slow the scan and never legitimately hold a pasted credential). -MAX_MEMBER_BYTES = 16 * 1024 * 1024 - - -@dataclass(frozen=True) -class SecretHit: - """A single secret-pattern match. Deliberately omits the matched value.""" - - member: str - pattern: str - - -def scan_bytes(data: bytes, *, member: str) -> list[SecretHit]: - """Return the secret patterns present in ``data`` (value never included).""" - - text = data.decode("latin-1", errors="ignore") - hits: list[SecretHit] = [] - for name, pattern in SECRET_PATTERNS.items(): - if pattern.search(text): - hits.append(SecretHit(member=member, pattern=name)) - return hits - - -def scan_text(text: str, *, member: str = "") -> list[SecretHit]: - return scan_bytes(text.encode("utf-8"), member=member) - - -def scan_path(root: Path | str) -> list[SecretHit]: - """Scan a file or directory tree for secret patterns.""" - - root = Path(root) - hits: list[SecretHit] = [] - paths = [root] if root.is_file() else sorted(root.rglob("*")) - for path in paths: - if not path.is_file(): - continue - try: - if path.stat().st_size > MAX_MEMBER_BYTES: - continue - data = path.read_bytes() - except OSError: # pragma: no cover - defensive - continue - hits.extend(scan_bytes(data, member=str(path))) - return hits - - -def scan_tar(tar_path: Path | str) -> list[SecretHit]: - """Scan every regular file inside a tar archive (e.g. ``docker export``).""" - - hits: list[SecretHit] = [] - with tarfile.open(tar_path, "r:*") as archive: - for member in archive: - if not member.isfile() or member.size > MAX_MEMBER_BYTES: - continue - handle = archive.extractfile(member) - if handle is None: # pragma: no cover - defensive - continue - with handle: - hits.extend(scan_bytes(handle.read(), member=member.name)) - return hits diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/attestation.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/attestation.py index 9782e9feb..f7ba5483e 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/attestation.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/attestation.py @@ -1,589 +1,141 @@ -"""Challenge-side acceptance gate for Phala-attested task results (M4). - -When the Phala attestation flag is ON the decentralized validator must not write -a task's score unless that task's result carries a Phala-tier attestation the -validator can VERIFY (architecture.md sec 4 C4). This module is the -self-contained, validator-owned verifier + decision layer the acceptance gate in -:mod:`agent_challenge.evaluation.validator_executor` consults: - -* :func:`extract_attestation_envelope` pulls the additive ``execution_proof`` + - ``attestation_binding`` blocks off a task's ``BASE_BENCHMARK_RESULT=`` line - (emitted by :mod:`agent_challenge.canonical.attested_result`), or reports the - result unattested. -* :class:`AttestationGate` verifies that envelope against the VALIDATOR's own - expectations -- fail-closed and conjunctive, mirroring base's Phala-tier - ``verify_execution_proof``: the TDX quote must be DCAP-valid with an acceptable - TCB, its event log must replay to the quote's signed RTMR3 yielding the - canonical ``compose_hash``, the reconstructed measurement must be in the - validator allowlist, ``report_data`` must equal the architecture-sec-6 binding - over ``(measurement, submission agent_hash, task_ids, scores_digest, nonce)``, - and the nonce must be a fresh, validator-issued, unconsumed one. -* The gate returns an :class:`AttestationDecision` carrying a distinguishable, - retrievable reason so a non-accepted result is observable (unattested vs - verification-failed vs verifier-unavailable/retryable), never a silent no-op. - -The trust root is the hardware-signed quote: the measurement and ``report_data`` -are read from the parsed TD report (not the untrusted attestation block). base's -models are not importable in this repo's venv, so the derivation is the shared -single-source :mod:`agent_challenge.canonical.report_data` (byte-identical to -base's ``phala_report_data``) and the quote primitives are agent-challenge's own -:mod:`agent_challenge.keyrelease.quote`. +"""Host-trust result admission shim (T40 — Phala AttestationGate removed). + +Product path is unattested/host-trust only. This module keeps a minimal +decision type so direct_result / validator_executor can admit plan-backed +results without TDX quote verification. Never claims TEE / independent +verification. """ from __future__ import annotations -import hashlib -import json -import secrets -import time -from collections.abc import Callable, Iterable, Mapping -from dataclasses import dataclass, field -from enum import Enum -from typing import Any, Protocol, runtime_checkable - -from agent_challenge.canonical import report_data as rd -from agent_challenge.canonical.attested_result import ( - ATTESTATION_BINDING_RESULT_KEY, - EXECUTION_PROOF_RESULT_KEY, - PHALA_TDX_TIER, -) -from agent_challenge.canonical.measurement import CANONICAL_MEASUREMENT_FIELDS -from agent_challenge.evaluation.own_runner.result_schema import RESULT_LINE_PREFIX -from agent_challenge.keyrelease.nonce import NonceState -from agent_challenge.keyrelease.quote import ( - QuoteStructureError, - QuoteVerificationError, - QuoteVerifier, - QuoteVerifierUnavailable, - decode_key_provider, - os_image_hash_from_registers, - parse_td_report, - parse_tdx_quote_v4, - replay_rtmr3, - validate_rtmr3_event_log, -) - -#: TCB statuses the acceptance gate accepts by default (architecture sec 7). -ACCEPTABLE_TCB_DEFAULT: frozenset[str] = frozenset({"UpToDate"}) - -# --------------------------------------------------------------------------- # -# Retrievable, distinguishable acceptance-outcome reasons (VAL-VERIFY-026). -# --------------------------------------------------------------------------- # -#: The result carried no (Phala-tier) attestation at all. -ATTESTATION_MISSING = "attestation_missing" -#: An attestation was present but its verification failed (permanent for this -#: result: forged/invalid quote, bad TCB, non-allowlisted measurement, bad -#: event-log replay, mis-bound report_data, or stale/unknown nonce). -ATTESTATION_VERIFICATION_FAILED = "attestation_verification_failed" -#: The quote-verification dependency was transiently unavailable/timed out; the -#: result is PARKED (retryable), neither accepted nor permanently rejected. -ATTESTATION_VERIFIER_UNAVAILABLE = "attestation_verifier_unavailable" - - -class AttestationVerifierUnavailable(Exception): - """Signals a transient quote-verifier outage so the result is parked (retryable). - - A :class:`~agent_challenge.keyrelease.quote.QuoteVerifier` raises this (rather - than :class:`~agent_challenge.keyrelease.quote.QuoteVerificationError`) when it - cannot reach its collateral/dependency, distinguishing a "cannot verify right - now" outage from a "quote is invalid" cryptographic rejection. - """ +from collections.abc import Mapping +from dataclasses import dataclass +from enum import StrEnum +from typing import Any -class AttestationOutcome(Enum): - """The gate's verdict for a task result under the Phala flag.""" +class AttestationOutcome(StrEnum): + """Admission outcome labels (host-trust; names kept for wire/DB compat).""" - #: Attestation present and fully verified -> the score may be written. VERIFIED = "verified" - #: No Phala-tier attestation on the result -> reject/park (not scored). - UNATTESTED = "unattested" - #: Attestation present but verification failed -> reject/park (not scored). VERIFICATION_FAILED = "verification_failed" - #: Quote verifier transiently unavailable -> park (retryable, not scored). VERIFIER_UNAVAILABLE = "verifier_unavailable" + REJECTED = "rejected" -_REASON_FOR: dict[AttestationOutcome, str | None] = { - AttestationOutcome.VERIFIED: None, - AttestationOutcome.UNATTESTED: ATTESTATION_MISSING, - AttestationOutcome.VERIFICATION_FAILED: ATTESTATION_VERIFICATION_FAILED, - AttestationOutcome.VERIFIER_UNAVAILABLE: ATTESTATION_VERIFIER_UNAVAILABLE, -} - - -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class AttestationDecision: - """The gate's decision: the outcome and a retrievable, distinguishable reason.""" - outcome: AttestationOutcome - reason: str | None + reason: str | None = None + + @classmethod + def of(cls, outcome: AttestationOutcome, reason: str | None = None) -> AttestationDecision: + return cls(outcome=outcome, reason=reason) @property def accepted(self) -> bool: - """Whether the result's attestation verified (its score may be written).""" - return self.outcome is AttestationOutcome.VERIFIED - @property - def retryable(self) -> bool: - """Whether the non-acceptance is a transient (retryable) park, not permanent.""" - - return self.outcome is AttestationOutcome.VERIFIER_UNAVAILABLE - @classmethod - def of(cls, outcome: AttestationOutcome) -> AttestationDecision: - return cls(outcome=outcome, reason=_REASON_FOR[outcome]) - - -# --------------------------------------------------------------------------- # -# Validator-owned measurement allowlist (canonical 6-register subset). -# --------------------------------------------------------------------------- # -@dataclass(frozen=True) class ResultMeasurementAllowlist: - """A validator-owned set of canonical measurements a result quote must match. - - Matching is exact across ALL canonical registers - (``mrtd, rtmr0, rtmr1, rtmr2, compose_hash, os_image_hash``; ``rtmr3`` is - runtime and excluded). An EMPTY allowlist matches nothing (fail closed) -- an - unconfigured validator never accepts a quote, never accept-any. - """ - - entries: tuple[dict[str, str], ...] = () - - @classmethod - def from_measurements( - cls, measurements: Iterable[Mapping[str, Any]] - ) -> ResultMeasurementAllowlist: - return cls(tuple(_canonical_measurement_mapping(m) for m in measurements)) - - def __bool__(self) -> bool: - return bool(self.entries) - - def contains(self, measurement: Mapping[str, Any]) -> bool: - candidate = _canonical_measurement_mapping(measurement) - return any(candidate == entry for entry in self.entries) - + """No-op allowlist stub — measurement allowlists removed with Phala.""" -def _canonical_measurement_mapping(measurement: Mapping[str, Any]) -> dict[str, str]: - return {field: str(measurement[field]) for field in CANONICAL_MEASUREMENT_FIELDS} + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass - -# --------------------------------------------------------------------------- # -# Validator-issued, single-use, TTL-bounded nonce ledger. -# --------------------------------------------------------------------------- # -@runtime_checkable -class NonceConsumer(Protocol): - """Consumes a validator-issued nonce, reporting its freshness state. - - ``consume`` is single-use: the first consume of a known, unexpired nonce - returns :attr:`~agent_challenge.keyrelease.nonce.NonceState.OK`; any later - consume returns :attr:`~agent_challenge.keyrelease.nonce.NonceState.CONSUMED`. - """ - - def consume(self, nonce: str) -> NonceState: # pragma: no cover - protocol - ... - - -def execution_proof_signing_payload(*, manifest_sha256: str, unit_id: str) -> bytes: - """Return the pinned tier-0 worker signature payload.""" - - return hashlib.sha256(f"{manifest_sha256}:{unit_id}".encode()).digest() - - -def verify_worker_signature( - worker_pubkey: str, - payload: bytes, - signature: str, -) -> bool: - """Verify the endpoint-owned tier-0 signature without trusting caller state.""" - - try: - import bittensor as bt - - return bool(bt.Keypair(ss58_address=worker_pubkey).verify(payload, signature)) - except Exception: # noqa: BLE001 - malformed/unavailable signer fails closed + def allows(self, *args: Any, **kwargs: Any) -> bool: return False -@dataclass -class InMemoryNonceLedger: - """A validator-local, single-use, TTL-bounded :class:`NonceConsumer`. - - An empty ledger consumes nothing (every nonce is UNKNOWN), so an unconfigured - validator fails closed. ``clock`` is injectable for deterministic expiry. - """ - - ttl_seconds: float = 120.0 - clock: Callable[[], float] = time.monotonic - _issued: dict[str, float] = field(default_factory=dict) - _consumed: set[str] = field(default_factory=set) - - def issue(self, nonce: str | None = None) -> str: - value = nonce if nonce is not None else secrets.token_urlsafe(32) - self._issued[value] = self.clock() - return value - - def is_outstanding(self, nonce: str) -> bool: - if not nonce or nonce in self._consumed or nonce not in self._issued: - return False - return (self.clock() - self._issued[nonce]) <= self.ttl_seconds - - def consume(self, nonce: str) -> NonceState: - if not nonce or nonce not in self._issued: - return NonceState.UNKNOWN - if nonce in self._consumed: - return NonceState.CONSUMED - if (self.clock() - self._issued[nonce]) > self.ttl_seconds: - del self._issued[nonce] - self._consumed.add(nonce) - return NonceState.EXPIRED - del self._issued[nonce] - self._consumed.add(nonce) - return NonceState.OK - - -# --------------------------------------------------------------------------- # -# Envelope extraction off the BASE_BENCHMARK_RESULT= line. -# --------------------------------------------------------------------------- # -def parse_benchmark_result_payload(stdout: str) -> dict[str, Any] | None: - """Return the parsed ``BASE_BENCHMARK_RESULT=`` JSON payload, or ``None``. - - Scans (last-wins, matching the runner's own parser) for the single result - line and parses its JSON object. Returns ``None`` when no parseable result - line is present. - """ - - if not stdout: - return None - for line in reversed(stdout.splitlines()): - if line.startswith(RESULT_LINE_PREFIX): - try: - parsed = json.loads(line[len(RESULT_LINE_PREFIX) :]) - except json.JSONDecodeError: - return None - return parsed if isinstance(parsed, dict) else None - return None - - -def extract_attestation_envelope( - stdout: str, -) -> tuple[dict[str, Any], dict[str, Any]] | None: - """Extract ``(execution_proof, attestation_binding)`` from a result's stdout. - - Returns ``None`` when the result carries no Phala-tier attestation (no result - line, no ``execution_proof`` block, a non-Phala tier, or a missing binding), - so the caller treats it as unattested. Only a well-formed Phala-tier envelope - with both additive blocks is returned for verification. - """ - - payload = parse_benchmark_result_payload(stdout) - if payload is None: - return None - execution_proof = payload.get(EXECUTION_PROOF_RESULT_KEY) - binding = payload.get(ATTESTATION_BINDING_RESULT_KEY) - if not isinstance(execution_proof, Mapping) or not isinstance(binding, Mapping): - return None - if execution_proof.get("tier") != PHALA_TDX_TIER: - return None - return dict(execution_proof), dict(binding) - - -# --------------------------------------------------------------------------- # -# The acceptance gate. -# --------------------------------------------------------------------------- # -@dataclass class AttestationGate: - """Verifies a task result's Phala attestation against validator expectations. + """Host-trust gate: admit plan-matching software checks only (no TDX). - Fail-closed and conjunctive: any missing dependency (no quote verifier, empty - allowlist, no nonce ledger) or any failing check yields a non-accepting - decision, so an unconfigured or misconfigured validator never accepts an - attested-looking result. A transient quote-verifier outage parks (retryable). + Quote verification always fails closed. Callers under unattested mode + should use :meth:`decide_host_trust` instead of quote paths. """ - quote_verifier: QuoteVerifier | None = None - allowlist: ResultMeasurementAllowlist = field(default_factory=ResultMeasurementAllowlist) - nonce_validator: NonceConsumer | None = None - acceptable_tcb: frozenset[str] = ACCEPTABLE_TCB_DEFAULT - - def decide(self, stdout: str, *, expected_agent_hash: str) -> AttestationDecision: - """Decide whether ``stdout``'s attested result may be written as a score.""" + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.quote_verifier = kwargs.get("quote_verifier") - envelope = extract_attestation_envelope(stdout) - if envelope is None: - return AttestationDecision.of(AttestationOutcome.UNATTESTED) - execution_proof, binding = envelope - outcome = self._verify(execution_proof, binding, expected_agent_hash) - return AttestationDecision.of(outcome) + def decide(self, *args: Any, **kwargs: Any) -> AttestationDecision: + return AttestationDecision.of( + AttestationOutcome.VERIFICATION_FAILED, + reason="phala_attestation_removed_use_host_trust", + ) def decide_eval_result( self, - result_request: Mapping[str, Any], + validated: Mapping[str, Any], *, - eval_plan: Mapping[str, Any], - expected_agent_hash: str, - nonce_outstanding: bool, - key_granted: bool, - consume_nonce: bool = False, + eval_plan: Mapping[str, Any] | None = None, + expected_agent_hash: str | None = None, + nonce_outstanding: bool = False, + key_granted: bool = False, endpoint_rebound: bool = False, - rebound_worker_signature: Mapping[str, Any] | None = None, + rebound_worker_signature: Mapping[str, str] | None = None, + **kwargs: Any, ) -> AttestationDecision: - """Verify one schema-v1 direct Eval result against immutable plan state. - - This is the production-result counterpart to :meth:`decide`. It never - derives expected identity from the submitted proof and does not use the - legacy stdout/binding envelope. Database nonce consumption is normally - performed by the surrounding transaction after this method returns - ``VERIFIED``. ``consume_nonce`` remains available for offline - single-use ledgers and is deliberately false for the HTTP route so the - receipt, nonce, and score transaction has one owner. + """Host-trust admit: plan agent_hash match + outstanding nonce. + + Never verifies TDX quotes. Marks path as host-trust only. """ - if endpoint_rebound: - proof = result_request.get("execution_proof") - if not isinstance(proof, Mapping): - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - signature = proof.get("worker_signature") - if not isinstance(signature, Mapping) or signature != { - "worker_pubkey": "", - "sig": "", - }: - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - if not isinstance(rebound_worker_signature, Mapping): - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - rebound_pubkey = rebound_worker_signature.get("worker_pubkey") - rebound_sig = rebound_worker_signature.get("sig") - if ( - not isinstance(rebound_pubkey, str) - or not rebound_pubkey - or not isinstance(rebound_sig, str) - or not rebound_sig - ): - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - try: - from agent_challenge.evaluation.plan_scoring import validate_eval_result_from_plan - - request = validate_eval_result_from_plan(eval_plan, result_request) - except (ValueError, KeyError, TypeError): - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - - if request["agent_hash"] != expected_agent_hash or not key_granted: - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - proof = request["execution_proof"] - if endpoint_rebound: - signature = { - "worker_pubkey": str(rebound_worker_signature["worker_pubkey"]), - "sig": str(rebound_worker_signature["sig"]), - } - proof = dict(proof) - proof["worker_signature"] = signature - request = dict(request) - request["execution_proof"] = proof - payload = execution_proof_signing_payload( - manifest_sha256=proof["manifest_sha256"], - unit_id=eval_plan["eval_run_id"], - ) - if not verify_worker_signature(signature["worker_pubkey"], payload, signature["sig"]): - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - attestation = proof["attestation"] - quote = attestation["tdx_quote"] - - try: - report = parse_tdx_quote_v4(quote) - validate_rtmr3_event_log(attestation["event_log"]) - verdict = self.quote_verifier.verify(quote) if self.quote_verifier else None - except (QuoteVerifierUnavailable, AttestationVerifierUnavailable): - return AttestationDecision.of(AttestationOutcome.VERIFIER_UNAVAILABLE) - except (QuoteStructureError, QuoteVerificationError, ValueError, TypeError): - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - except Exception: # noqa: BLE001 - unexpected verifier/backend failures are retryable - return AttestationDecision.of(AttestationOutcome.VERIFIER_UNAVAILABLE) - - if verdict is None or verdict.tcb_status not in self.acceptable_tcb: - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - - try: - replay = replay_rtmr3(attestation["event_log"]) - except (QuoteVerificationError, ValueError, TypeError): - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - if replay.rtmr3 != report.rtmr3 or replay.compose_hash is None: - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - if attestation["measurement"]["rtmr3"] != report.rtmr3: - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - if attestation["measurement"]["compose_hash"] != replay.compose_hash: - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - - expected = eval_plan["eval_app"]["measurement"] - measurement = { - "mrtd": report.mrtd, - "rtmr0": report.rtmr0, - "rtmr1": report.rtmr1, - "rtmr2": report.rtmr2, - "compose_hash": replay.compose_hash, - "os_image_hash": os_image_hash_from_registers(report.mrtd, report.rtmr1, report.rtmr2), - } - expected_measurement = { - "mrtd": expected["mrtd"], - "rtmr0": expected["rtmr0"], - "rtmr1": expected["rtmr1"], - "rtmr2": expected["rtmr2"], - "compose_hash": eval_plan["eval_app"]["compose_hash"], - "os_image_hash": expected["os_image_hash"], - } - if measurement != expected_measurement or not self.allowlist.contains(measurement): - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - # Live dstack RTMR3 key-provider events carry JSON (e.g. {"name":"kms",...}) - # as hex. Collapse onto the allowlist pin via the shared KR decoder so the - # score gate matches host KR / plan pin ``phala`` (not raw hex equality). - try: - decoded_provider = decode_key_provider(replay.key_provider) - except QuoteVerificationError: - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - if decoded_provider != str(expected["key_provider"]): - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - vm_image_hash = attestation["vm_config"]["os_image_hash"] - if vm_image_hash is not None and vm_image_hash != expected_measurement["os_image_hash"]: - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - if report.report_data.hex() != attestation["report_data"]: - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) if not nonce_outstanding: - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - if consume_nonce: - if self.nonce_validator is None: - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - if self.nonce_validator.consume(eval_plan["score_nonce"]) is not NonceState.OK: - return AttestationDecision.of(AttestationOutcome.VERIFICATION_FAILED) - return AttestationDecision.of(AttestationOutcome.VERIFIED) - - def _verify( - self, - execution_proof: Mapping[str, Any], - binding: Mapping[str, Any], - expected_agent_hash: str, - ) -> AttestationOutcome: - attestation = execution_proof.get("attestation") - if not isinstance(attestation, Mapping): - return AttestationOutcome.VERIFICATION_FAILED - - quote = attestation.get("tdx_quote") - event_log = attestation.get("event_log") - if not isinstance(quote, str) or not quote or not isinstance(event_log, list): - return AttestationOutcome.VERIFICATION_FAILED - - # -- the run identity the VALIDATOR expects (never trusted blindly) ---- # - agent_hash = binding.get("agent_hash") - task_ids = binding.get("task_ids") - scores = binding.get("scores") - scores_digest = binding.get("scores_digest") - nonce = binding.get("validator_nonce") - if ( - not isinstance(agent_hash, str) - or not isinstance(task_ids, list) - or not isinstance(scores, Mapping) - or not isinstance(scores_digest, str) - or not isinstance(nonce, str) - or not nonce - ): - return AttestationOutcome.VERIFICATION_FAILED - - # scores_digest must be the digest of the scores the result reports, so a - # score cannot be altered without breaking the report_data binding. - if rd.scores_digest(scores) != scores_digest: - return AttestationOutcome.VERIFICATION_FAILED - # the attested agent must be the submission actually under evaluation. - if agent_hash != expected_agent_hash: - return AttestationOutcome.VERIFICATION_FAILED - - # dependencies missing -> fail closed (never accept-any). - if self.quote_verifier is None or not self.allowlist or self.nonce_validator is None: - return AttestationOutcome.VERIFICATION_FAILED - - try: - report = parse_td_report(quote) - except QuoteStructureError: - return AttestationOutcome.VERIFICATION_FAILED - - # cryptographic signature/cert-chain + TCB posture (trust root). - try: - verdict = self.quote_verifier.verify(quote) - except (AttestationVerifierUnavailable, QuoteVerifierUnavailable): - return AttestationOutcome.VERIFIER_UNAVAILABLE - except QuoteVerificationError: - return AttestationOutcome.VERIFICATION_FAILED - except Exception: # noqa: BLE001 - any verifier error fails closed - return AttestationOutcome.VERIFICATION_FAILED - if verdict.tcb_status not in self.acceptable_tcb: - return AttestationOutcome.VERIFICATION_FAILED - - # RTMR3 validated by content: replay the event log to the signed RTMR3. - try: - replay = replay_rtmr3(event_log) - except QuoteVerificationError: - return AttestationOutcome.VERIFICATION_FAILED - if replay.rtmr3 != report.rtmr3 or replay.compose_hash is None: - return AttestationOutcome.VERIFICATION_FAILED - - # measurement reconstructed from the SIGNED registers must be allowlisted. - measurement = { - "mrtd": report.mrtd, - "rtmr0": report.rtmr0, - "rtmr1": report.rtmr1, - "rtmr2": report.rtmr2, - "compose_hash": replay.compose_hash, - "os_image_hash": os_image_hash_from_registers(report.mrtd, report.rtmr1, report.rtmr2), - } - if not self.allowlist.contains(measurement): - return AttestationOutcome.VERIFICATION_FAILED - - # report_data must bind exactly (measurement, agent, task set, scores, nonce). - expected_report_data = rd.report_data_hex( - canonical_measurement=measurement, - agent_hash=agent_hash, - task_ids=task_ids, - scores_digest=scores_digest, - validator_nonce=nonce, + return AttestationDecision.of( + AttestationOutcome.VERIFICATION_FAILED, + reason="score_nonce_not_outstanding", + ) + plan = eval_plan or {} + plan_hash = str(plan.get("agent_hash") or "").strip() + expected = (expected_agent_hash or plan_hash).strip() + got = "" + if isinstance(validated, Mapping): + proof = validated.get("execution_proof") + if isinstance(proof, Mapping): + got = str(proof.get("agent_hash") or "").strip() + if not got: + got = str(validated.get("agent_hash") or "").strip() + if expected and got and expected != got: + return AttestationDecision.of( + AttestationOutcome.VERIFICATION_FAILED, + reason="agent_hash_mismatch", + ) + # Host-trust software admit (unattested). Callers must mark envelopes. + return AttestationDecision.of( + AttestationOutcome.VERIFIED, + reason="host_trust_plan_admit_unattested", ) - if report.report_data.hex() != expected_report_data: - return AttestationOutcome.VERIFICATION_FAILED - # nonce consumed LAST (single-use); a rejection above never burns it. - if self.nonce_validator.consume(nonce) is not NonceState.OK: - return AttestationOutcome.VERIFICATION_FAILED + def decide_host_trust(self, *args: Any, **kwargs: Any) -> AttestationDecision: + return self.decide_eval_result(*args, **kwargs) - return AttestationOutcome.VERIFIED +def execution_proof_signing_payload(*, manifest_sha256: str, unit_id: str) -> bytes: + """Canonical bytes signed by the endpoint worker for result rebound.""" -def failclosed_gate() -> AttestationGate: - """A default, fully fail-closed gate (accepts nothing). + man = (manifest_sha256 or "").strip().lower() + uid = (unit_id or "").strip() + return f"agent-challenge-exec-proof-v1|{man}|{uid}".encode() - Used when the Phala flag is ON but no gate was injected: with no quote - verifier, an empty allowlist, and an empty nonce ledger, every attested - result is rejected -- an unconfigured validator never writes an unverified - score. - """ - return AttestationGate( - quote_verifier=None, - allowlist=ResultMeasurementAllowlist(), - nonce_validator=InMemoryNonceLedger(), - ) +def verify_worker_signature(*args: Any, **kwargs: Any) -> bool: + """Signature verify stub — TEE worker signatures not product path after T40.""" + + return False + + +def failclosed_gate() -> AttestationGate: + """Return a gate that never verifies TEE quotes (host-trust only).""" + + return AttestationGate() __all__ = [ - "ACCEPTABLE_TCB_DEFAULT", - "ATTESTATION_MISSING", - "ATTESTATION_VERIFICATION_FAILED", - "ATTESTATION_VERIFIER_UNAVAILABLE", "AttestationDecision", "AttestationGate", "AttestationOutcome", - "AttestationVerifierUnavailable", - "InMemoryNonceLedger", - "NonceConsumer", "ResultMeasurementAllowlist", - "extract_attestation_envelope", "execution_proof_signing_payload", "failclosed_gate", - "parse_benchmark_result_payload", "verify_worker_signature", ] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/authorization.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/authorization.py index 5bcb1de5d..a5d634db5 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/authorization.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/authorization.py @@ -391,13 +391,20 @@ def _build_plan( issued_at_ms = _milliseconds(now) expires_at_ms = _milliseconds(now + timedelta(seconds=settings.eval_run_ttl_seconds)) key_release_endpoint = settings.eval_key_release_endpoint + # T40: KR endpoint optional under unattested/no_phala host-trust path only. + # Legacy prepare without no_phala still requires a configured endpoint string + # (keyrelease package itself is untouched; score path no longer requires grant). + no_phala = bool( + getattr(settings, "no_phala", False) or getattr(settings, "unattested_execution", False) + ) if not isinstance(key_release_endpoint, str) or not key_release_endpoint.strip(): - # Empty endpoint used to residual as bare wire 500 via validate_eval_plan; - # surface a closed unavailable code instead. - raise EvalAuthorizationUnavailable( - "validator Eval key release endpoint is unavailable", - code="eval_key_release_endpoint_unavailable", - ) + if no_phala: + key_release_endpoint = "" + else: + raise EvalAuthorizationUnavailable( + "validator Eval key release endpoint is unavailable", + code="eval_key_release_endpoint_unavailable", + ) package_tree_sha = getattr(submission, "package_tree_sha", None) if not isinstance(package_tree_sha, str) or not package_tree_sha.strip(): raise EvalAuthorizationUnavailable( @@ -489,20 +496,21 @@ async def _authorized_review_digest( admit_eval_cvm_launch_from_assignment, ) - dual_on = True - require_residual = True + # T40: dual TEE flags permanently off. Defaults are host-trust. + dual_on = False + require_residual = False expected_tree: str | None = getattr(submission, "package_tree_sha", None) if isinstance(expected_tree, str): expected_tree = expected_tree.strip() or None else: expected_tree = None if settings is not None: - phala = bool(getattr(settings, "phala_attestation_enabled", True)) - review = bool(getattr(settings, "attested_review_enabled", True)) + phala = bool(getattr(settings, "phala_attestation_enabled", False)) + review = bool(getattr(settings, "attested_review_enabled", False)) dual_on = phala and review - # Residual is mandatory for production dual-flag prepare; flag-off still - # fails later emit gates but residual check runs when dual_on. - require_residual = dual_on + # Host-trust: residual optional at prepare (software policy when present). + # dual_on is always False after T40 settings force-off. + require_residual = bool(dual_on) try: decision = admit_eval_cvm_launch_from_assignment( diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/blackbox_attestation_chain.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/blackbox_attestation_chain.py deleted file mode 100644 index e1206b11d..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/blackbox_attestation_chain.py +++ /dev/null @@ -1,1457 +0,0 @@ -"""Black-box re-verifiable end-to-end attestation chain (VAL-ACAT-020, VAL-ACAT-X001..X012). - -Product freeze (library/ac-attestation.md): - -Miner script+ZIP → measured review under ``.rules`` with OpenRouter planned/observed -digests → attested times ≤24h → Eval CVM fresh re-verify → RA-TLS key-release → -optional measured eval-agent OR digests → score domain → raw weights into Base -aggregate path. - -This module **composes** existing fail-closed gates into one chronological package -that: - -1. Maps every cross-area hop with stable hop ids and codes. -2. Produces an independent re-verify surface from package artifacts alone. -3. Fails closed on mutilation (report_data, digests, times, domain mixups). -4. Asserts zero Base gateway critical-path dependency. -5. Routes weights via challenge raw-weight / master aggregate only (never - ``set_weights`` / burn recipe). - -Lab/disposable or fixture chain materials are acceptable when re-verifiable. -Production PATH honesty stays dual-label: REAL-PROVIDER TEE remains BLOCKED -without hard provider gates. Does **not** restore Base LLM gateway. -""" - -from __future__ import annotations - -import copy -import hashlib -import json -from collections.abc import Mapping, MutableMapping, Sequence -from dataclasses import dataclass, field -from datetime import UTC, datetime, timedelta -from typing import Any - -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.evaluation.eval_agent_llm import ( - MEASURED_EVAL_CVM_KIND, - MODE_MEASURED_OPENROUTER, - MODE_TOOLS_ONLY, - REFUSE_BASE_GATEWAY, - admit_eval_agent_llm_for_score, - bind_eval_agent_or_digests_into_score_materials, - build_eval_agent_observed_transport, - build_eval_agent_planned_request, -) -from agent_challenge.evaluation.fresh_review_gate import ( - admit_eval_cvm_fresh_review, -) -from agent_challenge.evaluation.score_chain_gate import ( - KEY_RELEASE_DOMAIN, - REVIEW_DOMAIN, - SCORE_DOMAIN, - admit_production_score_from_chain, - build_score_binding_from_plan_and_digest, - recompute_key_release_report_data_hex, -) -from agent_challenge.review.attested_times import FRESHNESS_WINDOW_MS -from agent_challenge.review.harness_entry import ( - PRODUCT_HARNESS_KIND, - ProductHarnessAdmissionError, - admit_product_review_entry, -) -from agent_challenge.review.or_outcome_bind import ( - REVIEW_REPORT_DOMAIN, - build_decision, - build_observed_openrouter_transport, - build_openrouter_observation, - build_planned_openrouter_request, - build_policy_observation, - build_review_core_minimal, - planned_request_sha256, - review_digest, - review_report_data_hex, - sha256_hex, -) - -# --------------------------------------------------------------------------- -# Stable hop ids (chronological index — VAL-ACAT-020 / X001..X012) -# --------------------------------------------------------------------------- - -HOP_MINER_SUBMIT = "hop.miner_submit_no_gateway" # X001 -HOP_RULES_LOAD = "hop.review_rules_measured" # X002 -HOP_OR_PLANNED = "hop.review_or_planned" # X003 -HOP_OR_OBSERVED = "hop.review_or_observed" # X004 -HOP_FRESHNESS_24H = "hop.freshness_24h" # X005 -HOP_EVAL_CONJUNCTION = "hop.eval_launch_conjunction" # X006 -HOP_KEY_RELEASE = "hop.eval_ra_tls_key_release" # X007 -HOP_EVAL_AGENT_OR = "hop.eval_agent_or_measured" # X008 -HOP_SCORE_DOMAIN = "hop.score_domain_attestation" # X009 -HOP_RAW_WEIGHTS = "hop.raw_weights_aggregate" # X010 -HOP_GATEWAY_FREE = "hop.zero_base_gateway" # X011 -HOP_HONESTY = "hop.fairness_honesty" # X012 - -HOP_ORDER: tuple[str, ...] = ( - HOP_MINER_SUBMIT, - HOP_RULES_LOAD, - HOP_OR_PLANNED, - HOP_OR_OBSERVED, - HOP_FRESHNESS_24H, - HOP_EVAL_CONJUNCTION, - HOP_KEY_RELEASE, - HOP_EVAL_AGENT_OR, - HOP_SCORE_DOMAIN, - HOP_RAW_WEIGHTS, - HOP_GATEWAY_FREE, - HOP_HONESTY, -) - -# Assertion map for evidence index -HOP_ASSERTIONS: dict[str, str] = { - HOP_MINER_SUBMIT: "VAL-ACAT-X001", - HOP_RULES_LOAD: "VAL-ACAT-X002", - HOP_OR_PLANNED: "VAL-ACAT-X003", - HOP_OR_OBSERVED: "VAL-ACAT-X004", - HOP_FRESHNESS_24H: "VAL-ACAT-X005", - HOP_EVAL_CONJUNCTION: "VAL-ACAT-X006", - HOP_KEY_RELEASE: "VAL-ACAT-X007", - HOP_EVAL_AGENT_OR: "VAL-ACAT-X008", - HOP_SCORE_DOMAIN: "VAL-ACAT-X009", - HOP_RAW_WEIGHTS: "VAL-ACAT-X010", - HOP_GATEWAY_FREE: "VAL-ACAT-X011", - HOP_HONESTY: "VAL-ACAT-X012", -} - -# Refuse codes unique to this composition surface -REFUSE_CHAIN_INCOMPLETE = "e2e_chain_incomplete" -REFUSE_CHAIN_MUTILATED = "e2e_chain_mutilated_reverify_failed" -REFUSE_GATEWAY_RESIDUAL = "e2e_base_gateway_residual" -REFUSE_SET_WEIGHTS = "e2e_set_weights_forbidden" -REFUSE_BURN_TRACKED = "e2e_burn_weights_tracked_forbidden" -REFUSE_HOP_ORDER = "e2e_hop_order_invalid" -REFUSE_PACKAGE_TAMPER = "e2e_package_tamper" - -BASE_GATEWAY_ENV_NAMES = frozenset( - { - "BASE_GATEWAY_TOKEN", - "BASE_LLM_GATEWAY_URL", - "GATEWAY_TOKEN", - "CENTRAL_GATEWAY_TOKEN", - "CHALLENGE_LLM_GATEWAY_TOKEN", - "CHALLENGE_LLM_GATEWAY_BASE_URL", - "CHALLENGE_LLM_GATEWAY_TOKEN_FILE", - "CHALLENGE_AGENT_GATEWAY_TOKEN", - } -) -BASE_GATEWAY_URL_MARKERS = ("/llm/v1", "BASE_LLM_GATEWAY", "BASE_GATEWAY_TOKEN") -FORBIDDEN_SIDE_EFFECTS = frozenset({"set_weights", "burn_weights_24h", "WeightSetter"}) - -# Fixture-stable seeds (deterministic package digests across same inputs) -T0_DEFAULT = 1_700_000_000_000 -MS_23H = FRESHNESS_WINDOW_MS - 60_000 -PRODUCT_ENTRY = "python -m agent_challenge.selfdeploy" -SAMPLE_ZIP = b"PK\x03\x04agent-challenge-e2e-fixture-agent-v1" -ENTRY_BYTES = b'#!/usr/bin/env python3\n"""selfdeploy entry for e2e chain"""\n' -SAMPLE_RULES: dict[str, bytes] = { - ".rules/acceptance.md": b"# acceptance\nMeasured review path only.\n", - ".rules/anti-cheat.md": b"# anti-cheat\nNo unmeasured shortcuts.\n", - ".rules/hardcoding.md": b"# hardcoding\nNo Base gateway hardcoding.\n", - ".rules/security.md": b"# security\nFail closed on missing attestation.\n", -} -ROUTING_DEFAULT = sha256_hex(b'{"order":["e2e-blackbox"]}') -BODY_DEFAULT = b'{"model":"x-ai/grok-4.5","messages":[{"role":"user","content":"review"}]}' -BODY_SHA_DEFAULT = sha256_hex(BODY_DEFAULT) -RESP_DEFAULT = ( - b'{"id":"gen-e2e","model":"x-ai/grok-4.5","choices":[{"message":{"content":"allow"}}]}' -) -RESP_SHA_DEFAULT = sha256_hex(RESP_DEFAULT) -META_DEFAULT = sha256_hex(b"meta-e2e-blackbox") -SPKI_DEFAULT = "aa" * 32 -REGS_DEFAULT = { - "mrtd": "11" * 48, - "rtmr0": "22" * 48, - "rtmr1": "33" * 48, - "rtmr2": "44" * 48, -} -COMPOSE_HASH_DEFAULT = "ab" * 32 -AGENT_HASH_DEFAULT = "55" * 32 -MINER_HOTKEY_DEFAULT = "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY" - - -# --------------------------------------------------------------------------- -# Dataclasses -# --------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class HopResult: - """One chronological hop outcome in the e2e evidence index.""" - - hop_id: str - assertion_id: str - status: str # pass | refuse | skip - reason_code: str - detail: str = "" - digests: Mapping[str, str] = field(default_factory=dict) - times_ms: Mapping[str, int] = field(default_factory=dict) - gateway_dependency: bool = False - set_weights_invoked: bool = False - - def as_dict(self) -> dict[str, Any]: - return { - "hop_id": self.hop_id, - "assertion_id": self.assertion_id, - "status": self.status, - "reason_code": self.reason_code, - "detail": self.detail, - "digests": dict(self.digests), - "times_ms": dict(self.times_ms), - "gateway_dependency": self.gateway_dependency, - "set_weights_invoked": self.set_weights_invoked, - } - - -@dataclass(frozen=True) -class ChainAdmission: - """Overall E2E chain decision.""" - - admitted: bool - reason_code: str - hops: tuple[HopResult, ...] - package: Mapping[str, Any] - package_digest: str - production_emit: bool - raw_weights: Mapping[str, float] - gateway_free: bool - reverify_ok: bool - tee_labels: Mapping[str, str] - side_effects: Mapping[str, int] - - def as_dict(self) -> dict[str, Any]: - return { - "admitted": self.admitted, - "reason_code": self.reason_code, - "hops": [h.as_dict() for h in self.hops], - "package_digest": self.package_digest, - "production_emit": self.production_emit, - "raw_weights": dict(self.raw_weights), - "gateway_free": self.gateway_free, - "reverify_ok": self.reverify_ok, - "tee_labels": dict(self.tee_labels), - "side_effects": dict(self.side_effects), - "chronological_hop_ids": [h.hop_id for h in self.hops], - "assertions_mapped": {h.assertion_id: h.status for h in self.hops if h.assertion_id}, - } - - -class BlackboxChainError(PermissionError): - """Fail-closed e2e chain refuse.""" - - def __init__(self, code: str, message: str | None = None) -> None: - self.code = code - super().__init__(message or code) - - -# --------------------------------------------------------------------------- -# Internal helpers -# --------------------------------------------------------------------------- - - -def canonical_json_v1(obj: Mapping[str, Any] | Sequence[Any] | Any) -> bytes: - return json.dumps(obj, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") - - -def package_digest(package: Mapping[str, Any]) -> str: - """Digest of re-verifiable materials only (exclude derived hop statuses).""" - - materials = { - "schema_version": package.get("schema_version"), - "identity": package.get("identity"), - "review_envelope": package.get("review_envelope"), - "key_release_grant": package.get("key_release_grant"), - "eval_plan": package.get("eval_plan"), - "score_binding": package.get("score_binding"), - "score_report_data_hex": package.get("score_report_data_hex"), - "scores_digest": package.get("scores_digest"), - "agent_or_materials": package.get("agent_or_materials"), - "raw_weights_payload": package.get("raw_weights_payload"), - "gateway_inventory": package.get("gateway_inventory"), - "honesty": package.get("honesty"), - "issued_at_ms": package.get("issued_at_ms"), - "received_at_ms": package.get("received_at_ms"), - "dual_flags_on": package.get("dual_flags_on"), - "agent_llm_mode": package.get("agent_llm_mode"), - } - return hashlib.sha256(canonical_json_v1(materials)).hexdigest() - - -def scan_gateway_residuals( - *, - env: Mapping[str, str] | None = None, - urls: Sequence[str] | None = None, - master_routes: Sequence[str] | None = None, - secrets: Mapping[str, str] | None = None, -) -> list[str]: - """Return residual Base gateway markers (empty => gateway-free).""" - - hits: list[str] = [] - for name in env or {}: - upper = str(name).upper() - if upper in BASE_GATEWAY_ENV_NAMES or "LLM_GATEWAY" in upper: - hits.append(f"env:{upper}") - for url in urls or (): - u = str(url) - for marker in BASE_GATEWAY_URL_MARKERS: - if marker.lower() in u.lower(): - hits.append(f"url:{marker}") - for route in master_routes or (): - r = str(route) - if "/llm/v1" in r: - hits.append(f"route:{r}") - for name in secrets or {}: - upper = str(name).upper() - if upper in BASE_GATEWAY_ENV_NAMES or "OPENROUTER" in upper and "MASTER" in upper: - hits.append(f"secret:{upper}") - return hits - - -def build_raw_weights_payload( - *, - hotkey_weights: Mapping[str, float], - challenge_slug: str = "agent-challenge", - epoch: int = 1, - revision: int = 1, - computed_at: datetime | None = None, - expires_at: datetime | None = None, - nonce: str = "e2e-raw-weight-nonce-1", -) -> dict[str, Any]: - """Closed raw-weight payload for master aggregate path (no set_weights). - - Uses the same canonical digest rule as Base ``RawWeightPushRequest`` so the - package is independently re-checkable against Base aggregation ingress - without invoking ops ``set_weights``. - """ - - if not hotkey_weights: - raise BlackboxChainError(REFUSE_CHAIN_INCOMPLETE, "empty hotkey weight map") - for hotkey, weight in hotkey_weights.items(): - if not isinstance(weight, float) or weight < 0 or weight != weight: - raise BlackboxChainError(REFUSE_CHAIN_INCOMPLETE, f"invalid weight for {hotkey!r}") - now = computed_at or datetime(2023, 11, 14, 22, 13, 20, tzinfo=UTC) - exp = expires_at or (now + timedelta(hours=1)) - if exp <= now: - raise BlackboxChainError(REFUSE_CHAIN_INCOMPLETE, "expires_at must be after computed_at") - - def _iso(dt: datetime) -> str: - return dt.astimezone(UTC).isoformat().replace("+00:00", "Z") - - body: dict[str, Any] = { - "protocol_version": "1.0", - "challenge_slug": challenge_slug, - "epoch": int(epoch), - "revision": int(revision), - "computed_at": _iso(now), - "expires_at": _iso(exp), - "nonce": nonce, - "weights": {k: float(v) for k, v in hotkey_weights.items()}, - } - digest = hashlib.sha256(canonical_json_v1(body)).hexdigest() - body["payload_digest"] = digest - body["path"] = f"/internal/v1/challenges/{challenge_slug}/raw-weights" - body["master_set_weights"] = False - body["ops_set_weights"] = False - body["burn_weights_invoked"] = False - return body - - -def _times(*, issued: int, received: int) -> dict[str, int]: - base = min(issued, received) - return { - "issued_at_ms": issued, - "started_at_ms": base, - "model_call_marked_at_ms": base + 1, - "request_started_at_ms": base + 2, - "request_finished_at_ms": base + 3, - "verifier_finished_at_ms": base + 4, - "report_finished_at_ms": base + 5, - "expires_at_ms": max(issued, received) + 3_600_000, - "submission_received_at_ms": received, - } - - -def _rules_observation_from_identity(identity: Mapping[str, Any]) -> dict[str, Any]: - return { - "rules_version": identity["rules_version"], - "rules_bundle_sha256": identity["rules_bundle_sha256"], - "rules_files": list(identity["rules_files"]), - "rules_file_digests": dict(identity["rules_file_digests"]), - "rules_policy_text_sha256": identity["rules_policy_text_sha256"], - } - - -def build_fixture_review_envelope( - *, - identity: Mapping[str, Any], - issued_at_ms: int, - received_at_ms: int, - session_id: str = "rs-e2e-blackbox", - assignment_id: str = "ra-e2e-blackbox", - submission_id: str = "sub-e2e-blackbox", - review_nonce: str = "nonce-e2e-review", - verdict: str = "allow", -) -> dict[str, Any]: - """Closed allow review envelope with OR digests + bound times.""" - - planned = build_planned_openrouter_request( - body_sha256=BODY_SHA_DEFAULT, - body_length=len(BODY_DEFAULT), - routing_sha256=ROUTING_DEFAULT, - ) - p_digest = planned_request_sha256(planned) - observed = build_observed_openrouter_transport( - planned_request_sha256_=p_digest, - response_body_sha256=RESP_SHA_DEFAULT, - response_body_length=len(RESP_DEFAULT), - metadata_sha256=META_DEFAULT, - ) - or_obs = build_openrouter_observation( - planned=planned, - observed=observed, - request_body_sha256=BODY_SHA_DEFAULT, - request_body_length=len(BODY_DEFAULT), - response_id="gen-e2e", - ) - policy = build_policy_observation( - prompt_version="review-policy-prompt-v1", - prompt_bytes=b"prompt-e2e", - tool_schema_version="review-policy-tool-v1", - tool_schema_bytes=b"tools-e2e", - verifier_version="review-policy-verifier-v1", - verifier_bytes=b"verifier-e2e", - routing_sha256=ROUTING_DEFAULT, - ) - core = build_review_core_minimal( - session_id=session_id, - assignment_id=assignment_id, - submission_id=submission_id, - review_nonce=review_nonce, - assignment_digest="aa" * 32, - rules_observation=_rules_observation_from_identity(identity), - policy_observation=policy, - openrouter_observation=or_obs, - decision=build_decision(verdict=verdict), - times=_times(issued=issued_at_ms, received=received_at_ms), - ) - return { - "schema_version": 1, - "domain": REVIEW_REPORT_DOMAIN, - "review_digest": review_digest(core), - "report_data_hex": review_report_data_hex(core), - "review_core": core, - "planned_openrouter": planned, - "observed_openrouter": observed, - } - - -def _os_image_hash() -> str: - from agent_challenge.keyrelease.quote import os_image_hash_from_registers - - return os_image_hash_from_registers( - REGS_DEFAULT["mrtd"], REGS_DEFAULT["rtmr1"], REGS_DEFAULT["rtmr2"] - ) - - -def build_fixture_eval_plan( - *, - authorizing_review_digest: str, - agent_hash: str = AGENT_HASH_DEFAULT, - eval_run_id: str = "eval-e2e-blackbox-1", -) -> dict[str, Any]: - os_hash = _os_image_hash() - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - return ew.validate_eval_plan( - { - "schema_version": 1, - "eval_run_id": eval_run_id, - "submission_id": "submission-e2e-blackbox-1", - "submission_version": 1, - "authorizing_review_digest": authorizing_review_digest, - "agent_hash": agent_hash, - "package_tree_sha": "b" * 64, - "selected_tasks": [ - { - "task_id": "task-a", - "image_ref": "registry.example/task@sha256:" + "77" * 32, - "task_config_sha256": "88" * 32, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "99" * 32, - "compose_hash": COMPOSE_HASH_DEFAULT, - "app_identity": "agent-challenge-eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "aa" * 32, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), - "measurement": { - **REGS_DEFAULT, - "os_image_hash": os_hash, - "key_provider": "phala", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": f"/evaluation/v1/runs/{eval_run_id}/result", - "key_release_nonce": "key-release-e2e-1", - "score_nonce": "score-nonce-e2e-1", - "run_token_sha256": "bb" * 32, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - ) - - -def build_key_release_grant(plan: Mapping[str, Any], *, spki: str = SPKI_DEFAULT) -> dict[str, Any]: - rd = recompute_key_release_report_data_hex( - eval_run_id=str(plan["eval_run_id"]), - key_release_nonce=str(plan["key_release_nonce"]), - ra_tls_spki_digest=spki, - ) - return { - "domain": KEY_RELEASE_DOMAIN, - "schema_version": 2, - "eval_run_id": plan["eval_run_id"], - "key_release_nonce": plan["key_release_nonce"], - "ra_tls_spki_digest": spki, - "report_data_hex": rd, - "agent_hash": plan.get("agent_hash"), - "base_gateway_token_minted": False, - } - - -def build_scores_digest(plan: Mapping[str, Any]) -> str: - from agent_challenge.evaluation.plan_scoring import build_score_record_from_eval_plan - - record = build_score_record_from_eval_plan(dict(plan), {"task-a": [1.0]}) - return ew.score_record_digest(record) - - -def build_eval_agent_materials( - *, - body: bytes = b'{"model":"openrouter/auto","messages":[]}', - model: str = "x-ai/grok-4.5", -) -> dict[str, Any]: - routing = sha256_hex(b'{"order":["eval-agent-e2e"]}') - body_sha = sha256_hex(body) - planned = build_eval_agent_planned_request( - body_sha256=body_sha, - body_length=len(body), - routing_sha256=routing, - model=model, - ) - resp = b'{"id":"gen-agent-e2e","choices":[]}' - observed = build_eval_agent_observed_transport( - planned=planned, - response_body_sha256=sha256_hex(resp), - response_body_length=len(resp), - metadata_sha256=sha256_hex(b"eval-agent-meta"), - ) - return bind_eval_agent_or_digests_into_score_materials( - planned=planned, - observed=observed, - ) - - -def build_complete_package( - *, - dual_flags_on: bool = True, - issued_at_ms: int = T0_DEFAULT, - received_at_ms: int | None = None, - agent_zip_bytes: bytes = SAMPLE_ZIP, - entry_script_identity: str = PRODUCT_ENTRY, - entry_script_bytes: bytes = ENTRY_BYTES, - rules_files: Mapping[str, bytes] | None = None, - agent_llm_mode: str = MODE_TOOLS_ONLY, - include_measured_agent_or: bool = False, - miner_hotkey: str = MINER_HOTKEY_DEFAULT, - raw_score: float = 1.0, - master_env: Mapping[str, str] | None = None, - master_routes: Sequence[str] | None = None, - set_weights_count: int = 0, - burn_invoked: bool = False, - real_provider_pass_claimed: bool = False, - verdict: str = "allow", - skip_review: bool = False, - skip_key_release: bool = False, - skip_score: bool = False, -) -> dict[str, Any]: - """Build a full re-verifiable artifact package for the e2e chain.""" - - received = T0_DEFAULT + MS_23H if received_at_ms is None else received_at_ms - try: - identity = admit_product_review_entry( - agent_zip_bytes=agent_zip_bytes, - entry_script_identity=entry_script_identity, - entry_script_bytes=entry_script_bytes, - rules_files=rules_files if rules_files is not None else SAMPLE_RULES, - ) - identity_dict = identity.as_dict() - identity_error: str | None = None - except ProductHarnessAdmissionError as exc: - identity_dict = { - "schema_version": 1, - "harness_kind": PRODUCT_HARNESS_KIND, - "error": exc.code, - } - identity_error = exc.code - - review_envelope: dict[str, Any] | None = None - if not skip_review and identity_error is None: - review_envelope = build_fixture_review_envelope( - identity=identity_dict, - issued_at_ms=issued_at_ms, - received_at_ms=received, - verdict=verdict, - ) - - eval_plan: dict[str, Any] | None = None - key_release_grant: dict[str, Any] | None = None - score_binding: dict[str, Any] | None = None - score_report_data_hex: str | None = None - scores_digest: str | None = None - - if review_envelope is not None and not skip_score: - eval_plan = build_fixture_eval_plan( - authorizing_review_digest=str(review_envelope["review_digest"]), - ) - if not skip_key_release: - key_release_grant = build_key_release_grant(eval_plan) - scores_digest = build_scores_digest(eval_plan) - score_binding = build_score_binding_from_plan_and_digest( - eval_plan=eval_plan, - scores_digest=scores_digest, - ) - score_report_data_hex = ew.score_report_data_hex(score_binding) - - agent_or_materials: dict[str, Any] | None = None - llm_mode = agent_llm_mode - if include_measured_agent_or: - llm_mode = MODE_MEASURED_OPENROUTER - agent_or_materials = build_eval_agent_materials() - - raw_payload = build_raw_weights_payload( - hotkey_weights={miner_hotkey: float(raw_score)}, - ) - if set_weights_count: - raw_payload = dict(raw_payload) - raw_payload["master_set_weights"] = True - raw_payload["ops_set_weights_count"] = set_weights_count - if burn_invoked: - raw_payload = dict(raw_payload) - raw_payload["burn_weights_invoked"] = True - - env = dict(master_env or {}) - residuals = scan_gateway_residuals( - env=env, - master_routes=master_routes - or ( - "/health", - "/version", - "/v1/registry", - "/challenges/agent-challenge/openapi.json", - "/internal/v1/challenges/agent-challenge/raw-weights", - ), - secrets={}, - ) - - honesty = { - "tee_local_fixture": "LOCAL-FIXTURE PASS" if dual_flags_on else "FAIL", - "tee_real_provider": "REAL-PROVIDER PASS" if real_provider_pass_claimed else "BLOCKED", - "real_provider_pass_is_possible": False, - "real_provider_claimed": bool(real_provider_pass_claimed), - "secrets_redacted": True, - "secondary_platform_network_502_ok": True, - "joinbase_authoritative": True, - "no_false_score_on_refuse": True, - } - - package: dict[str, Any] = { - "schema_version": 1, - "domain": "base-agent-challenge-e2e-blackbox-v1", - "dual_flags_on": dual_flags_on, - "issued_at_ms": issued_at_ms, - "received_at_ms": received, - "identity": identity_dict, - "identity_error": identity_error, - "review_envelope": review_envelope, - "key_release_grant": key_release_grant, - "eval_plan": eval_plan, - "score_binding": score_binding, - "score_report_data_hex": score_report_data_hex, - "scores_digest": scores_digest, - "score_nonce_state": "outstanding", - "agent_llm_mode": llm_mode, - "agent_or_materials": agent_or_materials, - "agent_llm_runtime_kind": MEASURED_EVAL_CVM_KIND if include_measured_agent_or else None, - "agent_llm_measurement": { - "compose_hash": COMPOSE_HASH_DEFAULT, - "os_image_hash": _os_image_hash(), - "mrtd": REGS_DEFAULT["mrtd"], - } - if include_measured_agent_or - else None, - "agent_llm_allowlist": [ - { - "compose_hash": COMPOSE_HASH_DEFAULT, - "os_image_hash": _os_image_hash(), - "mrtd": REGS_DEFAULT["mrtd"], - } - ] - if include_measured_agent_or - else None, - "claims_agent_model_call": include_measured_agent_or, - "raw_weights_payload": raw_payload, - "gateway_inventory": { - "master_env_keys": sorted(env.keys()), - "residuals": residuals, - "master_routes": list( - master_routes - or ( - "/health", - "/version", - "/v1/registry", - "/challenges/agent-challenge/openapi.json", - "/internal/v1/challenges/agent-challenge/raw-weights", - ) - ), - "openrouter_destination": "https://openrouter.ai:443", - "base_llm_v1_present": any("/llm/v1" in str(r) for r in (master_routes or ())), - }, - "side_effects": { - "set_weights": int(set_weights_count), - "burn_weights_24h": 1 if burn_invoked else 0, - "master_set_weights": 1 if set_weights_count else 0, - }, - "honesty": honesty, - "miner_hotkey": miner_hotkey, - "submit_path": "/challenges/agent-challenge/submissions", - "submit_requires_base_gateway_token": False, - } - package["package_digest"] = package_digest(package) - return package - - -def _pass_hop( - hop_id: str, - *, - reason_code: str = "ok", - detail: str = "", - digests: Mapping[str, str] | None = None, - times_ms: Mapping[str, int] | None = None, -) -> HopResult: - return HopResult( - hop_id=hop_id, - assertion_id=HOP_ASSERTIONS[hop_id], - status="pass", - reason_code=reason_code, - detail=detail, - digests=dict(digests or {}), - times_ms=dict(times_ms or {}), - gateway_dependency=False, - set_weights_invoked=False, - ) - - -def _refuse_hop( - hop_id: str, - reason_code: str, - *, - detail: str = "", - gateway_dependency: bool = False, - set_weights_invoked: bool = False, -) -> HopResult: - return HopResult( - hop_id=hop_id, - assertion_id=HOP_ASSERTIONS[hop_id], - status="refuse", - reason_code=reason_code, - detail=detail, - gateway_dependency=gateway_dependency, - set_weights_invoked=set_weights_invoked, - ) - - -def reverify_package(package: Mapping[str, Any]) -> ChainAdmission: - """Independently re-verify a previously built package from artifacts alone. - - This is the black-box surface for VAL-ACAT-020: given only the immutable - package materials + digests, re-run each product gate in chronology and - refuse on any hop failure or package digest mismatch. - """ - - hops: list[HopResult] = [] - dual = bool(package.get("dual_flags_on")) - reason_terminal = "e2e_chain_verified" - admitted = True - - # Digest integrity first - expected_digest = package.get("package_digest") - recomputed = package_digest(package) - if not isinstance(expected_digest, str) or expected_digest != recomputed: - hop = _refuse_hop(HOP_MINER_SUBMIT, REFUSE_PACKAGE_TAMPER, detail="package_digest mismatch") - hops.append(hop) - return ChainAdmission( - admitted=False, - reason_code=REFUSE_PACKAGE_TAMPER, - hops=tuple(hops), - package=dict(package), - package_digest=recomputed, - production_emit=False, - raw_weights={}, - gateway_free=False, - reverify_ok=False, - tee_labels={"REAL-PROVIDER": "BLOCKED"}, - side_effects={"set_weights": 0, "burn_weights_24h": 0}, - ) - - # --- X001 miner submit without gateway --- - identity_err = package.get("identity_error") - submit_requires_gw = bool(package.get("submit_requires_base_gateway_token")) - residuals = list((package.get("gateway_inventory") or {}).get("residuals") or []) - if submit_requires_gw or residuals: - hops.append( - _refuse_hop( - HOP_MINER_SUBMIT, - REFUSE_GATEWAY_RESIDUAL, - detail="submit depends on Base gateway residuals", - gateway_dependency=True, - ) - ) - admitted = False - reason_terminal = REFUSE_GATEWAY_RESIDUAL - elif identity_err: - hops.append(_refuse_hop(HOP_MINER_SUBMIT, str(identity_err))) - admitted = False - reason_terminal = str(identity_err) - else: - identity = package.get("identity") or {} - hops.append( - _pass_hop( - HOP_MINER_SUBMIT, - reason_code="submit_no_gateway", - digests={ - "zip_sha256": str(identity.get("zip_sha256", "")), - "session_identity_sha256": sha256_hex( - canonical_json_v1(identity) if isinstance(identity, Mapping) else b"{}" - ), - }, - ) - ) - - # --- X002 rules measured --- - identity = package.get("identity") or {} - if admitted and isinstance(identity, Mapping) and identity.get("rules_version"): - hops.append( - _pass_hop( - HOP_RULES_LOAD, - digests={ - "rules_version": str(identity["rules_version"]), - "rules_bundle_sha256": str(identity.get("rules_bundle_sha256", "")), - }, - ) - ) - elif admitted: - hops.append(_refuse_hop(HOP_RULES_LOAD, REFUSE_CHAIN_INCOMPLETE, detail="rules missing")) - admitted = False - reason_terminal = REFUSE_CHAIN_INCOMPLETE - else: - hops.append(_refuse_hop(HOP_RULES_LOAD, reason_terminal, detail="prior hop failed")) - - # --- X003 / X004 OR planned + observed --- - envelope = package.get("review_envelope") - if admitted and isinstance(envelope, Mapping): - core = envelope.get("review_core") - if not isinstance(core, Mapping): - hops.append(_refuse_hop(HOP_OR_PLANNED, REFUSE_CHAIN_INCOMPLETE)) - hops.append(_refuse_hop(HOP_OR_OBSERVED, REFUSE_CHAIN_INCOMPLETE)) - admitted = False - reason_terminal = REFUSE_CHAIN_INCOMPLETE - else: - or_obs = core.get("openrouter_observation") or {} - planned_sha = str(or_obs.get("planned_request_sha256", "")) - observed_sha = str(or_obs.get("transport_observation_sha256", "")) - if not planned_sha or planned_sha in {"00" * 32, "ff" * 32}: - hops.append( - _refuse_hop( - HOP_OR_PLANNED, - "review_or_planned_digest_missing", - ) - ) - admitted = False - reason_terminal = "review_or_planned_digest_missing" - else: - hops.append( - _pass_hop( - HOP_OR_PLANNED, - digests={"planned_request_sha256": planned_sha}, - ) - ) - if admitted: - if not observed_sha or observed_sha in {"00" * 32, "ff" * 32}: - hops.append( - _refuse_hop( - HOP_OR_OBSERVED, - "review_or_observed_digest_missing", - ) - ) - admitted = False - reason_terminal = "review_or_observed_digest_missing" - else: - hops.append( - _pass_hop( - HOP_OR_OBSERVED, - digests={"transport_observation_sha256": observed_sha}, - ) - ) - elif admitted: - hops.append(_refuse_hop(HOP_OR_PLANNED, REFUSE_CHAIN_INCOMPLETE)) - hops.append(_refuse_hop(HOP_OR_OBSERVED, REFUSE_CHAIN_INCOMPLETE)) - admitted = False - reason_terminal = REFUSE_CHAIN_INCOMPLETE - else: - hops.append(_refuse_hop(HOP_OR_PLANNED, reason_terminal)) - hops.append(_refuse_hop(HOP_OR_OBSERVED, reason_terminal)) - - # --- X005 freshness 24h --- - if admitted and isinstance(envelope, Mapping): - eval_decision = admit_eval_cvm_fresh_review(envelope=envelope) - times = { - "issued_at_ms": int(eval_decision.bound_issued_at_ms or 0), - "received_at_ms": int(eval_decision.bound_received_at_ms or 0), - } - if not eval_decision.may_launch and eval_decision.reason_code in { - "attestation_stale_over_24h", - "attestation_time_order_invalid", - "attestation_times_missing", - "attestation_times_invalid", - }: - hops.append( - _refuse_hop( - HOP_FRESHNESS_24H, - eval_decision.reason_code, - detail="age window fail-closed", - ) - ) - admitted = False - reason_terminal = eval_decision.reason_code - elif not eval_decision.may_launch: - # defer conjunct failures to X006; age itself can still be ok - age_ok = True - issued = package.get("issued_at_ms") - received = package.get("received_at_ms") - if ( - isinstance(issued, int) - and isinstance(received, int) - and (received - issued) > FRESHNESS_WINDOW_MS - ): - age_ok = False - if not age_ok: - hops.append(_refuse_hop(HOP_FRESHNESS_24H, "attestation_stale_over_24h")) - admitted = False - reason_terminal = "attestation_stale_over_24h" - else: - hops.append( - _pass_hop( - HOP_FRESHNESS_24H, - reason_code="age_within_24h", - times_ms=times, - ) - ) - else: - hops.append( - _pass_hop( - HOP_FRESHNESS_24H, - reason_code="age_within_24h", - times_ms=times, - ) - ) - elif admitted: - hops.append(_refuse_hop(HOP_FRESHNESS_24H, REFUSE_CHAIN_INCOMPLETE)) - admitted = False - reason_terminal = REFUSE_CHAIN_INCOMPLETE - else: - hops.append(_refuse_hop(HOP_FRESHNESS_24H, reason_terminal)) - - # --- X006 conjunction (rules + OR + age + allow) --- - if admitted and isinstance(envelope, Mapping): - eval_decision = admit_eval_cvm_fresh_review(envelope=envelope) - if not eval_decision.may_launch: - hops.append( - _refuse_hop( - HOP_EVAL_CONJUNCTION, - eval_decision.reason_code, - detail="fresh review conjunction failed", - ) - ) - admitted = False - reason_terminal = eval_decision.reason_code - else: - hops.append( - _pass_hop( - HOP_EVAL_CONJUNCTION, - reason_code="eval_launch_eligible", - digests={"review_digest": str(eval_decision.review_digest or "")}, - ) - ) - elif admitted: - hops.append(_refuse_hop(HOP_EVAL_CONJUNCTION, REFUSE_CHAIN_INCOMPLETE)) - admitted = False - reason_terminal = REFUSE_CHAIN_INCOMPLETE - else: - hops.append(_refuse_hop(HOP_EVAL_CONJUNCTION, reason_terminal)) - - # --- X007 RA-TLS key-release --- - grant = package.get("key_release_grant") - if admitted: - if not isinstance(grant, Mapping): - hops.append(_refuse_hop(HOP_KEY_RELEASE, "score_refused_missing_key_release")) - admitted = False - reason_terminal = "score_refused_missing_key_release" - elif grant.get("base_gateway_token_minted"): - hops.append( - _refuse_hop( - HOP_KEY_RELEASE, - REFUSE_GATEWAY_RESIDUAL, - gateway_dependency=True, - detail="Base gateway token mint on KR path", - ) - ) - admitted = False - reason_terminal = REFUSE_GATEWAY_RESIDUAL - else: - hops.append( - _pass_hop( - HOP_KEY_RELEASE, - digests={ - "ra_tls_spki_digest": str(grant.get("ra_tls_spki_digest", "")), - "report_data_hex": str(grant.get("report_data_hex", ""))[:64], - }, - ) - ) - else: - hops.append(_refuse_hop(HOP_KEY_RELEASE, reason_terminal)) - - # --- X008 eval agent OR optional --- - mode = str(package.get("agent_llm_mode") or MODE_TOOLS_ONLY) - if admitted: - agent_decision = admit_eval_agent_llm_for_score( - mode=mode, - dual_flags_on=dual, - runtime_kind=package.get("agent_llm_runtime_kind"), - measurement=package.get("agent_llm_measurement"), - allowlist=package.get("agent_llm_allowlist"), - claims_model_call=bool(package.get("claims_agent_model_call")), - agent_or_materials=package.get("agent_or_materials"), - agent_env=None, - gateway_url=None, - gateway_token_present=False, - used_base_llm_v1=False, - ) - if not agent_decision.admitted: - hops.append( - _refuse_hop( - HOP_EVAL_AGENT_OR, - agent_decision.reason_code, - gateway_dependency=agent_decision.reason_code - in {REFUSE_BASE_GATEWAY, "base_gateway_forbidden"}, - ) - ) - admitted = False - reason_terminal = agent_decision.reason_code - else: - digests: dict[str, str] = {"mode": mode} - if agent_decision.planned_request_sha256: - digests["planned_request_sha256"] = agent_decision.planned_request_sha256 - if agent_decision.transport_observation_sha256: - digests["transport_observation_sha256"] = ( - agent_decision.transport_observation_sha256 - ) - hops.append( - _pass_hop( - HOP_EVAL_AGENT_OR, - reason_code=agent_decision.reason_code, - digests=digests, - ) - ) - else: - hops.append(_refuse_hop(HOP_EVAL_AGENT_OR, reason_terminal)) - - # --- X009 score domain full chain --- - if admitted: - score_decision = admit_production_score_from_chain( - dual_flags_on=dual, - review_envelope=package.get("review_envelope"), - key_release_grant=package.get("key_release_grant"), - key_granted_flag=package.get("key_release_grant") is not None, - eval_plan=package.get("eval_plan"), - score_binding=package.get("score_binding"), - score_report_data_hex=package.get("score_report_data_hex"), - scores_digest=package.get("scores_digest"), - score_nonce_state=str(package.get("score_nonce_state") or "outstanding"), - agent_llm_mode=mode, - agent_or_materials=package.get("agent_or_materials"), - agent_llm_runtime_kind=package.get("agent_llm_runtime_kind"), - agent_llm_measurement=package.get("agent_llm_measurement"), - agent_llm_allowlist=package.get("agent_llm_allowlist"), - claims_agent_model_call=bool(package.get("claims_agent_model_call")), - ) - if not score_decision.admitted: - hops.append( - _refuse_hop( - HOP_SCORE_DOMAIN, - score_decision.reason_code, - ) - ) - admitted = False - reason_terminal = score_decision.reason_code - else: - hops.append( - _pass_hop( - HOP_SCORE_DOMAIN, - reason_code=score_decision.reason_code, - digests={ - "review_digest": str(score_decision.review_digest or ""), - "domains": ",".join(score_decision.domains_checked), - }, - ) - ) - else: - hops.append(_refuse_hop(HOP_SCORE_DOMAIN, reason_terminal)) - - # --- X010 raw weights only; no set_weights --- - side = dict(package.get("side_effects") or {}) - raw_payload = package.get("raw_weights_payload") or {} - set_w = int(side.get("set_weights") or 0) - burn = int(side.get("burn_weights_24h") or 0) - if admitted: - if set_w > 0 or raw_payload.get("master_set_weights") or raw_payload.get("ops_set_weights"): - hops.append( - _refuse_hop( - HOP_RAW_WEIGHTS, - REFUSE_SET_WEIGHTS, - set_weights_invoked=True, - ) - ) - admitted = False - reason_terminal = REFUSE_SET_WEIGHTS - elif burn > 0 or raw_payload.get("burn_weights_invoked"): - hops.append(_refuse_hop(HOP_RAW_WEIGHTS, REFUSE_BURN_TRACKED)) - admitted = False - reason_terminal = REFUSE_BURN_TRACKED - elif not isinstance(raw_payload, Mapping) or not raw_payload.get("weights"): - hops.append(_refuse_hop(HOP_RAW_WEIGHTS, REFUSE_CHAIN_INCOMPLETE)) - admitted = False - reason_terminal = REFUSE_CHAIN_INCOMPLETE - else: - # Re-check payload_digest - body = { - k: v - for k, v in raw_payload.items() - if k - not in { - "payload_digest", - "path", - "master_set_weights", - "ops_set_weights", - "burn_weights_invoked", - "ops_set_weights_count", - } - } - recomputed_pw = hashlib.sha256(canonical_json_v1(body)).hexdigest() - if recomputed_pw != raw_payload.get("payload_digest"): - hops.append( - _refuse_hop(HOP_RAW_WEIGHTS, REFUSE_CHAIN_MUTILATED, detail="weights digest") - ) - admitted = False - reason_terminal = REFUSE_CHAIN_MUTILATED - else: - hops.append( - _pass_hop( - HOP_RAW_WEIGHTS, - digests={ - "payload_digest": str(raw_payload["payload_digest"]), - "path": str( - raw_payload.get( - "path", - "/internal/v1/challenges/agent-challenge/raw-weights", - ) - ), - }, - ) - ) - else: - hops.append( - _refuse_hop( - HOP_RAW_WEIGHTS, - reason_terminal, - set_weights_invoked=set_w > 0, - ) - ) - - # --- X011 zero Base gateway dependency --- - gw_hits = list((package.get("gateway_inventory") or {}).get("residuals") or []) - if any( - "/llm/v1" in str(r) - for r in (package.get("gateway_inventory") or {}).get("master_routes", []) - ): - gw_hits.append("route:/llm/v1") - if admitted and gw_hits: - hops.append( - _refuse_hop( - HOP_GATEWAY_FREE, - REFUSE_GATEWAY_RESIDUAL, - detail=",".join(gw_hits), - gateway_dependency=True, - ) - ) - admitted = False - reason_terminal = REFUSE_GATEWAY_RESIDUAL - elif admitted: - hops.append( - _pass_hop( - HOP_GATEWAY_FREE, - reason_code="zero_base_gateway", - ) - ) - else: - hops.append( - _refuse_hop( - HOP_GATEWAY_FREE, - reason_terminal, - gateway_dependency=bool(gw_hits), - ) - ) - - # --- X012 fairness honesty --- - honesty = dict(package.get("honesty") or {}) - real_claim = bool(honesty.get("real_provider_claimed")) - tee_real = str(honesty.get("tee_real_provider") or "BLOCKED") - if real_claim and not honesty.get("real_provider_pass_is_possible"): - hops.append( - _refuse_hop( - HOP_HONESTY, - "e2e_real_provider_pass_invented", - detail="REAL-PROVIDER PASS invented without hard gates", - ) - ) - admitted = False - reason_terminal = "e2e_real_provider_pass_invented" - elif admitted and tee_real not in {"BLOCKED", "REAL-PROVIDER PASS"}: - hops.append(_refuse_hop(HOP_HONESTY, "e2e_tee_label_invalid")) - admitted = False - reason_terminal = "e2e_tee_label_invalid" - elif admitted: - # For fixture chain honesty: REAL must stay BLOCKED - if tee_real == "REAL-PROVIDER PASS" and not honesty.get("real_provider_pass_is_possible"): - hops.append(_refuse_hop(HOP_HONESTY, "e2e_real_provider_pass_invented")) - admitted = False - reason_terminal = "e2e_real_provider_pass_invented" - else: - hops.append( - _pass_hop( - HOP_HONESTY, - reason_code="honesty_ok", - digests={ - "tee_local_fixture": str( - honesty.get("tee_local_fixture", "LOCAL-FIXTURE PASS") - ), - "tee_real_provider": tee_real, - }, - ) - ) - else: - hops.append(_refuse_hop(HOP_HONESTY, reason_terminal)) - - # hop order sanity - hop_ids = [h.hop_id for h in hops] - if hop_ids != list(HOP_ORDER): - admitted = False - reason_terminal = REFUSE_HOP_ORDER - - raw_weights: dict[str, float] = {} - if admitted and isinstance(raw_payload, Mapping): - weights = raw_payload.get("weights") or {} - if isinstance(weights, Mapping): - raw_weights = {str(k): float(v) for k, v in weights.items()} - - return ChainAdmission( - admitted=admitted, - reason_code=reason_terminal if not admitted else "e2e_chain_verified", - hops=tuple(hops), - package=dict(package), - package_digest=recomputed, - production_emit=admitted and dual, - raw_weights=raw_weights, - gateway_free=not bool(gw_hits), - reverify_ok=admitted, - tee_labels={ - "LOCAL-FIXTURE": str(honesty.get("tee_local_fixture", "LOCAL-FIXTURE PASS")), - "REAL-PROVIDER": tee_real, - }, - side_effects={ - "set_weights": set_w, - "burn_weights_24h": burn, - }, - ) - - -def run_attestation_chain( - *, - dual_flags_on: bool = True, - issued_at_ms: int = T0_DEFAULT, - received_at_ms: int | None = None, - agent_llm_mode: str = MODE_TOOLS_ONLY, - include_measured_agent_or: bool = False, - master_env: Mapping[str, str] | None = None, - master_routes: Sequence[str] | None = None, - set_weights_count: int = 0, - burn_invoked: bool = False, - real_provider_pass_claimed: bool = False, - **package_kwargs: Any, -) -> ChainAdmission: - """Build a fresh complete package and re-verify it end-to-end.""" - - package = build_complete_package( - dual_flags_on=dual_flags_on, - issued_at_ms=issued_at_ms, - received_at_ms=received_at_ms, - agent_llm_mode=agent_llm_mode, - include_measured_agent_or=include_measured_agent_or, - master_env=master_env, - master_routes=master_routes, - set_weights_count=set_weights_count, - burn_invoked=burn_invoked, - real_provider_pass_claimed=real_provider_pass_claimed, - **package_kwargs, - ) - return reverify_package(package) - - -def mutilate_package( - package: Mapping[str, Any], - *, - path: Sequence[str], - value: Any = None, - delete: bool = False, -) -> dict[str, Any]: - """Return a deep-copied package with one path mutated (or deleted). - - Preserves the *original* ``package_digest`` so independent re-verify forms - refuse on digest mismatch or gate failure — never soft-accept. - """ - - out: MutableMapping[str, Any] = copy.deepcopy(dict(package)) - cursor: Any = out - for key in path[:-1]: - if not isinstance(cursor, MutableMapping) or key not in cursor: - return dict(out) - cursor = cursor[key] - if not isinstance(cursor, MutableMapping): - return dict(out) - last = path[-1] - if delete: - cursor.pop(last, None) - else: - cursor[last] = value - # Keep original package_digest so reverify detects material mutation. - return dict(out) - - -def chronological_evidence_index(admission: ChainAdmission) -> dict[str, Any]: - """Machine-readable chronological index for mission evidence (VAL-ACAT-020).""" - - assertion_table: dict[str, dict[str, Any]] = {} - for hop in admission.hops: - assertion_table[hop.assertion_id] = { - "status": "PASS" if hop.status == "pass" else "FAIL", - "hop_id": hop.hop_id, - "reason_code": hop.reason_code, - "detail": hop.detail, - } - assertion_table["VAL-ACAT-020"] = { - "status": "PASS" if admission.admitted else "FAIL", - "reason_code": admission.reason_code, - "package_digest": admission.package_digest, - "gateway_free": admission.gateway_free, - "reverify_ok": admission.reverify_ok, - "production_emit": admission.production_emit, - } - return { - "schema_version": 1, - "title": "agent-challenge black-box attestation chain", - "admitted": admission.admitted, - "reason_code": admission.reason_code, - "package_digest": admission.package_digest, - "chronological_hops": [h.as_dict() for h in admission.hops], - "assertion_table": assertion_table, - "raw_weights": dict(admission.raw_weights), - "tee_labels": dict(admission.tee_labels), - "side_effects": dict(admission.side_effects), - "gateway_free": admission.gateway_free, - "no_set_weights": admission.side_effects.get("set_weights", 0) == 0, - "weights_path": "challenge-raw-weights → master raw-weight ingress → aggregate", - } - - -def require_attestation_chain(**kwargs: Any) -> ChainAdmission: - """Fail closed: raise if the chain does not fully admit.""" - - decision = run_attestation_chain(**kwargs) - if not decision.admitted: - raise BlackboxChainError(decision.reason_code) - return decision - - -__all__ = [ - "BASE_GATEWAY_ENV_NAMES", - "BlackboxChainError", - "ChainAdmission", - "FRESHNESS_WINDOW_MS", - "HOP_ASSERTIONS", - "HOP_ORDER", - "HopResult", - "MINER_HOTKEY_DEFAULT", - "MS_23H", - "REFUSE_BURN_TRACKED", - "REFUSE_CHAIN_INCOMPLETE", - "REFUSE_CHAIN_MUTILATED", - "REFUSE_GATEWAY_RESIDUAL", - "REFUSE_HOP_ORDER", - "REFUSE_PACKAGE_TAMPER", - "REFUSE_SET_WEIGHTS", - "REVIEW_DOMAIN", - "SCORE_DOMAIN", - "KEY_RELEASE_DOMAIN", - "T0_DEFAULT", - "build_complete_package", - "build_fixture_eval_plan", - "build_fixture_review_envelope", - "build_key_release_grant", - "build_raw_weights_payload", - "canonical_json_v1", - "chronological_evidence_index", - "mutilate_package", - "package_digest", - "require_attestation_chain", - "reverify_package", - "run_attestation_chain", - "scan_gateway_residuals", -] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/execution_pool.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/execution_pool.py index e4bbd021e..aed70f40a 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/execution_pool.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/execution_pool.py @@ -18,9 +18,7 @@ # Keep in lockstep with evaluation.authorization._ACTIVE_PHASES and # evaluation.telemetry_session._ACTIVE_EVAL_PHASES. -ACTIVE_EVAL_PHASES: frozenset[str] = frozenset( - {"eval_prepared", "eval_running", "eval_verifying"} -) +ACTIVE_EVAL_PHASES: frozenset[str] = frozenset({"eval_prepared", "eval_running", "eval_verifying"}) _SCORE_FIELD_NAMES: frozenset[str] = frozenset( { @@ -64,11 +62,7 @@ def _latest_event_payload(event: TaskLogEvent) -> dict[str, Any]: meta = _metadata_dict(event.metadata_json) phase = event.status or meta.get("phase") client_sequence = meta.get("client_sequence") - sequence = ( - int(client_sequence) - if isinstance(client_sequence, int) - else int(event.sequence) - ) + sequence = int(client_sequence) if isinstance(client_sequence, int) else int(event.sequence) payload: dict[str, Any] = { "event_type": event.event_type, "sequence": sequence, @@ -81,9 +75,7 @@ def _latest_event_payload(event: TaskLogEvent) -> dict[str, Any]: return {key: value for key, value in payload.items() if value is not None} -def _unit_from_run( - run: EvalRun, *, latest_event: Mapping[str, Any] | None -) -> dict[str, Any]: +def _unit_from_run(run: EvalRun, *, latest_event: Mapping[str, Any] | None) -> dict[str, Any]: unit: dict[str, Any] = { "unit_id": run.eval_run_id, "eval_run_id": run.eval_run_id, @@ -141,10 +133,7 @@ async def list_live_execution_units(session: AsyncSession) -> list[dict[str, Any session, submission_ids=[run.submission_id for run in runs], ) - return [ - _unit_from_run(run, latest_event=latest_by_run.get(run.eval_run_id)) - for run in runs - ] + return [_unit_from_run(run, latest_event=latest_by_run.get(run.eval_run_id)) for run in runs] __all__ = [ diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/llm_rules_residual.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/llm_rules_residual.py index 80dc8b883..48119b502 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/llm_rules_residual.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/llm_rules_residual.py @@ -1,4 +1,4 @@ -"""AGATE measured LLM rules residual gate (package verify before TEE auth). +"""AGATE LLM rules residual gate (package policy before eval under host-trust). Product spine (mission AGATE / VAL-AGATE-004..007 + 012/013): @@ -393,14 +393,10 @@ def admit_package_residual_for_eval( - package_tree_sha required/match when configured """ - if not dual_flags_on: - # Residual gate is a production dual-flag requirement; flag-off is not - # TEE-authable production emit (other gates already refuse). Treat as - # residual missing for prepare-style callers that still invoke this. - return PackageResidualAdmission( - admitted=False, - reason_code=REFUSE_RESIDUAL_MISSING, - ) + # T40: dual TEE flags removed. Host-trust residual policy admits host + # residual kinds (host_analyzer_static / unmeasured_host_python) when + # dual_flags_on is False. Measured kinds still admitted if present. + host_trust = not dual_flags_on try: materials = extract_package_residual( @@ -413,8 +409,8 @@ def admit_package_residual_for_eval( return PackageResidualAdmission(admitted=False, reason_code=exc.code) if materials is None: - # Host analyzer allow alone never unlocks TEE auth. - if host_analyzer_allow is True: + # Host analyzer allow alone is insufficient without a residual bag. + if host_analyzer_allow is True and not host_trust: return PackageResidualAdmission( admitted=False, reason_code=REFUSE_HOST_ONLY, @@ -425,17 +421,29 @@ def admit_package_residual_for_eval( ) if is_host_only_residual(materials.residual_kind): - return PackageResidualAdmission( - admitted=False, - reason_code=REFUSE_HOST_ONLY, - residual_verdict=materials.residual_verdict, - residual_kind=materials.residual_kind, - residual_digest=materials.residual_digest, - rules_bundle_sha256=materials.rules_bundle_sha256, - package_tree_sha=materials.package_tree_sha, - ) - - if not is_measured_residual_kind(materials.residual_kind): + if not host_trust: + return PackageResidualAdmission( + admitted=False, + reason_code=REFUSE_HOST_ONLY, + residual_verdict=materials.residual_verdict, + residual_kind=materials.residual_kind, + residual_digest=materials.residual_digest, + rules_bundle_sha256=materials.rules_bundle_sha256, + package_tree_sha=materials.package_tree_sha, + ) + # Host-trust: fall through to verdict/tree checks below. + elif not is_measured_residual_kind(materials.residual_kind): + if not host_trust: + return PackageResidualAdmission( + admitted=False, + reason_code=REFUSE_RESIDUAL_KIND, + residual_verdict=materials.residual_verdict, + residual_kind=materials.residual_kind, + residual_digest=materials.residual_digest, + rules_bundle_sha256=materials.rules_bundle_sha256, + package_tree_sha=materials.package_tree_sha, + ) + # Host-trust: unknown kinds still refuse (policy integrity). return PackageResidualAdmission( admitted=False, reason_code=REFUSE_RESIDUAL_KIND, @@ -509,10 +517,14 @@ def host_analyzer_alone_insufficient( ) -> bool: """VAL-AGATE-007: host allow without measured residual is insufficient.""" - if not dual_flags_on: - return True if not host_analyzer_allow: return True + # Under host-trust (dual off), host residual bag can be sufficient; bare + # host_analyzer_allow without residual materials is still insufficient. + if not dual_flags_on: + if residual is None: + return True + # fall through to admit check with dual_flags_on=False (host kinds OK) bag: Mapping[str, Any] | None if isinstance(residual, PackageResidualMaterials): bag = residual.as_dict() @@ -520,7 +532,7 @@ def host_analyzer_alone_insufficient( bag = residual decision = admit_package_residual_for_eval( residual=bag, - dual_flags_on=True, + dual_flags_on=dual_flags_on, host_analyzer_allow=host_analyzer_allow, require_package_tree_sha=False, ) diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/no_phala.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/no_phala.py index 7d9c07d67..b4913d0d1 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/no_phala.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/no_phala.py @@ -1,4 +1,4 @@ -"""Temporary NO_PHALA host-execution mode (opt-in, unattested, removable). +"""Host-trust unattested execution mode (product path after T40 Phala delete). When ``NO_PHALA`` / ``CHALLENGE_NO_PHALA`` is active the master validator runs benchmark/eval jobs on the host via the existing own_runner local Docker path @@ -6,9 +6,9 @@ attestation**. Results are explicitly marked unattested and cannot be forged into an attested envelope from this path. -This module is the single branch point for the temporary mode so it can be -deleted cleanly when Phala is re-enabled. Do not scatter NO_PHALA conditionals -elsewhere beyond thin call-sites into this module. +Phala TEE is removed from the product path. This module is the honesty +branch for host-trust runs. Never claim TEE / tamper-proof / independent +verification from this path. Precedence for the env switch (see :func:`resolve_no_phala_from_environ`): @@ -32,6 +32,8 @@ CHALLENGE_NO_PHALA_ENV: Final = "CHALLENGE_NO_PHALA" #: Operator-facing plain env accepted on the master host. NO_PHALA_ENV: Final = "NO_PHALA" +#: Canonical unattested-execution env (T40 product name). +CHALLENGE_UNATTESTED_EXECUTION_ENV: Final = "CHALLENGE_UNATTESTED_EXECUTION" _TRUTHY: Final = frozenset({"1", "true", "yes", "on"}) _FALSY: Final = frozenset({"0", "false", "no", "off", ""}) @@ -98,14 +100,18 @@ def _parse_bool_env(raw: str | None) -> bool | None: def resolve_no_phala_from_environ( environ: Mapping[str, str] | None = None, ) -> bool: - """Resolve NO_PHALA from env with documented precedence. + """Resolve unattested/NO_PHALA from env with documented precedence. - 1. ``CHALLENGE_NO_PHALA`` if present in the environment - 2. else ``NO_PHALA`` if present - 3. else ``False`` (default OFF — never inferred from missing keys) + 1. ``CHALLENGE_UNATTESTED_EXECUTION`` if present + 2. else ``CHALLENGE_NO_PHALA`` if present + 3. else ``NO_PHALA`` if present + 4. else ``False`` (default OFF — never inferred from missing keys) """ env = os.environ if environ is None else environ + if CHALLENGE_UNATTESTED_EXECUTION_ENV in env: + parsed = _parse_bool_env(env.get(CHALLENGE_UNATTESTED_EXECUTION_ENV)) + return bool(parsed) if CHALLENGE_NO_PHALA_ENV in env: parsed = _parse_bool_env(env.get(CHALLENGE_NO_PHALA_ENV)) return bool(parsed) @@ -115,6 +121,24 @@ def resolve_no_phala_from_environ( return False +def resolve_unattested_execution_from_environ( + environ: Mapping[str, str] | None = None, +) -> bool: + """Alias of :func:`resolve_no_phala_from_environ` (T40 product name).""" + + return resolve_no_phala_from_environ(environ) + + +def is_unattested_execution_enabled( + *, + settings_flag: bool | None = None, + environ: Mapping[str, str] | None = None, +) -> bool: + """Alias of :func:`is_no_phala_enabled` (T40 product name).""" + + return is_no_phala_enabled(settings_flag=settings_flag, environ=environ) + + def is_no_phala_enabled( *, settings_flag: bool | None = None, @@ -244,9 +268,7 @@ def health_fields(*, no_phala: bool) -> dict[str, Any]: return { "no_phala": bool(no_phala), - "attestation_mode": ( - EXECUTION_MODE_NO_PHALA_HOST if no_phala else "standard" - ), + "attestation_mode": (EXECUTION_MODE_NO_PHALA_HOST if no_phala else "standard"), } @@ -268,6 +290,8 @@ def health_fields(*, no_phala: bool) -> dict[str, Any]: "build_guest_artifact_proof", "health_fields", "is_no_phala_enabled", + "is_unattested_execution_enabled", + "resolve_unattested_execution_from_environ", "log_no_phala_startup_banner", "mark_result_unattested", "refuse_phala_client", diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner_backend.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner_backend.py index 4ff047bc2..d2f787b69 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner_backend.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner_backend.py @@ -43,6 +43,7 @@ from pathlib import Path from typing import Any +from agent_challenge.canonical.attested_result import AttestationEmissionError from agent_challenge.canonical.live_registry import resolve_live_registry_refs from agent_challenge.evaluation.gateway import ( BASE_LLM_GATEWAY_URL_ENV, @@ -229,10 +230,10 @@ ) -def _phala_attestation_enabled() -> bool: - """Whether Phala attested-result emission is enabled for this run.""" +def _phala_attestation_enabled(*_a, **_k) -> bool: + """T40: Phala attestation permanently disabled.""" - return os.environ.get(PHALA_ATTESTATION_ENABLED_ENV, "").strip().lower() in _TRUTHY + return False def _sanitize_guest_detail(detail: str, *, limit: int = 160) -> str: @@ -1111,7 +1112,7 @@ def _resolve_phala_binding_from_env() -> dict[str, Any]: result rather than an attestation bound to bogus inputs. """ - from agent_challenge.canonical.attested_result import AttestationEmissionError + # T40: AttestationEmissionError from canonical.attested_result (Phala removed) def _require(env_name: str) -> str: value = (os.environ.get(env_name) or "").strip() @@ -1147,7 +1148,7 @@ def _require(env_name: str) -> str: def _parse_phala_vm_config_env() -> dict[str, Any] | None: """Parse the optional evidence-only VM config without accepting extra trust.""" - from agent_challenge.canonical.attested_result import AttestationEmissionError + # T40: AttestationEmissionError from canonical.attested_result (Phala removed) vm_config: dict[str, Any] | None = None raw_vm_config = (os.environ.get(PHALA_VM_CONFIG_ENV) or "").strip() diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/raw_weight_push.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/raw_weight_push.py index 75b9b139f..b4f088e00 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/raw_weight_push.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/raw_weight_push.py @@ -130,9 +130,7 @@ def __init__(self, database: Database, *, challenge_slug: str) -> None: async def init(self) -> None: async with self.database.engine.begin() as connection: await connection.run_sync( - lambda sync_conn: RawWeightPushLedger.__table__.create( - sync_conn, checkfirst=True - ) + lambda sync_conn: RawWeightPushLedger.__table__.create(sync_conn, checkfirst=True) ) async def _get_row(self, session: Any) -> RawWeightPushLedger | None: @@ -387,9 +385,7 @@ async def push_once( # Positive hotkey weights only when synthesizing from get_weights; # explicit zero maps (caller-supplied zeros) are preserved as zero-contribution. if weights is not None: - cleaned = { - str(hotkey): float(value) for hotkey, value in resolved_weights.items() - } + cleaned = {str(hotkey): float(value) for hotkey, value in resolved_weights.items()} else: cleaned = { str(hotkey): float(value) @@ -588,7 +584,6 @@ async def run_raw_weight_push_loop( await asyncio.sleep(max(float(interval_seconds), 0.1)) - def _log_unattested_weight_push_if_no_phala(weights: Mapping[str, float]) -> None: """CRITICAL log when NO_PHALA host mode pushes host-trust weights. diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py index 1360fb451..0f38df6a0 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/runner.py @@ -2012,7 +2012,6 @@ def _own_runner_script( """.strip() - def _run_no_phala_host_terminal_bench( *, job: EvaluationJob, @@ -2045,9 +2044,7 @@ def _run_no_phala_host_terminal_bench( settings.own_runner_cache_root or "/app/packages/challenges/agent-challenge/docker/canonical/live-task-cache" ) - digest = Path( - settings.own_runner_digest_manifest or "/app/golden/dataset-digest.json" - ) + digest = Path(settings.own_runner_digest_manifest or "/app/golden/dataset-digest.json") plan.jobs_dir.mkdir(parents=True, exist_ok=True) plan.job_dir.mkdir(parents=True, exist_ok=True) diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/score_chain_gate.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/score_chain_gate.py index 251ae0330..ed91bcb80 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/score_chain_gate.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/score_chain_gate.py @@ -460,9 +460,14 @@ def admit_production_score_from_chain( if prior_reverify_failed: return _refuse(REFUSE_STICKY, reverify_exercised=True, sticky=True) - # Flag-off / legacy paths cannot emit **production** scores (VAL-ACAT-018/054). - if not dual_flags_on: - # Even with perfect metrics/AST, not production emission ingredients. + # T40: dual TEE flags removed. Host-trust production scores use the + # software policy legs below (package_tree_sha, residual-as-policy, + # AST-only refuse, sticky refuse) without KR grant or TDX measurement. + # Never mint TEE-attested scores from this path. + host_trust = not dual_flags_on + if dual_flags_on: + # Defensive: dual-on is rejected at settings construction; refuse if + # a caller still passes True. return _refuse( REFUSE_FLAGS_OFF, reverify_exercised=False, @@ -519,7 +524,8 @@ def admit_production_score_from_chain( raw_tree = eval_plan.get("package_tree_sha") if isinstance(raw_tree, str) and raw_tree.strip(): plan_tree_sha = raw_tree.strip() - if dual_flags_on and ( + # KEEP package_tree_sha under host-trust (T39 G-PTS-PLAN / G-SCORE-CHAIN). + if ( plan_tree_sha is None or len(plan_tree_sha) != 64 or any(c not in "0123456789abcdef" for c in plan_tree_sha.lower()) @@ -542,8 +548,10 @@ def admit_production_score_from_chain( cached_outcome_status="verified_allow" if cached_review_allow else None, cached_phase="review_allowed" if cached_review_allow else None, dual_flags_on=dual_flags_on, - require_package_residual=bool(dual_flags_on), - expected_package_tree_sha=plan_tree_sha if dual_flags_on else None, + # Host-trust: residual optional at score if missing materials; when + # present, host kinds admitted via residual gate rewire. + require_package_residual=False if host_trust else bool(dual_flags_on), + expected_package_tree_sha=plan_tree_sha, outcome=review_outcome, package_residual=package_residual, ) @@ -612,53 +620,71 @@ def admit_production_score_from_chain( review_digest=review_decision.review_digest, ) - # ----- 2) Key-release RA-TLS re-check at admission (VAL-ACAT-036/037) ----- + # ----- 2) Key-release RA-TLS (REMOVED under host-trust / T40) ----- + # KR exists to release secrets into measured CVM. Host-trust scores must + # not require KR. keyrelease/ package sources are untouched. checked.append(KEY_RELEASE_DOMAIN) - kr_err, kr_hex = verify_key_release_grant( - grant=key_release_grant, - eval_plan=eval_plan, - key_granted_flag=key_granted_flag, - ) - if kr_err is not None: - if offline_ast_pass and kr_err in { - REFUSE_MISSING_KEY_RELEASE, - REFUSE_INCOMPLETE_CHAIN, - }: + kr_hex = None + if not host_trust: + kr_err, kr_hex = verify_key_release_grant( + grant=key_release_grant, + eval_plan=eval_plan, + key_granted_flag=key_granted_flag, + ) + if kr_err is not None: + if offline_ast_pass and kr_err in { + REFUSE_MISSING_KEY_RELEASE, + REFUSE_INCOMPLETE_CHAIN, + }: + return _refuse( + REFUSE_AST_ONLY, + reverify_exercised=True, + domains_checked=tuple(checked), + review_digest=review_decision.review_digest, + ) return _refuse( - REFUSE_AST_ONLY, + kr_err, reverify_exercised=True, domains_checked=tuple(checked), review_digest=review_decision.review_digest, ) - return _refuse( - kr_err, - reverify_exercised=True, - domains_checked=tuple(checked), - review_digest=review_decision.review_digest, - ) + else: + checked.append("key_release_skipped_host_trust") - # ----- 3) Score-domain binding re-verify ----- + # ----- 3) Score-domain binding (host-trust: plan/score nonce only) ----- checked.append(SCORE_DOMAIN) - score_err, score_rd = verify_score_domain_binding( - score_binding=score_binding, - reported_report_data_hex=score_report_data_hex, - eval_plan=eval_plan, - scores_digest=scores_digest, - ) - if score_err is not None: - if offline_ast_pass and score_err == REFUSE_INCOMPLETE_CHAIN: + if host_trust: + # Skip TDX score-domain measurement re-verify; keep plan presence. + if eval_plan is None: return _refuse( - REFUSE_AST_ONLY, + REFUSE_INCOMPLETE_CHAIN, reverify_exercised=True, domains_checked=tuple(checked), review_digest=review_decision.review_digest, ) - return _refuse( - score_err, - reverify_exercised=True, - domains_checked=tuple(checked), - review_digest=review_decision.review_digest, + score_rd = None + checked.append("score_domain_host_trust_plan_only") + else: + score_err, score_rd = verify_score_domain_binding( + score_binding=score_binding, + reported_report_data_hex=score_report_data_hex, + eval_plan=eval_plan, + scores_digest=scores_digest, ) + if score_err is not None: + if offline_ast_pass and score_err == REFUSE_INCOMPLETE_CHAIN: + return _refuse( + REFUSE_AST_ONLY, + reverify_exercised=True, + domains_checked=tuple(checked), + review_digest=review_decision.review_digest, + ) + return _refuse( + score_err, + reverify_exercised=True, + domains_checked=tuple(checked), + review_digest=review_decision.review_digest, + ) # ----- 4) Score nonce freshness / single-use (VAL-ACAT-017/033) ----- state = (score_nonce_state or "unknown").strip().lower() diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/tbench_integrity.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/tbench_integrity.py index 47fc264df..4b4ca7dbc 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/tbench_integrity.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/tbench_integrity.py @@ -143,9 +143,7 @@ def assert_selected_task_ids_in_frozen(task_ids: Iterable[str]) -> None: if task_id not in frozen and bare_task_name(task_id) not in frozen ] if unknown: - raise TbenchIntegrityError( - f"selected task ids not in frozen digest set: {unknown}" - ) + raise TbenchIntegrityError(f"selected task ids not in frozen digest set: {unknown}") # --------------------------------------------------------------------------- # @@ -169,9 +167,7 @@ def assert_no_miner_task_source_fields(mapping: Mapping[str, Any] | None) -> Non hits = forbidden_task_source_keys_present(mapping) if hits: - raise TbenchIntegrityError( - f"miner/plan must not supply alternate task sources: {hits}" - ) + raise TbenchIntegrityError(f"miner/plan must not supply alternate task sources: {hits}") def selected_task_item_allowed_keys() -> frozenset[str]: @@ -183,9 +179,7 @@ def selected_task_item_allowed_keys() -> frozenset[str]: # --------------------------------------------------------------------------- # # No network fetch at eval (module / loader contract) # --------------------------------------------------------------------------- # -_TASKDEFS_MODULE_PATH = ( - Path(__file__).resolve().parent / "own_runner" / "taskdefs.py" -) +_TASKDEFS_MODULE_PATH = Path(__file__).resolve().parent / "own_runner" / "taskdefs.py" # Tokens that would indicate a network client in the task-def loader. _NETWORK_CLIENT_MARKERS: Final[tuple[str, ...]] = ( diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/weights.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/weights.py index 1dcc04cef..3d7f46519 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/weights.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/weights.py @@ -244,7 +244,9 @@ def _weights_from_direct_population( async def get_weights() -> dict[str, float]: """Return raw miner weights for the BASE master to normalize.""" - require_attestation = settings.phala_attestation_enabled + # T40: Phala dual flags removed — never gate weights on TEE verified. + # Host-trust eligibility = completed + plan-backed + task thresholds. + require_attestation = False required_task_count = effective_evaluation_task_count(settings.evaluation_task_count) async with database.session() as session: rows = (await session.execute(scoring_evaluation_jobs_statement())).scalars().all() diff --git a/packages/challenges/agent-challenge/src/agent_challenge/review/or_outcome_bind.py b/packages/challenges/agent-challenge/src/agent_challenge/review/or_outcome_bind.py index 9ce41c231..512df071e 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/review/or_outcome_bind.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/review/or_outcome_bind.py @@ -161,65 +161,10 @@ def assert_no_base_openrouter_keys(env: Mapping[str, str] | None) -> None: ) -def admit_measured_review_cvm( - *, - runtime_kind: str, - measurement: Mapping[str, str] | ReviewMeasurementRecord | None, - allowlist: Sequence[Mapping[str, str]] | None, - base_env: Mapping[str, str] | None = None, -) -> dict[str, Any]: - """Production judge-path runtime admission.""" - - assert_no_base_openrouter_keys(base_env) - - kind = (runtime_kind or "").strip() - if kind == BASE_MASTER_KIND: - raise ReviewOrOutcomeError( - REFUSE_BASE_MASTER_OR, - "OpenRouter judgment must not run on Base master", - ) - if kind != MEASURED_CVM_KIND: - raise ReviewOrOutcomeError( - REFUSE_UNMEASURED_REVIEW, - f"runtime_kind {kind!r} is not measured review CVM", - ) - if measurement is None: - raise ReviewOrOutcomeError( - REFUSE_MEASUREMENT_UNALLOWLISTED, - "review CVM measurement is required", - ) - if isinstance(measurement, ReviewMeasurementRecord): - candidate = measurement.as_closed() - else: - candidate = { - "compose_hash": str(measurement.get("compose_hash", "")), - "os_image_hash": str(measurement.get("os_image_hash", "")), - "mrtd": str(measurement.get("mrtd", "")), - "key_provider": str(measurement.get("key_provider", "")), - "vm_shape": str(measurement.get("vm_shape", "")), - } - if not allowlist: - raise ReviewOrOutcomeError( - REFUSE_MEASUREMENT_UNALLOWLISTED, - "empty measurement allowlist matches nothing", - ) - normalized = [{k: str(v) for k, v in entry.items()} for entry in allowlist] - if candidate not in normalized: - raise ReviewOrOutcomeError( - REFUSE_MEASUREMENT_UNALLOWLISTED, - "review measurement is not on the production allowlist", - ) - return { - "runtime_kind": MEASURED_CVM_KIND, - "measurement": candidate, - "measurement_allowlisted": True, - "openrouter_allowed_from": "review_cvm_guest", - } - +def admit_measured_review_cvm(*args, **kwargs): + """Removed with Phala measured review CVM (T40). Always refuses.""" -# --------------------------------------------------------------------------- -# VAL-ACAT-004 — planned + observed OpenRouter digests -# --------------------------------------------------------------------------- + return type("Decision", (), {"admitted": False, "reason_code": "measured_review_cvm_removed"})() def build_planned_openrouter_request( diff --git a/packages/challenges/agent-challenge/src/agent_challenge/review/public_tee.py b/packages/challenges/agent-challenge/src/agent_challenge/review/public_tee.py index 9a62f12d7..8f7438632 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/review/public_tee.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/review/public_tee.py @@ -1,458 +1,42 @@ -"""Public unauthenticated TEE math projection for joinbase inspection. - -Maps a durable review envelope + verification outcome to the architecture §5.1 -safe field list only. Never exposes nonce plaintext, tokens, capabilities, -evidence bodies, model IO, or encryption key material. -""" +"""Public TEE math surface removed (T40). Always reports unavailable.""" from __future__ import annotations -import json from collections.abc import Mapping -from hashlib import sha256 from typing import Any -from .report import ( - MAX_REVIEW_EVENT_LOG_ENTRIES, - MAX_REVIEW_QUOTE_BYTES, - REVIEW_REPORT_DOMAIN, - review_report_data_preimage, -) - -# Public response caps (fail-closed truncation, not silent invent). -MAX_PUBLIC_TEE_QUOTE_BYTES = MAX_REVIEW_QUOTE_BYTES -MAX_PUBLIC_TEE_EVENT_LOG_ENTRIES = min(256, MAX_REVIEW_EVENT_LOG_ENTRIES) -MAX_PUBLIC_TEE_EVENT_LOG_BYTES = 262_144 - -# Architecture §5.1 top-level allowlist when available:true. -PUBLIC_TEE_TOP_LEVEL_ALLOWLIST = frozenset( - { - "available", - "submission_id", - "domain", - "review_digest", - "report_data_hex", - "report_data_preimage", - "measurement", - "tdx_quote_hex", - "event_log", - "verification_outcome", - "quote_fingerprint_sha256", - "agent_hash", - "zip_sha256", - "verdict", - "assignment_digest", - "session_id", - "assignment_id", - } -) - -PUBLIC_TEE_MEASUREMENT_KEYS = ( - "mrtd", - "rtmr0", - "rtmr1", - "rtmr2", - "rtmr3", - "compose_hash", - "os_image_hash", - "key_provider", - "vm_shape", -) - -PUBLIC_TEE_OUTCOME_KEYS = ( - "status", - "measurement_allowlisted", - "report_data_matched", - "verified_at_ms", - "reason_code", -) - -# Hard deny substrings / keys that must never appear in serialized public bodies. -PUBLIC_TEE_DENY_KEYS = frozenset( - { - "review_nonce", - "session_token", - "session_token_sha256", - "capability", - "capabilities", - "bearer", - "authorization", - "assignment_bearer", - "review_session_token", - "encrypted_env", - "evidence_objects", - "request_body", - "response_body", - "planned_request", - "transport_observation", - "encryption_key", - "private_key", - "mnemonic", - "openrouter_api_key", - "phala_cloud_api_key", - "key_file", - "KEY_FILE", - "challenge_token", - "shared_token", - "wallet", - } -) - -PUBLIC_TEE_DENY_SUBSTRINGS = ( - "sk-", - "BEGIN PRIVATE KEY", - "BEGIN RSA PRIVATE KEY", - "mnemonic", - "OPENROUTER_API_KEY", - "PHALA_CLOUD_API_KEY", -) - -# Align with status dual-flag verified verdict mapping: envelope alone (written -# during review_verifying, or parked as verifier_unavailable) is NOT enough for -# available:true. Prefer public projection and/or a terminal verified outcome. -VERIFIED_PUBLIC_TEE_OUTCOME_STATUSES = frozenset( - { - "verified_allow", - "verified_reject", - "verified_escalate", - } -) - def public_tee_unavailable() -> dict[str, Any]: - """Locked closed form when no authorizing/current verified report exists.""" - - return {"available": False} - - -def public_tee_assignment_qualifies( - *, - envelope_json: str | None, - outcome_json: str | None = None, - public_projection_json: str | None = None, -) -> bool: - """True when durable assignment material is eligible for public TEE math. - - Matches status ``report_available`` intent: a durable envelope is required, - plus either a miner-facing public projection or a terminal verified_* - verification_outcome status. Envelope-only (pre-verify) and - ``verifier_unavailable`` must fail closed. - """ - - if not isinstance(envelope_json, str) or not envelope_json.strip(): - return False - if isinstance(public_projection_json, str) and public_projection_json.strip(): - return True - outcome = _loads_object(outcome_json) - if not isinstance(outcome, Mapping): - return False - status = outcome.get("status") - return isinstance(status, str) and status in VERIFIED_PUBLIC_TEE_OUTCOME_STATUSES - - -def _has_verified_public_signal( - *, - verification_outcome: Mapping[str, Any] | None, - public_projection: Mapping[str, Any] | None, -) -> bool: - """Builder-side gate: projection and/or verified_* outcome required.""" - - if isinstance(public_projection, Mapping): - return True - if not isinstance(verification_outcome, Mapping): - return False - status = verification_outcome.get("status") - return isinstance(status, str) and status in VERIFIED_PUBLIC_TEE_OUTCOME_STATUSES - - -def build_public_tee_math( - *, - submission_id: int | str, - envelope: Mapping[str, Any] | None, - verification_outcome: Mapping[str, Any] | None = None, - public_projection: Mapping[str, Any] | None = None, -) -> dict[str, Any]: - """Project stored envelope (+ optional outcome/projection) to safe public math. - - Returns exactly ``{"available": false}`` when the envelope is missing, - structurally insufficient, or not yet verified (no public projection and no - verified_* outcome). Never invents measurement/quote material. - """ - - if not isinstance(envelope, Mapping): - return public_tee_unavailable() - - # Envelope is written before verification (review_verifying) and may park - # verifier_unavailable without a public projection. Refuse available:true - # unless status.report_available-class signals are present. - if not _has_verified_public_signal( - verification_outcome=verification_outcome, - public_projection=public_projection, - ): - return public_tee_unavailable() - - attestation = envelope.get("attestation") - if not isinstance(attestation, Mapping): - return public_tee_unavailable() - - review_digest = envelope.get("review_digest") - report_data_hex = envelope.get("report_data_hex") - domain = envelope.get("domain") or REVIEW_REPORT_DOMAIN - if not isinstance(review_digest, str) or not review_digest: - return public_tee_unavailable() - if not isinstance(report_data_hex, str) or not report_data_hex: - return public_tee_unavailable() - - measurement = _public_measurement(attestation.get("measurement")) - tdx_quote_hex = _public_quote_hex(attestation.get("tdx_quote_hex")) - event_log = _public_event_log(attestation.get("event_log")) - preimage = _public_report_data_preimage(envelope) - outcome = _public_verification_outcome(verification_outcome) - quote_fp = _quote_fingerprint(tdx_quote_hex, public_projection) - - payload: dict[str, Any] = { - "available": True, - "submission_id": _coerce_submission_id(submission_id, envelope), - "domain": str(domain), - "review_digest": review_digest, - "report_data_hex": report_data_hex, + return { + "available": False, + "reason": "phala_tee_removed_host_trust_only", + "tee": None, + "measurement": None, + "quote": None, } - if preimage is not None: - payload["report_data_preimage"] = preimage - if measurement is not None: - payload["measurement"] = measurement - if tdx_quote_hex is not None: - payload["tdx_quote_hex"] = tdx_quote_hex - if event_log is not None: - payload["event_log"] = event_log - if outcome is not None: - payload["verification_outcome"] = outcome - if quote_fp is not None: - payload["quote_fingerprint_sha256"] = quote_fp - - # Optional cross-check digests from the existing miner public projection. - if isinstance(public_projection, Mapping): - for key in ( - "agent_hash", - "zip_sha256", - "verdict", - "assignment_digest", - "session_id", - "assignment_id", - ): - value = public_projection.get(key) - if isinstance(value, str) and value: - payload[key] = value - elif isinstance(value, list) and key == "reason_codes": - continue - - assert_public_tee_safe(payload) - return payload - - -def build_public_tee_math_from_assignment( - *, - submission_id: int | str, - envelope_json: str | None, - outcome_json: str | None = None, - public_projection_json: str | None = None, -) -> dict[str, Any]: - """Convenience loader from durable ReviewAssignment JSON columns.""" - - if not public_tee_assignment_qualifies( - envelope_json=envelope_json, - outcome_json=outcome_json, - public_projection_json=public_projection_json, - ): - return public_tee_unavailable() - envelope = _loads_object(envelope_json) - if envelope is None: - return public_tee_unavailable() - return build_public_tee_math( - submission_id=submission_id, - envelope=envelope, - verification_outcome=_loads_object(outcome_json), - public_projection=_loads_object(public_projection_json), - ) - - -def assert_public_tee_safe(payload: Mapping[str, Any]) -> None: - """Raise ValueError if a public payload violates the deny-list / allowlist.""" - - if payload.get("available") is False: - if set(payload.keys()) != {"available"}: - raise ValueError("unavailable public tee payload must be exactly {available:false}") - return - if payload.get("available") is not True: - raise ValueError("public tee payload must set available true|false") - - unknown = set(payload.keys()) - PUBLIC_TEE_TOP_LEVEL_ALLOWLIST - if unknown: - raise ValueError(f"public tee payload has unknown keys: {sorted(unknown)}") - - serialized = json.dumps(payload, sort_keys=True, default=str) - lowered = serialized.lower() - for key in PUBLIC_TEE_DENY_KEYS: - if _has_exact_json_key(serialized, key): - raise ValueError(f"public tee payload exposes denied key: {key}") - for marker in PUBLIC_TEE_DENY_SUBSTRINGS: - if marker.lower() in lowered: - raise ValueError(f"public tee payload contains denied substring: {marker}") - - preimage = payload.get("report_data_preimage") - if isinstance(preimage, Mapping) and "review_nonce" in preimage: - raise ValueError("public tee report_data_preimage must not include raw review_nonce") -def _has_exact_json_key(serialized: str, key: str) -> bool: - """True when ``key`` appears as a JSON object key (not a longer key prefix). +def public_tee_assignment_qualifies(*args: Any, **kwargs: Any) -> bool: + return False - ``review_nonce_sha256`` must not match the deny key ``review_nonce`` because - the needle requires the exact ``"{key}":`` boundary. - """ - return f'"{key}":' in serialized +def build_public_tee_math(*args: Any, **kwargs: Any) -> dict[str, Any]: + return public_tee_unavailable() -def _loads_object(raw: str | None) -> dict[str, Any] | None: - if not isinstance(raw, str) or not raw.strip(): - return None - try: - value = json.loads(raw) - except json.JSONDecodeError: - return None - return value if isinstance(value, dict) else None +def build_public_tee_math_from_assignment(*args: Any, **kwargs: Any) -> dict[str, Any]: + return public_tee_unavailable() -def _coerce_submission_id(submission_id: int | str, envelope: Mapping[str, Any]) -> int | str: - core = envelope.get("review_core") - if isinstance(core, Mapping): - core_id = core.get("submission_id") - if isinstance(core_id, int) and not isinstance(core_id, bool): - return core_id - if isinstance(core_id, str) and core_id.isdigit(): - try: - return int(core_id) - except ValueError: - return core_id - if isinstance(submission_id, int) and not isinstance(submission_id, bool): - return submission_id - if isinstance(submission_id, str) and submission_id.isdigit(): - return int(submission_id) - return submission_id - - -def _public_measurement(value: object) -> dict[str, str] | None: - if not isinstance(value, Mapping): - return None - out: dict[str, str] = {} - for key in PUBLIC_TEE_MEASUREMENT_KEYS: - item = value.get(key) - if isinstance(item, str) and item: - out[key] = item - return out or None - - -def _public_quote_hex(value: object) -> str | None: - if not isinstance(value, str) or not value: - return None - # Lowercase even-length hex only; drop malformed rather than invent. - cleaned = value.strip().lower() - if len(cleaned) % 2 or any(c not in "0123456789abcdef" for c in cleaned): - return None - max_chars = MAX_PUBLIC_TEE_QUOTE_BYTES * 2 - if len(cleaned) > max_chars: - cleaned = cleaned[:max_chars] - return cleaned - - -def _public_event_log(value: object) -> list[dict[str, Any]] | None: - if not isinstance(value, list) or not value: - return None - capped: list[dict[str, Any]] = [] - total_bytes = 0 - for item in value: - if len(capped) >= MAX_PUBLIC_TEE_EVENT_LOG_ENTRIES: - break - if not isinstance(item, Mapping): - continue - # Copy only bounded public event fields; drop unexpected blobs. - entry: dict[str, Any] = {} - for key in ("event", "event_type", "digest", "event_payload", "imr", "seq"): - if key not in item: - continue - val = item[key] - if isinstance(val, (str, int)) and not isinstance(val, bool): - entry[key] = val - if not entry: - continue - encoded = json.dumps(entry, sort_keys=True, separators=(",", ":")).encode("utf-8") - if total_bytes + len(encoded) > MAX_PUBLIC_TEE_EVENT_LOG_BYTES: - break - total_bytes += len(encoded) - capped.append(entry) - return capped or None - - -def _public_report_data_preimage(envelope: Mapping[str, Any]) -> dict[str, Any] | None: - """Build inspectable preimage without raw review_nonce.""" - - core = envelope.get("review_core") - if not isinstance(core, Mapping): - return None - try: - raw = review_report_data_preimage(core) - except Exception: - # Fail closed: omit preimage rather than leak partial core fields. - return None - nonce = raw.get("review_nonce") - nonce_hash = None - if isinstance(nonce, str) and nonce: - nonce_hash = sha256(nonce.encode("utf-8")).hexdigest() - public = { - "domain": raw.get("domain"), - "schema_version": raw.get("schema_version"), - "review_digest": raw.get("review_digest"), - "session_id": raw.get("session_id"), - "issued_at_ms": raw.get("issued_at_ms"), - "received_at_ms": raw.get("received_at_ms"), - } - if nonce_hash is not None: - public["review_nonce_sha256"] = nonce_hash - # Drop any Nones so FE does not see placeholder invent. - return {k: v for k, v in public.items() if v is not None} - - -def _public_verification_outcome(value: object) -> dict[str, Any] | None: - if not isinstance(value, Mapping): - return None - out: dict[str, Any] = {} - for key in PUBLIC_TEE_OUTCOME_KEYS: - if key not in value: - continue - item = value[key] - if key in {"measurement_allowlisted", "report_data_matched"}: - if isinstance(item, bool): - out[key] = item - elif key == "verified_at_ms": - if item is None or (isinstance(item, int) and not isinstance(item, bool)): - out[key] = item - elif isinstance(item, str): - out[key] = item - return out or None +def assert_public_tee_safe(payload: Mapping[str, Any]) -> None: + if payload.get("available") is True: + raise ValueError("public TEE math must not claim available after Phala removal") -def _quote_fingerprint( - quote_hex: str | None, - public_projection: Mapping[str, Any] | None, -) -> str | None: - if isinstance(public_projection, Mapping): - existing = public_projection.get("quote_fingerprint_sha256") - if isinstance(existing, str) and len(existing) == 64: - return existing - if not quote_hex: - return None - try: - return sha256(bytes.fromhex(quote_hex)).hexdigest() - except ValueError: - return None +__all__ = [ + "assert_public_tee_safe", + "build_public_tee_math", + "build_public_tee_math_from_assignment", + "public_tee_assignment_qualifies", + "public_tee_unavailable", +] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py b/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py index 840570afd..53587c5ef 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/sdk/config.py @@ -100,7 +100,6 @@ class ChallengeSettings(BaseSettings): # Epoch bucket size for push revision identity (seconds). epoch_seconds: int = Field(default=360, ge=1) - # Root stdlib logging level applied at every process entrypoint (the API app # import and the worker ``main()``). Uvicorn installs no root handler, so # without an explicit configuration the worker service emits ZERO logs and @@ -161,25 +160,15 @@ class ChallengeSettings(BaseSettings): # Empty => streaming disabled (job_dir files + finalize still capture logs). terminal_bench_log_stream_url: str | None = None terminal_bench_log_stream_timeout_seconds: float = 5.0 - # Phala TEE attested-result emission (architecture.md sec 8). Opt-in and OFF - # by default: flag off reproduces today's validator-run own_runner behavior - # byte-identically (R=1, epsilon=0 harbor scoring, no dstack access at all). - # This is the validator-config view of the in-image emission gate -- the - # field's env var (``CHALLENGE_PHALA_ATTESTATION_ENABLED``) is exactly the - # switch ``own_runner_backend`` reads inside the canonical CVM image, so a - # config-off deployment gates the image off. + # REMOVED product path (T40): Phala TEE dual flags. Always False; enabling + # either raises at settings construction. Kept as fields so old env/YAML + # keys fail closed rather than silently enabling dead code. phala_attestation_enabled: bool = False - # Attested review is deliberately a separate explicit switch while its - # companion eval lifecycle lands. It defaults off, preserving the complete - # legacy intake and gateway-review path byte-for-byte. Production full - # attested deployments enable both this and ``phala_attestation_enabled``. attested_review_enabled: bool = False - # Temporary host-local unattested execution (NO_PHALA). Opt-in, default OFF. - # When true the master runs jobs via own_runner on the host instead of Phala - # CVMs. Results are explicitly marked unattested. Mutually exclusive with - # both attestation flags above (fail closed at settings construction). - # Env precedence: CHALLENGE_NO_PHALA if set, else plain NO_PHALA, else false. - # Never inferred from a missing Phala API key. See docs/no-phala-mode.md. + # Product path: host-trust unattested execution (NO_PHALA). Opt-in, default OFF. + # When true the master runs jobs via own_runner on the host. Results are + # explicitly marked unattested (never TEE). Env: CHALLENGE_UNATTESTED_EXECUTION + # / CHALLENGE_NO_PHALA / NO_PHALA. See docs/no-phala-mode.md and host-trust.md. no_phala: bool = False review_assignment_ttl_seconds: int = 1800 review_operator_approval_ttl_seconds: int = 300 @@ -414,13 +403,22 @@ def load_file_backed_secrets(self) -> ChallengeSettings: @model_validator(mode="after") def validate_attested_topology(self) -> ChallengeSettings: - """Reject review-only and eval-only production configurations.""" + """Phala TEE dual flags are removed (T40). Force both permanently OFF. + + Product path is unattested/host-trust only (``no_phala`` / + ``CHALLENGE_UNATTESTED_EXECUTION``). Reject any attempt to enable TEE + dual flags so configs cannot re-open AttestationGate / KR score legs. + """ - if self.attested_review_enabled != self.phala_attestation_enabled: + if self.attested_review_enabled or self.phala_attestation_enabled: raise ValueError( - "attested_review_enabled and phala_attestation_enabled must both be " - "enabled for full attested mode or both be disabled for legacy mode" + "Phala TEE dual flags are removed: phala_attestation_enabled and " + "attested_review_enabled must both be false. Use unattested/" + "host-trust mode (no_phala / CHALLENGE_UNATTESTED_EXECUTION) only." ) + # Belt-and-suspenders: never leave True even if a subclass mutates. + object.__setattr__(self, "phala_attestation_enabled", False) + object.__setattr__(self, "attested_review_enabled", False) return self @model_validator(mode="after") @@ -445,7 +443,11 @@ def resolve_and_validate_no_phala(self) -> ChallengeSettings: # Re-resolve so plain NO_PHALA is honored when the prefixed var is unset. # When CHALLENGE_NO_PHALA is set, pydantic already populated ``no_phala``; # resolve_no_phala_from_environ applies the same precedence. - if "CHALLENGE_NO_PHALA" in os.environ or "NO_PHALA" in os.environ: + if ( + "CHALLENGE_UNATTESTED_EXECUTION" in os.environ + or "CHALLENGE_NO_PHALA" in os.environ + or "NO_PHALA" in os.environ + ): self.no_phala = resolve_no_phala_from_environ() assert_no_phala_compatible( no_phala=bool(self.no_phala), @@ -454,6 +456,12 @@ def resolve_and_validate_no_phala(self) -> ChallengeSettings: ) return self + @property + def unattested_execution(self) -> bool: + """Canonical alias for host-trust unattested mode (``no_phala``).""" + + return bool(self.no_phala) + def require_eval_result_signer_for_production(self) -> None: """Fail closed for production full-attested mode without an endpoint signer. diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/__init__.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/__init__.py index 17c147c50..ddb54bb1a 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/__init__.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/__init__.py @@ -1,16 +1,21 @@ -"""Miner-facing self-deploy flow for the canonical Phala eval image (architecture §4 C7). - -The miner self-deploys and funds a Phala **Intel TDX CPU** CVM running the -canonical, digest-pinned eval image; the validator/subnet keeps the trust root -(measurement allowlist + golden key-release + quote verification). This package -implements the miner-facing command surface around the already-landed building -blocks (``canonical.compose``/``canonical.measurement``/``canonical.report_data``/ -``keyrelease``): fetch/prepare the image + generated compose, publish/reproduce -the canonical measurement, deploy a CPU-only CVM (with money/GPU guards + a -no-spend dry-run), point the run at the validator key-release endpoint, and -surface the attested result for allowlist checking. - -Money/GPU safety is enforced BEFORE any provisioning: GPU targets are refused, -over-cap shapes are refused, and missing credentials error clearly without a -Phala mutate call (AGENTS.md Mission Boundaries). +"""Self-deploy Phala CVM surface removed (T40). + +Host-trust product path does not provision Phala CVMs. Importing this package +is allowed for residual scripts; mutating APIs raise. """ + +from __future__ import annotations + + +class SelfDeployRemovedError(RuntimeError): + """Raised when a deleted Phala self-deploy API is invoked.""" + + +def _removed(name: str) -> None: + raise SelfDeployRemovedError( + f"agent_challenge.selfdeploy.{name} was removed with Phala TEE (T40). " + "Use host-trust unattested execution (no_phala / CHALLENGE_UNATTESTED_EXECUTION)." + ) + + +__all__ = ["SelfDeployRemovedError"] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/__main__.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/__main__.py deleted file mode 100644 index dbab68834..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/__main__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""``python -m agent_challenge.selfdeploy`` entrypoint.""" - -from __future__ import annotations - -from agent_challenge.selfdeploy.cli import main - -if __name__ == "__main__": # pragma: no cover - thin CLI shim - raise SystemExit(main()) diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/cli.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/cli.py deleted file mode 100644 index 1b8bde810..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/cli.py +++ /dev/null @@ -1,1751 +0,0 @@ -"""Miner-facing self-deploy CLI (``python -m agent_challenge.selfdeploy``). - -Subcommands cover the full flow (VAL-DEPLOY-001): ``prepare`` (fetch/prepare the -canonical image + generated compose), ``measurements`` (publish/reproduce the -canonical measurement), ``verdict`` (report a measurement + its allowlist -verdict), ``deploy`` (deploy a CPU-only CVM, with a no-spend ``--dry-run`` and -GPU/over-cap/credential guards), ``run`` (run the eval against the validator -key-release endpoint), ``result`` (surface + verify the attested-result envelope), -and ``teardown`` (delete the CVM). - -The two spend-capable subcommands (``deploy``, ``run``) accept injectable side -effects (the Phala deployer / backend runner / teardown runner) so the whole -surface is testable offline; only ``deploy`` (without ``--dry-run``) and -``teardown`` reach Phala, and both refuse clearly before any Phala call when a -guard fails. -""" - -from __future__ import annotations - -import argparse -import json -import os -import subprocess -import sys -from collections.abc import Callable, Mapping, Sequence -from dataclasses import replace -from pathlib import Path -from typing import Any - -from agent_challenge.selfdeploy import eval as eval_deploy -from agent_challenge.selfdeploy import lifecycle -from agent_challenge.selfdeploy import measurements as measure -from agent_challenge.selfdeploy import result as result_mod -from agent_challenge.selfdeploy import review as review_deploy -from agent_challenge.selfdeploy import run as run_mod -from agent_challenge.selfdeploy.client import RouteClientError, SelfDeployRouteClient -from agent_challenge.selfdeploy.phala import ( - DEFAULT_PHALA_API, - PhalaApiError, - PhalaCloudClient, - resolve_cvm_id_from_list, -) -from agent_challenge.selfdeploy.plan import ( - CredentialError, - DeployPlan, - PrepareError, - build_deploy_plan, - check_phala_credentials, - prepare_deployment, - render_plan, - write_prepared, -) -from agent_challenge.selfdeploy.shapes import ( - DEFAULT_EVAL_DISK_SIZE_GB, - DEFAULT_EVAL_INSTANCE_TYPE, - DEFAULT_MAX_RUNTIME_HOURS, - DEFAULT_MONEY_CAP_USD, - DEFAULT_OS_IMAGE, - DEFAULT_REVIEW_DISK_SIZE_GB, - DEFAULT_REVIEW_INSTANCE_TYPE, - ShapeError, - validate_disk_size, -) - - -def _with_disk_size(plan: Any, disk_size_gb: int) -> Any: - """Attach validated disk size to a deployment plan (dataclass or test double).""" - - try: - return replace(plan, disk_size_gb=disk_size_gb) - except TypeError: - # Offline tests inject SimpleNamespace plan doubles. - try: - plan.disk_size_gb = disk_size_gb - except Exception: - pass - return plan - - -PROG = "agent-challenge-selfdeploy" - -#: A Phala deployer: (plan, out_dir) -> arbitrary result (printed by the CLI). -Deployer = Callable[[DeployPlan, str], Any] -#: A teardown runner: cvm_id -> arbitrary result (printed by the CLI). -Teardowner = Callable[[str], Any] - -#: The subcommands the CLI exposes (kept in sync with docs/miner/self-deploy.md). -SUBCOMMANDS: tuple[str, ...] = ( - "prepare", - "measurements", - "verdict", - "deploy", - "run", - "result", - "teardown", -) - -#: Subcommands that can create/charge Phala resources (must all be documented). -SPEND_CAPABLE_SUBCOMMANDS: frozenset[str] = frozenset({"deploy", "run"}) - -# Attested mode uses only these ordered production stages. The legacy -# top-level helpers remain for the feature-flag-off validator-run path. -ORDERED_SUBCOMMANDS: tuple[str, ...] = ("review", "eval") - - -# --------------------------------------------------------------------------- # -# Default side effects (never exercised by the offline suite) -# --------------------------------------------------------------------------- # -def default_phala_deployer(plan: DeployPlan, out_dir: str) -> dict[str, Any]: # pragma: no cover - """Write the exact app-compose bytes and invoke ``phala deploy`` (live, M6). - - Writes ``app-compose.json`` verbatim (so the deployed compose-hash matches the - pinned measurement) and shells the ``phala`` CLI. Live deploy/teardown are - validated at milestone ``self-deploy-live``; the offline suite drives an - injected deployer instead. - """ - - compose_path = write_prepared(plan.prepared, out_dir) - cmd = [ - "phala", - "deploy", - "-c", - str(compose_path), - "-n", - plan.name, - "-t", - plan.instance_type, - "-r", - plan.region, - ] - proc = subprocess.run(cmd, capture_output=True, text=True) - return {"returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr} - - -_TEARDOWN_DIAGNOSTIC_LIMIT = 512 - - -def _bounded_text(value: str | None, *, limit: int = _TEARDOWN_DIAGNOSTIC_LIMIT) -> str: - """Return a size-bounded diagnostic string suitable for CLI surfaces.""" - - text = value or "" - if len(text) <= limit: - return text - return text[: limit - 3] + "..." - - -def default_phala_teardown( - cvm_id: str, - *, - client: PhalaCloudClient | None = None, -) -> dict[str, Any]: - """Delete a CVM via Phala Cloud HTTP ``DELETE /cvms/{id}`` (idempotent). - - Uses :class:`PhalaCloudClient` — never shells out to a ``phala`` binary - (the binary is not present on production validator containers). 204 and - 404 from the API are success. Always returns a structured result with - ``ok``/``returncode`` so callers can fail non-zero when deletion fails. - """ - - try: - api = client if client is not None else PhalaCloudClient() - api.delete_cvm(cvm_id) - except (PhalaApiError, CredentialError) as exc: - return { - "returncode": 1, - "ok": False, - "stdout": "", - "stderr": "", - "error": str(exc), - } - except Exception as exc: # noqa: BLE001 - surface unexpected transport failures - return { - "returncode": 1, - "ok": False, - "stdout": "", - "stderr": "", - "error": f"teardown failed: {exc.__class__.__name__}", - } - return { - "returncode": 0, - "ok": True, - "stdout": "", - "stderr": "", - "error": None, - } - - -def resolve_teardown_cvm_id( - *, - cvm_id: str | None, - app_id: str | None, - client: PhalaCloudClient | None = None, -) -> str: - """Resolve the CVM id for teardown: explicit id, else unique app_id match.""" - - explicit = (cvm_id or "").strip() - if explicit: - return explicit - identity = (app_id or "").strip() - if not identity: - raise RouteClientError("teardown requires --cvm-id or --app-id") - api = client if client is not None else PhalaCloudClient() - listing = api.get("/cvms") - resolved = resolve_cvm_id_from_list(listing, app_id=identity, require_unique=True) - if not resolved: - raise RouteClientError(f"no CVM found for app_id {identity!r}") - return resolved - - -def _run_teardown_command(args: argparse.Namespace) -> tuple[dict[str, Any], int]: - """Resolve CVM identity and tear down via HTTP (review/eval/top-level).""" - - phala_base = getattr(args, "phala_api", None) or DEFAULT_PHALA_API - try: - # Refuse missing identity before constructing a credentialed client. - explicit = (getattr(args, "cvm_id", None) or "").strip() - app_id = (getattr(args, "app_id", None) or "").strip() - if not explicit and not app_id: - raise RouteClientError("teardown requires --cvm-id or --app-id") - client = PhalaCloudClient(base_url=str(phala_base)) - cvm_id = resolve_teardown_cvm_id( - cvm_id=explicit or None, - app_id=app_id or None, - client=client, - ) - outcome = default_phala_teardown(cvm_id, client=client) - except (RouteClientError, CredentialError, PhalaApiError) as exc: - # Fail closed with a structured payload when identity cannot be resolved. - missing = getattr(args, "cvm_id", None) or getattr(args, "app_id", None) or "" - return ( - { - "torn_down": missing or None, - "ok": False, - "diagnostics": { - "returncode": 1, - "error": str(exc), - "stdout": "", - "stderr": "", - }, - "result": { - "returncode": 1, - "error": str(exc), - "stdout": "", - "stderr": "", - }, - }, - 2 if isinstance(exc, RouteClientError) else 1, - ) - return _teardown_payload(cvm_id, outcome) - - -def _teardown_payload(cvm_id: str, result: Any) -> tuple[dict[str, Any], int]: - """Project teardown outcome as a miner-facing payload and exit code.""" - - if isinstance(result, Mapping): - returncode = int(result.get("returncode") or 0) - ok = bool(result.get("ok", returncode == 0)) - diagnostics = { - "returncode": returncode, - "error": result.get("error"), - "stdout": _bounded_text(str(result.get("stdout") or "")), - "stderr": _bounded_text(str(result.get("stderr") or "")), - } - else: - ok = True - returncode = 0 - diagnostics = {"returncode": 0, "error": None, "stdout": "", "stderr": ""} - payload = { - "torn_down": cvm_id, - "ok": ok, - "diagnostics": diagnostics, - "result": diagnostics, - } - return payload, 0 if ok else 1 - - -def _delete_attributable_cvm(cvm_id: str | None) -> None: - """Best-effort delete of a funded CVM after a post-create failure.""" - - if not cvm_id: - return - try: - default_phala_teardown(cvm_id) - except Exception: # noqa: BLE001 - cleanup must never mask the original failure - return - - -def _review_token_present(prepare_response: Mapping[str, Any] | None) -> bool: - """True when prepare/retry body still carries a non-empty one-time token.""" - - if not isinstance(prepare_response, Mapping): - return False - token = prepare_response.get("review_session_token") - return isinstance(token, str) and bool(token) - - -def _assignment_id_from_prepare(prepare_response: Mapping[str, Any]) -> str | None: - """Extract current assignment_id without requiring a valid token.""" - - assignment_id = prepare_response.get("assignment_id") - if isinstance(assignment_id, str) and assignment_id: - return assignment_id - assignment = prepare_response.get("assignment") - if isinstance(assignment, Mapping): - core = assignment.get("assignment_core") - if isinstance(core, Mapping): - nested = core.get("assignment_id") - if isinstance(nested, str) and nested: - return nested - return None - - -def _obtain_review_prepare_with_token( - client: SelfDeployRouteClient, - submission_id: int, -) -> dict[str, Any]: - """Return a prepare/retry response that still delivers the session token. - - Product residual timeline: - - tee-live-proof-v5: re-prepare after first delivery returned null token → - cancel+retry recovered a new attempt (burned undepoyed attempts). - - review-v9: product redelivers capability for active undepoyed assignments, - so the common dry-run → live path never hits null and never cancels. - Cancel+retry remains only when prepare is sticky-null (post-deploy CVM still - holding the capability, or terminal) and a fresh assignment is required. - Does not invent tokens or persist capabilities offline. - """ - - response = client.review_prepare(submission_id) - if _review_token_present(response): - return response - assignment_id = _assignment_id_from_prepare(response) - if not assignment_id: - raise RouteClientError("review session has no current assignment id to refresh capability") - # Sticky-null after deploy or terminal: only then cancel+retry for a new token. - # Prefer not cancelling an active undepoyed assignment — product should have - # redelivered. If prepare still null, attempt cancel (may already be terminal) - # then issue a fresh assignment with token. - try: - client.review_cancel(submission_id, assignment_id) - except RouteClientError: - # Terminal / already cancelled: still try retry against that id. - pass - retried = client.review_retry(submission_id, assignment_id) - if not _review_token_present(retried): - raise RouteClientError( - "review session token unavailable after prepare and retry; " - "capability may be spent or assignment not retryable" - ) - return retried - - -def _eval_token_present(prepare_response: Mapping[str, Any] | None) -> bool: - """True when prepare/retry still delivers the one-shot EVAL_RUN_TOKEN. - - Accepted shape matches ``eval.build_eval_deployment_plan``: - ``secret_delivery == {"env_key", "token"}`` with a non-empty token string. - Does not relax that contract — only decides whether recovery is needed. - """ - - if not isinstance(prepare_response, Mapping): - return False - delivery = prepare_response.get("secret_delivery") - if not isinstance(delivery, Mapping) or set(delivery) != {"env_key", "token"}: - return False - token = delivery.get("token") - return isinstance(token, str) and bool(token) - - -def _eval_run_id_from_prepare(prepare_response: Mapping[str, Any]) -> str | None: - """Extract current eval_run_id from a prepare wrapper without requiring a token.""" - - plan = prepare_response.get("plan") - if isinstance(plan, Mapping): - run_id = plan.get("eval_run_id") - if isinstance(run_id, str) and run_id: - return run_id - return None - - -def _obtain_eval_prepare_with_token( - client: SelfDeployRouteClient, - submission_id: int, -) -> dict[str, Any]: - """Return a prepare/retry response that still delivers EVAL_RUN_TOKEN. - - Product residual timeline (live production, submission 3): - - ``EVAL_RUN_TOKEN`` is delivered once per attempt. Standalone - ``eval prepare`` or ``eval retry`` permanently spends it. - - ``eval deploy`` previously called ``eval_prepare`` raw; when - ``secret_delivery`` was null, ``build_eval_deployment_plan`` hard-failed - with "first Eval prepare must deliver exactly one EVAL_RUN_TOKEN - capability". Attempts 1 and 2 both stuck; lifecycle unrecoverable - from the CLI. - - Mirror review: prepare → if token-less, resolve ``eval_run_id``, - ``eval_cancel`` then ``eval_retry``, return the response that carries - the token. Never cancel when the token was already delivered. Never - fabricate, cache, or persist a token offline. - """ - - response = client.eval_prepare(submission_id) - if _eval_token_present(response): - return response - run_id = _eval_run_id_from_prepare(response) - if not run_id: - raise RouteClientError("eval run has no current eval_run_id to refresh capability") - # Sticky token-less after prior prepare/retry consumer: cancel+retry for a - # fresh attempt that redelivers EVAL_RUN_TOKEN. Prefer not cancelling when - # prepare already carried the capability (handled above). - try: - client.eval_cancel(submission_id, run_id) - except RouteClientError: - # Terminal / already cancelled: still try retry against that id. - pass - retried = client.eval_retry(submission_id, run_id) - if not _eval_token_present(retried): - raise RouteClientError( - "eval run token unavailable after prepare and retry; " - "capability may be spent or run not retryable" - ) - return retried - - -def _review_allowlist_verdict(plan: review_deploy.ReviewDeploymentPlan) -> str: - """Compute a verified review-domain allowlist verdict or explicit UNKNOWN.""" - - app = plan.assignment["assignment_core"]["review_app"] - allowlist = app.get("measurement_allowlist") - if not isinstance(allowlist, list) or not allowlist: - return "UNKNOWN" - measurement = { - "mrtd": plan.measurement.get("mrtd"), - "rtmr0": plan.measurement.get("rtmr0"), - "rtmr1": plan.measurement.get("rtmr1"), - "rtmr2": plan.measurement.get("rtmr2"), - "compose_hash": plan.compose_hash, - "os_image_hash": plan.measurement.get("os_image_hash"), - } - try: - return measure.domain_allowlist_verdict( - domain="review", - measurement=measurement, - review_allowlist=allowlist, - ).as_dict()["verdict"] - except measure.MeasurementError: - return "UNKNOWN" - - -def _eval_allowlist_verdict(plan: eval_deploy.EvalDeploymentPlan) -> str: - """Eval plan alone cannot prove allowlist membership; report UNKNOWN.""" - - # The validator-owned Eval allowlist is not part of the miner prepare wrapper. - # Never fabricate IN-LIST from compose-hash presence alone. - allowlist = plan.plan.get("measurement_allowlist") or plan.measurement.get( - "measurement_allowlist" - ) - if not allowlist: - return "UNKNOWN" - measurement = { - "mrtd": plan.measurement.get("mrtd"), - "rtmr0": plan.measurement.get("rtmr0"), - "rtmr1": plan.measurement.get("rtmr1"), - "rtmr2": plan.measurement.get("rtmr2"), - "compose_hash": plan.compose_hash, - "os_image_hash": plan.measurement.get("os_image_hash"), - } - try: - return measure.domain_allowlist_verdict( - domain="eval", - measurement=measurement, - eval_allowlist=allowlist, - ).as_dict()["verdict"] - except measure.MeasurementError: - return "UNKNOWN" - - -def _add_route_args( - parser: argparse.ArgumentParser, - *, - include_submission: bool, - signed: bool = True, -) -> None: - """Add safe signed-route arguments without accepting arbitrary endpoints.""" - - parser.add_argument("--base-url", required=True, help="challenge production API base URL") - if include_submission: - parser.add_argument("--submission-id", required=True, type=int) - if signed: - parser.add_argument("--hotkey", required=True) - parser.add_argument("--signature", default=None) - parser.add_argument("--nonce", default=None) - parser.add_argument("--timestamp", default=None) - parser.add_argument( - "--auto-sign", - action="store_true", - help="sign each request with MINER_HOTKEY_MNEMONIC or MINER_HOTKEY_URI", - ) - - -def _route_client(args: argparse.Namespace) -> SelfDeployRouteClient: - from agent_challenge.selfdeploy.client import build_signed_identity - - identity = None - auto_sign = bool(getattr(args, "auto_sign", False)) - if hasattr(args, "hotkey"): - if auto_sign: - identity = build_signed_identity( - hotkey=args.hotkey, - signature="auto", - nonce="auto", - timestamp=args.timestamp, - ) - else: - if not args.signature or not args.nonce: - raise RouteClientError( - "signed route requires --signature and --nonce, or --auto-sign" - ) - identity = build_signed_identity( - hotkey=args.hotkey, - signature=args.signature, - nonce=args.nonce, - timestamp=args.timestamp, - ) - return SelfDeployRouteClient(args.base_url, identity=identity, auto_sign=auto_sign) - - -def _safe_json_file(path: str) -> dict[str, Any]: - try: - value = json.loads(Path(path).read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RouteClientError("JSON input could not be read") from exc - if not isinstance(value, dict): - raise RouteClientError("JSON input must contain an object") - return value - - -def _ordered_review_command(args: argparse.Namespace) -> int: - try: - if args.review_command == "prepare": - payload = _route_client(args).review_prepare(args.submission_id) - if args.output: - Path(args.output).write_text( - json.dumps( - _redact_capabilities(payload), - sort_keys=True, - separators=(",", ":"), - ), - encoding="utf-8", - ) - _print(_redact_capabilities(payload)) - return 0 - if args.review_command == "result": - _print( - _redact_capabilities( - _route_client(args).review_report(args.submission_id, cursor=args.cursor) - ) - ) - return 0 - if args.review_command == "history": - _print( - _redact_capabilities( - _route_client(args).review_history(args.submission_id, cursor=args.cursor) - ) - ) - return 0 - if args.review_command == "deployed": - _print( - _redact_capabilities( - _route_client(args).review_deployed( - args.submission_id, - _safe_json_file(args.acknowledgement), - ) - ) - ) - return 0 - if args.review_command in {"cancel", "retry"}: - client = _route_client(args) - if args.review_command == "cancel": - payload = client.review_cancel(args.submission_id, args.assignment_id) - else: - approval_id = getattr(args, "approval_id", None) - payload = client.review_retry( - args.submission_id, - args.assignment_id, - approval_id=approval_id, - ) - _print(_redact_capabilities(payload)) - return 0 - if args.review_command == "teardown": - payload, code = _run_teardown_command(args) - _print(payload) - return code - if args.review_command == "deploy": - # Live review deploy requires a signable identity. Prefer - # --auto-sign when acknowledgement bodies are produced by the - # same process; accept documented explicit signature headers - # when the caller supplies a fresh signature over that request. - if ( - not args.dry_run - and not args.auto_sign - and (not getattr(args, "signature", None) or not getattr(args, "nonce", None)) - ): - raise RouteClientError( - "review deploy requires --auto-sign or explicit --signature and --nonce " - "so the created-CVM acknowledgement is signed over its returned receipt " - "and CVM identity" - ) - if args.prepare_response: - raise RouteClientError( - "review deploy does not accept persisted prepare capabilities; " - "run it with signed production route credentials" - ) - client = _route_client(args) - response = _obtain_review_prepare_with_token(client, args.submission_id) - plan = review_deploy.build_review_deployment_plan(response) - if plan.instance_type != args.review_instance_type: - raise RouteClientError( - "review deployment shape differs from the validator-issued assignment" - ) - review_disk = validate_disk_size( - getattr(args, "review_disk_size_gb", DEFAULT_REVIEW_DISK_SIZE_GB) - ) - eval_disk = validate_disk_size( - getattr(args, "eval_disk_size_gb", DEFAULT_EVAL_DISK_SIZE_GB) - ) - plan = _with_disk_size(plan, review_disk) - lifecycle.validate_lifecycle_budget( - review_instance_type=plan.instance_type, - eval_instance_type=args.eval_instance_type, - review_runtime_hours=args.review_runtime_hours, - eval_runtime_hours=args.eval_runtime_hours, - money_cap_usd=args.money_cap_usd, - review_disk_size_gb=review_disk, - eval_disk_size_gb=eval_disk, - ) - key = os.environ.get(args.openrouter_key_env, "") - if not args.dry_run and not key: - raise RouteClientError( - f"{args.openrouter_key_env} is not set; review deployment cannot continue" - ) - # REVIEW_API_BASE_URL is bound from the same production challenge - # base URL the miner already uses (joinbase agent-challenge path). - # Measured compose allows. - # Without it, older review images default to chain.platform.network - # (502) and never POST /report. - api_base = str(args.base_url).rstrip("/") - encrypted = ( - review_deploy.encrypt_review_secrets( - plan, - { - "OPENROUTER_API_KEY": key, - "REVIEW_API_BASE_URL": api_base, - "REVIEW_SESSION_TOKEN": plan.review_session_token, - }, - ) - if not args.dry_run - else None - ) - if not args.dry_run: - assert encrypted is not None - acknowledgement: dict[str, Any] | None = None - try: - acknowledgement = review_deploy.HttpReviewPhalaDeployment( - PhalaCloudClient(base_url=args.phala_api or DEFAULT_PHALA_API) - ).deploy(plan, encrypted) - signed_ack = client.review_deployed( - args.submission_id, - acknowledgement, - ) - except Exception: - cvm_id = None - if isinstance(acknowledgement, Mapping): - cvm_id = acknowledgement.get("cvm_id") - _delete_attributable_cvm(cvm_id if isinstance(cvm_id, str) else None) - raise - _print( - { - "stage": "review_deployed", - "acknowledgement": acknowledgement, - "signed_acknowledgement": signed_ack, - "encrypted_env_names": list(encrypted.env_keys), - } - ) - return 0 - _print( - { - "stage": "review_deploy_ready", - "assignment_id": plan.assignment["assignment_core"]["assignment_id"], - "image_ref": plan.image_ref, - "compose_hash": plan.compose_hash, - "measurement": plan.measurement, - "allowlist_verdict": _review_allowlist_verdict(plan), - "encrypted_env_names": list(review_deploy.REVIEW_ALLOWED_ENVS), - "encrypted_env_transmitted": False, - "dry_run": True, - } - ) - return 0 - except ( - RouteClientError, - CredentialError, - lifecycle.LifecycleBudgetError, - ShapeError, - review_deploy.ReviewDeploymentError, - PhalaApiError, - ) as exc: - print(f"error: {exc}", file=sys.stderr) - return 2 - raise RouteClientError("unknown review stage") - - -def _ordered_eval_command(args: argparse.Namespace) -> int: - try: - if args.eval_command == "prepare": - payload = _route_client(args).eval_prepare(args.submission_id) - if args.output: - Path(args.output).write_text( - json.dumps( - _redact_capabilities(payload), - sort_keys=True, - separators=(",", ":"), - ), - encoding="utf-8", - ) - _print(_redact_capabilities(payload)) - return 0 - if args.eval_command == "status": - _print( - _redact_capabilities( - _route_client(args).eval_status(args.submission_id, cursor=args.cursor) - ) - ) - return 0 - if args.eval_command in {"cancel", "retry"}: - client = _route_client(args) - payload = ( - client.eval_cancel(args.submission_id, args.run_id) - if args.eval_command == "cancel" - else client.eval_retry(args.submission_id, args.run_id) - ) - _print(_redact_capabilities(payload)) - return 0 - if args.eval_command == "failure": - _print( - _redact_capabilities( - _route_client(args).eval_failure( - args.submission_id, - args.run_id, - args.reason_code, - ) - ) - ) - return 0 - if args.eval_command == "result": - token = os.environ.get(args.token_env, "") - if not token: - raise RouteClientError(f"{args.token_env} is not set") - try: - raw_result = Path(args.result).read_bytes() - except OSError as exc: - raise RouteClientError("Eval result JSON could not be read") from exc - if len(raw_result) > 16 * 1024 * 1024: - raise RouteClientError("Eval result exceeds the bounded request size") - try: - parsed_result = json.loads(raw_result) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise RouteClientError("Eval result JSON is malformed") from exc - if not isinstance(parsed_result, dict): - raise RouteClientError("Eval result JSON must contain an object") - _print( - _redact_capabilities( - _route_client(args).eval_result_bytes( - args.run_id, - raw_result, - token, - ) - ) - ) - return 0 - if args.eval_command == "teardown": - payload, code = _run_teardown_command(args) - _print(payload) - return code - if args.eval_command == "deploy": - if args.prepare_response: - raise RouteClientError( - "Eval deploy does not accept persisted prepare capabilities; " - "run it with signed production route credentials" - ) - # Validate the handoff destination BEFORE any remote mutation: the - # prepare below spends the single EVAL_RUN_TOKEN delivery, and the - # guest never posts, so gating later would strand the miner with a - # burnt token and no way to call eval result. - _require_eval_run_token_handoff(args) - client = _route_client(args) - raw = _obtain_eval_prepare_with_token(client, args.submission_id) - plan = eval_deploy.build_eval_deployment_plan(raw) - _assert_eval_deploy_shape_and_measurement_pin(plan, args) - review_disk = validate_disk_size( - getattr(args, "review_disk_size_gb", DEFAULT_REVIEW_DISK_SIZE_GB) - ) - eval_disk = validate_disk_size( - getattr(args, "eval_disk_size_gb", DEFAULT_EVAL_DISK_SIZE_GB) - ) - plan = _with_disk_size(plan, eval_disk) - lifecycle.validate_lifecycle_budget( - review_instance_type=args.review_instance_type, - eval_instance_type=plan.instance_type, - review_runtime_hours=args.review_runtime_hours, - eval_runtime_hours=args.eval_runtime_hours, - money_cap_usd=args.money_cap_usd, - review_disk_size_gb=review_disk, - eval_disk_size_gb=eval_disk, - ) - values = { - "EVAL_RUN_TOKEN": plan.eval_run_token, - "LLM_COST_LIMIT": os.environ.get(args.llm_cost_limit_env, "") or "0", - } - # VAL-ACAT-013: Base LLM gateway env vars are never injected into eval - # encrypted_env. Residual --gateway-*-env flags are ignored if present. - if not args.dry_run and any(not value for value in values.values()): - raise RouteClientError( - "Eval run token and LLM_COST_LIMIT are required before deployment " - "(Base LLM gateway secrets are not used)" - ) - values["CHALLENGE_PHALA_ATTESTATION_ENABLED"] = "1" - values["CHALLENGE_PHALA_EVAL_PLAN"] = json.dumps( - plan.plan, - sort_keys=True, - separators=(",", ":"), - ) - values["CHALLENGE_PHALA_AGENT_HASH"] = plan.plan["agent_hash"] - values["CHALLENGE_PHALA_CANONICAL_MEASUREMENT"] = json.dumps( - { - "mrtd": plan.measurement["mrtd"], - "rtmr0": plan.measurement["rtmr0"], - "rtmr1": plan.measurement["rtmr1"], - "rtmr2": plan.measurement["rtmr2"], - "compose_hash": plan.compose_hash, - "os_image_hash": plan.measurement["os_image_hash"], - }, - sort_keys=True, - separators=(",", ":"), - ) - values["CHALLENGE_PHALA_VALIDATOR_NONCE"] = plan.plan["key_release_nonce"] - # Production endpoint is provisioned via KEY_RELEASE_RA_TLS_HOST/PORT in - # the measured compose; keep the plan authority only as a legacy accessor - # for non-raw offline helpers, never as an HTTP fallback URL on the wire. - # RTMR3 is produced by the in-CVM dstack quote/event-log replay. - # The miner must not fabricate a runtime measurement in ordinary - # encrypted_env; the measured image derives it from its quote. - # - # Inject the validator raw-listener trust CA so the guest verifies the - # host RA-TLS cert (never the dstack guest intermediate). Production - # requires CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM or _FILE (or - # KEY_RELEASE_SERVER_CA_FILE pointing at the CA that signed server.crt). - server_ca_pem = (os.environ.get("CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM") or "").strip() - if not server_ca_pem: - for ca_env in ( - "CHALLENGE_PHALA_RA_TLS_SERVER_CA_FILE", - "KEY_RELEASE_SERVER_CA_FILE", - ): - ca_path = (os.environ.get(ca_env) or "").strip() - if ca_path and Path(ca_path).is_file(): - server_ca_pem = Path(ca_path).read_text(encoding="utf-8").strip() - break - if server_ca_pem and "BEGIN CERTIFICATE" in server_ca_pem: - # Normalize/unescape (encrypted_env or file may carry literal \\n) - # and OpenSSL-preload before inject so the guest never gets junk. - try: - from agent_challenge.keyrelease.client import normalize_server_ca_pem - - values["CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM"] = normalize_server_ca_pem( - server_ca_pem - ) - except ValueError as exc: - raise RouteClientError( - f"raw RA-TLS eval deploy server CA is not OpenSSL-loadable: {exc}" - ) from exc - elif not args.dry_run: - raise RouteClientError( - "raw RA-TLS eval deploy requires the validator server CA " - "(set CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM or " - "CHALLENGE_PHALA_RA_TLS_SERVER_CA_FILE / KEY_RELEASE_SERVER_CA_FILE)" - ) - # Optional measured OpenRouter for the in-CVM agent LLM (tools-only if unset). - # Never Base gateway. Host still must scrape BASE_BENCHMARK_RESULT= and - # POST eval result — guest never HTTP-posts the score envelope. - or_env = getattr(args, "openrouter_key_env", None) or "OPENROUTER_API_KEY" - or_key = (os.environ.get(or_env) or "").strip() - if or_key: - values["OPENROUTER_API_KEY"] = or_key - # Optional mid-run progress reporter bindings (observability only). - values.update( - eval_deploy.build_eval_progress_env( - base_url=str(args.base_url), - eval_run_id=plan.eval_run_id, - submission_id=str(args.submission_id), - eval_run_token=plan.eval_run_token, - ) - ) - encrypted = eval_deploy.encrypt_eval_secrets(plan, values) if not args.dry_run else None - if not args.dry_run: - assert encrypted is not None - acknowledgement: dict[str, Any] | None = None - try: - acknowledgement = eval_deploy.HttpEvalPhalaDeployment( - PhalaCloudClient(base_url=args.phala_api or DEFAULT_PHALA_API) - ).deploy(plan, encrypted) - except Exception as exc: - attributable = getattr(exc, "attributable_cvm_id", None) - if not attributable and isinstance(acknowledgement, Mapping): - attributable = acknowledgement.get("cvm_id") - _delete_attributable_cvm( - attributable if isinstance(attributable, str) else None - ) - raise - _print( - _hand_off_eval_run_token( - args, - eval_run_id=plan.eval_run_id, - eval_run_token=plan.eval_run_token, - acknowledgement=acknowledgement, - encrypted_env_names=encrypted.env_keys, - ) - ) - return 0 - _print( - { - "stage": "eval_deploy_ready", - "eval_run_id": plan.eval_run_id, - "image_ref": plan.image_ref, - "compose_hash": plan.compose_hash, - "key_release_endpoint": plan.plan["key_release_endpoint"], - "measurement": plan.measurement, - "allowlist_verdict": _eval_allowlist_verdict(plan), - "encrypted_env_names": list(eval_deploy.EVAL_ALLOWED_ENVS), - "encrypted_env_transmitted": False, - "dry_run": True, - } - ) - return 0 - except ( - RouteClientError, - CredentialError, - lifecycle.LifecycleBudgetError, - ShapeError, - eval_deploy.EvalDeploymentError, - PhalaApiError, - ) as exc: - print(f"error: {exc}", file=sys.stderr) - return 2 - raise RouteClientError("unknown Eval stage") - - -_REDACTED_CAPABILITY_KEYS = frozenset( - { - "review_session_token", - "token", - "OPENROUTER_API_KEY", - "EVAL_RUN_TOKEN", - "RUNNER_HOTKEY_MNEMONIC", - "REVIEW_SESSION_TOKEN", - "BASE_GATEWAY_TOKEN", # residual key name only; not product eval secret - "golden_plaintext", - "golden_key", - "raw_response", - "raw_request", - "unrestricted_source", - } -) - - -def _redact_capabilities(value: Any) -> Any: - """Remove one-time capability bytes from CLI output and persisted plans.""" - - if isinstance(value, dict): - result = { - key: _redact_capabilities(item) - for key, item in value.items() - if key not in _REDACTED_CAPABILITY_KEYS - } - if "secret_delivery" in result and result["secret_delivery"] is not None: - delivery = result["secret_delivery"] - if isinstance(delivery, dict): - result["secret_delivery"] = {"env_key": delivery.get("env_key")} - return result - if isinstance(value, list): - return [_redact_capabilities(item) for item in value] - return value - - -def _assert_eval_deploy_shape_and_measurement_pin( - plan: eval_deploy.EvalDeploymentPlan, - args: argparse.Namespace, -) -> None: - """Fail closed before Phala create on shape or optional rtmr0 pin mismatch. - - Shape check needs the built plan, so it runs after prepare has spent the - one-shot EVAL_RUN_TOKEN delivery. Next deploy recovers via - ``_obtain_eval_prepare_with_token`` cancel+retry when prepare is sticky-null. - """ - - requested = str(getattr(args, "eval_instance_type", "") or "").strip() - if plan.instance_type != requested: - plan_vm_shape = plan.measurement.get("vm_shape") - plan_vm = ( - str(plan_vm_shape).replace("-", ".") - if isinstance(plan_vm_shape, str) and plan_vm_shape - else plan.instance_type - ) - plan_rtmr0 = plan.measurement.get("rtmr0") - raise RouteClientError( - measure.format_eval_shape_mismatch_error( - plan_instance_type=plan.instance_type, - requested_instance_type=requested, - plan_vm_shape=plan_vm, - plan_rtmr0=plan_rtmr0 if isinstance(plan_rtmr0, str) else None, - ) - ) - expected_path = getattr(args, "expected_measurement", None) - if isinstance(expected_path, str) and expected_path.strip(): - try: - expected = measure.load_expected_measurement_mapping(expected_path.strip()) - pin_error = measure.compare_plan_rtmr0_to_expected(plan.measurement, expected) - except measure.MeasurementError as exc: - raise RouteClientError(str(exc)) from exc - if pin_error is not None: - raise RouteClientError(pin_error) - - -def _require_eval_run_token_handoff(args: argparse.Namespace) -> None: - """Fail closed on live eval deploy unless the miner can recover the token. - - ``EVAL_RUN_TOKEN`` is a one-shot capability. prepare/status redact it, and - the guest only emits the attested envelope — the host must post via - ``eval result``. Without an explicit handoff at deploy time the miner has - no path to submit. Dry-run skips this gate (no spend, no post). - """ - - if getattr(args, "dry_run", False): - return - token_output = getattr(args, "token_output", None) - emit_run_token = bool(getattr(args, "emit_run_token", False)) - if token_output or emit_run_token: - return - raise RouteClientError( - "eval deploy requires --token-output PATH and/or --emit-run-token so the " - "miner can later call eval result with EVAL_RUN_TOKEN; the one-time token " - "is not recoverable from prepare/status output" - ) - - -def _write_eval_run_token_file(path: str, token: str) -> None: - """Write the one-time run token to PATH with mode 0o600 (create securely).""" - - # O_CREAT|O_WRONLY|O_TRUNC with mode 0o600 — never create world-readable then chmod. - fd = os.open( - path, - os.O_CREAT | os.O_WRONLY | os.O_TRUNC, - 0o600, - ) - try: - with os.fdopen(fd, "w", encoding="utf-8") as handle: - handle.write(token) - fd = -1 # fdopen owns it - finally: - if fd >= 0: - os.close(fd) - - -def _hand_off_eval_run_token( - args: argparse.Namespace, - *, - eval_run_id: str, - eval_run_token: str, - acknowledgement: Mapping[str, Any] | None, - encrypted_env_names: Sequence[str], -) -> dict[str, Any]: - """Build deploy-success payload and optionally surface the run token once. - - The token is never written to ``--output`` and never passes through - ``_redact_capabilities``. ``eval_run_id`` is always present for - ``eval result --run-id``. - """ - - token_output = getattr(args, "token_output", None) - if isinstance(token_output, str) and token_output: - _write_eval_run_token_file(token_output, eval_run_token) - payload: dict[str, Any] = { - "stage": "eval_deployed", - "eval_run_id": eval_run_id, - "acknowledgement": acknowledgement, - "encrypted_env_names": list(encrypted_env_names), - } - if bool(getattr(args, "emit_run_token", False)): - payload["eval_run_token"] = eval_run_token - output_path = getattr(args, "output", None) - if isinstance(output_path, str) and output_path: - # Persisted plan/metadata must never carry the one-time token. - safe = {key: value for key, value in payload.items() if key != "eval_run_token"} - Path(output_path).write_text( - json.dumps(safe, sort_keys=True, separators=(",", ":")), - encoding="utf-8", - ) - return payload - - -# --------------------------------------------------------------------------- # -# Parser -# --------------------------------------------------------------------------- # -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog=PROG, - description=( - "Miner self-deploy flow for the canonical Phala TEE eval image: prepare, " - "reproduce measurements, deploy a CPU-only CVM, run the eval against the " - "validator key-release endpoint, surface the attested result, and tear down." - ), - ) - sub = parser.add_subparsers(dest="command", required=True) - - prep = sub.add_parser( - "prepare", - help="fetch/prepare the canonical image + generated Phala compose", - description="Resolve the digest-pinned canonical image and write the deployable compose.", - ) - prep.add_argument("--image", required=True, help="canonical image ref (repo@sha256:<64hex>)") - prep.add_argument("--key-release-url", required=True, help="validator key-release endpoint URL") - prep.add_argument("--out", default=".", help="output directory for app-compose.json") - prep.add_argument("--name", default=None, help="app name (default: canonical app name)") - - meas = sub.add_parser( - "measurements", - help="publish/reproduce the canonical measurement record", - description="Deterministically recompute the pinnable {mrtd,rtmr0-2,compose_hash," - "os_image_hash} record.", - ) - meas.add_argument("--metadata", required=True, help="dstack image metadata.json path") - meas.add_argument("--cpu", type=int, required=True, help="vCPU count of the pinned VM shape") - meas.add_argument("--memory", required=True, help="memory of the pinned VM shape, e.g. 4G") - meas.add_argument("--compose", required=True, help="app-compose.json path to pin") - meas.add_argument("--dstack-mr", default=None, help="override the dstack-mr binary") - - verd = sub.add_parser( - "verdict", - help="report a measurement and its validator-allowlist verdict", - description="Report a measurement's canonical fields and whether it is in the allowlist.", - ) - verd.add_argument("--measurement", default=None, help="measurement JSON string or file path") - verd.add_argument("--from-result", default=None, help="a captured run output to read it from") - verd.add_argument("--allowlist", required=True, help="validator allowlist JSON string/file") - verd.add_argument("--domain", choices=("review", "eval"), default="eval") - - dep = sub.add_parser( - "deploy", - help="deploy a CPU-only Phala CVM (miner-funded); use --dry-run to plan", - description="Build a validated CPU-only deploy plan and (unless --dry-run) deploy it.", - ) - dep.add_argument("--image", required=True, help="canonical image ref (repo@sha256:<64hex>)") - dep.add_argument("--key-release-url", required=True, help="validator key-release endpoint URL") - dep.add_argument( - "--instance-type", - default=None, - help="CPU Intel TDX shape (default: smallest, tdx.small)", - ) - dep.add_argument("--os-image", default=DEFAULT_OS_IMAGE, help="dstack CPU OS image") - dep.add_argument( - "--region", - default=None, - help="Phala region (default: us-west-1; bare us-west remaps to us-west-1)", - ) - dep.add_argument("--name", default=None, help="app/CVM name") - dep.add_argument("--out", default=".", help="output directory for app-compose.json") - dep.add_argument( - "--max-runtime-hours", - type=float, - default=DEFAULT_MAX_RUNTIME_HOURS, - help="projected max runtime used for the cost-cap guard", - ) - dep.add_argument( - "--money-cap-usd", - type=float, - default=DEFAULT_MONEY_CAP_USD, - help="hard spend cap; a shape whose projected cost exceeds it is refused", - ) - dep.add_argument( - "--dry-run", - action="store_true", - help="print the full deploy plan and make zero CVM-creating calls", - ) - - runp = sub.add_parser( - "run", - help="run the eval against the validator key-release endpoint", - description="Run the canonical eval; fails closed with no result if key-release fails.", - ) - runp.add_argument("--job-dir", required=True, help="orchestrator job directory") - runp.add_argument( - "--task", - dest="task_ids", - action="append", - required=True, - metavar="TASK_ID", - help="task id to evaluate (repeatable)", - ) - runp.add_argument("--key-release-url", required=True, help="validator key-release endpoint URL") - - res = sub.add_parser( - "result", - help="surface + verify the attested-result envelope", - description="Parse a captured run output, surface the envelope, and verify its binding.", - ) - res.add_argument( - "--from", dest="from_path", default=None, help="captured run output (else stdin)" - ) - res.add_argument("--allowlist", default=None, help="also report the allowlist verdict") - res.add_argument( - "--quote-verified", - choices=["true", "false"], - default=None, - help="fold a quote-verification verdict (Phala verify / dcap-qvl) into the acceptance", - ) - res.add_argument( - "--nonce-state", - choices=["ok", "stale", "consumed", "unknown"], - default=None, - help="fold the validator nonce-ledger verdict into the acceptance", - ) - res.add_argument( - "--key-grant", - choices=["true", "false"], - default=None, - help="fold the matching validator key-grant outcome into the acceptance", - ) - - tear = sub.add_parser( - "teardown", - help="delete a deployed CVM (idempotent)", - description="Delete the CVM so no resource is left running (phala cvms delete -f).", - ) - tear.add_argument( - "--cvm-id", - default=None, - help="the CVM id to delete (optional if --app-id is set)", - ) - tear.add_argument( - "--app-id", - default=None, - help="resolve CVM id via GET /cvms exact app_id match when --cvm-id is omitted", - ) - - # Ordered production lifecycle. The older top-level helpers remain as - # compatibility shims for offline callers, but all new spend-capable work - # is organized under review/eval and requires the stage-specific identity. - review = sub.add_parser( - "review", - help="ordered review prepare/deploy/deployed/result/cancel/retry/teardown stages", - ) - review_sub = review.add_subparsers(dest="review_command", required=True) - review_prepare = review_sub.add_parser("prepare", help="request one signed review assignment") - _add_route_args(review_prepare, include_submission=True) - review_prepare.add_argument("--output", default=None, help="safe assignment output path") - review_deploy = review_sub.add_parser("deploy", help="deploy the signed review assignment") - _add_route_args(review_deploy, include_submission=True) - review_deploy.add_argument( - "--prepare-response", - default=None, - help=argparse.SUPPRESS, - ) - review_deploy.add_argument( - "--openrouter-key-env", - default="OPENROUTER_API_KEY", - help="environment variable holding the user key", - ) - review_deploy.add_argument("--phala-api", default=None, help="Phala Cloud API base URL") - review_deploy.add_argument( - "--review-instance-type", - default=DEFAULT_REVIEW_INSTANCE_TYPE, - help=f"review CVM shape (default: {DEFAULT_REVIEW_INSTANCE_TYPE})", - ) - review_deploy.add_argument( - "--eval-instance-type", - default=DEFAULT_EVAL_INSTANCE_TYPE, - help=f"eval CVM shape used for combined budget (default: {DEFAULT_EVAL_INSTANCE_TYPE})", - ) - review_deploy.add_argument( - "--review-disk-size-gb", - type=int, - default=DEFAULT_REVIEW_DISK_SIZE_GB, - help=f"review disk size GB (default: {DEFAULT_REVIEW_DISK_SIZE_GB})", - ) - review_deploy.add_argument( - "--eval-disk-size-gb", - type=int, - default=DEFAULT_EVAL_DISK_SIZE_GB, - help=f"eval disk size GB for combined budget (default: {DEFAULT_EVAL_DISK_SIZE_GB})", - ) - review_deploy.add_argument("--review-runtime-hours", type=float, default=6.0) - review_deploy.add_argument("--eval-runtime-hours", type=float, default=6.0) - review_deploy.add_argument("--money-cap-usd", type=float, default=20.0) - review_deploy.add_argument( - "--dry-run", - action="store_true", - help="validate and print safe deployment metadata without provisioning", - ) - review_ack = review_sub.add_parser("deployed", help="acknowledge the created review CVM") - _add_route_args(review_ack, include_submission=True) - review_ack.add_argument("--acknowledgement", required=True, help="acknowledgement JSON path") - review_result = review_sub.add_parser("result", help="surface the signed review audit bundle") - _add_route_args(review_result, include_submission=True) - review_result.add_argument("--cursor", default=None) - review_history = review_sub.add_parser("history", help="read safe review attempt history") - _add_route_args(review_history, include_submission=True) - review_history.add_argument("--cursor", default=None) - for name, help_text in ( - ("cancel", "cancel the expected review assignment"), - ("retry", "retry the expected review assignment"), - ): - item = review_sub.add_parser(name, help=help_text) - _add_route_args(item, include_submission=True) - item.add_argument("--assignment-id", required=True) - if name == "retry": - item.add_argument( - "--approval-id", - default=None, - help=( - "one-use operator approval_id required to retry after " - "policy reject/escalate (atomically consumed by the server)" - ), - ) - review_tear = review_sub.add_parser("teardown", help="delete the review CVM") - review_tear.add_argument( - "--cvm-id", - default=None, - help="CVM id (optional if --app-id is set)", - ) - review_tear.add_argument( - "--app-id", - default=None, - help="resolve CVM via GET /cvms exact app_id match when --cvm-id is omitted", - ) - review_tear.add_argument("--phala-api", default=None, help="Phala Cloud API base URL") - - evaluation = sub.add_parser( - "eval", - help="ordered eval prepare/deploy/result/status/cancel/retry/failure/teardown stages", - ) - eval_sub = evaluation.add_subparsers(dest="eval_command", required=True) - eval_prepare = eval_sub.add_parser( - "prepare", - help="request the immutable Eval plan after allow", - ) - _add_route_args(eval_prepare, include_submission=True) - eval_prepare.add_argument("--output", default=None, help="safe plan output path") - eval_deploy_parser = eval_sub.add_parser("deploy", help="deploy the signed Eval plan") - _add_route_args(eval_deploy_parser, include_submission=True) - eval_deploy_parser.add_argument( - "--prepare-response", - default=None, - help=argparse.SUPPRESS, - ) - # Residual CLI flags kept for operator scripts that still pass them; values - # are ignored. Base gateway secrets are never required for eval deploy. - eval_deploy_parser.add_argument( - "--gateway-token-env", - default="BASE_GATEWAY_TOKEN", - help=argparse.SUPPRESS, - ) - eval_deploy_parser.add_argument( - "--gateway-url-env", - default="BASE_LLM_GATEWAY_URL", - help=argparse.SUPPRESS, - ) - eval_deploy_parser.add_argument("--llm-cost-limit-env", default="LLM_COST_LIMIT") - eval_deploy_parser.add_argument( - "--openrouter-key-env", - default="OPENROUTER_API_KEY", - help=( - "optional env var holding measured OpenRouter key for in-CVM agent LLM " - "(tools-only if unset; never Base gateway)" - ), - ) - eval_deploy_parser.add_argument("--phala-api", default=None, help="Phala Cloud API base URL") - eval_deploy_parser.add_argument( - "--review-instance-type", - default=DEFAULT_REVIEW_INSTANCE_TYPE, - help=f"review CVM shape used for combined budget (default: {DEFAULT_REVIEW_INSTANCE_TYPE})", - ) - eval_deploy_parser.add_argument( - "--eval-instance-type", - default=DEFAULT_EVAL_INSTANCE_TYPE, - help=f"eval CVM shape (default: {DEFAULT_EVAL_INSTANCE_TYPE})", - ) - eval_deploy_parser.add_argument( - "--review-disk-size-gb", - type=int, - default=DEFAULT_REVIEW_DISK_SIZE_GB, - help=f"review disk size GB for combined budget (default: {DEFAULT_REVIEW_DISK_SIZE_GB})", - ) - eval_deploy_parser.add_argument( - "--eval-disk-size-gb", - type=int, - default=DEFAULT_EVAL_DISK_SIZE_GB, - help=f"eval disk size GB (default: {DEFAULT_EVAL_DISK_SIZE_GB})", - ) - eval_deploy_parser.add_argument("--review-runtime-hours", type=float, default=6.0) - eval_deploy_parser.add_argument("--eval-runtime-hours", type=float, default=6.0) - eval_deploy_parser.add_argument("--money-cap-usd", type=float, default=20.0) - eval_deploy_parser.add_argument( - "--dry-run", - action="store_true", - help="validate and print safe deployment metadata without provisioning", - ) - eval_deploy_parser.add_argument( - "--token-output", - default=None, - metavar="PATH", - help=( - "write the one-time EVAL_RUN_TOKEN to PATH with mode 0600 " - "(required for eval result unless --emit-run-token)" - ), - ) - eval_deploy_parser.add_argument( - "--emit-run-token", - action="store_true", - help=( - "include eval_run_token in deploy success JSON on stdout " - "(required for eval result unless --token-output)" - ), - ) - eval_deploy_parser.add_argument( - "--expected-measurement", - default=None, - metavar="PATH", - help=( - "optional JSON measurement pin; when present, plan rtmr0 must match " - "before Phala create (truncated prefix on mismatch; never logs full digests)" - ), - ) - eval_deploy_parser.add_argument( - "--output", - default=None, - help="write safe deploy metadata JSON (never includes EVAL_RUN_TOKEN)", - ) - eval_result_parser = eval_sub.add_parser( - "result", - help="post the exact result to the direct route", - ) - _add_route_args(eval_result_parser, include_submission=False, signed=False) - eval_result_parser.add_argument("--run-id", required=True) - eval_result_parser.add_argument("--result", required=True, help="exact result JSON path") - eval_result_parser.add_argument("--token-env", default="EVAL_RUN_TOKEN") - eval_status = eval_sub.add_parser("status", help="read signed Eval receipt/history") - _add_route_args(eval_status, include_submission=True) - eval_status.add_argument("--cursor", default=None) - for name, help_text in ( - ("cancel", "cancel the expected Eval run"), - ("retry", "retry the expected Eval run"), - ): - item = eval_sub.add_parser(name, help=help_text) - _add_route_args(item, include_submission=True) - item.add_argument("--run-id", required=True) - eval_failure = eval_sub.add_parser("failure", help="record a bounded pre-receipt failure") - _add_route_args(eval_failure, include_submission=True) - eval_failure.add_argument("--run-id", required=True) - eval_failure.add_argument( - "--reason-code", - required=True, - choices=( - "eval_deploy_failed", - "eval_tunnel_failed", - "eval_key_release_unavailable", - "eval_no_result", - ), - ) - eval_tear = eval_sub.add_parser("teardown", help="delete the Eval CVM") - eval_tear.add_argument( - "--cvm-id", - default=None, - help="CVM id (optional if --app-id is set)", - ) - eval_tear.add_argument( - "--app-id", - default=None, - help="resolve CVM via GET /cvms exact app_id match when --cvm-id is omitted", - ) - eval_tear.add_argument("--phala-api", default=None, help="Phala Cloud API base URL") - - return parser - - -# --------------------------------------------------------------------------- # -# Handlers -# --------------------------------------------------------------------------- # -def _print(payload: Any) -> None: - print(json.dumps(payload, indent=2, sort_keys=True)) - - -def _cmd_prepare(args: argparse.Namespace) -> int: - kwargs: dict[str, Any] = {"image": args.image, "key_release_url": args.key_release_url} - if args.name: - kwargs["app_name"] = args.name - try: - prepared = prepare_deployment(**kwargs) - except PrepareError as exc: - print(f"error: {exc}", file=sys.stderr) - return 2 - path = write_prepared(prepared, args.out) - _print( - { - "image": prepared.image, - "key_release_url": prepared.key_release_url, - "compose_hash": prepared.compose_hash, - "compose_path": str(path), - } - ) - return 0 - - -def _cmd_measurements(args: argparse.Namespace) -> int: - compose_text = Path(args.compose).read_text(encoding="utf-8") - record = measure.reproduce_measurement( - metadata_path=args.metadata, - cpu=args.cpu, - memory=args.memory, - compose=compose_text, - dstack_mr_bin=args.dstack_mr, - ) - print(record.to_json()) - return 0 - - -def _load_measurement_arg(args: argparse.Namespace) -> dict[str, Any]: - if args.measurement: - source = args.measurement.strip() - if source.startswith("{"): - return json.loads(source) - return json.loads(Path(source).read_text(encoding="utf-8")) - if args.from_result: - stdout = Path(args.from_result).read_text(encoding="utf-8") - surfaced = result_mod.surface_result(stdout) - attestation = surfaced.attestation - if attestation is None: - raise measure.MeasurementError("captured result carries no attested measurement") - return dict(attestation.get("measurement", {})) - raise measure.MeasurementError("provide --measurement or --from-result") - - -def _cmd_verdict(args: argparse.Namespace) -> int: - try: - measurement = _load_measurement_arg(args) - verdict = measure.domain_allowlist_verdict( - domain=args.domain, - measurement=measurement, - review_allowlist=args.allowlist if args.domain == "review" else None, - eval_allowlist=args.allowlist if args.domain == "eval" else None, - ) - except (measure.MeasurementError, result_mod.ResultSurfaceError) as exc: - print(f"error: {exc}", file=sys.stderr) - return 2 - _print(verdict.as_dict()) - return 0 if verdict.in_allowlist else 1 - - -def _cmd_deploy(args: argparse.Namespace, *, deployer: Deployer) -> int: - plan_kwargs: dict[str, Any] = { - "image": args.image, - "key_release_url": args.key_release_url, - "instance_type": args.instance_type, - "os_image": args.os_image, - "money_cap_usd": args.money_cap_usd, - "max_runtime_hours": args.max_runtime_hours, - } - if args.region: - plan_kwargs["region"] = args.region - if args.name: - plan_kwargs["name"] = args.name - try: - plan = build_deploy_plan(**plan_kwargs) - except (ShapeError, PrepareError) as exc: - print(f"error: refusing to deploy: {exc}", file=sys.stderr) - return 2 - - if args.dry_run: - _print(render_plan(plan)) - return 0 - - try: - check_phala_credentials() - except CredentialError as exc: - print(f"error: {exc}", file=sys.stderr) - return 2 - - outcome = deployer(plan, args.out) - _print({"deployed": True, "instance_type": plan.instance_type, "result": outcome}) - return 0 - - -def _cmd_run(args: argparse.Namespace, *, backend_main: run_mod.BackendMain | None) -> int: - try: - outcome = run_mod.run_eval( - job_dir=args.job_dir, - task_ids=args.task_ids, - key_release_url=args.key_release_url, - backend_main=backend_main, - ) - except run_mod.RunError as exc: - print(f"error: {exc}", file=sys.stderr) - return 2 - - if outcome.succeeded and outcome.surfaced is not None: - _print(outcome.surfaced.summary()) - return 0 - - # Fail closed: surface a clear error and NO attested result. - print(f"error: {outcome.clear_error}", file=sys.stderr) - if outcome.surfaced is not None: - _print( - { - "attested": False, - "status": outcome.surfaced.status, - "reason_code": outcome.surfaced.reason_code, - } - ) - return outcome.exit_code or 1 - - -def _cmd_result(args: argparse.Namespace) -> int: - if args.from_path: - stdout = Path(args.from_path).read_text(encoding="utf-8") - else: - stdout = sys.stdin.read() - try: - surfaced = result_mod.surface_result(stdout) - except result_mod.ResultSurfaceError as exc: - print(f"error: {exc}", file=sys.stderr) - return 2 - - allowlist_verdict = None - measurement_allowlisted: bool | None = None - if args.allowlist and surfaced.attestation is not None: - allowlist_verdict = measure.allowlist_verdict( - surfaced.attestation.get("measurement", {}), args.allowlist - ) - measurement_allowlisted = allowlist_verdict.in_allowlist - - quote_verified = None if args.quote_verified is None else args.quote_verified == "true" - key_grant_value = getattr(args, "key_grant", None) - key_grant_ok = None if key_grant_value is None else key_grant_value == "true" - # When the miner explicitly supplies the complete positive-signal set expected - # by the acceptance conjunction, require a known key-grant outcome. Without an - # explicit CLI flag, treat allowlist+quote+nonce arguments as implying a - # positive grant so the historical accepted-run control still works; callers - # that only pass partial signals remain pending rather than false-accept. - if ( - key_grant_ok is None - and quote_verified is True - and measurement_allowlisted is True - and args.nonce_state == "ok" - ): - key_grant_ok = True - verdict = result_mod.evaluate_acceptance( - surfaced, - quote_verified=quote_verified, - measurement_allowlisted=measurement_allowlisted, - nonce_state=args.nonce_state, - key_grant_ok=key_grant_ok, - ) - - if verdict.accepted is False: - # A rejected/unaccepted result: surface a clear, non-sensitive verdict - # with NO fabricated score and no golden/key/quote material (VAL-DEPLOY-026). - _print({"accepted": False, "reason": verdict.reason, "attested": surfaced.attested}) - return 1 - - summary = surfaced.summary() - if allowlist_verdict is not None: - summary["allowlist_verdict"] = allowlist_verdict.as_dict() - summary["acceptance"] = verdict.as_dict() - _print(summary) - return 0 - - -def _cmd_teardown(args: argparse.Namespace, *, teardowner: Teardowner) -> int: - # Injected teardowner (tests) still receives an explicit cvm id only. - if teardowner is not default_phala_teardown: - cvm_id = (getattr(args, "cvm_id", None) or "").strip() - if not cvm_id: - print( - "error: teardown requires --cvm-id when using a custom teardowner", - file=sys.stderr, - ) - return 2 - outcome = teardowner(cvm_id) - payload, code = _teardown_payload(cvm_id, outcome) - _print(payload) - return code - payload, code = _run_teardown_command(args) - _print(payload) - return code - - -def main( - argv: Sequence[str] | None = None, - *, - deployer: Deployer | None = None, - teardowner: Teardowner | None = None, - backend_main: run_mod.BackendMain | None = None, -) -> int: - parser = build_parser() - args = parser.parse_args(argv) - - if args.command == "prepare": - return _cmd_prepare(args) - if args.command == "review": - return _ordered_review_command(args) - if args.command == "eval": - return _ordered_eval_command(args) - if args.command == "measurements": - return _cmd_measurements(args) - if args.command == "verdict": - return _cmd_verdict(args) - if args.command == "deploy": - return _cmd_deploy(args, deployer=deployer or default_phala_deployer) - if args.command == "run": - return _cmd_run(args, backend_main=backend_main) - if args.command == "result": - return _cmd_result(args) - if args.command == "teardown": - return _cmd_teardown(args, teardowner=teardowner or default_phala_teardown) - parser.error(f"unknown command: {args.command}") # pragma: no cover - return 2 # pragma: no cover - - -__all__ = [ - "PROG", - "SPEND_CAPABLE_SUBCOMMANDS", - "SUBCOMMANDS", - "Deployer", - "Teardowner", - "build_parser", - "default_phala_deployer", - "default_phala_teardown", - "resolve_teardown_cvm_id", - "main", -] - - -if __name__ == "__main__": # pragma: no cover - thin CLI shim - raise SystemExit(main()) diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/eval.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/eval.py deleted file mode 100644 index cb6fd2120..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/eval.py +++ /dev/null @@ -1,504 +0,0 @@ -"""Ordered, encrypted deployment of the canonical Eval application. - -This module is deliberately independent from the legacy ``deploy`` helper. -Eval deployment accepts only the validator-issued Eval plan produced after a -verified review allow, derives the canonical compose from that plan, and sends -the resulting ciphertext to Phala. It never creates database state or invents -an authorization locally. -""" - -from __future__ import annotations - -import re -from collections.abc import Mapping -from dataclasses import dataclass, field -from hashlib import sha256 -from typing import Any, Protocol - -from dstack_sdk import EnvVar, encrypt_env_vars_sync - -from agent_challenge.canonical import eval_wire -from agent_challenge.canonical.compose import ( - DEFAULT_ALLOWED_ENVS, - MEASURED_ALLOWED_ENVS, - generate_app_compose, - render_app_compose, -) -from agent_challenge.canonical.key_release_endpoint import parse_key_release_authority -from agent_challenge.keyrelease.client import KEY_RELEASE_URL_ENV -from agent_challenge.selfdeploy.measurements import ( - ProvisionOsIdentityError, - verify_provision_os_identity, -) -from agent_challenge.selfdeploy.phala import ( - extract_cvm_id_from_create_response, - resolve_cvm_id_from_list, -) -from agent_challenge.selfdeploy.shapes import ( - DEFAULT_EVAL_DISK_SIZE_GB, - DEFAULT_INSTANCE_TYPE, - DEFAULT_OS_IMAGE, - validate_cpu_only, - validate_disk_size, -) - -#: Capacity-safe default (bare ``us-west`` → ERR-02-002 No teepod found). -DEFAULT_REGION = "us-west-1" -EVAL_ALLOWED_ENVS: tuple[str, ...] = MEASURED_ALLOWED_ENVS -# VAL-ACAT-013: production eval encrypted_env must NOT require Base LLM gateway -# secrets. Gateway routing is removed; only eval-run capability + attestation -# plan bindings (and optional cost limit) are required. -EVAL_REQUIRED_SECRET_ENVS: frozenset[str] = frozenset( - { - "CHALLENGE_PHALA_ATTESTATION_ENABLED", - "CHALLENGE_PHALA_EVAL_PLAN", - "EVAL_RUN_TOKEN", - "LLM_COST_LIMIT", - } -) - -#: Product moniker seeds measured compose ``name`` (compose_hash). Phala 40-hex -#: app_id is pinned separately as plan app_identity when using deterministic -#: provision (nonce). Never invent moniker→hex melt. -DEFAULT_EVAL_COMPOSE_NAME = "agent-challenge-eval-v1" -_APP_ID_HEX40_RE = re.compile(r"^[0-9a-f]{40}$") -#: Default nonce for the eval domain (disjoint from review's 0). -DEFAULT_EVAL_PHALA_APP_NONCE = 1 -#: Measure-time offline pin placeholder for ``key_release_url`` when the operator -#: pin pack was built without baking a live RA-TLS authority into the measured -#: app-compose. The live residual pin ``04011776…`` used this HTTPS value so the -#: compose_hash is stable across operator endpoint changes. Guest still resolves -#: the real endpoint from the signed plan / -#: ``CHALLENGE_PHALA_EVAL_PLAN.key_release_endpoint`` (never invent KR materials). -MEASURE_TIME_EVAL_KEY_RELEASE_PLACEHOLDER = "https://validator-kr.example.invalid:8701" - - -class EvalDeploymentError(ValueError): - """The validator-issued Eval plan or deployment request is unsafe.""" - - attributable_cvm_id: str | None = None - - -@dataclass(frozen=True) -class EvalDeploymentPlan: - """Canonical Eval deployment material. - - The run token is intentionally excluded from the normal representation. - Callers should encrypt it immediately and should never serialize this - object as evidence or status. - """ - - plan: dict[str, Any] - plan_sha256: str - compose: dict[str, Any] - compose_text: str - compose_hash: str - app_identity: str - image_ref: str - kms_public_key_hex: str - kms_public_key_sha256: str - measurement: dict[str, str] - eval_run_id: str - eval_run_token: str = field(repr=False) - instance_type: str = DEFAULT_INSTANCE_TYPE - region: str = DEFAULT_REGION - os_image: str = DEFAULT_OS_IMAGE - compose_name: str = DEFAULT_EVAL_COMPOSE_NAME - phala_app_nonce: int | None = None - disk_size_gb: int = DEFAULT_EVAL_DISK_SIZE_GB - - -@dataclass(frozen=True) -class EncryptedEvalSecrets: - """Ciphertext-only Eval secret delivery.""" - - ciphertext: str - env_keys: tuple[str, ...] - eval_run_id: str - app_identity: str - kms_public_key_sha256: str - - -class PhalaPost(Protocol): - def post(self, path: str, payload: Mapping[str, Any]) -> Mapping[str, Any]: - """POST one Phala Cloud API request.""" - - -def _plan_digest(plan: Mapping[str, Any]) -> str: - try: - canonical = eval_wire.canonical_json_v1(eval_wire.validate_eval_plan(plan)) - except eval_wire.EvalWireError as exc: - raise EvalDeploymentError("Eval plan is not canonical") from exc - return sha256(canonical).hexdigest() - - -def build_eval_deployment_plan( - prepare_response: Mapping[str, Any], -) -> EvalDeploymentPlan: - """Validate the exact signed Eval prepare wrapper and derive its compose. - - The response must be the first wrapper returned by the production signed - ``POST /submissions/{id}/eval/prepare`` route. That route is the sole - authorization gate and only returns after a persisted verified review allow. - This helper intentionally has no caller-controlled authorization boolean. - """ - - if not isinstance(prepare_response, Mapping): - raise EvalDeploymentError("Eval prepare response must be an object") - if set(prepare_response) != {"schema_version", "plan", "plan_sha256", "secret_delivery"}: - raise EvalDeploymentError("Eval prepare response has unexpected fields") - if prepare_response["schema_version"] != 1: - raise EvalDeploymentError("unsupported Eval prepare schema version") - plan_raw = prepare_response["plan"] - if not isinstance(plan_raw, Mapping): - raise EvalDeploymentError("Eval prepare response has no immutable plan") - try: - plan = eval_wire.validate_eval_plan(plan_raw) - except eval_wire.EvalWireError as exc: - raise EvalDeploymentError("Eval plan is invalid") from exc - expected_digest = _plan_digest(plan) - if prepare_response["plan_sha256"] != expected_digest: - raise EvalDeploymentError("Eval plan digest does not match canonical plan bytes") - if ( - not isinstance(plan["authorizing_review_digest"], str) - or not plan["authorizing_review_digest"] - ): - raise EvalDeploymentError("Eval plan is missing validator review authorization") - delivery = prepare_response["secret_delivery"] - if not isinstance(delivery, Mapping) or set(delivery) != {"env_key", "token"}: - raise EvalDeploymentError( - "first Eval prepare must deliver exactly one EVAL_RUN_TOKEN capability" - ) - if delivery["env_key"] != "EVAL_RUN_TOKEN" or not isinstance(delivery["token"], str): - raise EvalDeploymentError("Eval prepare delivered an invalid run capability") - token = delivery["token"] - if not token or sha256(token.encode("utf-8")).hexdigest() != plan["run_token_sha256"]: - raise EvalDeploymentError("Eval run token is not bound to the immutable plan") - - app = plan["eval_app"] - try: - shape_name = str(app["measurement"]["vm_shape"]).replace("-", ".") - shape = validate_cpu_only(instance_type=shape_name) - except (KeyError, TypeError, ValueError) as exc: - raise EvalDeploymentError("Eval plan does not identify a CPU Intel TDX shape") from exc - # The app identity, KMS key, measurement, and image all come from the - # validator-signed plan. Never accept a CLI override for any of them. - allowed = set(EVAL_ALLOWED_ENVS) - # The signed plan pins the exact compose_hash. Offline/default depends omit - # the live-registry side-manifest; live smoke pins it. Operator pin packs may - # also have been measured with a non-routable HTTPS key-release placeholder - # (compositionally stable; guest uses signed plan endpoint at runtime). - # Choose the generator mode whose rendered hash matches the signed plan - # fail-closed — never invent compose bytes / MRTD / KR roots. - live_registry_candidates = ( - None, - "/opt/agent-challenge/golden/live-registry-refs.json", - ) - # Prefer plan endpoint, then measure-time placeholder used for the live - # joinbase pin ``04011776…`` (tee-pin-pack / eval residual after KR). - plan_endpoint = str(plan.get("key_release_endpoint") or "").strip() or None - key_release_candidates: list[str | None] = [] - for candidate_url in ( - plan_endpoint, - MEASURE_TIME_EVAL_KEY_RELEASE_PLACEHOLDER, - ): - if candidate_url and candidate_url not in key_release_candidates: - key_release_candidates.append(candidate_url) - if not key_release_candidates: - key_release_candidates.append(None) - compose = None - compose_text = "" - compose_hash = "" - app_identity = str(app["app_identity"]) - if _APP_ID_HEX40_RE.fullmatch(app_identity.lower()): - app_identity = app_identity.lower() - compose_name = DEFAULT_EVAL_COMPOSE_NAME - phala_app_nonce: int | None = DEFAULT_EVAL_PHALA_APP_NONCE - else: - compose_name = app_identity - phala_app_nonce = None - name_candidates = (compose_name,) - # Also try signed identity as compost name for moniker-only legacy pins. - if compose_name != app_identity: - name_candidates = (compose_name, app_identity) - for live_path in live_registry_candidates: - for name in name_candidates: - for key_release_url in key_release_candidates: - candidate = generate_app_compose( - orchestrator_image=app["image_ref"], - name=name, - key_release_url=key_release_url, - allowed_envs=tuple(sorted(allowed)), - live_registry_manifest_path=live_path, - ) - candidate_text = render_app_compose(candidate) - candidate_hash = sha256(candidate_text.encode("utf-8")).hexdigest() - if candidate_hash == app["compose_hash"]: - compose = candidate - compose_text = candidate_text - compose_hash = candidate_hash - compose_name = name - break - if compose is not None: - break - if compose is not None: - break - if compose is None or compose_hash != app["compose_hash"]: - raise EvalDeploymentError("canonical Eval compose hash mismatches signed plan") - if app["kms_key_algorithm"] != "x25519": - raise EvalDeploymentError("Eval plan uses an unsupported KMS algorithm") - if sha256(bytes.fromhex(app["kms_public_key_hex"])).hexdigest() != app["kms_public_key_sha256"]: - raise EvalDeploymentError("Eval KMS public key digest mismatch") - return EvalDeploymentPlan( - plan=dict(plan), - plan_sha256=expected_digest, - compose=compose, - compose_text=compose_text, - compose_hash=compose_hash, - app_identity=app_identity, - image_ref=app["image_ref"], - kms_public_key_hex=app["kms_public_key_hex"], - kms_public_key_sha256=app["kms_public_key_sha256"], - measurement=dict(app["measurement"]), - eval_run_id=plan["eval_run_id"], - eval_run_token=token, - instance_type=shape.name, - os_image=DEFAULT_OS_IMAGE, - compose_name=compose_name, - phala_app_nonce=phala_app_nonce, - disk_size_gb=DEFAULT_EVAL_DISK_SIZE_GB, - ) - - -def encrypt_eval_secrets( - plan: EvalDeploymentPlan, - secrets: Mapping[str, str], -) -> EncryptedEvalSecrets: - """Encrypt the Eval run token and attestation plan bindings (no Base gateway).""" - - # Encrypt allowlist is the full DEFAULT_ALLOWED_ENVS (includes optional - # progress/telemetry names). Measured compose generation stays on - # EVAL_ALLOWED_ENVS / MEASURED_ALLOWED_ENVS so historical pins remain stable. - allowed = set(DEFAULT_ALLOWED_ENVS) - if not set(secrets) <= allowed or not EVAL_REQUIRED_SECRET_ENVS <= set(secrets): - raise EvalDeploymentError( - "Eval encrypted_env names must be scoped allowed names with the required run " - "and attestation plan capabilities (Base LLM gateway secrets are not allowed)" - ) - forbidden_gateway = { - "BASE_GATEWAY_TOKEN", - "BASE_LLM_GATEWAY_URL", - "GATEWAY_TOKEN", - "CENTRAL_GATEWAY_TOKEN", - } - if forbidden_gateway & set(secrets): - raise EvalDeploymentError( - "Eval encrypted_env must not include Base LLM gateway secrets " - "(BASE_GATEWAY_TOKEN / BASE_LLM_GATEWAY_URL / …)" - ) - # VAL-ACLOCK-009: free CHALLENGE_PHALA_KEY_RELEASE_URL is not a miner trust - # root. Prefer plan key_release_endpoint + KEY_RELEASE_RA_TLS_HOST/PORT. - # Name may remain in allowed_envs for measure-time pin hash stability, but - # any encrypted_env value must be the same RA-TLS authority as the signed - # plan (free HTTP(S) URLs always refuse). - if KEY_RELEASE_URL_ENV in secrets: - free_url = secrets[KEY_RELEASE_URL_ENV] - plan_endpoint = str(plan.plan.get("key_release_endpoint") or "").strip() - plan_auth = parse_key_release_authority(plan_endpoint) - free_auth = parse_key_release_authority(free_url if isinstance(free_url, str) else "") - if plan_auth is None or free_auth is None or free_auth != plan_auth: - raise EvalDeploymentError( - "Eval encrypted_env CHALLENGE_PHALA_KEY_RELEASE_URL is not miner-" - "authoritative; value must match plan key_release_endpoint RA-TLS " - "authority (prefer KEY_RELEASE_RA_TLS_HOST/PORT). Free HTTP(S) KR " - "URLs are refused." - ) - env_keys = tuple(name for name in DEFAULT_ALLOWED_ENVS if name in secrets) - values = {name: secrets[name] for name in env_keys} - if any(not isinstance(value, str) or not value for value in values.values()): - raise EvalDeploymentError("Eval encrypted_env values must be non-empty strings") - if values["EVAL_RUN_TOKEN"] != plan.eval_run_token: - raise EvalDeploymentError("Eval run token does not match signed prepare response") - try: - ciphertext = encrypt_env_vars_sync( - [EnvVar(key=name, value=values[name]) for name in env_keys], - plan.kms_public_key_hex, - ) - except Exception as exc: - raise EvalDeploymentError("Eval encrypted_env encryption failed") from exc - if not ciphertext: - raise EvalDeploymentError("Eval encrypted_env ciphertext is empty") - return EncryptedEvalSecrets( - ciphertext=ciphertext, - env_keys=env_keys, - eval_run_id=plan.eval_run_id, - app_identity=plan.app_identity, - kms_public_key_sha256=plan.kms_public_key_sha256, - ) - - -class HttpEvalPhalaDeployment: - """Transmit exact provision/create bytes to Phala Cloud.""" - - def __init__(self, api: PhalaPost) -> None: - self._api = api - - def deploy( - self, - plan: EvalDeploymentPlan, - encrypted: EncryptedEvalSecrets, - ) -> dict[str, str]: - from agent_challenge.evaluation.no_phala import refuse_phala_client - - refuse_phala_client("HttpEvalPhalaDeployment.deploy") - if ( - encrypted.eval_run_id != plan.eval_run_id - or encrypted.app_identity != plan.app_identity - or encrypted.kms_public_key_sha256 != plan.kms_public_key_sha256 - or not set(encrypted.env_keys) <= set(DEFAULT_ALLOWED_ENVS) - or not encrypted.ciphertext - ): - raise EvalDeploymentError("Eval encrypted_env is not bound to this run") - provision_request: dict[str, Any] = { - "app_id": plan.app_identity, - "name": plan.compose_name, - "instance_type": plan.instance_type, - "region": plan.region, - "compose_file": plan.compose, - "env_keys": list(encrypted.env_keys), - "image": plan.os_image, - # Sibling of compose_file — never mutate plan.compose. - "disk_size": validate_disk_size(plan.disk_size_gb), - } - if plan.phala_app_nonce is not None: - provision_request["nonce"] = plan.phala_app_nonce - provision = self._api.post("/cvms/provision", provision_request) - if provision.get("compose_hash") != plan.compose_hash: - raise EvalDeploymentError("Phala provision compose hash mismatches Eval plan") - # Honest: equality to pin only. Production pins are Phala 40-hex app_id - # (plus nonce) so Phala returns the same app_id and stable encrypt pubkey. - if provision.get("app_id") != plan.app_identity: - raise EvalDeploymentError("Phala provision app identity mismatches Eval plan") - if provision.get("app_env_encrypt_pubkey") != plan.kms_public_key_hex: - raise EvalDeploymentError("Phala provision KMS key mismatches Eval plan") - self._verify_provision_os_identity(plan, provision) - created = self._api.post( - "/cvms", - { - "app_id": plan.app_identity, - "compose_hash": plan.compose_hash, - "encrypted_env": encrypted.ciphertext, - "env_keys": list(encrypted.env_keys), - }, - ) - # Match review path: live Phala create uses numeric id; coerce + fallback. - try: - cvm_id = extract_cvm_id_from_create_response(created) - except ValueError: - cvm_id = None - getter = getattr(self._api, "get", None) - if callable(getter): - try: - listing = getter("/cvms") - except Exception: - listing = None - if isinstance(listing, Mapping): - cvm_id = resolve_cvm_id_from_list(listing, app_id=plan.app_identity) - if not isinstance(cvm_id, str) or not cvm_id: - raise EvalDeploymentError("Phala create response does not identify the Eval CVM") - try: - return { - "eval_run_id": plan.eval_run_id, - "cvm_id": cvm_id, - "app_identity": plan.app_identity, - "image_ref": plan.image_ref, - "compose_hash": plan.compose_hash, - "kms_public_key_sha256": plan.kms_public_key_sha256, - "phala_create_receipt_sha256": sha256( - repr(sorted(created.items())).encode("utf-8") - ).hexdigest(), - } - except Exception as exc: # pragma: no cover - defensive post-create binder - if isinstance(exc, EvalDeploymentError): - exc.attributable_cvm_id = cvm_id - else: - wrapped = EvalDeploymentError(str(exc)) - wrapped.attributable_cvm_id = cvm_id - raise wrapped from exc - raise - - @staticmethod - def _verify_provision_os_identity( - plan: EvalDeploymentPlan, - provision: Mapping[str, Any], - ) -> None: - try: - verify_provision_os_identity( - measurement=plan.measurement, - provision_os=provision.get("os_image_hash"), - mismatch_message=("Phala provision os_image_hash mismatches Eval plan measurement"), - ) - except ProvisionOsIdentityError as exc: - raise EvalDeploymentError(str(exc)) from exc - - -class EvalPhalaDeployment(HttpEvalPhalaDeployment): - """In-memory adapter used by contract tests.""" - - def __init__( - self, - *, - provision_response: Mapping[str, Any], - create_response: Mapping[str, Any], - ) -> None: - self.provision_response = dict(provision_response) - self.create_response = dict(create_response) - self.provision_requests: list[dict[str, Any]] = [] - self.create_requests: list[dict[str, Any]] = [] - super().__init__(self) - - def post(self, path: str, payload: Mapping[str, Any]) -> Mapping[str, Any]: - if path == "/cvms/provision": - self.provision_requests.append(dict(payload)) - return self.provision_response - if path == "/cvms": - self.create_requests.append(dict(payload)) - return self.create_response - raise AssertionError(f"unexpected Phala API path {path}") - - -def build_eval_progress_env( - *, - base_url: str, - eval_run_id: str, - submission_id: str, - eval_run_token: str, -) -> dict[str, str]: - """Bind progress-reporter env from plan ids + run token (no mnemonic).""" - - cleaned = base_url.strip().rstrip("/") - return { - "EVAL_PROGRESS_BASE_URL": cleaned, - "EVAL_RUN_ID": eval_run_id, - "EVAL_SUBMISSION_ID": str(submission_id), - "EVAL_RUN_TOKEN": eval_run_token, - } - - -__all__ = [ - "DEFAULT_EVAL_COMPOSE_NAME", - "DEFAULT_EVAL_PHALA_APP_NONCE", - "DEFAULT_OS_IMAGE", - "DEFAULT_REGION", - "EVAL_ALLOWED_ENVS", - "EVAL_REQUIRED_SECRET_ENVS", - "MEASURE_TIME_EVAL_KEY_RELEASE_PLACEHOLDER", - "EncryptedEvalSecrets", - "EvalDeploymentError", - "EvalDeploymentPlan", - "EvalPhalaDeployment", - "HttpEvalPhalaDeployment", - "build_eval_deployment_plan", - "build_eval_progress_env", - "encrypt_eval_secrets", -] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/lifecycle.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/lifecycle.py index 03a3d9c5c..587c55258 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/lifecycle.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/lifecycle.py @@ -1,122 +1,9 @@ -"""Shared self-deploy lifecycle safety guards.""" +"""Self-deploy lifecycle removed with Phala (T40).""" from __future__ import annotations -import math -from dataclasses import dataclass +from agent_challenge.selfdeploy import SelfDeployRemovedError -from agent_challenge.selfdeploy.shapes import ( - CPU_TDX_SHAPES, - DEFAULT_EVAL_DISK_SIZE_GB, - DEFAULT_MONEY_CAP_USD, - DEFAULT_REVIEW_DISK_SIZE_GB, - ShapeError, - projected_disk_cost_usd, - validate_cpu_only, - validate_disk_size, -) - -class LifecycleBudgetError(ShapeError): - """The combined review + Eval lifecycle exceeds the money cap.""" - - -@dataclass(frozen=True) -class LifecycleCost: - review_usd: float - eval_usd: float - total_usd: float - money_cap_usd: float - - -def projected_lifecycle_cost_usd( - *, - review_instance_type: str, - eval_instance_type: str, - review_runtime_hours: float, - eval_runtime_hours: float, - review_disk_size_gb: int = DEFAULT_REVIEW_DISK_SIZE_GB, - eval_disk_size_gb: int = DEFAULT_EVAL_DISK_SIZE_GB, -) -> float: - """Compute both CVM projections together (compute + disk), never budget each alone.""" - - for instance_type in (review_instance_type, eval_instance_type): - try: - validate_cpu_only(instance_type=instance_type) - except ShapeError as exc: - raise LifecycleBudgetError(str(exc)) from exc - try: - review_disk = validate_disk_size(review_disk_size_gb) - eval_disk = validate_disk_size(eval_disk_size_gb) - except ShapeError as exc: - raise LifecycleBudgetError(str(exc)) from exc - if ( - not math.isfinite(review_runtime_hours) - or not math.isfinite(eval_runtime_hours) - or review_runtime_hours < 0 - or eval_runtime_hours < 0 - ): - raise LifecycleBudgetError("runtime hours must be non-negative") - return ( - CPU_TDX_SHAPES[review_instance_type].usd_per_hour * review_runtime_hours - + projected_disk_cost_usd(review_disk, max_runtime_hours=review_runtime_hours) - + CPU_TDX_SHAPES[eval_instance_type].usd_per_hour * eval_runtime_hours - + projected_disk_cost_usd(eval_disk, max_runtime_hours=eval_runtime_hours) - ) - - -def validate_lifecycle_budget( - *, - review_instance_type: str, - eval_instance_type: str, - review_runtime_hours: float, - eval_runtime_hours: float, - money_cap_usd: float = 20.0, - review_disk_size_gb: int = DEFAULT_REVIEW_DISK_SIZE_GB, - eval_disk_size_gb: int = DEFAULT_EVAL_DISK_SIZE_GB, -) -> LifecycleCost: - """Refuse a combined lifecycle that could exceed the shared cap.""" - - if not math.isfinite(money_cap_usd) or money_cap_usd < 0: - raise LifecycleBudgetError("money cap must be a finite non-negative number") - if money_cap_usd > DEFAULT_MONEY_CAP_USD: - raise LifecycleBudgetError( - f"money cap cannot exceed the mission ${DEFAULT_MONEY_CAP_USD:.2f} cap" - ) - review_cost = CPU_TDX_SHAPES.get(review_instance_type) - eval_cost = CPU_TDX_SHAPES.get(eval_instance_type) - total = projected_lifecycle_cost_usd( - review_instance_type=review_instance_type, - eval_instance_type=eval_instance_type, - review_runtime_hours=review_runtime_hours, - eval_runtime_hours=eval_runtime_hours, - review_disk_size_gb=review_disk_size_gb, - eval_disk_size_gb=eval_disk_size_gb, - ) - if total > money_cap_usd: - raise LifecycleBudgetError( - f"projected review+eval cost ${total:.2f} exceeds the ${money_cap_usd:.2f} cap" - ) - assert review_cost is not None and eval_cost is not None - review_disk = validate_disk_size(review_disk_size_gb) - eval_disk = validate_disk_size(eval_disk_size_gb) - return LifecycleCost( - review_usd=( - review_cost.usd_per_hour * review_runtime_hours - + projected_disk_cost_usd(review_disk, max_runtime_hours=review_runtime_hours) - ), - eval_usd=( - eval_cost.usd_per_hour * eval_runtime_hours - + projected_disk_cost_usd(eval_disk, max_runtime_hours=eval_runtime_hours) - ), - total_usd=total, - money_cap_usd=money_cap_usd, - ) - - -__all__ = [ - "LifecycleBudgetError", - "LifecycleCost", - "projected_lifecycle_cost_usd", - "validate_lifecycle_budget", -] +def __getattr__(name: str): # noqa: ANN001 + raise SelfDeployRemovedError(f"selfdeploy.lifecycle.{name} removed with Phala TEE (T40)") diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/measurements.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/measurements.py deleted file mode 100644 index fc9f5df0d..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/measurements.py +++ /dev/null @@ -1,390 +0,0 @@ -"""Deterministic measurement reproduction + allowlist verdict for the miner CLI. - -The validator pins a canonical eval image by its measurement record -``{mrtd, rtmr0, rtmr1, rtmr2, compose_hash, os_image_hash}`` (architecture §6/§7). -The miner reproduces the *same* record from the same pinned image + compose so -both sides agree on the allowlist (VAL-DEPLOY-003/004), and the CLI reports a -run's measurement together with a correct in-allowlist verdict (VAL-DEPLOY-012). - -Reproduction wraps :mod:`agent_challenge.canonical.measurement` (``dstack-mr`` + -normalized compose-hash); the verdict compares the canonical six-field subset -against a validator-owned allowlist (a JSON list of entries, or an entries file in -the key-release allowlist format), ignoring any extra register such as -``key_provider`` that the run measurement does not carry. -""" - -from __future__ import annotations - -import json -import re -from collections.abc import Iterable, Mapping -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -from agent_challenge.canonical.measurement import ( - CANONICAL_MEASUREMENT_FIELDS, - CanonicalMeasurement, - build_canonical_measurement, - measurement_uses_product_os_identity, - product_os_image_hash, -) - -_SHA256_HEX_RE = re.compile(r"^[0-9a-f]{64}$") - - -class MeasurementError(ValueError): - """A measurement input (record or allowlist) is malformed.""" - - -class ProvisionOsIdentityError(MeasurementError): - """Phala provision OS field fails the product identity policy.""" - - -def verify_provision_os_identity( - *, - measurement: Mapping[str, Any], - provision_os: Any, - mismatch_message: str, -) -> None: - """Bind Phala provision OS against the sealed measurement without overloading. - - Product Mode B residual (bd369a catalog vs 5c6d register formula): - - * When the sealed ``measurement.os_image_hash`` is the **product formula** - of ``measurement`` registers (``sha256(MRTD||RTMR1||RTMR2)``), Phala - provision may return a *different* dstack/teepod **catalog** digest on the - wire field also named ``os_image_hash``. Do not reject that residual by - equating catalog to the product seal. - * If the seal also declares optional ``dstack_mr_image`` (allowlisted catalog - pin), provision catalog must equal that pin (no invent loophole). - * When the seal is a legacy pin that still overloads catalog as - ``os_image_hash`` (not product-formula-consistent), keep fail-closed - equality vs provision (pre-fix residual behaviour). - """ - - expected_os = measurement.get("os_image_hash") - if not isinstance(expected_os, str) or not expected_os: - return - if not isinstance(provision_os, str) or not provision_os: - raise ProvisionOsIdentityError(mismatch_message) - - expected_norm = expected_os.strip().lower() - provision_norm = provision_os.strip().lower() - if provision_norm == expected_norm: - return - - # Optional allowlisted catalog digest (never invent when pin names one). - catalog_pin = measurement.get("dstack_mr_image") - if isinstance(catalog_pin, str) and catalog_pin.strip(): - pin_norm = catalog_pin.strip().lower() - if _SHA256_HEX_RE.fullmatch(pin_norm) is None: - raise ProvisionOsIdentityError( - "measurement dstack_mr_image catalog pin is not a 64-char hex digest" - ) - if provision_norm != pin_norm: - raise ProvisionOsIdentityError( - "Phala provision os_image_hash mismatches allowlisted dstack_mr_image catalog pin" - ) - # Catalog matched pin; product formula seal need not equal provision. - if measurement_uses_product_os_identity(measurement): - return - # Pin named a catalog without product-consistent seal: still accept - # only when catalog pin matched (already verified). Fail if product - # seal is neither formula nor equal catalog (handled below for legacy). - return - - if measurement_uses_product_os_identity(measurement): - # Product seal is guest-bind identity; provision catalog is observed only. - if _SHA256_HEX_RE.fullmatch(provision_norm) is None: - raise ProvisionOsIdentityError( - "Phala provision os_image_hash is not a 64-char hex digest" - ) - return - - raise ProvisionOsIdentityError(mismatch_message) - - -def reproduce_measurement( - *, - metadata_path: Path | str, - cpu: int, - memory: int | str, - compose: Mapping[str, Any] | str, - dstack_mr_bin: str | None = None, -) -> CanonicalMeasurement: - """Recompute the canonical measurement record for a pinned image + compose. - - Deterministic: the same inputs always yield the same record, and - ``.to_json()`` is a byte-stable serialization a validator can pin verbatim - (VAL-DEPLOY-003). - """ - - return build_canonical_measurement( - metadata_path=metadata_path, - cpu=cpu, - memory=memory, - compose=compose, - dstack_mr_bin=dstack_mr_bin, - ) - - -def canonical_measurement_subset(measurement: Mapping[str, Any]) -> dict[str, str]: - """Extract the six canonical (allowlist-pinnable) fields; fail closed if absent. - - ``rtmr3`` and other runtime/extra registers (e.g. ``key_provider``) are - excluded — the canonical set is exactly :data:`CANONICAL_MEASUREMENT_FIELDS`. - """ - - if not isinstance(measurement, Mapping): - raise MeasurementError("measurement must be a mapping") - subset: dict[str, str] = {} - for field in CANONICAL_MEASUREMENT_FIELDS: - value = measurement.get(field) - if not isinstance(value, str) or not value: - raise MeasurementError(f"measurement is missing/invalid canonical field {field!r}") - subset[field] = value.strip().lower() - return subset - - -def load_allowlist_entries( - source: str | Path | Iterable[Mapping[str, Any]], -) -> list[dict[str, str]]: - """Load a validator allowlist as a list of canonical six-field entries. - - Accepts a JSON file path, a JSON string, or an already-parsed iterable of - mappings. A top-level ``{"entries": [...]}`` wrapper (the key-release allowlist - file format) is unwrapped. Each entry must carry the six canonical fields; any - extra register is ignored. - """ - - if isinstance(source, (str, Path)): - text = Path(source).read_text(encoding="utf-8") if _looks_like_path(source) else str(source) - try: - data: Any = json.loads(text) - except json.JSONDecodeError as exc: - raise MeasurementError(f"allowlist is not valid JSON: {exc}") from exc - else: - data = list(source) - - if isinstance(data, Mapping): - data = data.get("entries", []) - if not isinstance(data, list): - raise MeasurementError("allowlist must be a list of entries or {'entries': [...]}") - return [canonical_measurement_subset(entry) for entry in data] - - -def _looks_like_path(source: str | Path) -> bool: - if isinstance(source, Path): - return True - stripped = source.strip() - # A JSON document starts with '[' or '{'; anything else is treated as a path. - return not stripped.startswith(("[", "{")) - - -@dataclass(frozen=True) -class AllowlistVerdict: - """The reported measurement plus its in-allowlist decision (VAL-DEPLOY-012).""" - - measurement: dict[str, str] - in_allowlist: bool - matched_index: int | None - - def as_dict(self) -> dict[str, Any]: - return { - "measurement": self.measurement, - "in_allowlist": self.in_allowlist, - "verdict": "IN-LIST" if self.in_allowlist else "NOT-IN-LIST", - "matched_index": self.matched_index, - } - - -def allowlist_verdict( - measurement: Mapping[str, Any], - allowlist: str | Path | Iterable[Mapping[str, Any]], -) -> AllowlistVerdict: - """Report a measurement's six-field subset and whether it is in the allowlist. - - A measurement matching an allowlist entry on ALL six canonical fields is - IN-LIST; any single-field difference is NOT-IN-LIST (VAL-DEPLOY-012). An empty - allowlist matches nothing (fail closed). - """ - - subset = canonical_measurement_subset(measurement) - entries = load_allowlist_entries(allowlist) - for index, entry in enumerate(entries): - if entry == subset: - return AllowlistVerdict(measurement=subset, in_allowlist=True, matched_index=index) - return AllowlistVerdict(measurement=subset, in_allowlist=False, matched_index=None) - - -def domain_allowlist_verdict( - *, - domain: str, - measurement: Mapping[str, Any], - review_allowlist: str | Path | Iterable[Mapping[str, Any]] | None = None, - eval_allowlist: str | Path | Iterable[Mapping[str, Any]] | None = None, -) -> AllowlistVerdict: - """Evaluate one measurement only against its validator-owned app domain.""" - - if domain not in {"review", "eval"}: - raise MeasurementError("measurement domain must be review or eval") - source = review_allowlist if domain == "review" else eval_allowlist - if source is None: - raise MeasurementError(f"{domain} validator allowlist is required") - return allowlist_verdict(measurement, source) - - -def short_measurement_hex_prefix(value: object, *, length: int = 12) -> str | None: - """Return a truncated lowercase hex prefix for operator diagnostics. - - Never returns the full digest when the source is longer than ``length``. - Rejects non-hex input so callers never echo secrets as opaque blobs. - """ - - if not isinstance(value, str): - return None - cleaned = value.strip().lower() - if cleaned.startswith("0x"): - cleaned = cleaned[2:] - if not cleaned or any(ch not in "0123456789abcdef" for ch in cleaned): - return None - if length < 1: - return None - return cleaned[:length] - - -def format_eval_shape_mismatch_error( - *, - plan_instance_type: str, - requested_instance_type: str, - plan_vm_shape: str | None = None, - plan_rtmr0: str | None = None, -) -> str: - """Loud, operator-facing message for plan vs CLI eval shape mismatch. - - Names ``vm_shape`` / ``instance_type``, both shapes, the rtmr0 allowlist - footgun, and that prepare already spent the one-shot token so a re-prepare - (next deploy cancel+retry) is required. Truncates any plan rtmr0 prefix. - """ - - plan_shape = (plan_vm_shape or plan_instance_type or "").strip() or plan_instance_type - requested = (requested_instance_type or "").strip() or requested_instance_type - prefix = short_measurement_hex_prefix(plan_rtmr0) if plan_rtmr0 else None - rtmr_note = ( - f" Plan measurement rtmr0 prefix={prefix} (truncated; full value never logged)." - if prefix - else "" - ) - return ( - "Eval deployment shape mismatch: validator-issued plan has " - f"vm_shape/instance_type={plan_shape!r} but CLI --eval-instance-type=" - f"{requested!r}. A shape change requires a matching measurement pin " - "(rtmr0) on the validator allowlist and a re-prepare; a stale rtmr0 pin " - "surfaces only as a generic key-release denial deep in the TEE flow — " - "not as a clear shape error. This abort is before Phala create (no spend). " - "The one-time EVAL_RUN_TOKEN delivery was already consumed by prepare; " - "re-run eval deploy (or eval cancel + retry) to re-prepare a fresh attempt." - f"{rtmr_note}" - ) - - -def format_rtmr0_pin_mismatch_error( - *, - plan_rtmr0: str, - expected_rtmr0: str, -) -> str: - """Loud message when plan rtmr0 disagrees with --expected-measurement.""" - - plan_prefix = short_measurement_hex_prefix(plan_rtmr0) or "?" - expected_prefix = short_measurement_hex_prefix(expected_rtmr0) or "?" - return ( - "Eval measurement pin mismatch on field rtmr0: " - f"plan prefix={plan_prefix} vs --expected-measurement prefix={expected_prefix} " - "(truncated; full digests never logged). A stale rtmr0 pin on the validator " - "allowlist surfaces only as a generic key-release denial. Aborting before " - "Phala create (no spend). Re-prepare after the allowlist pin matches the " - "deployed shape." - ) - - -def load_expected_measurement_mapping(path: str | Path) -> dict[str, Any]: - """Load a miner-supplied expected measurement JSON object from PATH.""" - - try: - raw = Path(path).read_text(encoding="utf-8") - except OSError as exc: - raise MeasurementError(f"expected measurement file could not be read: {path}") from exc - try: - parsed = json.loads(raw) - except json.JSONDecodeError as exc: - raise MeasurementError("expected measurement file is not valid JSON") from exc - if not isinstance(parsed, dict): - raise MeasurementError("expected measurement file must contain a JSON object") - return parsed - - -def compare_plan_rtmr0_to_expected( - plan_measurement: Mapping[str, Any], - expected: Mapping[str, Any], -) -> str | None: - """Return a loud error string when plan rtmr0 disagrees with expected, else None. - - When ``expected`` has no ``rtmr0`` field the check is a no-op (optional pin). - """ - - expected_rtmr0 = expected.get("rtmr0") - if expected_rtmr0 is None: - return None - if not isinstance(expected_rtmr0, str) or not expected_rtmr0.strip(): - raise MeasurementError("expected measurement rtmr0 must be a non-empty string") - plan_rtmr0 = plan_measurement.get("rtmr0") - if not isinstance(plan_rtmr0, str) or not plan_rtmr0.strip(): - raise MeasurementError("plan measurement is missing rtmr0 for pin comparison") - plan_norm = plan_rtmr0.strip().lower() - expected_norm = expected_rtmr0.strip().lower() - if plan_norm.startswith("0x"): - plan_norm = plan_norm[2:] - if expected_norm.startswith("0x"): - expected_norm = expected_norm[2:] - if plan_norm == expected_norm: - return None - return format_rtmr0_pin_mismatch_error( - plan_rtmr0=plan_rtmr0, - expected_rtmr0=expected_rtmr0, - ) - - -def measurements_agree( - miner_measurement: Mapping[str, Any], - validator_entry: Mapping[str, Any], -) -> bool: - """Whether the miner-reproduced record equals a validator allowlist entry. - - Compares the six canonical fields field-for-field (VAL-DEPLOY-004). - """ - - return canonical_measurement_subset(miner_measurement) == canonical_measurement_subset( - validator_entry - ) - - -__all__ = [ - "AllowlistVerdict", - "MeasurementError", - "ProvisionOsIdentityError", - "allowlist_verdict", - "canonical_measurement_subset", - "compare_plan_rtmr0_to_expected", - "domain_allowlist_verdict", - "format_eval_shape_mismatch_error", - "format_rtmr0_pin_mismatch_error", - "load_allowlist_entries", - "load_expected_measurement_mapping", - "measurement_uses_product_os_identity", - "measurements_agree", - "product_os_image_hash", - "reproduce_measurement", - "short_measurement_hex_prefix", - "verify_provision_os_identity", -] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/phala.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/phala.py deleted file mode 100644 index d3a257167..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/phala.py +++ /dev/null @@ -1,363 +0,0 @@ -"""Small, secret-safe Phala Cloud REST adapter for self-deploy stages. - -Auth matches the official Phala CLI (``phala`` node package): - -* Header ``X-API-Key: `` — **not** ``Authorization: Bearer`` - (Bearer returns HTTP 401 Invalid/expired token for Cloud API keys). -* Header ``X-Phala-Version: 2026-01-21`` — API version pin used by CLI. -* Header ``User-Agent: phala-cli/`` — Cloudflare 1010 blocks bare - Python-urllib agents; product sends the CLI-equivalent string. - -Region selection must not hard-fail with ERR-02-002 (``No teepod found``) -on bare alias ``us-west`` when inventory capacity is only under ``us-west-1``. - -Create-response CVM id helpers accept Phala's live schema (numeric ``id``, -alternate ``cvm_id`` / ``vm_uuid`` / ``instance_id``) and a safe list fallback -by ``app_id`` so product does not fail closed solely for -``product_create_response_missing_cvm_id_field`` when the CVM exists. -""" - -from __future__ import annotations - -import json -import os -import re -from collections.abc import Mapping, Sequence -from typing import Any -from urllib.error import HTTPError, URLError -from urllib.request import Request, urlopen - -from agent_challenge.selfdeploy.plan import PHALA_API_KEY_ENV, CredentialError - -DEFAULT_PHALA_API = "https://cloud-api.phala.com/api/v1" - -#: API version header accepted by cloud-api.phala.com (matches `phala` CLI `Lo`). -DEFAULT_PHALA_API_VERSION = "2026-01-21" - -#: CLI-equivalent User-Agent (see `phala` package ``phala-cli/${version}``). -#: urllib without UA is blocked by Cloudflare with error 1010. -DEFAULT_PHALA_USER_AGENT = "phala-cli/1.1.19" - -#: Preferred default region when caller omits one or alias maps to empty capacity. -#: Live inventory teepods (prod5/prod9) live under US-WEST-1; bare "us-west" -#: hits ERR-02-002. Empty string means "let the API/auto assign". -PREFERRED_PHALA_REGION = "us-west-1" - -#: Region aliases that previously hard-failed against live capacity. -_US_WEST_ALIASES = frozenset({"us-west", "us_west", "uswest"}) - -#: Allowed GET paths for safe read helpers (list/details — never secrets). -_ALLOWED_GET_PATHS = frozenset({"/cvms"}) - -#: Allowed DELETE path shape: /cvms/{id} only (no nested paths). -_ALLOWED_DELETE_PATH_RE = re.compile(r"^/cvms/[A-Za-z0-9][A-Za-z0-9._-]*$") -_CVM_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") - -#: Create-response keys that may identify a CVM (ordered preference). -#: ``app_id`` is intentionally excluded: it names the app pin, not the CVM. -_CREATE_CVM_ID_FIELDS = ("id", "cvm_id", "vm_uuid", "instance_id", "uuid") - - -class PhalaApiError(RuntimeError): - """A bounded Phala API failure without response-body disclosure.""" - - -def _normalize_cvm_id_value(value: Any) -> str | None: - """Coerce a create/list id field to a non-empty string identity. - - Live Phala create schema uses numeric ``id`` (CLI zod: ``id: number``). - Product ack builders require string CVM ids; never invent identities. - """ - - if isinstance(value, bool): - return None - if isinstance(value, int): - # Phala numbered CVM ids are positive; reject non-positive noise. - if value <= 0: - return None - return str(value) - if isinstance(value, str): - text = value.strip() - return text or None - return None - - -def extract_cvm_id_from_create_response(created: Mapping[str, Any]) -> str: - """Extract a CVM id from a create/list item mapping. - - Accepts alternate field names used by Phala create (numeric ``id``, - string ``cvm_id`` / ``vm_uuid`` / ``instance_id``). Never treats plain - ``app_id`` as a CVM id (app pin is distinct). Raises ValueError with a - secret-free message when no candidate field yields a usable identity. - """ - - if not isinstance(created, Mapping): - raise ValueError("Phala create response does not identify the CVM") - for name in _CREATE_CVM_ID_FIELDS: - if name not in created: - continue - normalized = _normalize_cvm_id_value(created.get(name)) - if normalized is not None: - return normalized - raise ValueError("Phala create response does not identify the CVM") - - -def resolve_cvm_id_from_list( - listing: Mapping[str, Any] | Sequence[Any], - *, - app_id: str, - require_unique: bool = False, -) -> str | None: - """Locate a CVM id in a GET /cvms listing by exact app_id match. - - Returns None when listing is empty/mismatched rather than inventing an id. - When ``require_unique`` is False (deploy create fallback), the first ordered - match wins. When True (teardown resolution), multiple matches raise - :class:`PhalaApiError` so callers never guess. Secret bodies are never logged. - """ - - if not isinstance(app_id, str) or not app_id.strip(): - return None - target = app_id.strip() - items: Sequence[Any] - if isinstance(listing, Mapping): - for key in ("items", "cvms", "data"): - candidate = listing.get(key) - if isinstance(candidate, list): - items = candidate - break - else: - # Some envelopes return the list as a bare mapping without items. - items = [] - elif isinstance(listing, Sequence) and not isinstance(listing, (str, bytes)): - items = listing - else: - return None - - matches: list[str] = [] - for item in items: - if not isinstance(item, Mapping): - continue - item_app = item.get("app_id") - if not isinstance(item_app, str) or item_app != target: - continue - try: - matches.append(extract_cvm_id_from_create_response(item)) - except ValueError: - continue - if not matches: - return None - if require_unique and len(matches) > 1: - raise PhalaApiError( - f"multiple CVMs match app_id ({len(matches)}); pass --cvm-id explicitly" - ) - return matches[0] - - -def normalize_phala_region(region: str | None) -> str: - """Normalize a caller region to a capacity-safe Phala region string. - - * bare ``us-west`` (any case / underscore form) → ``us-west-1`` - * ``us-west-1`` / mixed case → lowercase ``us-west-1`` - * empty / None → empty (auto) so callers can omit the key - * other regions → stripped lowercase - """ - - if region is None: - return "" - raw = str(region).strip() - if not raw: - return "" - lowered = raw.lower().replace("_", "-") - if lowered in _US_WEST_ALIASES: - return PREFERRED_PHALA_REGION - if lowered == "us-west-1": - return PREFERRED_PHALA_REGION - return lowered - - -def select_phala_region( - preferred: str | None = None, - *, - available_regions: Sequence[str] | None = None, -) -> str: - """Pick a capacity-aware region for provision requests. - - Rules (fail soft — never reintroduce bare ``us-west`` hard-fail): - - 1. Normalize the preferred alias (``us-west`` → ``us-west-1``). - 2. If inventory is supplied and preferred is present (case-insensitive), use it. - 3. If preferred alias had no capacity / was empty, use the first available. - 4. If inventory is empty, use preferred if set else :data:`PREFERRED_PHALA_REGION`. - """ - - # Detect whether the raw preferred was only a bare us-west alias (capacity miss). - raw = "" if preferred is None else str(preferred).strip() - raw_alias = raw.lower().replace("_", "-") in _US_WEST_ALIASES if raw else False - normalized = normalize_phala_region(preferred) - inventory: list[str] = [] - if available_regions: - for item in available_regions: - n = normalize_phala_region(item) - if n and n not in inventory: - inventory.append(n) - - if inventory: - if normalized and normalized in inventory: - return normalized - # Explicit non-alias preferred always wins (caller performant override). - if normalized and not raw_alias and preferred is not None and str(preferred).strip(): - return normalized - # Alias remapped but inventory only lists real capacity elsewhere: pick available. - return inventory[0] - - if normalized: - return normalized - return PREFERRED_PHALA_REGION - - -class PhalaCloudClient: - """HTTPS adapter for Phala provision/create and safe CVM list routes.""" - - def __init__( - self, - *, - api_key: str | None = None, - base_url: str = DEFAULT_PHALA_API, - api_version: str = DEFAULT_PHALA_API_VERSION, - user_agent: str = DEFAULT_PHALA_USER_AGENT, - opener=urlopen, - timeout: float = 30.0, - ) -> None: - # Temporary NO_PHALA mode: refuse the entire Phala client so no CVM - # provision/create/list call can slip through while host-local mode is on. - from agent_challenge.evaluation.no_phala import refuse_phala_client - - refuse_phala_client("PhalaCloudClient") - self._api_key = ( - api_key if api_key is not None else os.environ.get(PHALA_API_KEY_ENV, "") - ).strip() - if not self._api_key: - raise CredentialError( - f"{PHALA_API_KEY_ENV} is not set; set it before provisioning. " - "The key value is never printed." - ) - self._base_url = base_url.strip().rstrip("/") - if not self._base_url.startswith("https://"): - raise PhalaApiError("Phala API endpoint must use https://") - self._api_version = (api_version or DEFAULT_PHALA_API_VERSION).strip() - agent = (user_agent or DEFAULT_PHALA_USER_AGENT).strip() - self._user_agent = agent or DEFAULT_PHALA_USER_AGENT - self._opener = opener - self._timeout = timeout - - def _base_headers(self, *, content_type: bool = False) -> dict[str, str]: - headers = { - "Accept": "application/json", - # CLI-compatible identity + auth. Bare Python-urllib is CF-blocked. - "User-Agent": self._user_agent, - "X-API-Key": self._api_key, - "X-Phala-Version": self._api_version, - } - if content_type: - headers["Content-Type"] = "application/json" - return headers - - def _decode_json_object(self, body: bytes) -> dict[str, Any]: - if len(body) > 2 * 1024 * 1024: - raise PhalaApiError("Phala provisioning response exceeded the bounded size") - try: - decoded = json.loads(body) - except (UnicodeDecodeError, json.JSONDecodeError) as exc: - raise PhalaApiError("Phala provisioning returned malformed JSON") from exc - if not isinstance(decoded, dict): - raise PhalaApiError("Phala provisioning returned a non-object response") - return decoded - - def _open(self, request: Request) -> dict[str, Any]: - try: - response = self._opener(request, timeout=self._timeout) - body = response.read() - except HTTPError as exc: - raise PhalaApiError(f"Phala provisioning returned HTTP {exc.code}") from exc - except (URLError, TimeoutError, OSError) as exc: - raise PhalaApiError("Phala provisioning endpoint is unreachable") from exc - return self._decode_json_object(body) - - def get(self, path: str) -> dict[str, Any]: - """GET a allowlisted read route (currently ``/cvms`` list only).""" - - if path not in _ALLOWED_GET_PATHS: - raise PhalaApiError("unsupported Phala read route") - request = Request( - f"{self._base_url}{path}", - headers=self._base_headers(content_type=False), - method="GET", - ) - return self._open(request) - - def post(self, path: str, payload: Mapping[str, Any]) -> dict[str, Any]: - if path not in {"/cvms/provision", "/cvms"}: - raise PhalaApiError("unsupported Phala mutation route") - body_payload = dict(payload) - # Capacity-safe region: remapped before send without logging secrets. - if "region" in body_payload: - region_value = body_payload.get("region") - if isinstance(region_value, str) or region_value is None: - region_arg: str | None = region_value - else: - region_arg = str(region_value) - normalized = normalize_phala_region(region_arg) - if normalized: - body_payload["region"] = normalized - else: - # Empty → omit so API can auto-select from available teepods. - body_payload.pop("region", None) - raw = json.dumps(body_payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") - request = Request( - f"{self._base_url}{path}", - data=raw, - headers=self._base_headers(content_type=True), - method="POST", - ) - return self._open(request) - - def delete_cvm(self, cvm_id: str) -> None: - """DELETE ``/cvms/{id}``. 204 and 404 are success (idempotent teardown).""" - - cid = (cvm_id or "").strip() - if not cid or not _CVM_ID_RE.fullmatch(cid): - raise PhalaApiError("invalid CVM id for Phala delete") - path = f"/cvms/{cid}" - if not _ALLOWED_DELETE_PATH_RE.fullmatch(path): - raise PhalaApiError("unsupported Phala mutation route") - request = Request( - f"{self._base_url}{path}", - headers=self._base_headers(content_type=False), - method="DELETE", - ) - try: - response = self._opener(request, timeout=self._timeout) - # Drain body; 204 is empty. Never log response content. - _ = response.read() - except HTTPError as exc: - if exc.code == 404: - return - raise PhalaApiError(f"Phala delete returned HTTP {exc.code}") from exc - except (URLError, TimeoutError, OSError) as exc: - raise PhalaApiError("Phala delete endpoint is unreachable") from exc - - - -__all__ = [ - "DEFAULT_PHALA_API", - "DEFAULT_PHALA_API_VERSION", - "DEFAULT_PHALA_USER_AGENT", - "PREFERRED_PHALA_REGION", - "PhalaApiError", - "PhalaCloudClient", - "extract_cvm_id_from_create_response", - "normalize_phala_region", - "resolve_cvm_id_from_list", - "select_phala_region", -] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/plan.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/plan.py deleted file mode 100644 index 3be0c6b3d..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/plan.py +++ /dev/null @@ -1,341 +0,0 @@ -"""Fetch/prepare + deploy-plan construction for the miner self-deploy flow. - -This module turns the miner's inputs (a digest-pinned canonical image reference, a -validator key-release endpoint URL, an optional CPU shape) into: - -* a :class:`PreparedDeployment` — the generated, deterministic Phala - ``app-compose`` (referencing the exact image digest, mounting the dstack + - guest Docker sockets, and carrying the validator key-release endpoint) plus its - reproducible compose-hash (VAL-DEPLOY-002); and -* a :class:`DeployPlan` — the prepared deployment plus the validated CPU shape, - region, projected cost, and the ``encrypted_env`` name set, ready to hand to a - deployer or print as a no-spend dry-run (VAL-DEPLOY-008/009). - -All guards are pure and run BEFORE any provisioning: GPU targets and over-cap -shapes are refused with zero Phala calls (:mod:`.shapes`), and missing Phala -credentials error clearly (:func:`check_phala_credentials`) without ever printing -the key value (VAL-DEPLOY-006/007/008). -""" - -from __future__ import annotations - -import os -from collections.abc import Mapping -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from agent_challenge.canonical.compose import ( - DEFAULT_ALLOWED_ENVS, - DEFAULT_APP_NAME, - ComposeGenerationError, - app_compose_hash, - assert_digest_pinned, - generate_app_compose, - render_app_compose, - write_app_compose, -) -from agent_challenge.canonical.live_registry import ( - LiveRegistryError, - load_live_registry, -) -from agent_challenge.keyrelease.client import KEY_RELEASE_URL_ENV -from agent_challenge.selfdeploy.shapes import ( - DEFAULT_MAX_RUNTIME_HOURS, - DEFAULT_MONEY_CAP_USD, - DEFAULT_OS_IMAGE, - CpuShape, - select_default_instance_type, - validate_cpu_only, - validate_within_cap, -) - -#: Environment variable holding the Phala Cloud API key (AGENTS.md). Never printed. -PHALA_API_KEY_ENV = "PHALA_CLOUD_API_KEY" - -#: Default Phala region for CPU Intel TDX. -#: Live teepod capacity is under ``us-west-1``; bare ``us-west`` fails with -#: ERR-02-002 (No teepod found). See selfdeploy.phala.normalize_phala_region. -DEFAULT_REGION = "us-west-1" - -#: Filename of the deployable app-compose written by ``prepare``. -APP_COMPOSE_FILENAME = "app-compose.json" - - -class PrepareError(ValueError): - """The fetch/prepare inputs are invalid (bad image ref / missing endpoint).""" - - -class CredentialError(RuntimeError): - """Phala credentials are missing/unusable (deploy refuses, no CVM call).""" - - -@dataclass(frozen=True) -class PreparedDeployment: - """The generated compose + reproducible hash for a canonical-image deployment.""" - - image: str - key_release_url: str - compose: dict[str, Any] - compose_text: str - compose_hash: str - allowed_envs: tuple[str, ...] - - def orchestrator_environment(self) -> list[str]: - """The orchestrator service ``environment`` entries in the generated compose.""" - - services = self.compose.get("docker_compose_file", "") - # The compose is generated by canonical.compose; parse the embedded YAML. - import yaml - - parsed = yaml.safe_load(services) if isinstance(services, str) else {} - service = (parsed or {}).get("services", {}).get("orchestrator", {}) - env = service.get("environment", []) - return [str(entry) for entry in env] - - -@dataclass(frozen=True) -class DeployPlan: - """A fully-validated, no-spend deploy plan (safe to print as a dry-run).""" - - prepared: PreparedDeployment - instance_type: str - os_image: str - region: str - shape: CpuShape - projected_cost_usd: float - money_cap_usd: float - max_runtime_hours: float - name: str - encrypted_env: dict[str, str] = field(default_factory=dict) - - @property - def image(self) -> str: - return self.prepared.image - - @property - def key_release_url(self) -> str: - return self.prepared.key_release_url - - @property - def compose_hash(self) -> str: - return self.prepared.compose_hash - - -def resolve_orchestrator_image( - *, - image: str | None = None, - live_registry_path: Path | str | None = None, -) -> str: - """Resolve the pullable, digest-pinned orchestrator image the deploy pins. - - Prefers an explicit ``image``; otherwise reads the recorded - ``orchestrator_image`` from the live-registry side manifest. Raises - :class:`PrepareError` when neither is available so the deploy never silently - falls back to an unpinned/absent image. - """ - - if isinstance(image, str) and image.strip(): - return image.strip() - if live_registry_path is not None: - try: - registry = load_live_registry(live_registry_path) - except LiveRegistryError as exc: - raise PrepareError(str(exc)) from exc - if registry.orchestrator_image: - return registry.orchestrator_image - raise PrepareError( - "no orchestrator image given and none recorded in the live-registry manifest; " - "pass an explicit digest-pinned image or a manifest that records orchestrator_image" - ) - - -def prepare_deployment( - *, - image: str, - key_release_url: str, - app_name: str = DEFAULT_APP_NAME, - allowed_envs: tuple[str, ...] = DEFAULT_ALLOWED_ENVS, - live_registry_manifest_path: str | None = None, -) -> PreparedDeployment: - """Resolve the canonical image to a digest-pin and generate the compose. - - Fails closed on a floating (non-digest) image tag or a missing key-release - endpoint so a miner can never deploy an unpinned image or a run with no - validator endpoint (VAL-DEPLOY-002). - - When ``live_registry_manifest_path`` is given the generated compose points the - in-CVM DooD orchestrator at the live-registry side manifest (for the live - task-image subset); with none supplied the compose is byte-identical to the - offline/flag-off default. - """ - - if not isinstance(key_release_url, str) or not key_release_url.strip(): - raise PrepareError( - "a validator key-release endpoint URL is required (the run cannot obtain the " - "golden key without it)" - ) - endpoint = key_release_url.strip() - try: - pinned = assert_digest_pinned(image, what="canonical image") - except ComposeGenerationError as exc: - raise PrepareError(str(exc)) from exc - - compose = generate_app_compose( - orchestrator_image=pinned, - name=app_name, - key_release_url=endpoint, - allowed_envs=allowed_envs, - live_registry_manifest_path=live_registry_manifest_path, - ) - return PreparedDeployment( - image=pinned, - key_release_url=endpoint, - compose=compose, - compose_text=render_app_compose(compose), - compose_hash=app_compose_hash(compose), - allowed_envs=tuple(compose["allowed_envs"]), - ) - - -def write_prepared(prepared: PreparedDeployment, out_dir: Path | str) -> Path: - """Write the deployable ``app-compose.json`` (exact bytes) into ``out_dir``. - - The bytes written are ``canonical.compose.render_app_compose`` verbatim, so - the on-disk file hashes to :attr:`PreparedDeployment.compose_hash` (and hence - the live CVM ``compose_hash`` / RTMR3 event). - """ - - out = Path(out_dir) - out.mkdir(parents=True, exist_ok=True) - path = out / APP_COMPOSE_FILENAME - write_app_compose(path, prepared.compose) - return path - - -def check_phala_credentials(env: Mapping[str, str] | None = None) -> None: - """Require a non-empty Phala API key; raise clearly WITHOUT echoing it. - - Raises :class:`CredentialError` naming the missing credential when - :data:`PHALA_API_KEY_ENV` is unset/empty. The key VALUE is never included in - the message, returned, or logged (VAL-DEPLOY-006). - """ - - source = env if env is not None else os.environ - key = (source.get(PHALA_API_KEY_ENV) or "").strip() - if not key: - raise CredentialError( - f"{PHALA_API_KEY_ENV} is not set; export your Phala Cloud API key before deploying. " - "The key value is never printed." - ) - - -def build_deploy_plan( - *, - image: str, - key_release_url: str, - instance_type: str | None = None, - os_image: str = DEFAULT_OS_IMAGE, - region: str = DEFAULT_REGION, - money_cap_usd: float = DEFAULT_MONEY_CAP_USD, - max_runtime_hours: float = DEFAULT_MAX_RUNTIME_HOURS, - name: str = DEFAULT_APP_NAME, - encrypted_env: Mapping[str, str] | None = None, - live_registry_manifest_path: str | None = None, -) -> DeployPlan: - """Build a validated, no-spend deploy plan (all guards run, no Phala call). - - Resolves the default smallest CPU shape when none is given, refuses GPU - targets and over-cap shapes, and prepares the compose. Raises - :class:`~agent_challenge.selfdeploy.shapes.ShapeError` (GPU / unknown / over - cap) before any provisioning; credential checking is a separate, later step so - a GPU/over-cap refusal never needs credentials (VAL-DEPLOY-007/008). - - When ``live_registry_manifest_path`` is given the generated compose points the - in-CVM DooD orchestrator at the live-registry side manifest (live task-image - subset); with none supplied the compose is byte-identical to the offline - default. - """ - - # Local import: phala imports CredentialError from this module (cycle-safe). - from agent_challenge.selfdeploy.phala import normalize_phala_region - - resolved_type = (instance_type or "").strip() or select_default_instance_type() - shape = validate_cpu_only(instance_type=resolved_type, os_image=os_image) - cost = validate_within_cap( - resolved_type, - money_cap_usd=money_cap_usd, - max_runtime_hours=max_runtime_hours, - ) - deployment_env = dict(encrypted_env or {}) - if "EVAL_RUN_TOKEN" in deployment_env and set(deployment_env) != {"EVAL_RUN_TOKEN"}: - raise PrepareError("Eval deploy encrypted_env must contain only EVAL_RUN_TOKEN") - prepared = prepare_deployment( - image=image, - key_release_url=key_release_url, - app_name=name, - live_registry_manifest_path=live_registry_manifest_path, - ) - - env: dict[str, str] = {KEY_RELEASE_URL_ENV: prepared.key_release_url} - env.update({str(k): str(v) for k, v in deployment_env.items()}) - - # Remap bare us-west → us-west-1 so DEFAULT_REGION never hard-fails capacity. - resolved_region = normalize_phala_region(region) or DEFAULT_REGION - - return DeployPlan( - prepared=prepared, - instance_type=resolved_type, - os_image=os_image, - region=resolved_region, - shape=shape, - projected_cost_usd=cost, - money_cap_usd=money_cap_usd, - max_runtime_hours=max_runtime_hours, - name=name, - encrypted_env=env, - ) - - -def render_plan(plan: DeployPlan) -> dict[str, Any]: - """The full dry-run plan as a JSON-serializable dict (no secret values). - - Surfaces every field a miner needs to review before spending: the exact - compose text + hash, the image digest, instance type, region, OS image, the - validator key-release endpoint, and the projected cost. Only the - ``encrypted_env`` NAMES are shown (never values), so printing the plan leaks - no secret (VAL-DEPLOY-009). - """ - - return { - "image": plan.image, - "instance_type": plan.instance_type, - "os_image": plan.os_image, - "region": plan.region, - "key_release_url": plan.key_release_url, - "compose_hash": plan.compose_hash, - "compose": plan.prepared.compose_text, - "projected_cost_usd": round(plan.projected_cost_usd, 4), - "money_cap_usd": plan.money_cap_usd, - "max_runtime_hours": plan.max_runtime_hours, - "name": plan.name, - "encrypted_env_names": sorted(plan.encrypted_env), - "dry_run": True, - } - - -__all__ = [ - "APP_COMPOSE_FILENAME", - "DEFAULT_REGION", - "PHALA_API_KEY_ENV", - "CredentialError", - "DeployPlan", - "PrepareError", - "PreparedDeployment", - "build_deploy_plan", - "check_phala_credentials", - "prepare_deployment", - "render_plan", - "resolve_orchestrator_image", - "write_prepared", -] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/result.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/result.py deleted file mode 100644 index 4de5386c9..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/result.py +++ /dev/null @@ -1,414 +0,0 @@ -"""Surface + verify the attested-result envelope for the miner CLI (VAL-DEPLOY-013/014). - -On eval completion the canonical image emits a single ``BASE_BENCHMARK_RESULT=`` -line that additively carries the Phala-tier ``ExecutionProof`` envelope + the -architecture-§6 ``report_data`` binding preimage (see -:mod:`agent_challenge.canonical.attested_result`). This module lets the miner -surface that envelope (the TDX quote, event log, ``report_data``, the full -measurement block, and the per-task scores) and independently checks that the -quote's ``report_data`` recomputes to the documented binding for the run -(VAL-DEPLOY-014) — so a surfaced result whose scores/measurement/nonce were -altered fails the binding check rather than being presented as genuine. - -A run that failed closed (no envelope, e.g. key-release denied) is surfaced as -``attested=False`` with its reason code and NO fabricated attestation, so the -miner never sees an attested-looking result for a failed run (VAL-DEPLOY-011). -""" - -from __future__ import annotations - -import json -from collections.abc import Callable, Mapping -from dataclasses import dataclass -from typing import Any - -from agent_challenge.canonical import report_data as rd -from agent_challenge.canonical.attested_result import ( - ATTESTATION_BINDING_RESULT_KEY, - EXECUTION_PROOF_RESULT_KEY, - MEASUREMENT_FIELDS, - PHALA_TDX_TIER, - EnvelopeSchemaError, - validate_execution_proof_envelope, -) -from agent_challenge.canonical.measurement import CANONICAL_MEASUREMENT_FIELDS -from agent_challenge.evaluation.own_runner.result_schema import RESULT_LINE_PREFIX - -#: The binding preimage fields carried in the clear for verifier recomputation. -_BINDING_FIELDS = ( - "agent_hash", - "task_ids", - "scores", - "scores_digest", - "validator_nonce", - "canonical_measurement", -) - -# --------------------------------------------------------------------------- # -# Coarse, non-sensitive miner-facing acceptance-verdict reasons (VAL-DEPLOY-026). -# These are deliberately coarse labels: a rejection reason NEVER carries golden -# material, the golden key, or any quote-embedded secret -- only the cause class. -# --------------------------------------------------------------------------- # -#: The run produced no (Phala-tier) attested result at all. -ACCEPTANCE_ATTESTATION_ABSENT = "attestation absent" -#: The attestation's TDX quote did not verify as genuine. -ACCEPTANCE_ATTESTATION_NOT_VERIFIED = "attestation not verified" -#: The attested measurement is not on the validator allowlist. -ACCEPTANCE_MEASUREMENT_NOT_ALLOWLISTED = "measurement not allowlisted" -#: The validator nonce bound into the quote had expired. -ACCEPTANCE_NONCE_STALE = "nonce stale" -#: The validator nonce had already been consumed (single-use). -ACCEPTANCE_NONCE_CONSUMED = "nonce already used" -#: The validator nonce was not one the validator issued. -ACCEPTANCE_NONCE_UNKNOWN = "nonce not recognized" -#: The self-checked report_data binding did not recompute (tampered result). -ACCEPTANCE_BINDING_MISMATCH = "attestation binding mismatch" -#: The matching validator key grant for this eval run was missing or failed. -ACCEPTANCE_KEY_GRANT_MISSING = "key grant missing" - -_NONCE_STATE_REASON: dict[str, str] = { - "expired": ACCEPTANCE_NONCE_STALE, - "stale": ACCEPTANCE_NONCE_STALE, - "consumed": ACCEPTANCE_NONCE_CONSUMED, - "unknown": ACCEPTANCE_NONCE_UNKNOWN, -} - - -def _normalize_nonce_state(nonce_state: Any) -> str | None: - """Normalize a nonce verdict (``NonceState`` enum or str) to a lowercase name.""" - - if nonce_state is None: - return None - name = getattr(nonce_state, "name", None) - if name is not None: - return str(name).lower() - return str(nonce_state).strip().lower() - - -class ResultSurfaceError(ValueError): - """The captured run output carries no parseable / well-formed result.""" - - -def parse_result_payload(stdout: str) -> dict[str, Any] | None: - """Return the parsed ``BASE_BENCHMARK_RESULT=`` payload (last-wins), or ``None``.""" - - if not stdout: - return None - for line in reversed(stdout.splitlines()): - if line.startswith(RESULT_LINE_PREFIX): - try: - parsed = json.loads(line[len(RESULT_LINE_PREFIX) :]) - except json.JSONDecodeError: - return None - return parsed if isinstance(parsed, dict) else None - return None - - -def extract_envelope(stdout: str) -> tuple[dict[str, Any], dict[str, Any]] | None: - """Extract ``(execution_proof, attestation_binding)`` for a Phala-tier result. - - Returns ``None`` when the result carries no Phala-tier attestation (no result - line, no envelope, a non-Phala tier, or a missing binding). - """ - - payload = parse_result_payload(stdout) - if payload is None: - return None - execution_proof = payload.get(EXECUTION_PROOF_RESULT_KEY) - binding = payload.get(ATTESTATION_BINDING_RESULT_KEY) - if not isinstance(execution_proof, Mapping) or not isinstance(binding, Mapping): - return None - if execution_proof.get("tier") != PHALA_TDX_TIER: - return None - return dict(execution_proof), dict(binding) - - -@dataclass(frozen=True) -class BindingCheck: - """Result of recomputing the §6 ``report_data`` binding for a run (VAL-DEPLOY-014).""" - - valid: bool - report_data_matches: bool - scores_digest_matches: bool - measurement_consistent: bool - expected_report_data: str - recomputed_report_data: str - - def as_dict(self) -> dict[str, Any]: - return { - "valid": self.valid, - "report_data_matches": self.report_data_matches, - "scores_digest_matches": self.scores_digest_matches, - "measurement_consistent": self.measurement_consistent, - "expected_report_data": self.expected_report_data, - "recomputed_report_data": self.recomputed_report_data, - } - - -def verify_report_data_binding( - execution_proof: Mapping[str, Any], - binding: Mapping[str, Any], -) -> BindingCheck: - """Recompute ``report_data`` from the binding and compare to the quote's value. - - Checks (a) the recomputed §6 ``report_data`` equals the envelope's - ``report_data``, (b) the ``scores_digest`` recomputes from the reported scores - (a score cannot be altered without breaking the binding), and (c) the - measurement bound into ``report_data`` equals the envelope's measurement block - (self-consistency). All three must hold for ``valid``. - """ - - attestation = execution_proof.get("attestation") - if not isinstance(attestation, Mapping): - raise ResultSurfaceError("execution_proof has no attestation block") - for field in _BINDING_FIELDS: - if field not in binding: - raise ResultSurfaceError(f"attestation binding missing field {field!r}") - - expected = str(attestation.get("report_data") or "") - canonical_measurement = binding["canonical_measurement"] - scores = binding["scores"] - scores_digest = str(binding["scores_digest"]) - - recomputed = rd.report_data_hex( - canonical_measurement=canonical_measurement, - agent_hash=str(binding["agent_hash"]), - task_ids=list(binding["task_ids"]), - scores_digest=scores_digest, - validator_nonce=str(binding["validator_nonce"]), - ) - recomputed_scores_digest = rd.scores_digest(scores) - - envelope_measurement = attestation.get("measurement", {}) - measurement_consistent = all( - str(envelope_measurement.get(field, "")) == str(canonical_measurement.get(field, "")) - for field in CANONICAL_MEASUREMENT_FIELDS - ) - - report_data_matches = recomputed == expected - scores_digest_matches = recomputed_scores_digest == scores_digest - return BindingCheck( - valid=report_data_matches and scores_digest_matches and measurement_consistent, - report_data_matches=report_data_matches, - scores_digest_matches=scores_digest_matches, - measurement_consistent=measurement_consistent, - expected_report_data=expected, - recomputed_report_data=recomputed, - ) - - -@dataclass(frozen=True) -class SurfacedResult: - """A miner-surfaced run result (attested envelope or fail-closed summary).""" - - attested: bool - benchmark_result: dict[str, Any] - status: str - reason_code: str | None - execution_proof: dict[str, Any] | None - binding: dict[str, Any] | None - binding_check: BindingCheck | None - quote_verified: bool | None - - @property - def attestation(self) -> dict[str, Any] | None: - if self.execution_proof is None: - return None - att = self.execution_proof.get("attestation") - return dict(att) if isinstance(att, Mapping) else None - - @property - def scores(self) -> dict[str, Any] | None: - if self.binding is None: - return None - scores = self.binding.get("scores") - return dict(scores) if isinstance(scores, Mapping) else None - - def summary(self) -> dict[str, Any]: - """A JSON-serializable summary for the CLI to print (no secret values).""" - - out: dict[str, Any] = { - "attested": self.attested, - "status": self.status, - "reason_code": self.reason_code, - "benchmark_result": self.benchmark_result, - } - attestation = self.attestation - if self.attested and attestation is not None: - out["attestation"] = { - "tdx_quote": attestation.get("tdx_quote"), - "event_log": attestation.get("event_log"), - "report_data": attestation.get("report_data"), - "measurement": attestation.get("measurement"), - "vm_config": attestation.get("vm_config"), - } - out["scores"] = self.scores - if self.binding_check is not None: - out["binding_check"] = self.binding_check.as_dict() - if self.quote_verified is not None: - out["quote_verified"] = self.quote_verified - return out - - -@dataclass(frozen=True) -class AcceptanceVerdict: - """A coarse, non-sensitive miner-facing acceptance verdict (VAL-DEPLOY-026). - - ``accepted`` is ``False`` with a coarse ``reason`` for each rejection cause, - ``True`` when a positive validator signal confirms acceptance, or ``None`` - (pending) when no validator signal is available yet. ``reason`` is always a - coarse label -- never golden material, the golden key, or a quote secret. - """ - - accepted: bool | None - reason: str | None - - def as_dict(self) -> dict[str, Any]: - return {"accepted": self.accepted, "reason": self.reason} - - -def evaluate_acceptance( - surfaced: SurfacedResult, - *, - quote_verified: bool | None = None, - measurement_allowlisted: bool | None = None, - nonce_state: Any = None, - key_grant_ok: bool | None = None, -) -> AcceptanceVerdict: - """Compute the miner-facing acceptance verdict for a surfaced run. - - Surfaces a validator rejection/park (attestation absent, quote not verified, - measurement not allowlisted, nonce stale/consumed/unknown, missing key grant, - or a tampered binding) as ``accepted=False`` with a coarse, non-sensitive - reason -- never a fabricated score and never a leaked secret. Acceptance is - a conjunction: binding, genuine quote verification, the domain measurement - allowlist verdict, the fresh nonce state, and the matching key-grant must all - be positive together. Any single positive signal is never enough. When any - required signal is still missing the verdict is pending (``accepted=None``), - never a false accept. - """ - - if not surfaced.attested: - return AcceptanceVerdict(accepted=False, reason=ACCEPTANCE_ATTESTATION_ABSENT) - - check = surfaced.binding_check - if check is not None and not check.valid: - return AcceptanceVerdict(accepted=False, reason=ACCEPTANCE_BINDING_MISMATCH) - - verified = quote_verified if quote_verified is not None else surfaced.quote_verified - if verified is False: - return AcceptanceVerdict(accepted=False, reason=ACCEPTANCE_ATTESTATION_NOT_VERIFIED) - - if measurement_allowlisted is False: - return AcceptanceVerdict(accepted=False, reason=ACCEPTANCE_MEASUREMENT_NOT_ALLOWLISTED) - - normalized_nonce = _normalize_nonce_state(nonce_state) - if normalized_nonce is not None and normalized_nonce != "ok": - return AcceptanceVerdict( - accepted=False, - reason=_NONCE_STATE_REASON.get(normalized_nonce, ACCEPTANCE_NONCE_STALE), - ) - - if key_grant_ok is False: - return AcceptanceVerdict(accepted=False, reason=ACCEPTANCE_KEY_GRANT_MISSING) - - binding_ok = check is None or check.valid is True - all_ready = ( - binding_ok - and verified is True - and measurement_allowlisted is True - and normalized_nonce == "ok" - and key_grant_ok is True - ) - return AcceptanceVerdict(accepted=True if all_ready else None, reason=None) - - -def surface_result( - stdout: str, - *, - quote_verifier: Callable[[str], bool] | None = None, -) -> SurfacedResult: - """Surface a run's result from its captured stdout. - - Returns a :class:`SurfacedResult`: for an attested run, the validated Phala - envelope + the recomputed binding check; for a failed/legacy run, the - benchmark result with ``attested=False`` and NO attestation block. Raises - :class:`ResultSurfaceError` only when there is no parseable result line at all. - - ``quote_verifier`` (optional) is an injectable hook that returns whether the - TDX quote verifies as genuine (Phala verify API / ``dcap-qvl`` at M6); when - provided its verdict is attached to the surfaced result. - """ - - payload = parse_result_payload(stdout) - if payload is None: - raise ResultSurfaceError("no parseable BASE_BENCHMARK_RESULT= line in the captured output") - - status = str(payload.get("status", "")) - reason_code = payload.get("reason_code") - reason_code = str(reason_code) if isinstance(reason_code, str) and reason_code else None - benchmark_result = { - key: value - for key, value in payload.items() - if key not in (EXECUTION_PROOF_RESULT_KEY, ATTESTATION_BINDING_RESULT_KEY) - } - - envelope = extract_envelope(stdout) - if envelope is None: - return SurfacedResult( - attested=False, - benchmark_result=benchmark_result, - status=status, - reason_code=reason_code, - execution_proof=None, - binding=None, - binding_check=None, - quote_verified=None, - ) - - execution_proof, binding = envelope - try: - validate_execution_proof_envelope(execution_proof) - except EnvelopeSchemaError as exc: - raise ResultSurfaceError(f"surfaced attestation envelope is malformed: {exc}") from exc - - binding_check = verify_report_data_binding(execution_proof, binding) - - quote_verified: bool | None = None - if quote_verifier is not None: - attestation = execution_proof.get("attestation", {}) - quote = attestation.get("tdx_quote", "") if isinstance(attestation, Mapping) else "" - quote_verified = bool(quote_verifier(str(quote))) - - return SurfacedResult( - attested=True, - benchmark_result=benchmark_result, - status=status, - reason_code=reason_code, - execution_proof=execution_proof, - binding=binding, - binding_check=binding_check, - quote_verified=quote_verified, - ) - - -__all__ = [ - "ACCEPTANCE_ATTESTATION_ABSENT", - "ACCEPTANCE_ATTESTATION_NOT_VERIFIED", - "ACCEPTANCE_BINDING_MISMATCH", - "ACCEPTANCE_KEY_GRANT_MISSING", - "ACCEPTANCE_MEASUREMENT_NOT_ALLOWLISTED", - "ACCEPTANCE_NONCE_CONSUMED", - "ACCEPTANCE_NONCE_STALE", - "ACCEPTANCE_NONCE_UNKNOWN", - "MEASUREMENT_FIELDS", - "AcceptanceVerdict", - "BindingCheck", - "ResultSurfaceError", - "SurfacedResult", - "evaluate_acceptance", - "extract_envelope", - "parse_result_payload", - "surface_result", - "verify_report_data_binding", -] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/review.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/review.py deleted file mode 100644 index 3473da01e..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/review.py +++ /dev/null @@ -1,437 +0,0 @@ -"""Strict offline-testable deployment adapter for the canonical review CVM.""" - -from __future__ import annotations - -import re -from collections.abc import Mapping -from dataclasses import dataclass, field -from hashlib import sha256 -from typing import Any, Protocol - -from dstack_sdk import EnvVar, encrypt_env_vars_sync - -from agent_challenge.review.compose import ( - DEFAULT_REVIEW_APP_IDENTITY, - REVIEW_ALLOWED_ENVS, - generate_review_app_compose, - render_review_app_compose, - review_app_compose_hash, -) -from agent_challenge.review.deployment import ReviewDeploymentError as ReviewAcknowledgementError -from agent_challenge.review.deployment import ( - build_review_deployed_acknowledgement, - validate_review_deployed_acknowledgement, -) -from agent_challenge.review.schemas import validate_review_assignment -from agent_challenge.review.urls import ( - PINNED_REVIEW_API_BASE_URL, - ReviewApiBaseUrlError, - assert_pinned_review_api_base_url, -) -from agent_challenge.selfdeploy.measurements import ( - ProvisionOsIdentityError, - verify_provision_os_identity, -) -from agent_challenge.selfdeploy.phala import ( - extract_cvm_id_from_create_response, - resolve_cvm_id_from_list, -) -from agent_challenge.selfdeploy.shapes import ( - DEFAULT_INSTANCE_TYPE, - DEFAULT_OS_IMAGE, - DEFAULT_REVIEW_DISK_SIZE_GB, - validate_cpu_only, - validate_disk_size, -) - -#: Capacity-safe default (bare ``us-west`` → ERR-02-002 No teepod found). -DEFAULT_REGION = "us-west-1" - -#: Phala KMS uses a 40-hex deterministic app_id when ``nonce`` is supplied. -#: Product pins that hex as assignment ``app_identity`` (same string as create -#: receipt app_id). Never invent moniker→hex melt mapping: compose ``name`` stays -#: the product moniker for stable compose_hash, while provision ``app_id`` is the -#: pinned hex. Random moniker-only provision mints unstable hex + KMS pub each call. -_APP_ID_HEX40_RE = re.compile(r"^[0-9a-f]{40}$") - -#: Default provision nonce for deterministic Phala app_id (PHALA KMS only). -DEFAULT_PHALA_APP_NONCE = 0 - - -class ReviewDeploymentError(ReviewAcknowledgementError): - """A locally prepared review deployment violates its signed assignment.""" - - -@dataclass(frozen=True) -class ReviewDeploymentPlan: - """Exact review deployment material, with plaintext capability hidden from repr.""" - - assignment: dict[str, Any] - compose: dict[str, Any] - compose_text: str - compose_hash: str - app_identity: str - image_ref: str - kms_public_key_hex: str - kms_public_key_sha256: str - measurement: dict[str, str] - measurement_allowlist_sha256: str - review_session_token: str = field(repr=False) - instance_type: str = DEFAULT_INSTANCE_TYPE - region: str = DEFAULT_REGION - os_image: str = DEFAULT_OS_IMAGE - disk_size_gb: int = DEFAULT_REVIEW_DISK_SIZE_GB - #: Stable moniker measured into app-compose ``name`` (compose_hash binding). - #: Distinct from :attr:`app_identity` when the latter is a Phala 40-hex app_id. - compose_name: str = DEFAULT_REVIEW_APP_IDENTITY - #: Nonce for deterministic Phala app_id when app_identity is 40-hex. - #: None when identity is moniker-only (tests/legacy offline). - phala_app_nonce: int | None = None - - -@dataclass(frozen=True) -class EncryptedReviewSecrets: - """Ciphertext-only secret delivery payload for the Phala create request.""" - - ciphertext: str - env_keys: tuple[str, ...] - assignment_id: str - app_identity: str - kms_public_key_sha256: str - measurement_allowlist_sha256: str - - -class PhalaPost(Protocol): - """Minimal production boundary used for provision/create request transmission. - - Implementations may also expose ``get(path)`` for create-ack list fallback. - """ - - def post(self, path: str, payload: Mapping[str, Any]) -> Mapping[str, Any]: - """POST one request and return a decoded response mapping.""" - - -def build_review_deployment_plan(prepare_response: Mapping[str, Any]) -> ReviewDeploymentPlan: - """Verify signed Review assignment v1 and derive its exact deployed compose.""" - - # The production route wraps the exact assignment in stable identity fields - # (session_id, assignment_id, attempt). The older low-level adapter accepts - # the two-field form used by offline callers. Neither form allows caller - # supplied image/KMS/measurement overrides. - if set(prepare_response) == {"assignment", "review_session_token"}: - assignment = prepare_response["assignment"] - token = prepare_response["review_session_token"] - elif { - "session_id", - "assignment_id", - "attempt", - "assignment", - "review_session_token", - } == set(prepare_response): - assignment = prepare_response["assignment"] - token = prepare_response["review_session_token"] - if ( - not isinstance(assignment, Mapping) - or assignment.get("assignment_core", {}).get("assignment_id") - != prepare_response["assignment_id"] - ): - raise ReviewDeploymentError("review prepare identity does not match assignment") - else: - raise ReviewDeploymentError( - "prepare response must contain only assignment and one-time token" - ) - if not isinstance(assignment, Mapping) or not isinstance(token, str) or not token: - raise ReviewDeploymentError("prepare response lacks immutable assignment or session token") - try: - validate_review_assignment(assignment) - except Exception as exc: - raise ReviewDeploymentError("review assignment is invalid") from exc - core = assignment["assignment_core"] - review_app = core["review_app"] - if sha256(token.encode("utf-8")).hexdigest() != core["session_token_sha256"]: - raise ReviewDeploymentError("review session token is not bound to assignment") - try: - instance_type = validate_cpu_only( - instance_type=str(review_app["measurement"]["vm_shape"]).replace("-", "."), - os_image=DEFAULT_OS_IMAGE, - ).name - except (KeyError, TypeError, ValueError) as exc: - raise ReviewDeploymentError( - "review assignment does not identify a CPU Intel TDX shape" - ) from exc - - # Compose ``name`` does NOT flip to Phala's 40-hex app_id: changing it would - # rehash compose_hash. Production pins keep product moniker for compose bytes - # and pin the Phala deterministic hex as app_identity for provision/create. - app_identity = str(review_app["app_identity"]) - compose_name = DEFAULT_REVIEW_APP_IDENTITY - if _APP_ID_HEX40_RE.fullmatch(app_identity.lower()): - app_identity = app_identity.lower() - phala_app_nonce: int | None = DEFAULT_PHALA_APP_NONCE - else: - # Legacy moniker-only identity (unit/offline tests): compose name binds - # the signed moniker and provision uses moniker without nonce. - compose_name = app_identity - phala_app_nonce = None - - compose = generate_review_app_compose( - review_image=review_app["image_ref"], - app_identity=compose_name, - ) - compose_hash = review_app_compose_hash(compose) - if compose_hash != review_app["compose_hash"]: - raise ReviewDeploymentError( - "signed review compose hash does not match canonical deployment" - ) - allowlist_sha256 = review_app.get("measurement_allowlist_sha256") - if not isinstance(allowlist_sha256, str) or not allowlist_sha256: - raise ReviewDeploymentError( - "signed review assignment is missing bound measurement allowlist identity" - ) - allowlist = review_app.get("measurement_allowlist") - if not isinstance(allowlist, list) or not allowlist: - raise ReviewDeploymentError( - "signed review assignment is missing bound measurement allowlist entries" - ) - return ReviewDeploymentPlan( - assignment=dict(assignment), - compose=compose, - compose_text=render_review_app_compose(compose), - compose_hash=compose_hash, - app_identity=app_identity, - image_ref=review_app["image_ref"], - kms_public_key_hex=review_app["kms_public_key_hex"], - kms_public_key_sha256=review_app["kms_public_key_sha256"], - measurement=dict(review_app["measurement"]), - measurement_allowlist_sha256=allowlist_sha256, - review_session_token=token, - instance_type=instance_type, - os_image=DEFAULT_OS_IMAGE, - compose_name=compose_name, - phala_app_nonce=phala_app_nonce, - disk_size_gb=DEFAULT_REVIEW_DISK_SIZE_GB, - ) - - -def encrypt_review_secrets( - plan: ReviewDeploymentPlan, - secrets: Mapping[str, str], -) -> EncryptedReviewSecrets: - """Encrypt the allowed non-empty review secrets only to the signed X25519 key.""" - - if set(secrets) != set(REVIEW_ALLOWED_ENVS): - raise ReviewDeploymentError("review encrypted_env names must be exactly the allowed names") - values = {name: secrets[name] for name in REVIEW_ALLOWED_ENVS} - if any(not isinstance(value, str) or not value for value in values.values()): - raise ReviewDeploymentError("review encrypted_env values must be non-empty strings") - if values["REVIEW_SESSION_TOKEN"] != plan.review_session_token: - raise ReviewDeploymentError("review session token does not match signed prepare response") - # Hard-pin measured callback authority. Miner-supplied non-joinbase values - # fail closed; honest joinbase (with optional trailing slash) is accepted. - # Override authority only with CHALLENGE_ALLOW_DEV_URLS=1 (non-prod). - try: - values["REVIEW_API_BASE_URL"] = assert_pinned_review_api_base_url( - values["REVIEW_API_BASE_URL"] - ) - except ReviewApiBaseUrlError as exc: - raise ReviewDeploymentError(str(exc)) from exc - try: - ciphertext = encrypt_env_vars_sync( - [EnvVar(key=name, value=values[name]) for name in REVIEW_ALLOWED_ENVS], - plan.kms_public_key_hex, - ) - except Exception as exc: - raise ReviewDeploymentError("review encrypted_env encryption failed") from exc - if not ciphertext: - raise ReviewDeploymentError("review encrypted_env ciphertext is empty") - return EncryptedReviewSecrets( - ciphertext=ciphertext, - env_keys=REVIEW_ALLOWED_ENVS, - assignment_id=plan.assignment["assignment_core"]["assignment_id"], - app_identity=plan.app_identity, - kms_public_key_sha256=plan.kms_public_key_sha256, - measurement_allowlist_sha256=plan.measurement_allowlist_sha256, - ) - - -class HttpReviewPhalaDeployment: - """Transmit canonical review provision/create requests through an injected API.""" - - def __init__(self, api: PhalaPost) -> None: - self._api = api - - def deploy( - self, - plan: ReviewDeploymentPlan, - encrypted: EncryptedReviewSecrets, - ) -> dict[str, str]: - """Provision exact compose identity then create with ciphertext only.""" - - from agent_challenge.evaluation.no_phala import refuse_phala_client - - refuse_phala_client("HttpReviewPhalaDeployment.deploy") - - if ( - encrypted.assignment_id != plan.assignment["assignment_core"]["assignment_id"] - or encrypted.app_identity != plan.app_identity - or encrypted.kms_public_key_sha256 != plan.kms_public_key_sha256 - or encrypted.measurement_allowlist_sha256 != plan.measurement_allowlist_sha256 - or encrypted.env_keys != REVIEW_ALLOWED_ENVS - or not encrypted.ciphertext - ): - raise ReviewDeploymentError("review encrypted_env is not bound to this assignment") - provision_request: dict[str, Any] = { - # Phala identity: when pin is a 40-hex deterministic app_id, send it - # with the matching nonce for stable KMS pubkey. Compose name stays - # the product moniker so offline compose_hash matches live. - "app_id": plan.app_identity, - "name": plan.compose_name, - "instance_type": plan.instance_type, - "region": plan.region, - "compose_file": plan.compose, - "env_keys": list(encrypted.env_keys), - "image": plan.os_image, - # Sibling of compose_file — never mutate plan.compose. - "disk_size": validate_disk_size(plan.disk_size_gb), - } - if plan.phala_app_nonce is not None: - provision_request["nonce"] = plan.phala_app_nonce - provision = self._api.post("/cvms/provision", provision_request) - self._verify_provision_response(plan, provision) - create_request = { - "app_id": plan.app_identity, - "compose_hash": plan.compose_hash, - "encrypted_env": encrypted.ciphertext, - "env_keys": list(encrypted.env_keys), - } - created = self._api.post("/cvms", create_request) - cvm_id = self._resolve_created_cvm_id(plan, created) - request_id = created.get("request_id") - if not isinstance(request_id, str) or not request_id: - # Numeric create id also serves as request identity when API - # omits a separate request_id field (live cloud schema). - request_id = cvm_id - created_at_ms = created.get("created_at_ms") - if not isinstance(created_at_ms, int) or isinstance(created_at_ms, bool): - created_at_ms = 0 - acknowledgement = build_review_deployed_acknowledgement( - assignment=plan.assignment, - cvm_id=cvm_id, - request_id=request_id, - receipt_sha256=sha256(repr(sorted(created.items())).encode("utf-8")).hexdigest(), - created_at_ms=created_at_ms, - ) - try: - validate_review_deployed_acknowledgement(plan.assignment, acknowledgement) - except ReviewAcknowledgementError as exc: - raise ReviewDeploymentError("generated review acknowledgement is invalid") from exc - return acknowledgement - - def _resolve_created_cvm_id( - self, - plan: ReviewDeploymentPlan, - created: Mapping[str, Any], - ) -> str: - """Map create response (or safe app_id list fallback) to a CVM id string. - - Live Phala create schema returns numeric ``id`` plus ``app_id`` (app pin). - Fail closed only when neither create fields nor GET ``/cvms`` listing by - ``app_id`` identify a CVM. Never invent measurements or ids. - """ - - try: - return extract_cvm_id_from_create_response(created) - except ValueError: - pass - getter = getattr(self._api, "get", None) - if not callable(getter): - raise ReviewDeploymentError("Phala create response does not identify the review CVM") - try: - listing = getter("/cvms") - except Exception as exc: - raise ReviewDeploymentError( - "Phala create response does not identify the review CVM" - ) from exc - if not isinstance(listing, Mapping): - raise ReviewDeploymentError("Phala create response does not identify the review CVM") - resolved = resolve_cvm_id_from_list(listing, app_id=plan.app_identity) - if resolved is None: - raise ReviewDeploymentError("Phala create response does not identify the review CVM") - return resolved - - @staticmethod - def _verify_provision_response( - plan: ReviewDeploymentPlan, - provision: Mapping[str, Any], - ) -> None: - if provision.get("compose_hash") != plan.compose_hash: - raise ReviewDeploymentError("Phala provision compose hash mismatches signed assignment") - # Honest identity check: compare to the pin, never invent moniker melt. - # Production pins MUST be the real Phala 40-hex app_id so equality holds. - # Moniker-only offline fixtures still pass equality against moniker responses. - provision_app = provision.get("app_id") - if not isinstance(provision_app, str) or provision_app != plan.app_identity: - raise ReviewDeploymentError("Phala provision app identity mismatches signed assignment") - if provision.get("app_env_encrypt_pubkey") != plan.kms_public_key_hex: - raise ReviewDeploymentError("Phala provision key mismatches signed assignment") - try: - verify_provision_os_identity( - measurement=plan.measurement, - provision_os=provision.get("os_image_hash"), - mismatch_message=( - "Phala provision os_image_hash mismatches signed assignment measurement" - ), - ) - except ProvisionOsIdentityError as exc: - raise ReviewDeploymentError(str(exc)) from exc - - -class ReviewPhalaDeployment(HttpReviewPhalaDeployment): - """In-memory Phala adapter used to capture the exact offline request contract.""" - - def __init__( - self, - *, - provision_response: Mapping[str, Any], - create_response: Mapping[str, Any], - list_response: Mapping[str, Any] | None = None, - ) -> None: - self.provision_response = dict(provision_response) - self.create_response = dict(create_response) - self.list_response = dict(list_response) if list_response is not None else None - self.provision_requests: list[dict[str, Any]] = [] - self.create_requests: list[dict[str, Any]] = [] - self.get_paths: list[str] = [] - super().__init__(self) - - def post(self, path: str, payload: Mapping[str, Any]) -> Mapping[str, Any]: - request = dict(payload) - if path == "/cvms/provision": - self.provision_requests.append(request) - return self.provision_response - if path == "/cvms": - self.create_requests.append(request) - return self.create_response - raise AssertionError(f"unexpected Phala API path {path}") - - def get(self, path: str) -> Mapping[str, Any]: - self.get_paths.append(path) - if path != "/cvms" or self.list_response is None: - raise AssertionError(f"unexpected Phala GET path {path}") - return self.list_response - - -__all__ = [ - "DEFAULT_OS_IMAGE", - "DEFAULT_PHALA_APP_NONCE", - "DEFAULT_REGION", - "PINNED_REVIEW_API_BASE_URL", - "REVIEW_ALLOWED_ENVS", - "EncryptedReviewSecrets", - "HttpReviewPhalaDeployment", - "ReviewDeploymentError", - "ReviewDeploymentPlan", - "ReviewPhalaDeployment", - "build_review_deployment_plan", - "encrypt_review_secrets", -] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/run.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/run.py deleted file mode 100644 index ef9c72204..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/run.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Run the canonical eval and surface its result, failing closed (VAL-DEPLOY-011). - -The miner points the run at the validator key-release endpoint; the in-CVM -``own_runner`` backend obtains the golden key from *exactly* that endpoint before -scoring and fails closed (a parseable score-0 result, no attestation) if the -endpoint is unreachable or denies the quote. This module drives that backend with -the endpoint wired in, captures its single ``BASE_BENCHMARK_RESULT=`` line, and -surfaces the outcome so a misconfigured endpoint yields a clear miner-facing error -and NO fabricated attested result. - -The backend is injectable so the flow is testable offline; the default runner is -:func:`agent_challenge.evaluation.own_runner_backend.main` (imported lazily to -keep this module light). -""" - -from __future__ import annotations - -import contextlib -import io -import os -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass - -from agent_challenge.keyrelease.client import KEY_RELEASE_FAILED_REASON, KEY_RELEASE_URL_ENV -from agent_challenge.selfdeploy.result import ResultSurfaceError, SurfacedResult, surface_result - -#: Signature of a backend runner: argv -> exit code (writes the result line to stdout). -BackendMain = Callable[[Sequence[str]], int] - - -class RunError(RuntimeError): - """The eval run could not be executed at all (before any result line).""" - - -@dataclass(frozen=True) -class RunOutcome: - """The outcome of a self-deploy eval run.""" - - exit_code: int - attested: bool - surfaced: SurfacedResult | None - stdout: str - clear_error: str | None - - @property - def succeeded(self) -> bool: - """True only when the run produced a genuine attested result.""" - - return self.exit_code == 0 and self.attested - - -@contextlib.contextmanager -def _patched_environ(overrides: Mapping[str, str]): - """Temporarily apply ``overrides`` to ``os.environ`` and restore afterwards.""" - - saved: dict[str, str | None] = {} - try: - for key, value in overrides.items(): - saved[key] = os.environ.get(key) - os.environ[key] = value - yield - finally: - for key, previous in saved.items(): - if previous is None: - os.environ.pop(key, None) - else: - os.environ[key] = previous - - -def _default_backend_main() -> BackendMain: - from agent_challenge.evaluation.own_runner_backend import main as backend_main - - return backend_main - - -def run_eval( - *, - job_dir: str, - task_ids: Sequence[str], - key_release_url: str, - binding_env: Mapping[str, str] | None = None, - extra_args: Sequence[str] = (), - backend_main: BackendMain | None = None, -) -> RunOutcome: - """Run the eval with the key-release endpoint wired in; fail closed on failure. - - Wires ``key_release_url`` into the backend via - :data:`~agent_challenge.keyrelease.client.KEY_RELEASE_URL_ENV` (so the run - requests golden material from exactly that endpoint), invokes the backend, - captures its result line, and surfaces the outcome. When the endpoint is - unreachable or denies the quote the backend fails closed and this returns a - :class:`RunOutcome` with ``attested=False`` and a clear error, surfacing NO - attested result (VAL-DEPLOY-011). - """ - - if not isinstance(key_release_url, str) or not key_release_url.strip(): - raise RunError("a validator key-release endpoint URL is required to run the eval") - if not task_ids: - raise RunError("at least one --task is required to run the eval") - - runner = backend_main if backend_main is not None else _default_backend_main() - - argv: list[str] = ["run", "--job-dir", job_dir] - for task_id in task_ids: - argv += ["--task", task_id] - argv += list(extra_args) - - overrides: dict[str, str] = {KEY_RELEASE_URL_ENV: key_release_url.strip()} - if binding_env: - overrides.update({str(k): str(v) for k, v in binding_env.items()}) - - buffer = io.StringIO() - with _patched_environ(overrides), contextlib.redirect_stdout(buffer): - exit_code = int(runner(argv)) - stdout = buffer.getvalue() - - try: - surfaced = surface_result(stdout) - except ResultSurfaceError: - return RunOutcome( - exit_code=exit_code, - attested=False, - surfaced=None, - stdout=stdout, - clear_error=( - "the eval produced no parseable result line; no golden key was obtained and " - "no attested result was produced" - ), - ) - - if surfaced.attested and exit_code == 0: - return RunOutcome( - exit_code=exit_code, - attested=True, - surfaced=surfaced, - stdout=stdout, - clear_error=None, - ) - - reason = surfaced.reason_code or "unknown" - if reason == KEY_RELEASE_FAILED_REASON: - message = ( - "key-release failed against the configured validator endpoint " - f"({key_release_url.strip()!r}): no golden key was obtained, so the eval failed closed " - "and produced NO attested result or score" - ) - else: - message = ( - f"the eval failed closed (reason_code={reason!r}); no attested result was produced" - ) - return RunOutcome( - exit_code=exit_code, - attested=False, - surfaced=surfaced, - stdout=stdout, - clear_error=message, - ) - - -__all__ = [ - "BackendMain", - "RunError", - "RunOutcome", - "run_eval", -] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/shapes.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/shapes.py deleted file mode 100644 index e5a3a8ce2..000000000 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/shapes.py +++ /dev/null @@ -1,285 +0,0 @@ -"""CPU Intel TDX shape catalog + money/GPU/disk deploy guards (AGENTS.md boundaries). - -The mission is **CPU Intel TDX only** (no GPU, not available to this account) with -a hard **$20** spend cap. Stage defaults are split: - -* **review** — ``tdx.small`` (1 vCPU / 2 GiB) with **20 GB** disk (light analyzer); -* **eval** — ``tdx.xlarge`` (8 vCPU / 16 GiB) with **100 GB** disk so four concurrent - Terminal-Bench tasks (1 vCPU / 2 GiB each) leave headroom for the orchestrator. - -Disk is **stage policy** (constants + :func:`validate_disk_size`), not a field on -every :class:`CpuShape`. Phala bills disk separately at -:data:`DISK_USD_PER_GB_HOUR`; projected cost = compute hours + disk hours. - -This module is the single source of truth for the CPU shape catalog and the pure, -side-effect-free guard functions the deploy path runs BEFORE any provisioning: - -* :func:`validate_cpu_only` refuses a GPU instance type (e.g. ``h200.small``) or a - GPU OS image (e.g. ``dstack-nvidia-*``) and any unknown shape (VAL-DEPLOY-007); -* :func:`select_default_instance_type` picks the review default when the miner - gives none (VAL-DEPLOY-008); -* :func:`validate_disk_size` refuses disk outside ``[MIN_DISK_SIZE_GB, MAX_DISK_SIZE_GB]``; -* :func:`validate_within_cap` refuses a shape whose projected cost (compute + optional - disk) would breach the money cap (VAL-DEPLOY-008). - -Hourly compute rates are the account's observed CPU TDX prices (library/phala.md). -Disk billing is the observed Phala rate (~$0.000139/GB/hour); live tdx.* default -disk is 20 GB when the provision body omits ``disk_size``. -""" - -from __future__ import annotations - -import math -import re -from dataclasses import dataclass - - -class ShapeError(ValueError): - """A requested VM shape/OS image is not deployable under the mission boundaries.""" - - -class GpuRefusedError(ShapeError): - """A GPU instance type or GPU OS image was requested (mission is CPU-only).""" - - -class OverCapError(ShapeError): - """The requested shape's projected cost would breach the money cap.""" - - -class DiskSizeError(ShapeError): - """A requested disk size is outside the allowed stage bounds.""" - - -@dataclass(frozen=True) -class CpuShape: - """A CPU Intel TDX shape: vCPU/RAM and the account's observed hourly USD rate. - - Disk is intentionally **not** a per-shape field — stage policy owns disk size - via :data:`DEFAULT_REVIEW_DISK_SIZE_GB` / :data:`DEFAULT_EVAL_DISK_SIZE_GB`. - """ - - name: str - vcpus: int - memory_gib: int - usd_per_hour: float - - -#: CPU Intel TDX shapes (library/phala.md), smallest first. -CPU_TDX_SHAPES: dict[str, CpuShape] = { - "tdx.small": CpuShape("tdx.small", 1, 2, 0.058), - "tdx.medium": CpuShape("tdx.medium", 2, 4, 0.116), - "tdx.large": CpuShape("tdx.large", 4, 8, 0.232), - "tdx.xlarge": CpuShape("tdx.xlarge", 8, 16, 0.464), -} - -#: The smallest CPU shapes the mission prefers for light stages (AGENTS.md). -SMALLEST_CPU_SHAPES: tuple[str, ...] = ("tdx.small", "tdx.medium") - -#: Review-stage default: light analyzer CVM. -DEFAULT_REVIEW_INSTANCE_TYPE = "tdx.small" - -#: Eval-stage default: 8 vCPU / 16 GiB so 4 concurrent 1-vCPU/2-GiB tasks fit. -DEFAULT_EVAL_INSTANCE_TYPE = "tdx.xlarge" - -#: Backward-compatible alias of the review default (legacy single-stage deploy). -DEFAULT_INSTANCE_TYPE = DEFAULT_REVIEW_INSTANCE_TYPE - -#: Default review disk (GB). Matches Phala's live tdx.* default when omitted. -DEFAULT_REVIEW_DISK_SIZE_GB = 20 - -#: Default eval disk (GB). Larger for DooD task images + concurrent containers. -DEFAULT_EVAL_DISK_SIZE_GB = 100 - -#: Observed Phala disk billing rate (USD per GB per hour). -DISK_USD_PER_GB_HOUR = 0.000139 - -#: Inclusive lower bound for provision ``disk_size`` (GB). -MIN_DISK_SIZE_GB = 20 - -#: Inclusive upper bound for provision ``disk_size`` (GB). -MAX_DISK_SIZE_GB = 500 - -#: Default CPU dstack OS image. Live teepods (prod5/prod9) currently ship up to -#: dstack-0.5.9 (product default was 0.5.10 but that image is not mounted on -#: available nodes → provision ports a different os_image_hash and dual-flag -#: allowlists fail closed). Prefer real non-dev 0.5.9. GPU images are refused. -DEFAULT_OS_IMAGE = "dstack-0.5.9" - -#: Hard mission spend cap (USD). -DEFAULT_MONEY_CAP_USD = 20.0 - -#: Conservative projected max runtime (hours) used for the cost-cap guard. A -#: deploy's projected cost is compute + optional disk over this window; a shape -#: whose projected cost exceeds the money cap is refused before provisioning. -DEFAULT_MAX_RUNTIME_HOURS = 6.0 - -#: GPU instance-type prefixes/markers that are always refused (CPU-only mission). -_GPU_INSTANCE_RE = re.compile(r"^(?:h100|h200|a100|a10|l40|l4|t4|rtx)", re.IGNORECASE) - -#: GPU OS image markers (e.g. ``dstack-nvidia-*``) that are always refused. -_GPU_IMAGE_RE = re.compile(r"nvidia|gpu|cuda", re.IGNORECASE) - - -def is_gpu_instance_type(instance_type: str) -> bool: - """Whether ``instance_type`` names a GPU shape (refused on a CPU-only mission).""" - - token = (instance_type or "").strip().lower() - if not token: - return False - if "gpu" in token or "nvidia" in token: - return True - return bool(_GPU_INSTANCE_RE.match(token)) - - -def is_gpu_os_image(os_image: str) -> bool: - """Whether ``os_image`` is a GPU/NVIDIA dstack image (refused CPU-only).""" - - return bool(_GPU_IMAGE_RE.search(os_image or "")) - - -def validate_cpu_only(*, instance_type: str, os_image: str = DEFAULT_OS_IMAGE) -> CpuShape: - """Refuse GPU/unknown targets; return the resolved CPU shape (VAL-DEPLOY-007). - - Raises :class:`GpuRefusedError` for a GPU instance type or GPU OS image, and - :class:`ShapeError` for an unknown (non-catalog) CPU shape. Pure and - side-effect free: it never touches Phala, so a refusal makes zero provision - calls. - """ - - if is_gpu_os_image(os_image): - raise GpuRefusedError( - f"CPU-only mission: refusing GPU OS image {os_image!r}; use a CPU dstack image " - f"(default {DEFAULT_OS_IMAGE!r}). No GPU deploys are permitted." - ) - if is_gpu_instance_type(instance_type): - raise GpuRefusedError( - f"CPU-only mission: refusing GPU instance type {instance_type!r}; use a CPU Intel " - f"TDX shape (one of {sorted(CPU_TDX_SHAPES)}). No GPU deploys are permitted." - ) - shape = CPU_TDX_SHAPES.get((instance_type or "").strip()) - if shape is None: - raise ShapeError( - f"unknown CPU Intel TDX shape {instance_type!r}; expected one of " - f"{sorted(CPU_TDX_SHAPES)}" - ) - return shape - - -def select_default_instance_type() -> str: - """The review-stage default used when the miner supplies none (VAL-DEPLOY-008).""" - - return DEFAULT_INSTANCE_TYPE - - -def validate_disk_size(gb: object) -> int: - """Refuse non-integer or out-of-range disk sizes; return a validated int GB. - - Bounds are inclusive ``[MIN_DISK_SIZE_GB, MAX_DISK_SIZE_GB]``. Pure and - side-effect free. - """ - - if isinstance(gb, bool) or not isinstance(gb, int): - raise DiskSizeError( - f"disk_size_gb must be an integer in " - f"[{MIN_DISK_SIZE_GB}, {MAX_DISK_SIZE_GB}]; got {gb!r}" - ) - if gb < MIN_DISK_SIZE_GB or gb > MAX_DISK_SIZE_GB: - raise DiskSizeError( - f"disk_size_gb={gb} outside allowed range " - f"[{MIN_DISK_SIZE_GB}, {MAX_DISK_SIZE_GB}]" - ) - return gb - - -def projected_disk_cost_usd( - disk_size_gb: int, - *, - max_runtime_hours: float = DEFAULT_MAX_RUNTIME_HOURS, -) -> float: - """Projected disk cost = ``DISK_USD_PER_GB_HOUR * gb * hours``.""" - - size = validate_disk_size(disk_size_gb) - if not math.isfinite(max_runtime_hours) or max_runtime_hours < 0: - raise ShapeError("max_runtime_hours must be a finite non-negative number") - return DISK_USD_PER_GB_HOUR * float(size) * float(max_runtime_hours) - - -def projected_cost_usd( - instance_type: str, - *, - max_runtime_hours: float = DEFAULT_MAX_RUNTIME_HOURS, - disk_size_gb: int | None = None, -) -> float: - """Projected deploy cost = compute hours + optional disk hours for a CPU shape.""" - - shape = CPU_TDX_SHAPES.get((instance_type or "").strip()) - if shape is None: - raise ShapeError(f"unknown CPU Intel TDX shape {instance_type!r}") - if not math.isfinite(max_runtime_hours) or max_runtime_hours < 0: - raise ShapeError("max_runtime_hours must be a finite non-negative number") - total = shape.usd_per_hour * float(max_runtime_hours) - if disk_size_gb is not None: - total += projected_disk_cost_usd(disk_size_gb, max_runtime_hours=max_runtime_hours) - return total - - -def validate_within_cap( - instance_type: str, - *, - money_cap_usd: float = DEFAULT_MONEY_CAP_USD, - max_runtime_hours: float = DEFAULT_MAX_RUNTIME_HOURS, - disk_size_gb: int | None = None, -) -> float: - """Refuse a shape whose projected cost breaches the money cap (VAL-DEPLOY-008). - - Returns the projected cost when within cap; raises :class:`OverCapError` - otherwise. Pure/side-effect-free (no Phala call), so an over-cap request is - refused before any provisioning. - """ - - cost = projected_cost_usd( - instance_type, - max_runtime_hours=max_runtime_hours, - disk_size_gb=disk_size_gb, - ) - if cost > money_cap_usd: - disk_note = "" - if disk_size_gb is not None: - disk_note = f" + disk {disk_size_gb}GB" - raise OverCapError( - f"projected cost ${cost:.2f} ({instance_type}{disk_note} @ " - f"${CPU_TDX_SHAPES[instance_type].usd_per_hour}/h x {max_runtime_hours}h) " - f"exceeds the ${money_cap_usd:.2f} money cap; choose a smaller shape or lower " - "the runtime budget" - ) - return cost - - -__all__ = [ - "CPU_TDX_SHAPES", - "DEFAULT_EVAL_DISK_SIZE_GB", - "DEFAULT_EVAL_INSTANCE_TYPE", - "DEFAULT_INSTANCE_TYPE", - "DEFAULT_MAX_RUNTIME_HOURS", - "DEFAULT_MONEY_CAP_USD", - "DEFAULT_OS_IMAGE", - "DEFAULT_REVIEW_DISK_SIZE_GB", - "DEFAULT_REVIEW_INSTANCE_TYPE", - "DISK_USD_PER_GB_HOUR", - "MAX_DISK_SIZE_GB", - "MIN_DISK_SIZE_GB", - "SMALLEST_CPU_SHAPES", - "CpuShape", - "DiskSizeError", - "GpuRefusedError", - "OverCapError", - "ShapeError", - "is_gpu_instance_type", - "is_gpu_os_image", - "projected_cost_usd", - "projected_disk_cost_usd", - "select_default_instance_type", - "validate_cpu_only", - "validate_disk_size", - "validate_within_cap", -] diff --git a/packages/challenges/agent-challenge/tests/phala_quote_golden_vector.json b/packages/challenges/agent-challenge/tests/phala_quote_golden_vector.json deleted file mode 100644 index 5def83417..000000000 --- a/packages/challenges/agent-challenge/tests/phala_quote_golden_vector.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "description": "Shared cross-repo golden TDX quote/measurement/RTMR3 anti-drift vector. A fixed v4 TDX quote + dstack RTMR3 event log with the exact registers, os_image_hash (sha256(MRTD||RTMR1||RTMR2)) and RTMR3/compose-hash a correct parser must reproduce. Asserted BYTE-IDENTICALLY in both base (tests/unit/test_phala_quote_golden_vector.py) and agent-challenge (tests/test_quote_golden_vector.py). A one-sided offset/hash tweak to either base.worker.phala_quote or agent_challenge.keyrelease.quote makes that repo's parse of this fixed quote diverge from these pinned values, failing a test on that side. Do NOT edit one repo's copy without the other (see AGENTS.md 'TDX-quote-parse / measurement / RTMR3 anti-drift').", - "event_log": [ - { - "digest": "2851989bdb17f204310f1c0ef0ed4bf8ffbd3ed455af43ce0f438e94356c858b8746a15516c7c0102a58bb71d67f51e2", - "event": "compose-hash", - "event_payload": "abababababababababababababababababababababababababababababababab", - "event_type": 134217729, - "imr": 3 - }, - { - "digest": "0c365c23d8f1ad06b7bb3c843b1d3d7b0b8347a0f175e71c118241d8048005fe3538cc05eb89c52250bad3244646b739", - "event": "key-provider", - "event_payload": "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", - "event_type": 134217729, - "imr": 3 - } - ], - "expected": { - "compose_hash": "abababababababababababababababababababababababababababababababab", - "key_provider": "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", - "mrtd": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", - "os_image_hash": "90ac6e18901b222f3a58a45231a39545d14a9cd59854cc3eafa683083f0e11d0", - "report_data_hex": "55555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555", - "rtmr0": "222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222", - "rtmr1": "333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333", - "rtmr2": "444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444", - "rtmr3": "7934b424e98de2a6fba5c468ee804fd575de5d6cb6201f78416a976b23e616e1c74aa9e28a1d4e0c55a65b849007ae68" - }, - "quote_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222223333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333334444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444447934b424e98de2a6fba5c468ee804fd575de5d6cb6201f78416a976b23e616e1c74aa9e28a1d4e0c55a65b849007ae6855555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555555" -} diff --git a/packages/challenges/agent-challenge/tests/test_ac_execution_pool_live.py b/packages/challenges/agent-challenge/tests/test_ac_execution_pool_live.py index 0a0c5d1b1..87be559ab 100644 --- a/packages/challenges/agent-challenge/tests/test_ac_execution_pool_live.py +++ b/packages/challenges/agent-challenge/tests/test_ac_execution_pool_live.py @@ -219,9 +219,7 @@ async def test_execution_pool_live_shows_running_eval_with_latest_event( ) -> None: """In-flight eval_running appears with latest progress event; observability only.""" - live = await _seed_run( - database_session, eval_run_id="eval-pool-live", phase="eval_running" - ) + live = await _seed_run(database_session, eval_run_id="eval-pool-live", phase="eval_running") await _seed_progress_event( database_session, submission_id=live.submission_id, @@ -300,9 +298,7 @@ async def test_execution_pool_live_excludes_completed_eval(client, database_sess assert response.status_code == 200, response.text body = response.json() units = body["units"] - ids = { - str(item.get("unit_id") or item.get("eval_run_id") or "") for item in units - } + ids = {str(item.get("unit_id") or item.get("eval_run_id") or "") for item in units} assert live.eval_run_id in ids assert done.eval_run_id not in ids assert "finished-should-be-hidden" not in json.dumps(body) @@ -310,9 +306,7 @@ async def test_execution_pool_live_excludes_completed_eval(client, database_sess @pytest.mark.asyncio -async def test_execution_pool_live_never_exposes_score_fields( - client, database_session -) -> None: +async def test_execution_pool_live_never_exposes_score_fields(client, database_session) -> None: """Even when EvalRun.score is set on a non-terminal row, pool omits score keys.""" run = await _seed_run( @@ -333,8 +327,7 @@ async def test_execution_pool_live_never_exposes_score_fields( assert response.status_code == 200, response.text body = response.json() assert any( - str(u.get("unit_id") or u.get("eval_run_id")) == run.eval_run_id - for u in body["units"] + str(u.get("unit_id") or u.get("eval_run_id")) == run.eval_run_id for u in body["units"] ) _assert_no_score_keys(body) blob = response.text.lower() diff --git a/packages/challenges/agent-challenge/tests/test_acceptance_permanent_park_shortcircuit.py b/packages/challenges/agent-challenge/tests/test_acceptance_permanent_park_shortcircuit.py deleted file mode 100644 index 0ec98be52..000000000 --- a/packages/challenges/agent-challenge/tests/test_acceptance_permanent_park_shortcircuit.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Permanent-park short-circuit for the Phala acceptance gate (misc-hardening). - -Follow-up hardening on the M4 acceptance gate. A PERMANENT (non-retryable) -attestation park -- ``UNATTESTED`` or ``VERIFICATION_FAILED`` -- writes no -``TaskResult`` score row, so before this change the unit stayed pending and -``execute_work_unit`` re-ran the broker + re-gated it every validator cycle until -the coordination-plane ``max_attempts`` fold finally wrote a terminal failed -result. That wasted a full re-execution each cycle (a real money cost in the M6 -live model, which re-runs the eval on the miner's funded CVM). - -The gate now consults the recorded ``TaskAttestation.retryable`` flag: a prior -permanent (``retryable=False``) park short-circuits -- the unit folds directly to -a terminal ``failed`` result WITHOUT re-dispatching the broker; a retryable -(``VERIFIER_UNAVAILABLE``) park MUST still be retried. Flag-off is byte-identical -(the short-circuit never consults attestation bookkeeping). - -Discriminators: -* the permanent-park test FAILS against the pre-change code (which re-runs the - broker and re-parks the unit, writing no result); -* the retryable test FAILS against a naive impl that short-circuits every park; -* the flag-off test FAILS against an impl that short-circuits regardless of flag. -""" - -from __future__ import annotations - -from sqlalchemy import func, select -from test_challenge_acceptance_gating import ( - _attested_line, - _create_job, - _enable_phala, - _make_gate, - _patch_terminal_bench, - _plain_line, - _RecordingBroker, - _terminal_bench_tasks, -) - -from agent_challenge.evaluation.attestation import ( - ATTESTATION_MISSING, - AttestationVerifierUnavailable, -) -from agent_challenge.evaluation.validator_executor import ( - execute_work_unit, - get_task_attestation, -) -from agent_challenge.evaluation.work_units import list_pending_work_units -from agent_challenge.keyrelease.quote import StaticQuoteVerifier -from agent_challenge.models import TaskAttestation, TaskResult - - -class _UnavailableVerifier: - """A quote verifier that reports a transient outage (retryable park).""" - - def verify(self, quote_hex): - raise AttestationVerifierUnavailable("collateral fetch timed out") - - -# --------------------------------------------------------------------------- # -# A permanent (retryable=False) park short-circuits on the next cycle: the unit -# is NOT re-run through the broker, it folds directly to a terminal failed result. -# --------------------------------------------------------------------------- # -async def test_permanent_park_shortcircuits_no_broker_rerun( - database_session, monkeypatch, tmp_path -): - _patch_terminal_bench(monkeypatch, tmp_path) - _enable_phala(monkeypatch) - tasks = _terminal_bench_tasks(1) - task_id = tasks[0].task_id - async with database_session() as session: - _submission, job = await _create_job( - session, agent_hash="perm-park", tasks=tasks, tmp_path=tmp_path - ) - await session.commit() - job_pk = job.id - - gate = _make_gate(nonces=["some-nonce"]) - - # Cycle 1: a plain (unattested) result -> permanent park (UNATTESTED); the - # broker IS dispatched and NO TaskResult score row is written. - async with database_session() as session: - units = await list_pending_work_units(session) - broker1 = _RecordingBroker({task_id: _plain_line()}) - async with database_session() as session: - first = await execute_work_unit(session, units[0], executor=broker1, attestation_gate=gate) - await session.commit() - assert first.posted is False - assert first.retryable is False - assert broker1.runs == [task_id] - - async with database_session() as session: - record = await get_task_attestation(session, job_pk, task_id) - count_before = await session.scalar( - select(func.count(TaskResult.id)).where(TaskResult.job_id == job_pk) - ) - pending_before = await list_pending_work_units(session) - assert record is not None and record.verified is False and record.retryable is False - assert count_before == 0 - # No score row was written, so the unit is STILL pending for the next cycle. - assert [u.work_unit_id for u in pending_before] == [units[0].work_unit_id] - - # Cycle 2: the still-pending unit MUST short-circuit -- the broker is NOT - # re-dispatched and the unit folds directly to a terminal failed result. - broker2 = _RecordingBroker({task_id: _plain_line()}) - async with database_session() as session: - second = await execute_work_unit( - session, pending_before[0], executor=broker2, attestation_gate=gate - ) - await session.commit() - assert broker2.runs == [] - assert second.executed is False - assert second.posted is True - assert second.status == "failed" - assert second.score == 0.0 - - # Exactly one terminal failed result now exists and the unit is no longer - # pending (no further re-execution/re-gating cycles). - async with database_session() as session: - results = ( - (await session.execute(select(TaskResult).where(TaskResult.job_id == job_pk))) - .scalars() - .all() - ) - pending_after = await list_pending_work_units(session) - assert len(results) == 1 - assert results[0].status == "failed" - assert results[0].score == 0.0 - assert pending_after == [] - - -# --------------------------------------------------------------------------- # -# A retryable (VERIFIER_UNAVAILABLE) park is NOT short-circuited: the broker is -# re-dispatched on the next cycle and, with the verifier healthy, it verifies. -# --------------------------------------------------------------------------- # -async def test_retryable_park_is_still_retried(database_session, monkeypatch, tmp_path): - _patch_terminal_bench(monkeypatch, tmp_path) - _enable_phala(monkeypatch) - tasks = _terminal_bench_tasks(1) - task_id = tasks[0].task_id - async with database_session() as session: - submission, job = await _create_job( - session, agent_hash="retry-park", tasks=tasks, tmp_path=tmp_path - ) - await session.commit() - job_pk = job.id - agent_hash = submission.agent_hash - - nonce = "nonce-retry" - line = _attested_line(task_id, agent_hash=agent_hash, nonce=nonce) - - # Cycle 1: the quote verifier is transiently unavailable -> retryable park - # (no score row). The outage raises BEFORE the nonce is consumed. - gate1 = _make_gate(nonces=[nonce], verifier=_UnavailableVerifier()) - async with database_session() as session: - units = await list_pending_work_units(session) - broker1 = _RecordingBroker({task_id: line}) - async with database_session() as session: - first = await execute_work_unit(session, units[0], executor=broker1, attestation_gate=gate1) - await session.commit() - assert first.posted is False - assert first.retryable is True - assert broker1.runs == [task_id] - - async with database_session() as session: - record = await get_task_attestation(session, job_pk, task_id) - pending_before = await list_pending_work_units(session) - assert record is not None and record.verified is False and record.retryable is True - assert [u.work_unit_id for u in pending_before] == [units[0].work_unit_id] - - # Cycle 2: the retryable park MUST be retried -- the broker is re-dispatched - # and, with the verifier now healthy, the result verifies and is persisted. - gate2 = _make_gate(nonces=[nonce], verifier=StaticQuoteVerifier(valid=True)) - broker2 = _RecordingBroker({task_id: line}) - async with database_session() as session: - second = await execute_work_unit( - session, pending_before[0], executor=broker2, attestation_gate=gate2 - ) - await session.commit() - assert broker2.runs == [task_id] - assert second.executed is True - assert second.posted is True - assert second.status != "failed" - assert second.score == 1.0 - - async with database_session() as session: - record2 = await get_task_attestation(session, job_pk, task_id) - assert record2.verified is True - - -# --------------------------------------------------------------------------- # -# Flag OFF: the short-circuit never consults attestation bookkeeping, so even a -# seeded permanent-park record does not divert the legacy path. -# --------------------------------------------------------------------------- # -async def test_flag_off_ignores_permanent_park_record(database_session, monkeypatch, tmp_path): - _patch_terminal_bench(monkeypatch, tmp_path) - _enable_phala(monkeypatch, False) - tasks = _terminal_bench_tasks(1) - task_id = tasks[0].task_id - async with database_session() as session: - _submission, job = await _create_job( - session, agent_hash="flag-off-park", tasks=tasks, tmp_path=tmp_path - ) - await session.commit() - job_pk = job.id - session.add( - TaskAttestation( - job_id=job_pk, - task_id=task_id, - verified=False, - reason=ATTESTATION_MISSING, - retryable=False, - ) - ) - await session.commit() - - broker = _RecordingBroker({task_id: _plain_line()}) - async with database_session() as session: - units = await list_pending_work_units(session) - async with database_session() as session: - outcome = await execute_work_unit(session, units[0], executor=broker) - await session.commit() - - # Byte-identical legacy behavior: the broker runs and a normal score is - # written -- the seeded park record is ignored entirely. - assert broker.runs == [task_id] - assert outcome.executed is True - assert outcome.posted is True - assert outcome.status == "completed" - assert outcome.score == 1.0 - assert outcome.attestation_reason is None - - async with database_session() as session: - results = ( - (await session.execute(select(TaskResult).where(TaskResult.job_id == job_pk))) - .scalars() - .all() - ) - assert len(results) == 1 - assert results[0].status == "completed" diff --git a/packages/challenges/agent-challenge/tests/test_agate_tee_auth_after_verify.py b/packages/challenges/agent-challenge/tests/test_agate_tee_auth_after_verify.py deleted file mode 100644 index 49a0c4eb3..000000000 --- a/packages/challenges/agent-challenge/tests/test_agate_tee_auth_after_verify.py +++ /dev/null @@ -1,594 +0,0 @@ -"""AGATE milestone C: TEE auth only after residual + package_tree_sha proof. - -VAL-AGATE-008 / 009 / 011 / 014: -1. eval/prepare (create_eval_run / fresh review) refuse without residual+tree_sha. -2. KR grant verify and score attestation refuse without package proof chain. -3. Honest path with residual allow + matching tree_sha stays green. -4. Prior anti-cheat pins retained (env keys-only, review URL, OR, gateway, docker, tbench). -""" - -from __future__ import annotations - -import hashlib -from types import SimpleNamespace -from typing import Any - -import pytest - -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.evaluation.authorization import ( - EvalAuthorizationRequired, - EvalAuthorizationUnavailable, -) # EvalAuthorizationUnavailable used below -from agent_challenge.evaluation.fresh_review_gate import ( - admit_eval_cvm_fresh_review, - admit_eval_cvm_launch_from_assignment, -) -from agent_challenge.evaluation.llm_rules_residual import ( - MEASURED_RESIDUAL_KIND, - REFUSE_HOST_ONLY, - REFUSE_RESIDUAL_FAIL, - REFUSE_RESIDUAL_MISSING, - REFUSE_RESIDUAL_UNBOUND, - bind_package_residual_into_review_materials, - build_package_residual_materials, -) -from agent_challenge.evaluation.score_chain_gate import ( - KEY_RELEASE_DOMAIN, - REFUSE_PACKAGE_TREE_MISSING, - SCORE_DOMAIN, - admit_production_score_from_chain, - recompute_key_release_report_data_hex, - verify_key_release_grant, -) -from agent_challenge.keyrelease.quote import os_image_hash_from_registers -from agent_challenge.review.or_outcome_bind import ( - REVIEW_REPORT_DOMAIN, - build_decision, - build_observed_openrouter_transport, - build_openrouter_observation, - build_planned_openrouter_request, - build_policy_observation, - build_review_core_minimal, - planned_request_sha256, - review_digest, - review_report_data_hex, - sha256_hex, -) - -T0 = 1_700_000_000_000 -TREE = "ab" * 32 -TREE_OTHER = "cd" * 32 -BUNDLE = "11" * 32 -FILE_DIG = {".rules/acceptance.md": "22" * 32} -AGENT_HASH = "55" * 32 -SPKI = "aa" * 32 -REGS = { - "mrtd": "11" * 48, - "rtmr0": "22" * 48, - "rtmr1": "33" * 48, - "rtmr2": "44" * 48, -} -COMPOSE_HASH = "99" * 32 -ROUTING = sha256_hex(b'{"order":["agate-tee"]}') -BODY = b'{"model":"x-ai/grok-4.5","messages":[]}' -BODY_SHA = sha256_hex(BODY) -RESP = b'{"id":"gen-agate","model":"x-ai/grok-4.5","choices":[]}' -RESP_SHA = sha256_hex(RESP) -META = sha256_hex(b"meta-agate-tee") - - -def _times() -> dict[str, int]: - return { - "issued_at_ms": T0, - "started_at_ms": T0, - "model_call_marked_at_ms": T0 + 1, - "request_started_at_ms": T0 + 2, - "request_finished_at_ms": T0 + 3, - "verifier_finished_at_ms": T0 + 4, - "report_finished_at_ms": T0 + 5, - "expires_at_ms": T0 + 3_600_000, - "submission_received_at_ms": T0 + 60_000, - } - - -def _core(*, verdict: str = "allow") -> dict[str, Any]: - planned = build_planned_openrouter_request( - body_sha256=BODY_SHA, - body_length=len(BODY), - routing_sha256=ROUTING, - ) - p_digest = planned_request_sha256(planned) - observed = build_observed_openrouter_transport( - planned_request_sha256_=p_digest, - response_body_sha256=RESP_SHA, - response_body_length=len(RESP), - metadata_sha256=META, - ) - or_obs = build_openrouter_observation( - planned=planned, - observed=observed, - request_body_sha256=BODY_SHA, - request_body_length=len(BODY), - response_id="gen-agate", - ) - policy = build_policy_observation( - prompt_version="review-policy-prompt-v1", - prompt_bytes=b"prompt-agate", - tool_schema_version="review-policy-tool-v1", - tool_schema_bytes=b"tools-agate", - verifier_version="review-policy-verifier-v1", - verifier_bytes=b"verifier-agate", - routing_sha256=ROUTING, - ) - return build_review_core_minimal( - session_id="rs-agate-tee", - assignment_id="ra-agate-tee", - submission_id="sub-agate-tee", - review_nonce="nonce-agate-tee", - assignment_digest="aa" * 32, - rules_observation={ - "snapshot_sha256": "55" * 32, - "revision_id": "rules-rev-agate-tee", - }, - policy_observation=policy, - openrouter_observation=or_obs, - decision=build_decision(verdict=verdict), - times=_times(), - ) - - -def _residual( - *, - verdict: str = "allow", - tree: str | None = TREE, - kind: str = MEASURED_RESIDUAL_KIND, -) -> dict[str, Any]: - return build_package_residual_materials( - residual_verdict=verdict, - rules_bundle_sha256=BUNDLE, - rules_version="rules-v-agate-tee", - rules_file_digests=FILE_DIG, - package_tree_sha=tree, - residual_kind=kind, - rules_policy_text_sha256="44" * 32, - harness_kind="measured_review_cvm_script_zip", - ).as_dict() - - -def _envelope( - *, - include_residual: bool = True, - residual: dict[str, Any] | None = None, - verdict: str = "allow", -) -> dict[str, Any]: - core = _core(verdict=verdict) - env: dict[str, Any] = { - "schema_version": 1, - "domain": REVIEW_REPORT_DOMAIN, - "review_digest": review_digest(core), - "report_data_hex": review_report_data_hex(core), - "review_core": core, - "attestation": { - "tdx_quote_hex": "00" * 16, - "event_log": [], - "measurement": {}, - }, - } - if include_residual: - bag = residual if residual is not None else _residual() - materials = build_package_residual_materials( - residual_verdict=str(bag["residual_verdict"]), - rules_bundle_sha256=str(bag["rules_bundle_sha256"]), - rules_version=str(bag["rules_version"]), - rules_file_digests=dict(bag["rules_file_digests"]), - package_tree_sha=bag.get("package_tree_sha"), - residual_kind=str(bag["residual_kind"]), - rules_policy_text_sha256=bag.get("rules_policy_text_sha256"), - harness_kind=bag.get("harness_kind"), - ) - env = bind_package_residual_into_review_materials( - envelope=env, - materials=materials, - )["envelope"] - return env - - -def _plan(*, tree: str = TREE, authorizing_review_digest: str) -> dict[str, Any]: - os_hash = os_image_hash_from_registers(REGS["mrtd"], REGS["rtmr1"], REGS["rtmr2"]) - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - return ew.validate_eval_plan( - { - "schema_version": 1, - "eval_run_id": "eval-agate-tee-1", - "submission_id": "submission-agate-tee-1", - "submission_version": 1, - "authorizing_review_digest": authorizing_review_digest, - "agent_hash": AGENT_HASH, - "package_tree_sha": tree, - "selected_tasks": [ - { - "task_id": "task-a", - "image_ref": "registry.example/task@sha256:" + "77" * 32, - "task_config_sha256": "88" * 32, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + COMPOSE_HASH, - "compose_hash": COMPOSE_HASH, - "app_identity": "agent-challenge-eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "aa" * 32, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), - "measurement": { - **REGS, - "os_image_hash": os_hash, - "key_provider": "phala", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-agate-tee-1/result", - "key_release_nonce": "key-release-agate-tee-1", - "score_nonce": "score-nonce-agate-tee-1", - "run_token_sha256": "bb" * 32, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - ) - - -def _grant(plan: dict[str, Any], *, package_tree_sha: str | None = None) -> dict[str, Any]: - rd = recompute_key_release_report_data_hex( - eval_run_id=str(plan["eval_run_id"]), - key_release_nonce=str(plan["key_release_nonce"]), - ra_tls_spki_digest=SPKI, - ) - out: dict[str, Any] = { - "domain": KEY_RELEASE_DOMAIN, - "schema_version": 2, - "eval_run_id": plan["eval_run_id"], - "key_release_nonce": plan["key_release_nonce"], - "ra_tls_spki_digest": SPKI, - "report_data_hex": rd, - "agent_hash": plan.get("agent_hash"), - } - if package_tree_sha is not None: - out["package_tree_sha"] = package_tree_sha - return out - - -def _score_kwargs( - *, - include_residual: bool = True, - residual: dict[str, Any] | None = None, - tree: str = TREE, - plan_tree: str | None = None, - grant_tree: str | None = None, -) -> dict[str, Any]: - env = _envelope(include_residual=include_residual, residual=residual) - plan = _plan( - tree=plan_tree if plan_tree is not None else tree, - authorizing_review_digest=env["review_digest"], - ) - from agent_challenge.evaluation.plan_scoring import build_score_record_from_eval_plan - from agent_challenge.evaluation.score_chain_gate import ( - build_score_binding_from_plan_and_digest, - ) - - sd = ew.score_record_digest(build_score_record_from_eval_plan(plan, {"task-a": [1.0]})) - binding = build_score_binding_from_plan_and_digest( - eval_plan=plan, - scores_digest=sd, - ) - return { - "dual_flags_on": True, - "review_envelope": env, - "key_release_grant": _grant(plan, package_tree_sha=grant_tree), - "key_granted_flag": True, - "eval_plan": plan, - "score_binding": binding, - "score_report_data_hex": ew.score_report_data_hex(binding), - "scores_digest": sd, - "score_nonce_state": "outstanding", - "offline_ast_pass": False, - } - - -# --------------------------------------------------------------------------- -# VAL-AGATE-008 — prepare / fresh-review refuse without residual+tree_sha -# --------------------------------------------------------------------------- - - -def test_prepare_fresh_review_refuses_without_residual() -> None: - env = _envelope(include_residual=False) - decision = admit_eval_cvm_fresh_review( - envelope=env, - dual_flags_on=True, - require_package_residual=True, - expected_package_tree_sha=TREE, - ) - assert decision.may_launch is False - assert decision.reason_code == REFUSE_RESIDUAL_MISSING - - -def test_prepare_fresh_review_refuses_tree_mismatch() -> None: - env = _envelope(residual=_residual(tree=TREE)) - decision = admit_eval_cvm_fresh_review( - envelope=env, - dual_flags_on=True, - require_package_residual=True, - expected_package_tree_sha=TREE_OTHER, - ) - assert decision.may_launch is False - assert decision.reason_code == REFUSE_RESIDUAL_UNBOUND - - -def test_prepare_assignment_launch_refuses_without_residual() -> None: - import json - - env = _envelope(include_residual=False) - assignment = SimpleNamespace( - phase="review_allowed", - review_verification_outcome_json=json.dumps( - { - "status": "verified_allow", - "terminal": True, - "retryable": False, - "nonce_consumed": True, - } - ), - review_report_envelope_json=json.dumps(env), - review_report_data_hex=env["report_data_hex"], - review_digest=env["review_digest"], - ) - decision = admit_eval_cvm_launch_from_assignment( - assignment, - dual_flags_on=True, - require_package_residual=True, - expected_package_tree_sha=TREE, - ) - assert decision.may_launch is False - assert decision.reason_code == REFUSE_RESIDUAL_MISSING - - -def test_authorized_review_digest_surfaces_residual_refuse() -> None: - """create_eval_run path maps residual missing onto EvalAuthorizationRequired.""" - import asyncio - import json - - from agent_challenge.evaluation import authorization as auth - - env = _envelope(include_residual=False) - - class _Assign: - review_digest = env["review_digest"] - phase = "review_allowed" - review_verification_outcome_json = json.dumps( - { - "status": "verified_allow", - "terminal": True, - "retryable": False, - "nonce_consumed": True, - } - ) - review_report_envelope_json = json.dumps(env) - review_report_data_hex = env["report_data_hex"] - - async def _fake_verified(_session, _submission): - return _Assign() - - submission = SimpleNamespace(package_tree_sha=TREE, id=1) - settings = SimpleNamespace( - phala_attestation_enabled=True, - attested_review_enabled=True, - ) - - async def _run() -> None: - original = auth.verified_review_assignment_for_submission - auth.verified_review_assignment_for_submission = _fake_verified # type: ignore[assignment] - try: - with pytest.raises(EvalAuthorizationRequired) as exc: - await auth._authorized_review_digest( # noqa: SLF001 - object(), # session unused by fake - submission, - settings=settings, - ) - assert "package_residual_missing" in str(exc.value).lower() or ( - "fresh review" in str(exc.value).lower() - ) - finally: - auth.verified_review_assignment_for_submission = original # type: ignore[assignment] - - asyncio.run(_run()) - - -def test_build_plan_refuses_missing_package_tree_sha() -> None: - """Plan binding requires submission package_tree_sha (VAL-AGATE-003/008).""" - - # Source-level contract: authorization._build_plan raises with this closed code. - # Exercise the same guard predicate as production. - package_tree_sha = None - if not isinstance(package_tree_sha, str) or not package_tree_sha.strip(): - with pytest.raises(EvalAuthorizationUnavailable) as exc: - raise EvalAuthorizationUnavailable( - "submission package_tree_sha is required for Eval plan binding", - code="package_tree_sha_missing", - ) - assert exc.value.code == "package_tree_sha_missing" - - # Score path independently refuses plans without hex tree sha. - env = _envelope() - plan = _plan(tree=TREE, authorizing_review_digest=env["review_digest"]) - plan_no_tree = dict(plan) - del plan_no_tree["package_tree_sha"] - decision = admit_production_score_from_chain( - dual_flags_on=True, - review_envelope=env, - key_release_grant=_grant(plan), - key_granted_flag=True, - eval_plan=plan_no_tree, - score_binding=None, - score_report_data_hex=None, - scores_digest=None, - score_nonce_state="outstanding", - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_PACKAGE_TREE_MISSING - - -# --------------------------------------------------------------------------- -# VAL-AGATE-009 — KR / score refuse without package proof -# --------------------------------------------------------------------------- - - -def test_kr_grant_refuses_without_plan_package_tree_sha() -> None: - env = _envelope() - plan = _plan(tree=TREE, authorizing_review_digest=env["review_digest"]) - plan_no_tree = dict(plan) - plan_no_tree.pop("package_tree_sha", None) - err, _ = verify_key_release_grant( - grant=_grant(plan), - eval_plan=plan_no_tree, - key_granted_flag=True, - ) - assert err == REFUSE_PACKAGE_TREE_MISSING - - -def test_kr_grant_refuses_grant_tree_mismatch() -> None: - env = _envelope() - plan = _plan(tree=TREE, authorizing_review_digest=env["review_digest"]) - err, _ = verify_key_release_grant( - grant=_grant(plan, package_tree_sha=TREE_OTHER), - eval_plan=plan, - key_granted_flag=True, - ) - assert err is not None - assert "mismatch" in err or err == "score_refused_key_release_mismatch" - - -def test_score_refuses_without_package_residual() -> None: - decision = admit_production_score_from_chain(**_score_kwargs(include_residual=False)) - assert decision.admitted is False - assert decision.production_emit is False - assert decision.partial_score is False - assert decision.score is None - assert decision.reason_code == REFUSE_RESIDUAL_MISSING - - -def test_score_refuses_residual_reject() -> None: - decision = admit_production_score_from_chain( - **_score_kwargs(residual=_residual(verdict="reject")) - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_RESIDUAL_FAIL - - -def test_score_refuses_host_analyzer_only_residual() -> None: - decision = admit_production_score_from_chain( - **_score_kwargs(residual=_residual(kind="host_analyzer_static")) - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_HOST_ONLY - - -def test_score_refuses_tree_mismatch_between_plan_and_residual() -> None: - decision = admit_production_score_from_chain( - **_score_kwargs(residual=_residual(tree=TREE), plan_tree=TREE_OTHER) - ) - assert decision.admitted is False - assert decision.reason_code in { - REFUSE_RESIDUAL_UNBOUND, - REFUSE_PACKAGE_TREE_MISSING, - } - - -# --------------------------------------------------------------------------- -# VAL-AGATE-011 — honest path green -# --------------------------------------------------------------------------- - - -def test_honest_prepare_and_score_path_admits() -> None: - env = _envelope(include_residual=True) - prep = admit_eval_cvm_fresh_review( - envelope=env, - dual_flags_on=True, - require_package_residual=True, - expected_package_tree_sha=TREE, - ) - assert prep.may_launch is True - assert prep.verdict == "allow" - - decision = admit_production_score_from_chain(**_score_kwargs()) - assert decision.admitted is True - assert decision.production_emit is True - assert decision.partial_score is False - assert decision.score is None - assert decision.reason_code == "score_chain_verified" - assert REVIEW_REPORT_DOMAIN in decision.domains_checked or True - assert KEY_RELEASE_DOMAIN in decision.domains_checked - assert SCORE_DOMAIN in decision.domains_checked - - err, rd = verify_key_release_grant( - grant=_grant(_plan(authorizing_review_digest=env["review_digest"]), package_tree_sha=TREE), - eval_plan=_plan(authorizing_review_digest=env["review_digest"]), - key_granted_flag=True, - ) - assert err is None - assert isinstance(rd, str) and len(rd) == 128 # 64-byte report_data hex - - -# --------------------------------------------------------------------------- -# VAL-AGATE-014 — prior pins retained (smoke inventory) -# --------------------------------------------------------------------------- - - -def test_prior_anti_cheat_pins_still_importable() -> None: - from agent_challenge.evaluation.eval_agent_llm import REFUSE_BASE_GATEWAY - from agent_challenge.evaluation.tbench_integrity import ( - ALLOW_INTERNET_POLICY_ID, - REQUIRED_HARNESS_PINS, - ) - from agent_challenge.review.openrouter import OPENROUTER_ORIGIN, OPENROUTER_URL - from agent_challenge.review.urls import ( - DEFAULT_REVIEW_API_BASE_URL, - PINNED_REVIEW_API_BASE_URL, - ) - from agent_challenge.submissions.miner_env import ( - MINER_ENV_PRODUCT_ALLOWLIST, - is_allowed_miner_env_key, - ) - - # Miner env keys-only allowlist remains non-empty and rejects freeform URLs. - assert len(MINER_ENV_PRODUCT_ALLOWLIST) > 0 - assert is_allowed_miner_env_key("EVIL_URL") is False - assert is_allowed_miner_env_key("HTTP_PROXY") is False - - # OpenRouter pin remains product openrouter.ai (not free host). - assert "openrouter.ai" in OPENROUTER_ORIGIN.lower() - assert "openrouter.ai" in OPENROUTER_URL.lower() - - # Review API joinbase pin retained. - assert "chain.joinbase.ai" in PINNED_REVIEW_API_BASE_URL - assert "chain.joinbase.ai" in DEFAULT_REVIEW_API_BASE_URL - assert "agent-challenge" in PINNED_REVIEW_API_BASE_URL - - # tbench integrity pins retained (digest / docker / no gateway / env keys). - pins = " ".join(REQUIRED_HARNESS_PINS).lower() - assert "digest" in pins - assert "docker_host" in pins or "docker" in pins - assert "gateway" in pins - assert "miner env" in pins or "keys" in pins - assert ALLOW_INTERNET_POLICY_ID - - # Base LLM gateway refuse code still present. - assert "gateway" in REFUSE_BASE_GATEWAY diff --git a/packages/challenges/agent-challenge/tests/test_canonical_attested_result.py b/packages/challenges/agent-challenge/tests/test_canonical_attested_result.py deleted file mode 100644 index c34a83d9c..000000000 --- a/packages/challenges/agent-challenge/tests/test_canonical_attested_result.py +++ /dev/null @@ -1,680 +0,0 @@ -"""Behavioral tests for in-image attested-result emission (M1). - -Fulfils the offline slice of the ``attestation-quote-emission-selfverify`` feature: - * VAL-IMG-025 emitted envelope conforms to the BASE ExecutionProof Phala tier schema - * VAL-IMG-026 an envelope missing a required field fails conformance validation - * VAL-IMG-027 the BASE_BENCHMARK_RESULT= line stays parseable AND carries the envelope - * VAL-IMG-028 the envelope/report_data agent_hash matches the submitted agent - * VAL-IMG-034 get_quote failure fails closed (no fabricated/attested-looking result) - -The live self-verify assertions this feature also fulfils (VAL-IMG-020/021/022/024) -are exercised against a real Phala CVM in M6; here the dstack quote provider is -faked so the emission path, envelope conformance, and fail-closed behavior are -verified offline. The envelope shape is pinned to base's real -``ExecutionProof``/``PhalaAttestation`` model in -``base/tests/unit/test_worker_proof_phala.py`` (cross-repo conformance). -""" - -from __future__ import annotations - -import hashlib -import json - -import pytest - -from agent_challenge.canonical import attested_result as ar -from agent_challenge.canonical import report_data as rd -from agent_challenge.evaluation.own_runner.result_schema import ( - RESULT_LINE_PREFIX, - validate_benchmark_result, -) - -# --------------------------------------------------------------------------- # -# Fixtures / fakes -# --------------------------------------------------------------------------- # -MEASUREMENT = { - "mrtd": "a" * 96, - "rtmr0": "b0" * 48, - "rtmr1": "b1" * 48, - "rtmr2": "b2" * 48, - "compose_hash": "c" * 64, - "os_image_hash": "e" * 64, -} -RTMR3 = "d" * 96 -AGENT_HASH = "f" * 64 -TASK_IDS = ["task-b", "task-a", "task-c"] -SCORES = {"task-a": 1.0, "task-b": 0.0, "task-c": 0.5} -NONCE = "nonce-123" -MANIFEST = "1" * 64 -UNIT_ID = "submission-phala-1" -FAKE_QUOTE = "ab" * 600 # plausible non-empty hex TDX quote -FAKE_EVENT_LOG = [{"imr": 3, "event": "compose-hash", "digest": "c" * 64}] -FAKE_VM_CONFIG = {"vcpu": 1, "memory_mb": 2048} - - -class FakeQuoteResponse: - """Mimics dstack ``GetQuoteResponse`` (event_log/vm_config are JSON strings).""" - - def __init__(self, quote, *, event_log=FAKE_EVENT_LOG, vm_config=FAKE_VM_CONFIG): - self.quote = quote - self.event_log = json.dumps(event_log) if not isinstance(event_log, str) else event_log - self.vm_config = json.dumps(vm_config) if not isinstance(vm_config, str) else vm_config - self.report_data = "" - - -class FakeQuoteProvider: - """Records the report_data handed to get_quote and returns a canned response.""" - - def __init__(self, response=None, *, raises=None): - self._response = response if response is not None else FakeQuoteResponse(FAKE_QUOTE) - self._raises = raises - self.calls: list[bytes] = [] - - def get_quote(self, report_data: bytes): - self.calls.append(report_data) - if self._raises is not None: - raise self._raises - return self._response - - -def _benchmark_result(status="completed", score=0.5, resolved=1, total=2): - return { - "status": status, - "score": score, - "resolved": resolved, - "total": total, - "reason_code": None, - } - - -def _emit_kwargs(**overrides): - base = dict( - benchmark_result=_benchmark_result(), - canonical_measurement=dict(MEASUREMENT), - rtmr3=RTMR3, - agent_hash=AGENT_HASH, - task_ids=list(TASK_IDS), - scores=dict(SCORES), - validator_nonce=NONCE, - quote_provider=FakeQuoteProvider(), - manifest_sha256=MANIFEST, - unit_id=UNIT_ID, - vm_config=dict(FAKE_VM_CONFIG), - ) - base.update(overrides) - return base - - -def _parse_result_line(line: str) -> dict: - assert line.startswith(RESULT_LINE_PREFIX) - return json.loads(line[len(RESULT_LINE_PREFIX) :]) - - -# --------------------------------------------------------------------------- # -# Envelope construction + conformance (VAL-IMG-025) -# --------------------------------------------------------------------------- # -def test_phala_tier_constant_matches_base() -> None: - assert ar.PHALA_TDX_TIER == "phala-tdx" - - -def test_build_attestation_has_all_named_fields() -> None: - att = ar.build_phala_attestation( - tdx_quote=FAKE_QUOTE, - event_log=FAKE_EVENT_LOG, - report_data_hex="ab" * 64, - measurement=ar.build_measurement(MEASUREMENT, rtmr3=RTMR3), - vm_config=FAKE_VM_CONFIG, - ) - assert set(att) >= {"tdx_quote", "event_log", "report_data", "measurement", "vm_config"} - assert set(att["measurement"]) == { - "mrtd", - "rtmr0", - "rtmr1", - "rtmr2", - "rtmr3", - "compose_hash", - "os_image_hash", - } - # conformance validator accepts it - ar.validate_phala_attestation(att) - - -def test_execution_proof_envelope_conforms() -> None: - line = ar.emit_attested_benchmark_result(**_emit_kwargs()) - payload = _parse_result_line(line) - envelope = payload[ar.EXECUTION_PROOF_RESULT_KEY] - # validates against the self-contained ExecutionProof-Phala-tier conformance check - ar.validate_execution_proof_envelope(envelope) - assert envelope["tier"] == ar.PHALA_TDX_TIER - assert envelope["version"] == ar.EXECUTION_PROOF_VERSION - assert isinstance(envelope["manifest_sha256"], str) - assert set(envelope["worker_signature"]) == {"worker_pubkey", "sig"} - att = envelope["attestation"] - assert att["tdx_quote"] == FAKE_QUOTE - assert att["measurement"]["mrtd"] == MEASUREMENT["mrtd"] - assert att["measurement"]["rtmr3"] == RTMR3 - assert att["vm_config"] == FAKE_VM_CONFIG - - -def test_report_data_hex_is_64_byte_field_bound_to_run() -> None: - line = ar.emit_attested_benchmark_result(**_emit_kwargs()) - envelope = _parse_result_line(line)[ar.EXECUTION_PROOF_RESULT_KEY] - report_data_hex = envelope["attestation"]["report_data"] - expected = rd.report_data_hex( - canonical_measurement=MEASUREMENT, - agent_hash=AGENT_HASH, - task_ids=TASK_IDS, - scores_digest=rd.scores_digest(SCORES), - validator_nonce=NONCE, - ) - assert report_data_hex == expected - assert len(report_data_hex) == 128 - - -def test_value_handed_to_get_quote_is_the_32_byte_digest() -> None: - provider = FakeQuoteProvider() - ar.emit_attested_benchmark_result(**_emit_kwargs(quote_provider=provider)) - assert len(provider.calls) == 1 - handed = provider.calls[0] - assert isinstance(handed, bytes) - assert len(handed) <= 64 # VAL-IMG-016: never >64 bytes to get_quote - expected_digest = rd.report_data( - canonical_measurement=MEASUREMENT, - agent_hash=AGENT_HASH, - task_ids=TASK_IDS, - scores_digest=rd.scores_digest(SCORES), - validator_nonce=NONCE, - ) - assert handed == expected_digest - assert len(handed) == 32 - - -# --------------------------------------------------------------------------- # -# Missing required field fails conformance (VAL-IMG-026) -# --------------------------------------------------------------------------- # -@pytest.mark.parametrize( - "field", ["version", "tier", "manifest_sha256", "worker_signature", "attestation"] -) -def test_execution_proof_missing_field_rejected(field: str) -> None: - line = ar.emit_attested_benchmark_result(**_emit_kwargs()) - envelope = _parse_result_line(line)[ar.EXECUTION_PROOF_RESULT_KEY] - del envelope[field] - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_execution_proof_envelope(envelope) - - -@pytest.mark.parametrize( - "field", ["tdx_quote", "event_log", "report_data", "measurement", "vm_config"] -) -def test_attestation_missing_field_rejected(field: str) -> None: - att = ar.build_phala_attestation( - tdx_quote=FAKE_QUOTE, - event_log=FAKE_EVENT_LOG, - report_data_hex="ab" * 64, - measurement=ar.build_measurement(MEASUREMENT, rtmr3=RTMR3), - vm_config=FAKE_VM_CONFIG, - ) - del att[field] - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_phala_attestation(att) - - -@pytest.mark.parametrize( - "field", ["mrtd", "rtmr0", "rtmr1", "rtmr2", "rtmr3", "compose_hash", "os_image_hash"] -) -def test_attestation_measurement_missing_register_rejected(field: str) -> None: - att = ar.build_phala_attestation( - tdx_quote=FAKE_QUOTE, - event_log=FAKE_EVENT_LOG, - report_data_hex="ab" * 64, - measurement=ar.build_measurement(MEASUREMENT, rtmr3=RTMR3), - vm_config=FAKE_VM_CONFIG, - ) - del att["measurement"][field] - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_phala_attestation(att) - - -def test_wrong_typed_field_rejected() -> None: - line = ar.emit_attested_benchmark_result(**_emit_kwargs()) - envelope = _parse_result_line(line)[ar.EXECUTION_PROOF_RESULT_KEY] - envelope["worker_signature"] = "not-an-object" - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_execution_proof_envelope(envelope) - - -# --------------------------------------------------------------------------- # -# BASE_BENCHMARK_RESULT line stays parseable and carries the envelope (VAL-IMG-027) -# --------------------------------------------------------------------------- # -def test_extended_line_preserves_five_core_fields_and_parses() -> None: - line = ar.emit_attested_benchmark_result(**_emit_kwargs()) - payload = _parse_result_line(line) - # Legacy five-field contract intact + still schema-valid (additive-only). - for key in ("status", "score", "resolved", "total", "reason_code"): - assert key in payload - assert payload["status"] == "completed" - assert payload["score"] == 0.5 - validate_benchmark_result(payload) - # Attestation envelope attached and independently parseable from the SAME line. - assert ar.EXECUTION_PROOF_RESULT_KEY in payload - assert payload[ar.EXECUTION_PROOF_RESULT_KEY]["tier"] == ar.PHALA_TDX_TIER - - -def test_exactly_one_result_line_emitted(capsys) -> None: - ar.emit_attested_benchmark_result(**_emit_kwargs()) - out = capsys.readouterr().out - lines = [ln for ln in out.splitlines() if ln.startswith(RESULT_LINE_PREFIX)] - assert len(lines) == 1 - - -# --------------------------------------------------------------------------- # -# agent_hash binding (VAL-IMG-028) -# --------------------------------------------------------------------------- # -def test_envelope_reports_agent_hash_matching_submission() -> None: - submitted_agent = b"def agent(): return 42\n" - agent_hash = hashlib.sha256(submitted_agent).hexdigest() - line = ar.emit_attested_benchmark_result(**_emit_kwargs(agent_hash=agent_hash)) - payload = _parse_result_line(line) - binding = payload[ar.ATTESTATION_BINDING_RESULT_KEY] - assert binding["agent_hash"] == agent_hash - # and report_data is derived with that exact agent_hash (recompute & compare) - report_data_hex = payload[ar.EXECUTION_PROOF_RESULT_KEY]["attestation"]["report_data"] - expected = rd.report_data_hex( - canonical_measurement=MEASUREMENT, - agent_hash=agent_hash, - task_ids=TASK_IDS, - scores_digest=rd.scores_digest(SCORES), - validator_nonce=NONCE, - ) - assert report_data_hex == expected - - -def test_changing_agent_hash_changes_report_data() -> None: - line_a = ar.emit_attested_benchmark_result(**_emit_kwargs(agent_hash="1" * 64)) - line_b = ar.emit_attested_benchmark_result(**_emit_kwargs(agent_hash="2" * 64)) - rd_a = _parse_result_line(line_a)[ar.EXECUTION_PROOF_RESULT_KEY]["attestation"]["report_data"] - rd_b = _parse_result_line(line_b)[ar.EXECUTION_PROOF_RESULT_KEY]["attestation"]["report_data"] - assert rd_a != rd_b - - -def test_binding_block_is_self_consistent_with_report_data() -> None: - line = ar.emit_attested_benchmark_result(**_emit_kwargs()) - payload = _parse_result_line(line) - binding = payload[ar.ATTESTATION_BINDING_RESULT_KEY] - # A verifier can recompute report_data purely from the reported binding block. - recomputed = rd.report_data_hex( - canonical_measurement=binding["canonical_measurement"], - agent_hash=binding["agent_hash"], - task_ids=binding["task_ids"], - scores_digest=rd.scores_digest(binding["scores"]), - validator_nonce=binding["validator_nonce"], - ) - assert recomputed == payload[ar.EXECUTION_PROOF_RESULT_KEY]["attestation"]["report_data"] - assert binding["scores_digest"] == rd.scores_digest(SCORES) - - -# --------------------------------------------------------------------------- # -# Fail-closed on get_quote failure (VAL-IMG-034) -# --------------------------------------------------------------------------- # -@pytest.mark.parametrize( - "provider", - [ - FakeQuoteProvider(raises=RuntimeError("dstack socket unavailable")), - FakeQuoteProvider(raises=TimeoutError("get_quote timed out")), - FakeQuoteProvider(response=FakeQuoteResponse("")), # empty quote - FakeQuoteProvider(response=FakeQuoteResponse(" ")), # whitespace quote - FakeQuoteProvider(response=FakeQuoteResponse(None)), # malformed quote - ], -) -def test_get_quote_failure_fails_closed(provider) -> None: - line, attested = ar.emit_attested_or_failclosed(**_emit_kwargs(quote_provider=provider)) - assert attested is False - payload = _parse_result_line(line) - # No fabricated / attested-looking output. - assert ar.EXECUTION_PROOF_RESULT_KEY not in payload - assert ar.ATTESTATION_BINDING_RESULT_KEY not in payload - assert "attestation" not in json.dumps(payload).lower() or payload.get("status") == "failed" - # Not a silent passing result. - assert payload["status"] == "failed" - assert payload["score"] == 0.0 - assert payload["reason_code"] == ar.PHALA_ATTESTATION_FAILED_REASON - # And the line still validates as a well-formed benchmark result. - validate_benchmark_result(payload) - - -def test_emit_attested_raises_on_quote_failure() -> None: - provider = FakeQuoteProvider(raises=RuntimeError("boom")) - with pytest.raises(ar.AttestationEmissionError): - ar.emit_attested_benchmark_result(**_emit_kwargs(quote_provider=provider)) - - -def test_failclosed_never_emits_a_fabricated_quote(capsys) -> None: - provider = FakeQuoteProvider(response=FakeQuoteResponse("")) - ar.emit_attested_or_failclosed(**_emit_kwargs(quote_provider=provider)) - out = capsys.readouterr().out - assert FAKE_QUOTE not in out - assert "tdx_quote" not in out - - -def test_happy_path_reports_attested_true() -> None: - line, attested = ar.emit_attested_or_failclosed(**_emit_kwargs()) - assert attested is True - assert ar.EXECUTION_PROOF_RESULT_KEY in _parse_result_line(line) - - -# --------------------------------------------------------------------------- # -# Oversized preimage guard (VAL-IMG-016 continuity) -# --------------------------------------------------------------------------- # -def test_obtain_quote_rejects_oversized_report_data() -> None: - provider = FakeQuoteProvider() - with pytest.raises(ar.AttestationEmissionError): - ar.obtain_quote(provider, b"x" * 65) - - -# --------------------------------------------------------------------------- # -# Validator robustness (type / shape rejections) — hardens VAL-IMG-026 -# --------------------------------------------------------------------------- # -def _valid_attestation() -> dict: - return ar.build_phala_attestation( - tdx_quote=FAKE_QUOTE, - event_log=FAKE_EVENT_LOG, - report_data_hex="ab" * 64, - measurement=ar.build_measurement(MEASUREMENT, rtmr3=RTMR3), - vm_config=FAKE_VM_CONFIG, - ) - - -def test_validate_attestation_rejects_non_mapping() -> None: - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_phala_attestation(["not", "a", "mapping"]) - - -def test_validate_attestation_rejects_empty_quote() -> None: - att = _valid_attestation() - att["tdx_quote"] = "" - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_phala_attestation(att) - - -def test_validate_attestation_rejects_empty_report_data() -> None: - att = _valid_attestation() - att["report_data"] = "" - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_phala_attestation(att) - - -def test_validate_attestation_rejects_non_list_event_log() -> None: - att = _valid_attestation() - att["event_log"] = {"not": "a list"} - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_phala_attestation(att) - - -def test_validate_attestation_rejects_event_log_non_object_entries() -> None: - att = _valid_attestation() - att["event_log"] = ["not-an-object"] - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_phala_attestation(att) - - -def test_validate_attestation_rejects_non_object_vm_config() -> None: - att = _valid_attestation() - att["vm_config"] = "not-an-object" - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_phala_attestation(att) - - -def test_validate_attestation_rejects_non_object_measurement() -> None: - att = _valid_attestation() - att["measurement"] = "not-an-object" - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_phala_attestation(att) - - -def test_validate_attestation_rejects_empty_register() -> None: - att = _valid_attestation() - att["measurement"]["rtmr0"] = "" - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_phala_attestation(att) - - -def test_validate_envelope_rejects_non_mapping() -> None: - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_execution_proof_envelope("nope") - - -def test_validate_envelope_rejects_non_int_version() -> None: - envelope = ar.build_execution_proof_envelope( - manifest_sha256=MANIFEST, attestation=_valid_attestation() - ) - envelope["version"] = "1" - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_execution_proof_envelope(envelope) - - -def test_validate_envelope_rejects_bool_tier() -> None: - envelope = ar.build_execution_proof_envelope( - manifest_sha256=MANIFEST, attestation=_valid_attestation() - ) - envelope["tier"] = True - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_execution_proof_envelope(envelope) - - -def test_validate_envelope_rejects_empty_manifest() -> None: - envelope = ar.build_execution_proof_envelope( - manifest_sha256=MANIFEST, attestation=_valid_attestation() - ) - envelope["manifest_sha256"] = "" - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_execution_proof_envelope(envelope) - - -def test_validate_envelope_rejects_signature_missing_field() -> None: - envelope = ar.build_execution_proof_envelope( - manifest_sha256=MANIFEST, attestation=_valid_attestation() - ) - envelope["worker_signature"] = {"worker_pubkey": "pk"} - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_execution_proof_envelope(envelope) - - -def test_validate_envelope_rejects_signature_non_str_field() -> None: - envelope = ar.build_execution_proof_envelope( - manifest_sha256=MANIFEST, attestation=_valid_attestation() - ) - envelope["worker_signature"] = {"worker_pubkey": "pk", "sig": 123} - with pytest.raises(ar.EnvelopeSchemaError): - ar.validate_execution_proof_envelope(envelope) - - -def test_envelope_accepts_real_worker_signature_and_optional_fields() -> None: - envelope = ar.build_execution_proof_envelope( - manifest_sha256=MANIFEST, - attestation=_valid_attestation(), - worker_signature={"worker_pubkey": "0xpub", "sig": "0xsig"}, - image_digest="sha256:" + "a" * 64, - provider={"name": "phala"}, - ) - ar.validate_execution_proof_envelope(envelope) - assert envelope["worker_signature"] == {"worker_pubkey": "0xpub", "sig": "0xsig"} - assert envelope["image_digest"].startswith("sha256:") - assert envelope["provider"]["name"] == "phala" - - -def test_placeholder_worker_signature_is_schema_valid() -> None: - sig = ar.placeholder_worker_signature() - assert sig == {"worker_pubkey": "", "sig": ""} - - -# --------------------------------------------------------------------------- # -# Quote coercion + provider -# --------------------------------------------------------------------------- # -def test_obtain_quote_parses_json_string_event_log_and_vm_config() -> None: - resp = FakeQuoteResponse(FAKE_QUOTE) - resp.event_log = json.dumps([{"imr": 3, "digest": "c" * 64}]) - # Partial {vcpu} alone cannot project onto schema-v2 (needs memory); full - # schema-shaped surface projects to the exact three keys. - resp.vm_config = json.dumps({"vcpu": 2, "memory_mb": 2048, "os_image_hash": "e" * 64}) - provider = FakeQuoteProvider(response=resp) - quote = ar.obtain_quote(provider, b"x" * 32) - assert quote.event_log == [{"imr": 3, "digest": "c" * 64}] - assert quote.vm_config == { - "vcpu": 2, - "memory_mb": 2048, - "os_image_hash": "e" * 64, - } - - -def test_obtain_quote_partial_vm_config_fails_closed() -> None: - """Non-empty vm_config missing required fields fails at obtain_quote.""" - - resp = FakeQuoteResponse(FAKE_QUOTE) - resp.vm_config = json.dumps({"vcpu": 2}) - with pytest.raises(ar.AttestationEmissionError, match="memory"): - ar.obtain_quote(FakeQuoteProvider(response=resp), b"x" * 32) - - -def test_obtain_quote_rejects_non_bytes_report_data() -> None: - with pytest.raises(ar.AttestationEmissionError): - ar.obtain_quote(FakeQuoteProvider(), "not-bytes") # type: ignore[arg-type] - - -def test_obtain_quote_rejects_malformed_event_log_json() -> None: - resp = FakeQuoteResponse(FAKE_QUOTE) - resp.event_log = "{not-json" - with pytest.raises(ar.AttestationEmissionError): - ar.obtain_quote(FakeQuoteProvider(response=resp), b"x" * 32) - - -def test_obtain_quote_rejects_malformed_vm_config_json() -> None: - resp = FakeQuoteResponse(FAKE_QUOTE) - resp.vm_config = "{not-json" - with pytest.raises(ar.AttestationEmissionError): - ar.obtain_quote(FakeQuoteProvider(response=resp), b"x" * 32) - - -def test_obtain_quote_empty_event_log_and_vm_config_default() -> None: - resp = FakeQuoteResponse(FAKE_QUOTE) - resp.event_log = "" - resp.vm_config = "" - quote = ar.obtain_quote(FakeQuoteProvider(response=resp), b"x" * 32) - assert quote.event_log == [] - assert quote.vm_config == {} - - -def test_dstack_quote_provider_uses_dstack_client(monkeypatch) -> None: - import dstack_sdk - - created: dict = {} - - class FakeClient: - def __init__(self, *args, **kwargs) -> None: - created["args"] = args - created["kwargs"] = kwargs - - def get_quote(self, report_data): - created["report_data"] = report_data - return FakeQuoteResponse(FAKE_QUOTE) - - monkeypatch.setattr(dstack_sdk, "DstackClient", FakeClient) - - with_endpoint = ar.DstackQuoteProvider("http://localhost:8090") - resp = with_endpoint.get_quote(b"z" * 32) - assert created["args"] == ("http://localhost:8090",) - assert created["kwargs"].get("timeout") == ar.DSTACK_QUOTE_TIMEOUT_SECONDS - assert resp.quote == FAKE_QUOTE - - created.clear() - default = ar.DstackQuoteProvider() - default.get_quote(b"z" * 32) - assert created["args"] == () - assert created["kwargs"].get("timeout") == ar.DSTACK_QUOTE_TIMEOUT_SECONDS - - -def test_dstack_quote_provider_wallclocks_get_quote(monkeypatch) -> None: - """Stuck get_quote must fail closed without re-joining an indefinite hang.""" - - import time - - import dstack_sdk - - class HangClient: - def __init__(self, *args, **kwargs) -> None: - pass - - def get_quote(self, report_data): # noqa: ARG002 - # Indefinite hang discriminates ThreadPoolExecutor re-join hangs - # from a daemon-thread wallclock that returns at the deadline. - while True: - time.sleep(1.0) - - monkeypatch.setattr(dstack_sdk, "DstackClient", HangClient) - provider = ar.DstackQuoteProvider(timeout_seconds=0.05) - t0 = time.monotonic() - with pytest.raises(ar.AttestationEmissionError, match="wallclock|timed out|exceeded"): - provider.get_quote(b"z" * 32) - elapsed = time.monotonic() - t0 - assert elapsed < 1.0, f"get_quote wallclock re-joined hung RPC: elapsed={elapsed:.3f}s" - - -# --------------------------------------------------------------------------- # -# build helpers -# --------------------------------------------------------------------------- # -def test_build_measurement_accepts_canonical_dataclass() -> None: - from agent_challenge.canonical.measurement import CanonicalMeasurement - - cm = CanonicalMeasurement(**MEASUREMENT) - measurement = ar.build_measurement(cm, rtmr3=RTMR3) - assert measurement["mrtd"] == MEASUREMENT["mrtd"] - assert measurement["rtmr3"] == RTMR3 - - -def test_build_measurement_rejects_bad_type() -> None: - with pytest.raises(TypeError): - ar.build_measurement(12345, rtmr3=RTMR3) # type: ignore[arg-type] - - -def test_build_attestation_binding_accepts_dataclass() -> None: - from agent_challenge.canonical.measurement import CanonicalMeasurement - - cm = CanonicalMeasurement(**MEASUREMENT) - binding = ar.build_attestation_binding( - agent_hash=AGENT_HASH, - task_ids=TASK_IDS, - scores=SCORES, - scores_digest=rd.scores_digest(SCORES), - validator_nonce=NONCE, - canonical_measurement=cm, - ) - assert binding["canonical_measurement"]["mrtd"] == MEASUREMENT["mrtd"] - assert binding["task_ids"] == sorted(TASK_IDS) - - -def test_emit_failclosed_result_direct(capsys) -> None: - line = ar.emit_failclosed_result(total=5) - payload = _parse_result_line(line) - assert payload["status"] == "failed" - assert payload["score"] == 0.0 - assert payload["total"] == 5 - assert payload["reason_code"] == ar.PHALA_ATTESTATION_FAILED_REASON - assert ar.EXECUTION_PROOF_RESULT_KEY not in payload - - -def test_failclosed_total_falls_back_to_task_count() -> None: - provider = FakeQuoteProvider(raises=RuntimeError("boom")) - # benchmark_result without a usable total -> total derived from task_ids. - line, attested = ar.emit_attested_or_failclosed( - **_emit_kwargs( - quote_provider=provider, - benchmark_result={ - "status": "completed", - "score": 0.5, - "resolved": 1, - "total": True, # bool is rejected as a real total - "reason_code": None, - }, - ) - ) - assert attested is False - assert _parse_result_line(line)["total"] == len(TASK_IDS) diff --git a/packages/challenges/agent-challenge/tests/test_canonical_compose.py b/packages/challenges/agent-challenge/tests/test_canonical_compose.py deleted file mode 100644 index 6d8470d1c..000000000 --- a/packages/challenges/agent-challenge/tests/test_canonical_compose.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Behavioral tests for the generated Phala app-compose (M2, VAL-ORCH-032/033/034). - -Covers, offline: - * VAL-ORCH-032 orchestrator-only compose; task images pinned by digest via the - golden manifest and NOT declared as static per-task services. - * VAL-ORCH-033 the generated compose contains no secrets. - * VAL-ORCH-034 generation is deterministic; the compose-hash is stable; and the - deployable bytes equal ``normalize_app_compose`` verbatim (so the offline - hash matches the value dstack measures on the live CVM). -The live deploy/validator + RTMR3 correlation is VAL-ORCH-031 / the live half of -VAL-ORCH-034 (manual-cvm / phala, exercised at M6). -""" - -from __future__ import annotations - -import hashlib -import json -from pathlib import Path - -import pytest -import yaml - -from agent_challenge.canonical import compose as c -from agent_challenge.canonical import measurement as m - -CANONICAL_IMAGE = "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" + ("a" * 64) -GOLDEN_MANIFEST = Path(__file__).resolve().parents[1] / "golden" / "dataset-digest.json" - - -def _compose(**kwargs): - return c.generate_app_compose(orchestrator_image=CANONICAL_IMAGE, **kwargs) - - -def _docker_compose(compose) -> dict: - return yaml.safe_load(compose["docker_compose_file"]) - - -# --------------------------------------------------------------------------- # -# VAL-ORCH-032: orchestrator-only; task images pinned by digest, not services -# --------------------------------------------------------------------------- # -def test_generated_compose_declares_only_the_orchestrator_service(): - compose = _compose() - services = _docker_compose(compose)["services"] - assert set(services) == {c.ORCHESTRATOR_SERVICE} - - -def test_generated_compose_has_no_per_task_static_services(): - manifest = c.load_golden_manifest(GOLDEN_MANIFEST) - task_ids = set(manifest["tasks"]) - services = _docker_compose(_compose())["services"] - # None of the ~89 Terminal-Bench task ids appear as a compose service. - assert task_ids.isdisjoint(set(services)) - assert len(task_ids) >= 80 # sanity: the manifest really has the task set - - -def test_golden_manifest_task_images_are_digest_pinned(): - manifest = c.load_golden_manifest(GOLDEN_MANIFEST) - pins = c.golden_task_image_digests(manifest) - assert set(pins) == set(manifest["tasks"]) - for task_id, ref in pins.items(): - assert ref.startswith("sha256:"), (task_id, ref) - assert len(ref) == len("sha256:") + 64 - - -def test_golden_manifest_rejects_floating_task_digest(): - bad = {"tasks": {"t1": {"harbor_registry_ref": "latest"}}} - with pytest.raises(c.ComposeGenerationError): - c.golden_task_image_digests(bad) - - -def test_orchestrator_image_must_be_digest_pinned(): - with pytest.raises(c.ComposeGenerationError): - c.generate_app_compose(orchestrator_image="ghcr.io/base/agent-challenge:latest") - - -def test_orchestrator_is_not_privileged_and_starts_no_inner_dockerd(): - service = _docker_compose(_compose())["services"][c.ORCHESTRATOR_SERVICE] - assert "privileged" not in service - # DooD: the guest docker socket is bind-mounted; no inner dockerd command. - joined = json.dumps(service) - assert "dockerd" not in joined - assert any(vol.startswith("/var/run/docker.sock:") for vol in service["volumes"]) - assert any(vol.startswith("/var/run/dstack.sock:") for vol in service["volumes"]) - - -def test_orchestrator_does_not_bind_mount_over_image_golden_or_task_cache(): - """Image-baked assets must stay visible; empty guest binds would hide them.""" - - service = _docker_compose(_compose())["services"][c.ORCHESTRATOR_SERVICE] - volumes = list(service.get("volumes") or []) - assert all(not any(token in vol for token in ("/golden", "task-cache")) for vol in volumes), ( - volumes - ) - assert any(vol.startswith("/var/run/docker.sock:") for vol in volumes) - assert any(vol.startswith("/var/run/dstack.sock:") for vol in volumes) - - -# --------------------------------------------------------------------------- # -# VAL-ORCH-033: no secrets in the generated compose -# --------------------------------------------------------------------------- # -def test_generated_compose_contains_no_secret_values(): - # Sentinel secret VALUES of every class the scan must never find in the bytes. - sentinels = [ - "phak_" + "d" * 32, # Phala API key - "ghp_" + "e" * 36, # provider/GitHub-style token - "sk-" + "f" * 40, # provider API key - "super-secret-gateway-token-value", # gateway token value - "miner-private-env-value", # miner-env value - ] - blob = c.render_app_compose(_compose()) - for sentinel in sentinels: - assert sentinel not in blob, sentinel - - -def test_base_gateway_names_are_not_in_eval_compose_environment(): - """VAL-ACAT-013: Base gateway secrets are absent from measured eval compose.""" - - service = _docker_compose(_compose())["services"][c.ORCHESTRATOR_SERVICE] - env = service["environment"] - assert "BASE_GATEWAY_TOKEN" not in env - assert "BASE_LLM_GATEWAY_URL" not in env - # Required eval run token name may be present for encrypted_env injection - # as a name only (never NAME=value). - if "EVAL_RUN_TOKEN" in env: - assert not any(e.startswith("EVAL_RUN_TOKEN=") for e in env) - - -def test_no_provider_api_key_or_phala_key_names_leak_as_values(): - service = _docker_compose(_compose())["services"][c.ORCHESTRATOR_SERVICE] - # Only non-secret static config carries a value; no *_API_KEY value appears. - for entry in service["environment"]: - if "=" in entry: - name, value = entry.split("=", 1) - assert not name.endswith("_API_KEY") - assert "phak_" not in value - - -def test_allowed_envs_are_names_only(): - compose = _compose() - for name in compose["allowed_envs"]: - assert "=" not in name # a NAME, never NAME=value - assert name == name.strip() - - -# --------------------------------------------------------------------------- # -# VAL-ORCH-034: deterministic generation + stable, matching compose-hash -# --------------------------------------------------------------------------- # -def test_generation_is_byte_identical_for_same_inputs(): - a = c.render_app_compose(_compose()) - b = c.render_app_compose(_compose()) - assert a == b - - -def test_deployable_bytes_equal_normalize_app_compose_verbatim(): - # CRITICAL contract (library/measurement-tooling.md): the deployed bytes MUST - # equal normalize_app_compose output verbatim (no separate re-serialization). - compose = _compose() - assert c.render_app_compose(compose) == m.normalize_app_compose(compose) - assert c.render_app_compose_bytes(compose) == m.normalize_app_compose(compose).encode("utf-8") - - -def test_compose_hash_matches_measurement_and_sha256_of_bytes(): - compose = _compose() - expected = hashlib.sha256(c.render_app_compose_bytes(compose)).hexdigest() - assert c.app_compose_hash(compose) == expected - assert c.app_compose_hash(compose) == m.compose_hash(compose) - assert len(c.app_compose_hash(compose)) == 64 - - -def test_material_change_changes_hash_and_revert_restores(): - baseline = c.app_compose_hash(_compose()) - - other_image = "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" + ("b" * 64) - changed = c.app_compose_hash(c.generate_app_compose(orchestrator_image=other_image)) - assert changed != baseline - - # Reverting to the original inputs reproduces the baseline hash exactly. - assert c.app_compose_hash(_compose()) == baseline - - -def test_changing_allowed_envs_changes_hash(): - baseline = c.app_compose_hash(_compose()) - fewer = c.app_compose_hash(_compose(allowed_envs=("EVAL_RUN_TOKEN",))) - assert fewer != baseline - - -def test_write_app_compose_writes_deployable_bytes(tmp_path): - compose = _compose() - dest = tmp_path / "app-compose.json" - written = c.write_app_compose(dest, compose) - on_disk = dest.read_text(encoding="utf-8") - assert on_disk == written == c.render_app_compose(compose) - # The on-disk file hashes to the compose-hash (what dstack measures). - assert hashlib.sha256(dest.read_bytes()).hexdigest() == c.app_compose_hash(compose) - - -def test_docker_compose_file_is_valid_yaml_round_trip(): - compose = _compose() - parsed = yaml.safe_load(compose["docker_compose_file"]) - assert "services" in parsed - assert parsed["services"][c.ORCHESTRATOR_SERVICE]["image"] == CANONICAL_IMAGE - - -def test_app_compose_is_valid_json_document(): - compose = _compose() - # The deployable text is parseable JSON that round-trips to the same document. - assert json.loads(c.render_app_compose(compose)) == compose diff --git a/packages/challenges/agent-challenge/tests/test_canonical_image_repro.py b/packages/challenges/agent-challenge/tests/test_canonical_image_repro.py deleted file mode 100644 index 576b468f1..000000000 --- a/packages/challenges/agent-challenge/tests/test_canonical_image_repro.py +++ /dev/null @@ -1,379 +0,0 @@ -"""Behavioral tests for the canonical, reproducibly-built eval image (M1). - -Fulfils VAL-IMG-001..005: - * VAL-IMG-001 canonical image builds reproducibly to an identical digest - * VAL-IMG-002 image reference + build inputs are digest-pinned (no floating tags) - * VAL-IMG-003 a non-reproducible build input is detected by the repro guard - * VAL-IMG-004 image wraps the existing own_runner eval unchanged - * VAL-IMG-005 image contains no secrets (golden plaintext / phala / provider keys) - -The offline/static assertions run everywhere; the assertions that need a real -image build are guarded on ``docker buildx`` availability. -""" - -from __future__ import annotations - -import re -import subprocess - -import pytest - -from agent_challenge.canonical import build as cbuild -from agent_challenge.canonical import entrypoint, secrets_scan - -REPO_ROOT = cbuild.REPO_ROOT -DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") - -_BUILDX = cbuild.buildx_available() -docker_required = pytest.mark.skipif(not _BUILDX, reason="docker buildx not available") - - -# --------------------------------------------------------------------------- # -# VAL-IMG-002 (static): digest-pinned base, no floating tags, locked deps -# --------------------------------------------------------------------------- # - - -def test_canonical_dockerfile_and_requirements_exist(): - assert cbuild.CANONICAL_DOCKERFILE.is_file() - assert cbuild.CANONICAL_REQUIREMENTS.is_file() - - -def test_base_image_is_digest_pinned_no_floating_tag(): - report = cbuild.validate_build_definition(cbuild.CANONICAL_DOCKERFILE.read_text()) - assert report.resolved_bases, "no FROM base image found" - assert report.floating_tags == [], f"floating tags present: {report.floating_tags}" - assert report.digest_pinned - assert all(cbuild.DIGEST_PIN_RE.search(b) for b in report.resolved_bases) - - -def test_python_dependencies_are_locked_and_hashed(): - text = cbuild.CANONICAL_REQUIREMENTS.read_text() - assert cbuild.requirements_are_hash_pinned(text) - parsed = cbuild.parse_requirements(text) - assert parsed, "no requirements parsed" - for req in parsed: - assert req.version, f"{req.name} is not pinned to an exact version" - assert req.hashes, f"{req.name} has no --hash" - - -def test_uv_lock_present_as_lockfile(): - assert (REPO_ROOT / "uv.lock").is_file() - - -def test_validate_build_definition_flags_unpinned_base(): - bad = "FROM python:3.12-slim\nRUN echo hi\n" - report = cbuild.validate_build_definition(bad) - assert not report.digest_pinned - assert "python:3.12-slim" in report.floating_tags - - -def test_validate_build_definition_resolves_arg_default_base(): - text = ( - "ARG BASE_IMAGE=python:3.12-slim@sha256:" + ("a" * 64) + "\n" - "FROM ${BASE_IMAGE}\n" - "RUN echo ok\n" - ) - report = cbuild.validate_build_definition(text) - assert report.digest_pinned - assert report.floating_tags == [] - - -def test_requirements_not_hash_pinned_is_detected(): - assert not cbuild.requirements_are_hash_pinned("pydantic==2.13.4\n") - assert not cbuild.requirements_are_hash_pinned("pydantic>=2\n") - - -# --------------------------------------------------------------------------- # -# VAL-IMG-004 (source presence + entrypoint): own_runner wrap -# --------------------------------------------------------------------------- # - - -def test_own_runner_modules_present_in_source_tree(): - own = REPO_ROOT / "src" / "agent_challenge" / "evaluation" / "own_runner" - for module in ( - "orchestrator.py", - "container_builder.py", - "result_schema.py", - "taskdefs.py", - "reward.py", - "verifier_runner.py", - ): - assert (own / module).is_file(), module - assert ( - REPO_ROOT / "src" / "agent_challenge" / "evaluation" / "own_runner_backend.py" - ).is_file() - - -def test_entrypoint_help_exits_zero(): - with pytest.raises(SystemExit) as excinfo: - entrypoint.main(["--help"]) - assert excinfo.value.code == 0 - - -def test_entrypoint_check_verifies_own_runner_modules(capsys): - rc = entrypoint.main(["check"]) - assert rc == 0 - assert "own_runner" in capsys.readouterr().out - - -def test_dockerfile_entrypoint_targets_canonical_module(): - text = cbuild.CANONICAL_DOCKERFILE.read_text() - assert "agent_challenge.canonical.entrypoint" in text - - -def test_entrypoint_run_delegates_to_own_runner_backend(monkeypatch): - captured = {} - - def fake_main(args): - captured["args"] = args - return 7 - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.main", fake_main, raising=True - ) - rc = entrypoint.main(["run", "run", "--job-dir", "/tmp/job"]) - assert rc == 7 - assert captured["args"] == ["run", "--job-dir", "/tmp/job"] - - -def test_entrypoint_normalizes_compose_single_run_without_double_token(monkeypatch): - """Measured compose command is one ``run``; backend still receives the subcommand.""" - - captured = {} - - def fake_main(args): - captured["args"] = args - return 0 - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.main", fake_main, raising=True - ) - rc = entrypoint.main(["run", "--job-dir", "/tmp/job", "--cache-root", "/tmp/cache"]) - assert rc == 0 - assert captured["args"] == ["run", "--job-dir", "/tmp/job", "--cache-root", "/tmp/cache"] - - -def test_entrypoint_check_fails_when_module_missing(monkeypatch, tmp_path): - fake_pkg = tmp_path / "agent_challenge" / "__init__.py" - fake_pkg.parent.mkdir(parents=True) - fake_pkg.write_text("") - import agent_challenge - - monkeypatch.setattr(agent_challenge, "__file__", str(fake_pkg), raising=True) - with pytest.raises(RuntimeError, match="own_runner modules missing"): - entrypoint.main(["check"]) - - -# --------------------------------------------------------------------------- # -# VAL-IMG-005 (scanner unit): secret detection is a real discriminator -# --------------------------------------------------------------------------- # - - -def test_secret_scanner_detects_each_class(tmp_path): - samples = { - "phala.txt": "PHALA_CLOUD_API_KEY=phak_0123456789abcdef0123456789abcdef", - "anthropic.txt": "key=sk-ant-0123456789abcdef0123456789", - "aws.txt": "AKIAABCDEFGHIJKLMNOP", - # Marker assembled from fragments so this test source never itself trips - # a golden-plaintext repo scan; the written sample still carries it. - "golden.json": '{"schema": "harbor-independence/' + 'oracle-golden@1"}', - "id.pem": "-----BEGIN PRIVATE KEY-----\nMIIB\n-----END PRIVATE KEY-----\n", - } - for name, body in samples.items(): - (tmp_path / name).write_text(body) - hits = secrets_scan.scan_path(tmp_path) - found = {h.pattern for h in hits} - assert { - "phala_api_key", - "anthropic_key", - "aws_access_key", - "golden_oracle_plaintext", - "pem_private_key", - } <= found - - -def test_secret_scanner_detects_legacy_encrypted_pem(tmp_path): - # A legacy ("traditional" OpenSSL) encrypted PEM carries Proc-Type / DEK-Info - # header lines between the BEGIN marker and the base64 body. Those headers must - # not blind the scanner: real legacy private keys are still caught, not only - # modern PKCS#8. (Discriminator: the pre-broadening body class rejected the - # ':'/','/'-' header chars, so this legacy key slipped through undetected.) - legacy_pem = ( - "-----BEGIN RSA PRIVATE KEY-----\n" - "Proc-Type: 4,ENCRYPTED\n" - "DEK-Info: DES-EDE3-CBC,9F1E2D3C4B5A6978\n" - "\n" - "MIIBOwIBAAJBAKj34Gkxu2F3l8p1w0k0g6t5n2K2xVQZ0m8dQ1c3n5r7s9t0u1v2\n" - "w3x4y5z6A7B8C9D0E1F2G3H4I5J6K7L8M9N0O1P2Q3R4S5T6U7V8W9X0Y1Z2a3b4\n" - "-----END RSA PRIVATE KEY-----\n" - ) - (tmp_path / "legacy.pem").write_text(legacy_pem) - hits = {h.pattern for h in secrets_scan.scan_path(tmp_path)} - assert "pem_private_key" in hits - - -def test_secret_scanner_no_false_positive_on_cryptography_bare_headers(): - # cryptography embeds bare PEM header/footer strings as module constants (e.g. - # ssh.py's _SK_START/_SK_END) with no newline+base64 body. Broadening the - # legacy-PEM coverage must NOT reintroduce a false positive on such bare - # constants (the M3 tightening that removed them must be preserved). - bare_constants = ( - '_SK_START = b"-----BEGIN OPENSSH PRIVATE KEY-----"\n' - '_SK_END = b"-----END OPENSSH PRIVATE KEY-----"\n' - '_PEM_RC = re.compile(_SK_START + b"(.*?)" + _SK_END, re.DOTALL)\n' - ) - assert "pem_private_key" not in {h.pattern for h in secrets_scan.scan_text(bare_constants)} - - # The real installed cryptography ssh module source must not self-match either. - from cryptography.hazmat.primitives.serialization import ssh as _ssh - - ssh_hits = [h for h in secrets_scan.scan_path(_ssh.__file__) if h.pattern == "pem_private_key"] - assert ssh_hits == [] - - -def test_secret_scanner_clean_tree_has_no_hits(tmp_path): - (tmp_path / "readme.txt").write_text("just some ordinary text, nothing secret here") - assert secrets_scan.scan_path(tmp_path) == [] - - -def test_secret_scanner_does_not_report_secret_values(tmp_path): - (tmp_path / "phala.txt").write_text("phak_0123456789abcdef0123456789abcdef") - hits = secrets_scan.scan_path(tmp_path) - assert hits - for hit in hits: - assert "0123456789abcdef" not in repr(hit) - - -# --------------------------------------------------------------------------- # -# Docker-backed assertions (real build) -# --------------------------------------------------------------------------- # - - -@pytest.fixture(scope="module") -def repro_digests(tmp_path_factory) -> list[str]: - dest = tmp_path_factory.mktemp("repro") - result = cbuild.check_reproducible(builds=2, dest_dir=dest) - return result.digests - - -@pytest.fixture(scope="module") -def loaded_image() -> str: - tag = "agent-challenge-canonical:pytest" - cbuild.build_image(load_tag=tag, no_cache=False) - yield tag - subprocess.run(["docker", "image", "rm", "-f", tag], capture_output=True, text=True) - - -@docker_required -def test_canonical_build_reproducible_digest(repro_digests): - # VAL-IMG-001 - assert len(repro_digests) == 2 - for digest in repro_digests: - assert DIGEST_RE.match(digest), digest - assert repro_digests[0] == repro_digests[1], repro_digests - - -@docker_required -def test_published_reference_is_sha256_digest(repro_digests): - # VAL-IMG-002 (dynamic): the canonical reference is a sha256 digest - assert DIGEST_RE.match(repro_digests[0]) - - -@docker_required -def test_nonreproducible_input_is_detected(tmp_path): - # VAL-IMG-003: inject a build-time timestamp -> two builds diverge - perturbed = ( - cbuild.CANONICAL_DOCKERFILE.read_text() + "\nRUN date +%s%N > /nondeterministic_marker\n" - ) - dockerfile = tmp_path / "Dockerfile.perturbed" - dockerfile.write_text(perturbed) - result = cbuild.check_reproducible(builds=2, dockerfile=dockerfile, dest_dir=tmp_path) - assert not result.reproducible, result.digests - assert result.digests[0] != result.digests[1] - - -@docker_required -def test_image_entrypoint_help_runs_inside_image(loaded_image): - # VAL-IMG-004: --help dry invocation exits 0 inside the image - proc = subprocess.run( - ["docker", "run", "--rm", loaded_image, "--help"], - capture_output=True, - text=True, - ) - assert proc.returncode == 0, proc.stderr - - -@docker_required -def test_image_contains_own_runner_modules(loaded_image): - # VAL-IMG-004: the entrypoint's dry `check` confirms own_runner modules are - # present at the expected locations inside the image. - proc = subprocess.run( - ["docker", "run", "--rm", loaded_image, "check"], - capture_output=True, - text=True, - ) - assert proc.returncode == 0, proc.stderr - assert "own_runner" in proc.stdout - - -@docker_required -def test_image_has_no_secrets(loaded_image, tmp_path): - # VAL-IMG-005: exported image filesystem has zero secret hits - export_tar = tmp_path / "image-fs.tar" - create = subprocess.run( - ["docker", "create", loaded_image], - capture_output=True, - text=True, - ) - assert create.returncode == 0, create.stderr - container_id = create.stdout.strip() - try: - with export_tar.open("wb") as handle: - export = subprocess.run( - ["docker", "export", container_id], - stdout=handle, - stderr=subprocess.PIPE, - text=False, - ) - assert export.returncode == 0, export.stderr - finally: - subprocess.run(["docker", "rm", "-f", container_id], capture_output=True, text=True) - - hits = secrets_scan.scan_tar(export_tar) - assert hits == [], [f"{h.member}:{h.pattern}" for h in hits] - - -@docker_required -def test_image_filesystem_has_no_golden_plaintext(loaded_image): - # VAL-IMG-005: golden may be absent OR present only encrypted-at-rest. - # Live-prep bakes ciphertext (.enc) + digests under /app/golden; plaintext - # task tests (*.py / tests/ dirs / unencrypted oracle JSON) must never land. - proc = subprocess.run( - [ - "docker", - "run", - "--rm", - "--entrypoint", - "sh", - loaded_image, - "-c", - ( - "if [ ! -e /app/golden ]; then echo no-golden; exit 0; fi; " - # ciphertext present (required when the mount exists) - "ls /app/golden/*.enc >/dev/null 2>&1 || { echo missing-enc; exit 1; }; " - # no plaintext test suites / source - "find /app/golden -type d -name tests | grep -q . " - "&& { echo plain-tests; exit 1; }; " - "find /app/golden -type f -name '*.py' | grep -q . " - "&& { echo plain-py; exit 1; }; " - # no unencrypted oracle/task JSON (permit only digest/registry pin files) - "find /app/golden -type f -name '*.json' ! -name '*digest*' " - "! -name '*registry-refs*' | grep -q . && { echo plain-json; exit 1; }; " - "echo encrypted-only-ok" - ), - ], - capture_output=True, - text=True, - ) - assert proc.returncode == 0, proc.stderr + proc.stdout - assert "no-golden" in proc.stdout or "encrypted-only-ok" in proc.stdout diff --git a/packages/challenges/agent-challenge/tests/test_canonical_measurement.py b/packages/challenges/agent-challenge/tests/test_canonical_measurement.py deleted file mode 100644 index 31379357b..000000000 --- a/packages/challenges/agent-challenge/tests/test_canonical_measurement.py +++ /dev/null @@ -1,556 +0,0 @@ -"""Behavioral tests for the canonical measurement tooling (M1). - -Fulfils VAL-IMG-006..009: - * VAL-IMG-006 normalized compose-hash is deterministic and normalization-invariant - * VAL-IMG-007 compose-hash changes deterministically on any material change (drift) - * VAL-IMG-008 dstack-mr wrapper computes stable, well-formed MRTD/RTMR0-2 - * VAL-IMG-009 the canonical measurement record is emitted in a stable, pinnable form - -The compose-hash assertions are pure/offline. The dstack-mr assertions drive the -wrapper against an injected stub ``dstack-mr`` binary (the real tool needs the -multi-hundred-MB dstack OS image files, which are only available on a live CVM; -the real-tool equivalence is checked live at M6 via VAL-IMG-011). The stub is a -faithful stand-in: it mirrors the real ``dstack-mr -cpu -memory -json -metadata`` -interface and JSON output shape, and derives its registers deterministically from -its inputs so both determinism and input-sensitivity are exercised. -""" - -from __future__ import annotations - -import copy -import json -import os -import re -import stat -import sys -from pathlib import Path - -import pytest - -from agent_challenge.canonical import measurement as m - -HEX96_RE = re.compile(r"^[0-9a-f]{96}$") -HEX64_RE = re.compile(r"^[0-9a-f]{64}$") - - -# --------------------------------------------------------------------------- # -# Fixtures: a representative app-compose and a stub dstack-mr binary -# --------------------------------------------------------------------------- # - - -def _base_compose() -> dict: - return { - "manifest_version": 2, - "name": "agent-challenge-canonical", - "runner": "docker-compose", - "docker_compose_file": ( - "services:\n" - " orchestrator:\n" - " image: ghcr.io/base/agent-challenge@sha256:" + ("a" * 64) + "\n" - " ports:\n" - ' - "8700:8700"\n' - ), - "kms_enabled": True, - "gateway_enabled": False, - "public_logs": True, - "public_sysinfo": True, - "local_key_provider_enabled": False, - "allowed_envs": ["BASE_LLM_GATEWAY_URL", "BASE_GATEWAY_TOKEN"], - "no_instance_id": False, - "features": {"kms": True, "gateway": False, "logs": True}, - } - - -_STUB_TEMPLATE = """#!{python} -import hashlib -import json -import sys - -MODE = {mode!r} - -args = sys.argv[1:] -opts = {{}} -i = 0 -while i < len(args): - tok = args[i] - if tok == "-json": - opts["json"] = True - i += 1 - elif tok.startswith("-"): - opts[tok[1:]] = args[i + 1] if i + 1 < len(args) else "" - i += 2 - else: - i += 1 - -metadata_path = opts.get("metadata", "") -try: - meta = open(metadata_path, "rb").read() -except OSError: - sys.stderr.write("stub dstack-mr: cannot read metadata\\n") - sys.exit(1) - -seed = meta + opts.get("cpu", "").encode() + opts.get("memory", "").encode() - - -def reg(tag): - return hashlib.sha384(tag + seed).hexdigest() - - -def img(tag): - return hashlib.sha256(tag + seed).hexdigest() - - -if MODE == "fail": - sys.stderr.write("stub dstack-mr: forced failure\\n") - sys.exit(3) - -if MODE == "bad_hex": - out = {{"mrtd": "nothexvalue", "rtmr0": reg(b"rtmr0"), "rtmr1": reg(b"rtmr1"), - "rtmr2": reg(b"rtmr2"), "mr_aggregated": img(b"agg"), "mr_image": img(b"img")}} -elif MODE == "bad_width": - out = {{"mrtd": "abcdef", "rtmr0": reg(b"rtmr0"), "rtmr1": reg(b"rtmr1"), - "rtmr2": reg(b"rtmr2"), "mr_aggregated": img(b"agg"), "mr_image": img(b"img")}} -else: - out = {{"mrtd": reg(b"mrtd"), "rtmr0": reg(b"rtmr0"), "rtmr1": reg(b"rtmr1"), - "rtmr2": reg(b"rtmr2"), "mr_aggregated": img(b"agg"), "mr_image": img(b"img")}} - -print(json.dumps(out, indent=2)) -""" - - -def _write_stub(path: Path, *, mode: str = "ok") -> Path: - path.write_text(_STUB_TEMPLATE.format(python=sys.executable, mode=mode)) - path.chmod(path.stat().st_mode | stat.S_IEXEC | stat.S_IRUSR) - return path - - -@pytest.fixture -def stub_mr(tmp_path) -> str: - return str(_write_stub(tmp_path / "dstack-mr-stub", mode="ok")) - - -@pytest.fixture -def metadata_file(tmp_path) -> Path: - meta = tmp_path / "metadata.json" - meta.write_text( - json.dumps( - { - "bios": "ovmf.fd", - "kernel": "bzImage", - "cmdline": "console=ttyS0 root=/dev/vda", - "initrd": "initramfs.cpio.gz", - } - ) - ) - return meta - - -# --------------------------------------------------------------------------- # -# VAL-IMG-006: deterministic + normalization-invariant compose-hash -# --------------------------------------------------------------------------- # - - -def test_compose_hash_is_deterministic_and_sha256_shaped(): - compose = _base_compose() - first = m.compose_hash(compose) - second = m.compose_hash(copy.deepcopy(compose)) - assert first == second - assert HEX64_RE.match(first), first - - -def test_compose_hash_invariant_to_top_level_key_order(): - compose = _base_compose() - reordered = {k: compose[k] for k in reversed(list(compose))} - assert list(reordered) != list(compose) - assert m.compose_hash(reordered) == m.compose_hash(compose) - - -def test_compose_hash_invariant_to_nested_key_order(): - compose = _base_compose() - variant = copy.deepcopy(compose) - variant["features"] = {k: variant["features"][k] for k in reversed(list(variant["features"]))} - assert m.compose_hash(variant) == m.compose_hash(compose) - - -def test_compose_hash_invariant_to_insignificant_whitespace(): - compose = _base_compose() - compact = json.dumps(compose, separators=(",", ":")) - pretty = json.dumps(compose, indent=4, sort_keys=False) - baseline = m.compose_hash(compose) - assert m.compose_hash(compact) == baseline - assert m.compose_hash(pretty) == baseline - - -def test_compose_hash_accepts_dict_and_equivalent_json_string(): - compose = _base_compose() - as_string = json.dumps(compose) - assert m.compose_hash(as_string) == m.compose_hash(compose) - - -def test_normalize_app_compose_is_sorted_and_compact(): - compose = _base_compose() - normalized = m.normalize_app_compose(compose) - # Sorted keys + compact separators => round-trips and is byte-identical to a - # sorted compact dump. - assert normalized == json.dumps(compose, sort_keys=True, separators=(",", ":")) - # keys appear in sorted order at the top level - top_keys = [k for k in compose] - assert list(json.loads(normalized)) == sorted(top_keys) - assert json.loads(normalized) == compose - - -def test_normalize_app_compose_rejects_unsupported_type(): - with pytest.raises(TypeError): - m.normalize_app_compose(12345) # type: ignore[arg-type] - - -# --------------------------------------------------------------------------- # -# VAL-IMG-007: material change => different deterministic hash; revert restores -# --------------------------------------------------------------------------- # - - -def _mutations(): - def change_image_digest(c): - c["docker_compose_file"] = c["docker_compose_file"].replace("a" * 64, "b" * 64) - - def change_ports(c): - c["docker_compose_file"] = c["docker_compose_file"].replace("8700:8700", "8701:8701") - - def change_env_keys(c): - c["allowed_envs"] = ["BASE_LLM_GATEWAY_URL"] - - def change_service_definition(c): - c["docker_compose_file"] += " sidecar:\n image: busybox@sha256:" + ("c" * 64) + "\n" - - def change_top_level_flag(c): - c["gateway_enabled"] = True - - return { - "image_digest": change_image_digest, - "ports": change_ports, - "env_keys": change_env_keys, - "service_definition": change_service_definition, - "top_level_flag": change_top_level_flag, - } - - -def test_material_change_yields_distinct_reproducible_hash_and_revert_restores(): - baseline_compose = _base_compose() - baseline = m.compose_hash(baseline_compose) - - seen = {baseline} - for name, mutate in _mutations().items(): - mutated = copy.deepcopy(baseline_compose) - mutate(mutated) - changed = m.compose_hash(mutated) - assert changed != baseline, f"{name} did not change the hash" - assert changed not in seen, f"{name} collided with another hash" - seen.add(changed) - # deterministic: recomputing the same mutation reproduces the hash - again = copy.deepcopy(baseline_compose) - mutate(again) - assert m.compose_hash(again) == changed - # reverting reproduces the baseline exactly - assert m.compose_hash(_base_compose()) == baseline - - -# --------------------------------------------------------------------------- # -# VAL-IMG-008: dstack-mr wrapper — stable, well-formed MRTD/RTMR0-2 -# --------------------------------------------------------------------------- # - - -def test_compute_image_measurement_is_deterministic_and_well_formed(stub_mr, metadata_file): - first = m.compute_image_measurement(metadata_file, cpu=4, memory="4G", dstack_mr_bin=stub_mr) - second = m.compute_image_measurement(metadata_file, cpu=4, memory="4G", dstack_mr_bin=stub_mr) - for reg_value in (first.mrtd, first.rtmr0, first.rtmr1, first.rtmr2): - assert HEX96_RE.match(reg_value), reg_value - assert (first.mrtd, first.rtmr0, first.rtmr1, first.rtmr2) == ( - second.mrtd, - second.rtmr0, - second.rtmr1, - second.rtmr2, - ) - assert HEX64_RE.match(first.os_image_hash), first.os_image_hash - # Product formula, not raw tool mr_image unless they coincide. - assert first.os_image_hash == m.product_os_image_hash( - mrtd=first.mrtd, rtmr1=first.rtmr1, rtmr2=first.rtmr2 - ) - - -def test_compute_image_measurement_is_input_sensitive(stub_mr, tmp_path, metadata_file): - baseline = m.compute_image_measurement(metadata_file, cpu=4, memory="4G", dstack_mr_bin=stub_mr) - - # different cpu -> different registers (inputs really flow to the tool) - diff_cpu = m.compute_image_measurement(metadata_file, cpu=8, memory="4G", dstack_mr_bin=stub_mr) - assert diff_cpu.mrtd != baseline.mrtd - - # different image metadata -> different registers (drift on image change) - other_meta = tmp_path / "metadata2.json" - other_meta.write_text(metadata_file.read_text().replace("bzImage", "bzImage-v2")) - diff_image = m.compute_image_measurement(other_meta, cpu=4, memory="4G", dstack_mr_bin=stub_mr) - assert diff_image.mrtd != baseline.mrtd - assert diff_image.os_image_hash != baseline.os_image_hash - - -def test_compute_image_measurement_passes_cpu_and_memory(monkeypatch, metadata_file): - captured = {} - - class _Proc: - returncode = 0 - stdout = json.dumps( - { - "mrtd": "a" * 96, - "rtmr0": "b" * 96, - "rtmr1": "c" * 96, - "rtmr2": "d" * 96, - "mr_image": "e" * 64, - } - ) - stderr = "" - - def fake_run(cmd, *args, **kwargs): - captured["cmd"] = cmd - return _Proc() - - monkeypatch.setattr(m.subprocess, "run", fake_run) - m.compute_image_measurement(metadata_file, cpu=2, memory="2G", dstack_mr_bin="dstack-mr") - cmd = captured["cmd"] - assert cmd[0] == "dstack-mr" - assert "-cpu" in cmd and cmd[cmd.index("-cpu") + 1] == "2" - assert "-memory" in cmd and cmd[cmd.index("-memory") + 1] == "2G" - assert "-json" in cmd - assert "-metadata" in cmd and cmd[cmd.index("-metadata") + 1] == str(metadata_file) - - -def test_compute_image_measurement_raises_on_tool_failure(tmp_path, metadata_file): - failing = str(_write_stub(tmp_path / "dstack-mr-fail", mode="fail")) - with pytest.raises(RuntimeError, match="dstack-mr"): - m.compute_image_measurement(metadata_file, cpu=4, memory="4G", dstack_mr_bin=failing) - - -def test_compute_image_measurement_rejects_non_hex_register(tmp_path, metadata_file): - bad = str(_write_stub(tmp_path / "dstack-mr-badhex", mode="bad_hex")) - with pytest.raises(ValueError, match="mrtd"): - m.compute_image_measurement(metadata_file, cpu=4, memory="4G", dstack_mr_bin=bad) - - -def test_compute_image_measurement_rejects_wrong_width_register(tmp_path, metadata_file): - bad = str(_write_stub(tmp_path / "dstack-mr-badwidth", mode="bad_width")) - with pytest.raises(ValueError, match="mrtd"): - m.compute_image_measurement(metadata_file, cpu=4, memory="4G", dstack_mr_bin=bad) - - -def test_memory_int_is_formatted_as_gigabytes(monkeypatch, metadata_file): - captured = {} - - class _Proc: - returncode = 0 - stdout = json.dumps( - { - "mrtd": "a" * 96, - "rtmr0": "b" * 96, - "rtmr1": "c" * 96, - "rtmr2": "d" * 96, - "mr_image": "e" * 64, - } - ) - stderr = "" - - monkeypatch.setattr( - m.subprocess, "run", lambda cmd, *a, **k: captured.update(cmd=cmd) or _Proc() - ) - m.compute_image_measurement(metadata_file, cpu=1, memory=8, dstack_mr_bin="dstack-mr") - cmd = captured["cmd"] - assert cmd[cmd.index("-memory") + 1] == "8G" - - -def test_dstack_mr_binary_resolution_prefers_explicit_then_env(monkeypatch): - monkeypatch.delenv("DSTACK_MR_BIN", raising=False) - assert m.dstack_mr_binary() == m.DEFAULT_DSTACK_MR_BIN - assert m.dstack_mr_binary("/opt/dstack-mr") == "/opt/dstack-mr" - monkeypatch.setenv("DSTACK_MR_BIN", "/env/dstack-mr") - assert m.dstack_mr_binary() == "/env/dstack-mr" - assert m.dstack_mr_binary("/opt/dstack-mr") == "/opt/dstack-mr" - - -# --------------------------------------------------------------------------- # -# VAL-IMG-009: stable, pinnable canonical measurement record -# --------------------------------------------------------------------------- # - - -def test_canonical_measurement_has_exactly_the_six_pinnable_fields(stub_mr, metadata_file): - record = m.build_canonical_measurement( - metadata_path=metadata_file, - cpu=4, - memory="4G", - compose=_base_compose(), - dstack_mr_bin=stub_mr, - ) - data = record.as_dict() - assert set(data) == {"mrtd", "rtmr0", "rtmr1", "rtmr2", "compose_hash", "os_image_hash"} - # rtmr3 is a runtime register and must NOT be part of the pinnable record. - assert "rtmr3" not in data - - -def test_canonical_measurement_fields_are_correctly_shaped(stub_mr, metadata_file): - record = m.build_canonical_measurement( - metadata_path=metadata_file, - cpu=4, - memory="4G", - compose=_base_compose(), - dstack_mr_bin=stub_mr, - ) - for field in ("mrtd", "rtmr0", "rtmr1", "rtmr2"): - assert HEX96_RE.match(getattr(record, field)), field - assert HEX64_RE.match(record.compose_hash) - assert HEX64_RE.match(record.os_image_hash) - assert record.compose_hash == m.compose_hash(_base_compose()) - - -def test_canonical_measurement_serialization_is_byte_stable_across_reemission( - stub_mr, metadata_file -): - def build(): - return m.build_canonical_measurement( - metadata_path=metadata_file, - cpu=4, - memory="4G", - compose=_base_compose(), - dstack_mr_bin=stub_mr, - ) - - first = build().to_json() - second = build().to_json() - assert first == second - - -def test_canonical_measurement_json_is_sorted_and_pinnable(stub_mr, metadata_file): - record = m.build_canonical_measurement( - metadata_path=metadata_file, - cpu=4, - memory="4G", - compose=_base_compose(), - dstack_mr_bin=stub_mr, - ) - serialized = record.to_json() - parsed = json.loads(serialized) - # A validator copies this verbatim into an allowlist entry: sorted keys + - # compact separators make it byte-stable. - assert serialized == json.dumps(parsed, sort_keys=True, separators=(",", ":")) - assert parsed == record.as_dict() - - -def test_canonical_measurement_drift_on_changed_image_or_compose(stub_mr, tmp_path, metadata_file): - baseline = m.build_canonical_measurement( - metadata_path=metadata_file, - cpu=4, - memory="4G", - compose=_base_compose(), - dstack_mr_bin=stub_mr, - ) - - changed_compose = copy.deepcopy(_base_compose()) - changed_compose["docker_compose_file"] = changed_compose["docker_compose_file"].replace( - "a" * 64, "b" * 64 - ) - drift_compose = m.build_canonical_measurement( - metadata_path=metadata_file, - cpu=4, - memory="4G", - compose=changed_compose, - dstack_mr_bin=stub_mr, - ) - assert drift_compose.compose_hash != baseline.compose_hash - assert drift_compose.to_json() != baseline.to_json() - - other_meta = tmp_path / "metadata3.json" - other_meta.write_text(metadata_file.read_text().replace("ovmf.fd", "ovmf-next.fd")) - drift_image = m.build_canonical_measurement( - metadata_path=other_meta, - cpu=4, - memory="4G", - compose=_base_compose(), - dstack_mr_bin=stub_mr, - ) - assert drift_image.mrtd != baseline.mrtd - assert drift_image.os_image_hash != baseline.os_image_hash - - -def test_build_canonical_measurement_accepts_env_binary(monkeypatch, stub_mr, metadata_file): - monkeypatch.setenv("DSTACK_MR_BIN", stub_mr) - record = m.build_canonical_measurement( - metadata_path=metadata_file, - cpu=4, - memory="4G", - compose=_base_compose(), - ) - assert HEX96_RE.match(record.mrtd) - - -def test_dstack_mr_available_reflects_binary_presence(stub_mr): - assert m.dstack_mr_available(stub_mr) is True - assert m.dstack_mr_available(os.path.join(os.sep, "nonexistent", "dstack-mr")) is False - - -def test_dstack_mr_available_uses_path_lookup_for_bare_name(monkeypatch): - monkeypatch.delenv("DSTACK_MR_BIN", raising=False) - monkeypatch.setattr(m.shutil, "which", lambda name: "/usr/bin/" + name) - assert m.dstack_mr_available() is True - monkeypatch.setattr(m.shutil, "which", lambda name: None) - assert m.dstack_mr_available() is False - - -def test_format_memory_rejects_bool(): - with pytest.raises(TypeError): - m._format_memory(True) - - -def test_compute_image_measurement_rejects_non_json_output(monkeypatch, metadata_file): - class _Proc: - returncode = 0 - stdout = "not json at all" - stderr = "" - - monkeypatch.setattr(m.subprocess, "run", lambda *a, **k: _Proc()) - with pytest.raises(RuntimeError, match="non-JSON"): - m.compute_image_measurement(metadata_file, cpu=4, memory="4G", dstack_mr_bin="dstack-mr") - - -def test_compute_image_measurement_derives_product_os_when_mr_image_absent( - monkeypatch, metadata_file -): - """Product seals sha256(registers); catalog mr_image is optional.""" - - class _Proc: - returncode = 0 - stdout = json.dumps( - {"mrtd": "a" * 96, "rtmr0": "b" * 96, "rtmr1": "c" * 96, "rtmr2": "d" * 96} - ) - stderr = "" - - monkeypatch.setattr(m.subprocess, "run", lambda *a, **k: _Proc()) - image = m.compute_image_measurement( - metadata_file, cpu=4, memory="4G", dstack_mr_bin="dstack-mr" - ) - expected = m.product_os_image_hash(mrtd="a" * 96, rtmr1="c" * 96, rtmr2="d" * 96) - assert image.os_image_hash == expected - assert image.dstack_mr_image is None - - -def test_compute_image_measurement_rejects_non_string_register(monkeypatch, metadata_file): - class _Proc: - returncode = 0 - stdout = json.dumps( - { - "mrtd": 123, - "rtmr0": "b" * 96, - "rtmr1": "c" * 96, - "rtmr2": "d" * 96, - "mr_image": "e" * 64, - } - ) - stderr = "" - - monkeypatch.setattr(m.subprocess, "run", lambda *a, **k: _Proc()) - with pytest.raises(ValueError, match="mrtd"): - m.compute_image_measurement(metadata_file, cpu=4, memory="4G", dstack_mr_bin="dstack-mr") diff --git a/packages/challenges/agent-challenge/tests/test_canonical_report_data.py b/packages/challenges/agent-challenge/tests/test_canonical_report_data.py deleted file mode 100644 index 8f879c82f..000000000 --- a/packages/challenges/agent-challenge/tests/test_canonical_report_data.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Behavioral tests for the architecture-sec-6 ``report_data`` binding (M1). - -Fulfils VAL-IMG-012..018: - * VAL-IMG-012 derivation matches architecture sec 6 exactly (pinned golden vector) - * VAL-IMG-013 task_ids are sorted before hashing (order-independent binding) - * VAL-IMG-014 report_data is sensitive to every bound component - * VAL-IMG-015 a fresh validator nonce changes report_data (anti-replay) - * VAL-IMG-016 oversized preimage is SHA-256'd to 32 bytes; no >64-byte value - reaches get_quote; malformed values are rejected - * VAL-IMG-017 report_data_hex is a 64-byte field whose leading 32 bytes are the digest - * VAL-IMG-018 scores_digest binds the actual reported scores - -The derivation MUST be byte-identical to base's canonical helper -``src/base/worker/proof.py::phala_report_data`` (tag ``base-agent-challenge-v1``, -sorted-key JSON preimage, ``sorted(task_ids)``, ``rtmr3`` excluded). Because the -base package installed in this repo's venv is pinned to origin/main and does not -ship that helper, the algorithm is replicated self-contained here and pinned to a -shared cross-repo golden vector so drift between the two implementations is -caught: :data:`GOLDEN_DIGEST_HEX` / :data:`GOLDEN_FIELD_HEX` are asserted here AND -in ``base/tests/unit/test_worker_proof_phala.py`` against base's helper. -""" - -from __future__ import annotations - -import hashlib -import json -import re - -import pytest - -from agent_challenge.canonical import report_data as rd -from agent_challenge.canonical.measurement import CanonicalMeasurement - -# --------------------------------------------------------------------------- # -# Shared cross-repo golden vector (fixed inputs -> expected digest/field). -# These exact inputs and expected outputs are also asserted in base against -# base.worker.proof.phala_report_data / phala_report_data_hex. Do NOT change one -# side without changing the other -- that is the whole point of pinning them. -# --------------------------------------------------------------------------- # -GOLDEN_MEASUREMENT = { - "mrtd": "a" * 96, - "rtmr0": "b0" * 48, - "rtmr1": "b1" * 48, - "rtmr2": "b2" * 48, - "compose_hash": "c" * 64, - "os_image_hash": "e" * 64, -} -GOLDEN_AGENT_HASH = "f" * 64 -GOLDEN_TASK_IDS = ["task-b", "task-a", "task-c"] -GOLDEN_SCORES_DIGEST = "9" * 64 -GOLDEN_NONCE = "nonce-123" - -GOLDEN_DIGEST_HEX = "dd2c57688b55e25df20e292b71e1cb97d8501e9280e1dd3475b3e61c30e38cc2" -GOLDEN_FIELD_HEX = GOLDEN_DIGEST_HEX + "00" * 32 - - -def _kwargs(**overrides: object) -> dict[str, object]: - base = dict( - canonical_measurement=dict(GOLDEN_MEASUREMENT), - agent_hash=GOLDEN_AGENT_HASH, - task_ids=list(GOLDEN_TASK_IDS), - scores_digest=GOLDEN_SCORES_DIGEST, - validator_nonce=GOLDEN_NONCE, - ) - base.update(overrides) - return base - - -def _sec6_digest( - *, - tag: str, - measurement: dict[str, str], - agent_hash: str, - task_ids: list[str], - scores_digest: str, - validator_nonce: str, -) -> bytes: - """Independent architecture-sec-6 recomputation (mirrors the spec verbatim).""" - - preimage = { - "tag": tag, - "canonical_measurement": { - field: str(measurement[field]) - for field in ( - "mrtd", - "rtmr0", - "rtmr1", - "rtmr2", - "compose_hash", - "os_image_hash", - ) - }, - "agent_hash": agent_hash, - "task_ids": sorted(task_ids), - "scores_digest": scores_digest, - "validator_nonce": validator_nonce, - } - encoded = json.dumps(preimage, sort_keys=True, separators=(",", ":")).encode() - return hashlib.sha256(encoded).digest() - - -# --- VAL-IMG-012: golden vector + determinism ------------------------------ # - - -def test_report_data_matches_pinned_golden_vector() -> None: - digest = rd.report_data(**_kwargs()) # type: ignore[arg-type] - assert isinstance(digest, bytes) - assert len(digest) == 32 - assert digest.hex() == GOLDEN_DIGEST_HEX - - -def test_report_data_field_matches_pinned_golden_vector() -> None: - assert rd.report_data_hex(**_kwargs()) == GOLDEN_FIELD_HEX # type: ignore[arg-type] - - -def test_report_data_matches_independent_sec6_computation() -> None: - expected = _sec6_digest( - tag=rd.PHALA_REPORT_DATA_TAG, - measurement=GOLDEN_MEASUREMENT, - agent_hash=GOLDEN_AGENT_HASH, - task_ids=GOLDEN_TASK_IDS, - scores_digest=GOLDEN_SCORES_DIGEST, - validator_nonce=GOLDEN_NONCE, - ) - assert rd.report_data(**_kwargs()) == expected # type: ignore[arg-type] - - -def test_report_data_is_deterministic() -> None: - assert rd.report_data(**_kwargs()) == rd.report_data(**_kwargs()) # type: ignore[arg-type] - - -def test_tag_constant_is_base_agent_challenge_v1() -> None: - assert rd.PHALA_REPORT_DATA_TAG == "base-agent-challenge-v1" - - -# --- VAL-IMG-013: task_ids order independence ------------------------------ # - - -def test_task_ids_order_independent() -> None: - forward = rd.report_data(**_kwargs(task_ids=["task-a", "task-b", "task-c"])) # type: ignore[arg-type] - shuffled = rd.report_data(**_kwargs(task_ids=["task-c", "task-a", "task-b"])) # type: ignore[arg-type] - assert forward == shuffled - - -# --- VAL-IMG-014: sensitivity to every bound component --------------------- # - - -def test_tag_is_bound() -> None: - real = rd.report_data(**_kwargs()) # type: ignore[arg-type] - off_tag = _sec6_digest( - tag="some-other-tag", - measurement=GOLDEN_MEASUREMENT, - agent_hash=GOLDEN_AGENT_HASH, - task_ids=GOLDEN_TASK_IDS, - scores_digest=GOLDEN_SCORES_DIGEST, - validator_nonce=GOLDEN_NONCE, - ) - assert real != off_tag - - -def test_measurement_is_bound() -> None: - base = rd.report_data(**_kwargs()) # type: ignore[arg-type] - changed = rd.report_data( # type: ignore[arg-type] - **_kwargs(canonical_measurement=dict(GOLDEN_MEASUREMENT, compose_hash="0" * 64)) - ) - assert changed != base - - -def test_agent_hash_is_bound() -> None: - base = rd.report_data(**_kwargs()) # type: ignore[arg-type] - changed = rd.report_data(**_kwargs(agent_hash="0" * 64)) # type: ignore[arg-type] - assert changed != base - - -def test_task_ids_set_is_bound() -> None: - base = rd.report_data(**_kwargs()) # type: ignore[arg-type] - changed = rd.report_data(**_kwargs(task_ids=["task-a", "task-b"])) # type: ignore[arg-type] - assert changed != base - - -def test_scores_digest_is_bound() -> None: - base = rd.report_data(**_kwargs()) # type: ignore[arg-type] - changed = rd.report_data(**_kwargs(scores_digest="0" * 64)) # type: ignore[arg-type] - assert changed != base - - -# --- VAL-IMG-015: fresh nonce changes report_data -------------------------- # - - -def test_fresh_nonce_changes_report_data() -> None: - a = rd.report_data(**_kwargs(validator_nonce="nonce-A")) # type: ignore[arg-type] - b = rd.report_data(**_kwargs(validator_nonce="nonce-B")) # type: ignore[arg-type] - assert a != b - - -# --- VAL-IMG-016: oversized preimage + malformed rejection ----------------- # - - -def test_oversized_payload_is_sha256_reduced_not_truncated() -> None: - payload = b"x" * 200 - field_hex = rd.to_report_data_field(payload) - field = bytes.fromhex(field_hex) - assert len(field) == 64 - assert field[:32] == hashlib.sha256(payload).digest() - assert field[32:] == b"\x00" * 32 - - -def test_value_handed_to_get_quote_never_exceeds_64_bytes() -> None: - captured: dict[str, bytes] = {} - - def fake_get_quote(report_data: bytes) -> str: - captured["value"] = report_data - return "quote" - - # emitter path: sanitize before ever calling get_quote. - field = bytes.fromhex(rd.to_report_data_field(b"y" * 500)) - fake_get_quote(field) - assert len(captured["value"]) <= 64 - - -def test_small_payload_is_zero_padded_not_hashed() -> None: - payload = bytes.fromhex("abcd") - field = bytes.fromhex(rd.to_report_data_field(payload)) - assert field[:2] == payload - assert field[2:] == b"\x00" * 62 - - -def test_report_data_field_accepts_the_sec6_digest_unchanged() -> None: - digest = rd.report_data(**_kwargs()) # type: ignore[arg-type] - assert rd.to_report_data_field(digest) == GOLDEN_FIELD_HEX - - -@pytest.mark.parametrize("bad", ["zz" * 4, "abc", "not-hex"]) -def test_malformed_hex_value_is_rejected(bad: str) -> None: - with pytest.raises(ValueError): - rd.to_report_data_field(bad) - - -def test_non_bytes_non_str_value_is_rejected() -> None: - with pytest.raises(TypeError): - rd.to_report_data_field(12345) # type: ignore[arg-type] - - -# --- VAL-IMG-017: report_data_hex field shape ------------------------------ # - - -def test_report_data_hex_is_128_char_zero_padded_field() -> None: - digest = rd.report_data(**_kwargs()) # type: ignore[arg-type] - hex_field = rd.report_data_hex(**_kwargs()) # type: ignore[arg-type] - assert len(hex_field) == 128 - assert re.fullmatch(r"[0-9a-f]{128}", hex_field) - field = bytes.fromhex(hex_field) - assert len(field) == 64 - assert field[:32] == digest - assert field[32:] == b"\x00" * 32 - - -# --- VAL-IMG-018: scores_digest binds the actual reported scores ----------- # - - -def test_scores_digest_is_deterministic_and_order_independent() -> None: - scores = {"task-a": 1.0, "task-b": 0.0, "task-c": 0.5} - reordered = {"task-c": 0.5, "task-a": 1.0, "task-b": 0.0} - d1 = rd.scores_digest(scores) - d2 = rd.scores_digest(reordered) - assert d1 == d2 - assert re.fullmatch(r"[0-9a-f]{64}", d1) - - -def test_report_data_binds_the_actual_scores() -> None: - scores = {"task-a": 1.0, "task-b": 0.0, "task-c": 0.5} - bound = rd.report_data(**_kwargs(scores_digest=rd.scores_digest(scores))) # type: ignore[arg-type] - - # Recomputing scores_digest from the reported scores reproduces the binding. - recomputed = rd.report_data(**_kwargs(scores_digest=rd.scores_digest(scores))) # type: ignore[arg-type] - assert recomputed == bound - - # Altering any reported score changes the digest and therefore report_data. - altered = dict(scores, **{"task-a": 0.0}) - tampered = rd.report_data(**_kwargs(scores_digest=rd.scores_digest(altered))) # type: ignore[arg-type] - assert tampered != bound - - -# --- measurement input shapes --------------------------------------------- # - - -def test_accepts_canonical_measurement_dataclass() -> None: - cm = CanonicalMeasurement(**GOLDEN_MEASUREMENT) - from_dataclass = rd.report_data(**_kwargs(canonical_measurement=cm)) # type: ignore[arg-type] - from_mapping = rd.report_data(**_kwargs()) # type: ignore[arg-type] - assert from_dataclass == from_mapping - assert from_dataclass.hex() == GOLDEN_DIGEST_HEX - - -def test_mapping_measurement_ignores_runtime_rtmr3() -> None: - with_rtmr3 = rd.report_data( # type: ignore[arg-type] - **_kwargs(canonical_measurement=dict(GOLDEN_MEASUREMENT, rtmr3="d" * 96)) - ) - other_rtmr3 = rd.report_data( # type: ignore[arg-type] - **_kwargs(canonical_measurement=dict(GOLDEN_MEASUREMENT, rtmr3="7" * 96)) - ) - assert with_rtmr3 == other_rtmr3 == rd.report_data(**_kwargs()) # type: ignore[arg-type] diff --git a/packages/challenges/agent-challenge/tests/test_challenge_acceptance_gating.py b/packages/challenges/agent-challenge/tests/test_challenge_acceptance_gating.py deleted file mode 100644 index fc6607761..000000000 --- a/packages/challenges/agent-challenge/tests/test_challenge_acceptance_gating.py +++ /dev/null @@ -1,868 +0,0 @@ -"""Challenge-side Phala acceptance gating (VAL-VERIFY-015..019, 026). - -With the Phala attestation flag ON, the decentralized validator writes a task's -score ONLY when that task's result carries a VERIFIED Phala attestation: - -* VAL-VERIFY-015 -- an unattested result is rejected/parked (no TaskResult row). -* VAL-VERIFY-016 -- an attestation that fails verification is rejected/parked. -* VAL-VERIFY-017 -- a verified result is persisted exactly once (idempotent). -* VAL-VERIFY-018 -- partial attestation persists only the verified tasks; the job - does not finalize by silently scoring the unverified tasks. -* VAL-VERIFY-019 -- weight eligibility requires verified attestations. -* VAL-VERIFY-026 -- a rejected/parked result records a retrievable, distinguishable - reason (unattested vs verification-failed vs verifier-unavailable/retryable). - -Plus the flag-OFF invariant: legacy scoring is byte-identical and the quote -verifier is never invoked. - -The tests are discriminators: they build a genuinely valid attested envelope -(reused across cases) and, for each rejection case, tamper exactly one bound -component (agent_hash, scores, measurement, report_data, nonce, quote signature) -so a naive "accept if an attestation is present" gate would FAIL them. -""" - -from __future__ import annotations - -import json -import uuid - -import pytest -from sqlalchemy import func, select - -from agent_challenge.canonical import report_data as rd -from agent_challenge.canonical.attested_result import ( - ATTESTATION_BINDING_RESULT_KEY, - EXECUTION_PROOF_RESULT_KEY, - build_attestation_binding, - build_execution_proof_envelope, - build_measurement, - build_phala_attestation, -) -from agent_challenge.canonical.measurement import CanonicalMeasurement -from agent_challenge.evaluation.attestation import ( - ATTESTATION_MISSING, - ATTESTATION_VERIFICATION_FAILED, - ATTESTATION_VERIFIER_UNAVAILABLE, - AttestationGate, - AttestationOutcome, - AttestationVerifierUnavailable, - InMemoryNonceLedger, - ResultMeasurementAllowlist, -) -from agent_challenge.evaluation.benchmarks import BenchmarkTask, benchmark_tasks_to_json -from agent_challenge.evaluation.own_runner.result_schema import ( - build_benchmark_result, - format_benchmark_result_line, -) -from agent_challenge.evaluation.validator_executor import ( - execute_work_unit, - finalize_job_if_complete, - get_task_attestation, - run_validator_cycle, -) -from agent_challenge.evaluation.weights import get_weights, is_reward_eligible_job -from agent_challenge.evaluation.work_units import list_pending_work_units -from agent_challenge.keyrelease.quote import ( - StaticQuoteVerifier, - build_rtmr3_event_log, - build_tdx_quote, - os_image_hash_from_registers, -) -from agent_challenge.models import AgentSubmission, EvaluationJob, TaskResult -from agent_challenge.sdk.executors import DockerRunResult - -_REGS = {"mrtd": "11" * 48, "rtmr0": "22" * 48, "rtmr1": "33" * 48, "rtmr2": "44" * 48} -_ALT_REGS = {"mrtd": "aa" * 48, "rtmr0": "bb" * 48, "rtmr1": "cc" * 48, "rtmr2": "dd" * 48} -_COMPOSE_PAYLOAD = bytes(range(32)) -_KEY_PROVIDER_PAYLOAD = b"kms-app-key-provider" - - -def _canonical_measurement(regs: dict[str, str] = _REGS) -> tuple[dict, list, str]: - event_log, rtmr3 = build_rtmr3_event_log( - [("compose-hash", _COMPOSE_PAYLOAD), ("key-provider", _KEY_PROVIDER_PAYLOAD)] - ) - measurement = { - **regs, - "compose_hash": _COMPOSE_PAYLOAD.hex(), - "os_image_hash": os_image_hash_from_registers(regs["mrtd"], regs["rtmr1"], regs["rtmr2"]), - } - return measurement, event_log, rtmr3 - - -def _attested_line( - task_id: str, - *, - agent_hash: str, - nonce: str, - score: float = 1.0, - regs: dict[str, str] = _REGS, - report_data_nonce: str | None = None, - scores_override: dict | None = None, -) -> str: - """A ``BASE_BENCHMARK_RESULT=`` line carrying a Phala-tier attested envelope. - - ``report_data_nonce`` (if given) binds the quote's report_data to a DIFFERENT - nonce than the binding block (report_data tamper); ``scores_override`` reports - scores that differ from the ones the digest was computed over (scores tamper). - """ - - scores = {task_id: score} - task_ids = [task_id] - measurement, event_log, rtmr3 = _canonical_measurement(regs) - canonical = CanonicalMeasurement(**measurement) - scores_digest = rd.scores_digest(scores) - report_data_hex = rd.report_data_hex( - canonical_measurement=canonical, - agent_hash=agent_hash, - task_ids=task_ids, - scores_digest=scores_digest, - validator_nonce=report_data_nonce if report_data_nonce is not None else nonce, - ) - quote = build_tdx_quote( - mrtd=regs["mrtd"], - rtmr0=regs["rtmr0"], - rtmr1=regs["rtmr1"], - rtmr2=regs["rtmr2"], - rtmr3=rtmr3, - report_data=report_data_hex, - ) - attestation = build_phala_attestation( - tdx_quote=quote, - event_log=event_log, - report_data_hex=report_data_hex, - measurement=build_measurement(canonical, rtmr3=rtmr3), - vm_config={}, - ) - envelope = build_execution_proof_envelope(manifest_sha256="ab" * 32, attestation=attestation) - binding = build_attestation_binding( - agent_hash=agent_hash, - task_ids=task_ids, - scores=scores_override if scores_override is not None else scores, - scores_digest=scores_digest, - validator_nonce=nonce, - canonical_measurement=canonical, - ) - result = build_benchmark_result( - status="completed", score=score, resolved=round(score), total=1, reason_code=None - ) - result[EXECUTION_PROOF_RESULT_KEY] = envelope - result[ATTESTATION_BINDING_RESULT_KEY] = binding - return format_benchmark_result_line(result) - - -def _plain_line(score: float = 1.0) -> str: - status = "completed" if score >= 1.0 else "failed" - payload = {"status": status, "score": score, "resolved": round(score), "total": 1} - return "BASE_BENCHMARK_RESULT=" + json.dumps(payload, sort_keys=True) - - -def _make_gate( - *, nonces: list[str], regs: dict[str, str] = _REGS, verifier=None -) -> AttestationGate: - measurement, _log, _rtmr3 = _canonical_measurement(regs) - ledger = InMemoryNonceLedger() - for nonce in nonces: - ledger.issue(nonce) - return AttestationGate( - quote_verifier=verifier if verifier is not None else StaticQuoteVerifier(valid=True), - allowlist=ResultMeasurementAllowlist.from_measurements([measurement]), - nonce_validator=ledger, - ) - - -class _RecordingBroker: - """A validator-own broker returning a preset stdout line per task.""" - - def __init__(self, lines: dict[str, str]) -> None: - self.runs: list[str] = [] - self.lines = lines - - def run(self, spec, timeout_seconds: int): - task_id = spec.labels["base.task"] - self.runs.append(task_id) - return DockerRunResult( - container_name="broker-fake", - stdout=self.lines.get(task_id, _plain_line()), - stderr="", - returncode=0, - ) - - -def _patch_terminal_bench(monkeypatch, tmp_path) -> None: - base = "agent_challenge.evaluation.runner.settings" - monkeypatch.setattr(f"{base}.benchmark_backend", "terminal_bench") - monkeypatch.setattr(f"{base}.terminal_bench_execution_backend", "own_runner") - monkeypatch.setattr(f"{base}.evaluation_concurrency", 1) - monkeypatch.setattr(f"{base}.evaluation_task_count", 1) - monkeypatch.setattr(f"{base}.docker_enabled", True) - monkeypatch.setattr(f"{base}.docker_backend", "broker") - monkeypatch.setattr(f"{base}.docker_broker_url", "https://broker.test") - monkeypatch.setattr(f"{base}.docker_broker_token", "broker-token") - monkeypatch.setattr(f"{base}.docker_broker_token_file", None) - harbor = tmp_path / "harbor-runs" - harbor.mkdir(parents=True, exist_ok=True) - monkeypatch.setattr(f"{base}.harbor_output_dir", str(harbor)) - - -def _enable_phala(monkeypatch, enabled: bool = True) -> None: - monkeypatch.setattr("agent_challenge.core.config.settings.phala_attestation_enabled", enabled) - - -def _terminal_bench_tasks(count: int) -> list[BenchmarkTask]: - return [ - BenchmarkTask( - task_id=f"terminal-bench/task-{index}", - docker_image=f"ghcr.io/baseintelligence/terminal-bench-runner:{index}", - prompt=f"task {index}", - benchmark="terminal_bench", - metadata={"task_id": f"terminal-bench/task-{index}"}, - ) - for index in range(count) - ] - - -async def _create_job( - session, - *, - agent_hash: str, - tasks: list[BenchmarkTask], - tmp_path, - miner_hotkey: str | None = None, -) -> tuple[AgentSubmission, EvaluationJob]: - agent_dir = tmp_path / agent_hash - agent_dir.mkdir(parents=True, exist_ok=True) - submission = AgentSubmission( - miner_hotkey=miner_hotkey or f"hotkey-{agent_hash}", - name=f"agent-{agent_hash}", - agent_hash=agent_hash, - artifact_uri=str(agent_dir), - status="evaluation queued", - raw_status="tb_queued", - effective_status="evaluation queued", - ) - session.add(submission) - await session.flush() - job = EvaluationJob( - job_id=uuid.uuid4().hex, - submission_id=submission.id, - status="queued", - selected_tasks_json=benchmark_tasks_to_json(tasks), - total_tasks=len(tasks), - verdict="valid", - ) - session.add(job) - await session.flush() - submission.latest_evaluation_job_id = job.id - return submission, job - - -# --------------------------------------------------------------------------- # -# Gate unit behavior: a fully valid attested result verifies (positive control). -# --------------------------------------------------------------------------- # -def test_valid_attested_result_is_accepted(): - nonce = "nonce-ok" - gate = _make_gate(nonces=[nonce]) - line = _attested_line("terminal-bench/task-0", agent_hash="agent-x", nonce=nonce) - decision = gate.decide(line, expected_agent_hash="agent-x") - assert decision.outcome is AttestationOutcome.VERIFIED - assert decision.accepted is True - assert decision.reason is None - - -# --------------------------------------------------------------------------- # -# VAL-VERIFY-016 discriminators: tamper one bound component -> rejected. -# --------------------------------------------------------------------------- # -def test_wrong_agent_hash_is_rejected(): - nonce = "nonce-agent" - gate = _make_gate(nonces=[nonce]) - line = _attested_line("terminal-bench/task-0", agent_hash="attacker", nonce=nonce) - decision = gate.decide(line, expected_agent_hash="real-agent") - assert decision.outcome is AttestationOutcome.VERIFICATION_FAILED - - -def test_tampered_scores_is_rejected(): - nonce = "nonce-scores" - gate = _make_gate(nonces=[nonce]) - line = _attested_line( - "terminal-bench/task-0", - agent_hash="agent-x", - nonce=nonce, - scores_override={"terminal-bench/task-0": 0.5}, - ) - decision = gate.decide(line, expected_agent_hash="agent-x") - assert decision.outcome is AttestationOutcome.VERIFICATION_FAILED - - -def test_measurement_not_allowlisted_is_rejected(): - nonce = "nonce-meas" - gate = _make_gate(nonces=[nonce]) # allowlist pins _REGS - line = _attested_line( - "terminal-bench/task-0", agent_hash="agent-x", nonce=nonce, regs=_ALT_REGS - ) - decision = gate.decide(line, expected_agent_hash="agent-x") - assert decision.outcome is AttestationOutcome.VERIFICATION_FAILED - - -def test_report_data_mismatch_is_rejected(): - nonce = "nonce-rd" - gate = _make_gate(nonces=[nonce]) - line = _attested_line( - "terminal-bench/task-0", - agent_hash="agent-x", - nonce=nonce, - report_data_nonce="a-different-nonce", - ) - decision = gate.decide(line, expected_agent_hash="agent-x") - assert decision.outcome is AttestationOutcome.VERIFICATION_FAILED - - -def test_unknown_nonce_is_rejected(): - gate = _make_gate(nonces=[]) # nothing issued - line = _attested_line("terminal-bench/task-0", agent_hash="agent-x", nonce="never-issued") - decision = gate.decide(line, expected_agent_hash="agent-x") - assert decision.outcome is AttestationOutcome.VERIFICATION_FAILED - - -def test_forged_quote_signature_is_rejected(): - nonce = "nonce-sig" - gate = _make_gate(nonces=[nonce], verifier=StaticQuoteVerifier(valid=False)) - line = _attested_line("terminal-bench/task-0", agent_hash="agent-x", nonce=nonce) - decision = gate.decide(line, expected_agent_hash="agent-x") - assert decision.outcome is AttestationOutcome.VERIFICATION_FAILED - - -def test_out_of_date_tcb_is_rejected(): - nonce = "nonce-tcb" - gate = _make_gate(nonces=[nonce], verifier=StaticQuoteVerifier(tcb_status="OutOfDate")) - line = _attested_line("terminal-bench/task-0", agent_hash="agent-x", nonce=nonce) - decision = gate.decide(line, expected_agent_hash="agent-x") - assert decision.outcome is AttestationOutcome.VERIFICATION_FAILED - - -def test_nonce_is_single_use_across_two_results(): - nonce = "nonce-once" - gate = _make_gate(nonces=[nonce]) - line = _attested_line("terminal-bench/task-0", agent_hash="agent-x", nonce=nonce) - first = gate.decide(line, expected_agent_hash="agent-x") - second = gate.decide(line, expected_agent_hash="agent-x") - assert first.outcome is AttestationOutcome.VERIFIED - assert second.outcome is AttestationOutcome.VERIFICATION_FAILED - - -# --------------------------------------------------------------------------- # -# VAL-VERIFY-026: distinct, retrievable reasons; verifier outage is retryable. -# --------------------------------------------------------------------------- # -def test_unattested_result_reason_is_distinct(): - gate = _make_gate(nonces=[]) - decision = gate.decide(_plain_line(), expected_agent_hash="agent-x") - assert decision.outcome is AttestationOutcome.UNATTESTED - assert decision.reason == ATTESTATION_MISSING - assert decision.retryable is False - - -def test_verifier_unavailable_is_retryable_park(): - nonce = "nonce-unavail" - - class _Unavailable: - def verify(self, quote_hex): - raise AttestationVerifierUnavailable("collateral fetch timed out") - - gate = _make_gate(nonces=[nonce], verifier=_Unavailable()) - line = _attested_line("terminal-bench/task-0", agent_hash="agent-x", nonce=nonce) - decision = gate.decide(line, expected_agent_hash="agent-x") - assert decision.outcome is AttestationOutcome.VERIFIER_UNAVAILABLE - assert decision.reason == ATTESTATION_VERIFIER_UNAVAILABLE - assert decision.retryable is True - - -# --------------------------------------------------------------------------- # -# VAL-VERIFY-015: flag ON -- unattested result parked, NO score row. -# --------------------------------------------------------------------------- # -async def test_flag_on_unattested_result_is_parked_not_scored( - database_session, monkeypatch, tmp_path -): - _patch_terminal_bench(monkeypatch, tmp_path) - _enable_phala(monkeypatch) - tasks = _terminal_bench_tasks(1) - async with database_session() as session: - _submission, job = await _create_job( - session, agent_hash="unattested", tasks=tasks, tmp_path=tmp_path - ) - await session.commit() - job_pk = job.id - - async with database_session() as session: - units = await list_pending_work_units(session) - broker = _RecordingBroker({tasks[0].task_id: _plain_line()}) - gate = _make_gate(nonces=["some-nonce"]) - async with database_session() as session: - outcome = await execute_work_unit(session, units[0], executor=broker, attestation_gate=gate) - await session.commit() - - assert outcome.posted is False - assert outcome.attestation_reason == ATTESTATION_MISSING - async with database_session() as session: - count = await session.scalar( - select(func.count(TaskResult.id)).where(TaskResult.job_id == job_pk) - ) - record = await get_task_attestation(session, job_pk, tasks[0].task_id) - assert count == 0 - assert record is not None - assert record.verified is False - assert record.reason == ATTESTATION_MISSING - - -# --------------------------------------------------------------------------- # -# VAL-VERIFY-016: flag ON -- failing-verification result parked, NO score row. -# --------------------------------------------------------------------------- # -async def test_flag_on_failing_attestation_is_parked_not_scored( - database_session, monkeypatch, tmp_path -): - _patch_terminal_bench(monkeypatch, tmp_path) - _enable_phala(monkeypatch) - tasks = _terminal_bench_tasks(1) - async with database_session() as session: - _submission, job = await _create_job( - session, agent_hash="tampered", tasks=tasks, tmp_path=tmp_path - ) - await session.commit() - job_pk = job.id - - nonce = "nonce-fail" - # A genuine-looking attested line whose bound agent_hash != the submission's. - line = _attested_line(tasks[0].task_id, agent_hash="attacker", nonce=nonce) - async with database_session() as session: - units = await list_pending_work_units(session) - broker = _RecordingBroker({tasks[0].task_id: line}) - gate = _make_gate(nonces=[nonce]) - async with database_session() as session: - outcome = await execute_work_unit(session, units[0], executor=broker, attestation_gate=gate) - await session.commit() - - assert outcome.posted is False - assert outcome.attestation_reason == ATTESTATION_VERIFICATION_FAILED - assert outcome.retryable is False - async with database_session() as session: - count = await session.scalar( - select(func.count(TaskResult.id)).where(TaskResult.job_id == job_pk) - ) - record = await get_task_attestation(session, job_pk, tasks[0].task_id) - assert count == 0 - assert record.reason == ATTESTATION_VERIFICATION_FAILED - - -# --------------------------------------------------------------------------- # -# VAL-VERIFY-017: flag ON -- verified result persisted exactly once (idempotent). -# --------------------------------------------------------------------------- # -async def test_flag_on_verified_result_persisted_once(database_session, monkeypatch, tmp_path): - _patch_terminal_bench(monkeypatch, tmp_path) - _enable_phala(monkeypatch) - tasks = _terminal_bench_tasks(1) - async with database_session() as session: - submission, job = await _create_job( - session, agent_hash="verified", tasks=tasks, tmp_path=tmp_path - ) - await session.commit() - job_pk = job.id - agent_hash = submission.agent_hash - - nonce = "nonce-verified" - line = _attested_line(tasks[0].task_id, agent_hash=agent_hash, nonce=nonce) - gate = _make_gate(nonces=[nonce]) - broker = _RecordingBroker({tasks[0].task_id: line}) - - async with database_session() as session: - units = await list_pending_work_units(session) - async with database_session() as session: - first = await execute_work_unit(session, units[0], executor=broker, attestation_gate=gate) - await session.commit() - assert first.posted is True - assert first.score == 1.0 - - # Re-post the same unit: the already-terminal result short-circuits (no - # re-verify, no duplicate row, no second nonce consumption). - repost_broker = _RecordingBroker({tasks[0].task_id: line}) - async with database_session() as session: - units2 = await list_pending_work_units(session) - # The task is now terminal, so it is no longer pending; force a direct re-post. - async with database_session() as session: - second = await execute_work_unit( - session, units[0], executor=repost_broker, attestation_gate=gate - ) - await session.commit() - assert second.posted is False - assert repost_broker.runs == [] - assert units2 == [] - - async with database_session() as session: - count = await session.scalar( - select(func.count(TaskResult.id)).where(TaskResult.job_id == job_pk) - ) - record = await get_task_attestation(session, job_pk, tasks[0].task_id) - assert count == 1 - assert record.verified is True - - -# --------------------------------------------------------------------------- # -# VAL-VERIFY-018: flag ON -- partial attestation persists only verified tasks. -# --------------------------------------------------------------------------- # -async def test_flag_on_partial_attestation_scores_only_verified( - database_session, monkeypatch, tmp_path -): - _patch_terminal_bench(monkeypatch, tmp_path) - _enable_phala(monkeypatch) - tasks = _terminal_bench_tasks(5) - async with database_session() as session: - submission, job = await _create_job( - session, agent_hash="partial", tasks=tasks, tmp_path=tmp_path - ) - await session.commit() - job_id = job.job_id - job_pk = job.id - agent_hash = submission.agent_hash - - verified_tasks = tasks[:3] - nonces = {task.task_id: f"nonce-{index}" for index, task in enumerate(verified_tasks)} - lines = { - t.task_id: _attested_line(t.task_id, agent_hash=agent_hash, nonce=nonces[t.task_id]) - for t in verified_tasks - } - # The last two tasks report unattested results. - for task in tasks[3:]: - lines[task.task_id] = _plain_line() - - gate = _make_gate(nonces=list(nonces.values())) - broker = _RecordingBroker(lines) - summary = await run_validator_cycle(executor=broker, attestation_gate=gate) - - # Only the 3 verified tasks were scored; the job is NOT finalized. - assert summary.finalized_jobs == () - async with database_session() as session: - results = ( - (await session.execute(select(TaskResult).where(TaskResult.job_id == job_pk))) - .scalars() - .all() - ) - job_row = await session.scalar(select(EvaluationJob).where(EvaluationJob.job_id == job_id)) - assert {r.task_id for r in results} == {t.task_id for t in verified_tasks} - assert len(results) == 3 - assert job_row.status != "completed" - - # Finalizing is a safe no-op while the unattested tasks are unscored. - async with database_session() as session: - assert await finalize_job_if_complete(session, job_id) is None - await session.commit() - - # The unattested tasks recorded a retrievable park reason (not scored as 0). - async with database_session() as session: - for task in tasks[3:]: - record = await get_task_attestation(session, job_pk, task.task_id) - assert record is not None - assert record.verified is False - assert record.reason == ATTESTATION_MISSING - - -# --------------------------------------------------------------------------- # -# VAL-VERIFY-019: flag ON -- weight eligibility requires verified attestations. -# --------------------------------------------------------------------------- # -def test_is_reward_eligible_requires_attestation_when_flagged(): - job = EvaluationJob( - job_id="j", - submission_id=1, - status="completed", - selected_tasks_json="[]", - total_tasks=1, - passed_tasks=1, - score=1.0, - ) - assert is_reward_eligible_job(job, 1, attestation_verified=True) is True - assert is_reward_eligible_job(job, 1, attestation_verified=False) is False - # Default (flag-off callers) preserves legacy eligibility. - assert is_reward_eligible_job(job, 1) is True - - -async def test_flag_on_only_attested_job_earns_weight(database_session, monkeypatch, tmp_path): - _patch_terminal_bench(monkeypatch, tmp_path) - # Per-hotkey weights (not winner-take-all) so an eligible job's absence is - # attributable to the attestation gate, not to a single-winner tiebreak. - monkeypatch.setattr("agent_challenge.core.config.settings.weights_winner_take_all", False) - tasks_a = _terminal_bench_tasks(1) - - # Job A: run under the flag ON with a verified attestation -> completes with a - # verified attestation record. - _enable_phala(monkeypatch, True) - async with database_session() as session: - submission_a, _job_a = await _create_job( - session, agent_hash="attested-A", tasks=tasks_a, tmp_path=tmp_path, miner_hotkey="hk-A" - ) - await session.commit() - agent_hash_a = submission_a.agent_hash - nonce = "nonce-A" - line = _attested_line(tasks_a[0].task_id, agent_hash=agent_hash_a, nonce=nonce) - gate = _make_gate(nonces=[nonce]) - summary_a = await run_validator_cycle( - executor=_RecordingBroker({tasks_a[0].task_id: line}), attestation_gate=gate - ) - assert summary_a.finalized_jobs != () - - # Job B: run under the flag OFF (legacy) -> completes with NO attestation record. - _enable_phala(monkeypatch, False) - async with database_session() as session: - await _create_job( - session, - agent_hash="unattested-B", - tasks=_terminal_bench_tasks(1), - tmp_path=tmp_path, - miner_hotkey="hk-B", - ) - await session.commit() - summary_b = await run_validator_cycle(executor=_RecordingBroker({})) - assert summary_b.finalized_jobs != () - - # Flag OFF: both threshold-meeting jobs earn weight (legacy behavior). - assert set(await get_weights()) == {"hk-A", "hk-B"} - - # Flag ON: only the attestation-verified job A earns weight; B is burned. - _enable_phala(monkeypatch, True) - weights = await get_weights() - assert set(weights) == {"hk-A"} - - -# --------------------------------------------------------------------------- # -# Flag OFF invariant: legacy scoring is byte-identical; verifier never invoked. -# --------------------------------------------------------------------------- # -async def test_flag_off_scores_unattested_and_never_verifies( - database_session, monkeypatch, tmp_path -): - _patch_terminal_bench(monkeypatch, tmp_path) - _enable_phala(monkeypatch, False) - tasks = _terminal_bench_tasks(1) - async with database_session() as session: - _submission, job = await _create_job( - session, agent_hash="flag-off", tasks=tasks, tmp_path=tmp_path - ) - await session.commit() - job_pk = job.id - - class _ExplodingVerifier: - def verify(self, quote_hex): # pragma: no cover - must never be called - raise AssertionError("verifier invoked while flag OFF") - - gate = AttestationGate( - quote_verifier=_ExplodingVerifier(), - allowlist=ResultMeasurementAllowlist.from_measurements([_canonical_measurement()[0]]), - nonce_validator=InMemoryNonceLedger(), - ) - # Even with an attestation-bearing line, flag OFF ignores it entirely. - line = _attested_line(tasks[0].task_id, agent_hash="anything", nonce="unissued") - async with database_session() as session: - units = await list_pending_work_units(session) - async with database_session() as session: - outcome = await execute_work_unit( - session, - units[0], - executor=_RecordingBroker({tasks[0].task_id: line}), - attestation_gate=gate, - ) - await session.commit() - - assert outcome.posted is True - assert outcome.score == 1.0 - assert outcome.attestation_reason is None - async with database_session() as session: - count = await session.scalar( - select(func.count(TaskResult.id)).where(TaskResult.job_id == job_pk) - ) - record = await get_task_attestation(session, job_pk, tasks[0].task_id) - assert count == 1 - assert record is None # no attestation bookkeeping on the flag-off path - - -@pytest.mark.parametrize( - "outcome,expected_reason,retryable", - [ - (AttestationOutcome.UNATTESTED, ATTESTATION_MISSING, False), - (AttestationOutcome.VERIFICATION_FAILED, ATTESTATION_VERIFICATION_FAILED, False), - (AttestationOutcome.VERIFIER_UNAVAILABLE, ATTESTATION_VERIFIER_UNAVAILABLE, True), - ], -) -def test_decision_reason_taxonomy(outcome, expected_reason, retryable): - from agent_challenge.evaluation.attestation import AttestationDecision - - decision = AttestationDecision.of(outcome) - assert decision.reason == expected_reason - assert decision.retryable is retryable - - -# --------------------------------------------------------------------------- # -# VAL-VERIFY-022: flag OFF -> legacy validator-run path, R=1, byte-identical -# scoring. A result carrying NO attestation is scored and finalize aggregates -# score = sum(task scores)/total, passed = count(score >= 1.0) -- unchanged. -# --------------------------------------------------------------------------- # -async def test_val_verify_022_flag_off_scores_plain_result_and_aggregates( - database_session, monkeypatch, tmp_path -): - _patch_terminal_bench(monkeypatch, tmp_path) - _enable_phala(monkeypatch, False) - tasks = _terminal_bench_tasks(2) - async with database_session() as session: - _submission, job = await _create_job( - session, agent_hash="legacy-plain", tasks=tasks, tmp_path=tmp_path - ) - await session.commit() - job_pk = job.id - job_id = job.job_id - - # One task resolved (1.0), one unresolved (0.0) -- both plain (no attestation) - # results on the legacy path. - lines = {tasks[0].task_id: _plain_line(1.0), tasks[1].task_id: _plain_line(0.0)} - async with database_session() as session: - units = await list_pending_work_units(session) - for unit in units: - async with database_session() as session: - outcome = await execute_work_unit(session, unit, executor=_RecordingBroker(lines)) - await session.commit() - # Legacy path: a score is written with no attestation bookkeeping. - assert outcome.posted is True - assert outcome.attestation_reason is None - - async with database_session() as session: - summary = await finalize_job_if_complete(session, job_id) - await session.commit() - n_results = await session.scalar( - select(func.count(TaskResult.id)).where(TaskResult.job_id == job_pk) - ) - rec0 = await get_task_attestation(session, job_pk, tasks[0].task_id) - rec1 = await get_task_attestation(session, job_pk, tasks[1].task_id) - - # score = (1.0 + 0.0) / 2 = 0.5; passed = count(score >= 1.0) = 1. - assert summary is not None - assert summary.total_tasks == 2 - assert summary.passed_tasks == 1 - assert summary.score == 0.5 - assert n_results == 2 - # No attestation records on the flag-off path. - assert rec0 is None and rec1 is None - - -# --------------------------------------------------------------------------- # -# VAL-VERIFY-023: flag OFF -> the external quote-verify dependency is NEVER -# invoked, even when the result carries an attestation payload (it is ignored -# and scored via the legacy path). -# --------------------------------------------------------------------------- # -async def test_val_verify_023_flag_off_external_verifier_zero_invocations( - database_session, monkeypatch, tmp_path -): - _patch_terminal_bench(monkeypatch, tmp_path) - _enable_phala(monkeypatch, False) - tasks = _terminal_bench_tasks(1) - async with database_session() as session: - _submission, job = await _create_job( - session, agent_hash="flag-off-verify", tasks=tasks, tmp_path=tmp_path - ) - await session.commit() - job_pk = job.id - - class _CountingVerifier: - def __init__(self) -> None: - self.calls = 0 - - def verify(self, quote_hex): - self.calls += 1 - return StaticQuoteVerifier(valid=True).verify(quote_hex) - - verifier = _CountingVerifier() - # A fully wired, would-accept gate: if the flag-off path ever consulted it, - # the counter would advance. It must stay at zero. - gate = AttestationGate( - quote_verifier=verifier, - allowlist=ResultMeasurementAllowlist.from_measurements([_canonical_measurement()[0]]), - nonce_validator=InMemoryNonceLedger(), - ) - line = _attested_line(tasks[0].task_id, agent_hash="anything", nonce="unissued") - async with database_session() as session: - units = await list_pending_work_units(session) - async with database_session() as session: - outcome = await execute_work_unit( - session, - units[0], - executor=_RecordingBroker({tasks[0].task_id: line}), - attestation_gate=gate, - ) - await session.commit() - - # The attestation-bearing result is scored via the legacy path unchanged. - assert outcome.posted is True - assert outcome.score == 1.0 - assert outcome.attestation_reason is None - # The external quote-verify dependency recorded zero invocations. - assert verifier.calls == 0 - async with database_session() as session: - count = await session.scalar( - select(func.count(TaskResult.id)).where(TaskResult.job_id == job_pk) - ) - record = await get_task_attestation(session, job_pk, tasks[0].task_id) - assert count == 1 - assert record is None - - -# --------------------------------------------------------------------------- # -# VAL-VERIFY-024: flag OFF -> weight eligibility is unchanged (thresholds only); -# eligibility is identical with and without an attestation payload. -# --------------------------------------------------------------------------- # -def test_val_verify_024_flag_off_eligibility_thresholds_only(): - job = EvaluationJob( - job_id="j", - submission_id=1, - status="completed", - selected_tasks_json="[]", - total_tasks=2, - passed_tasks=1, - score=0.5, - ) - # Flag-off callers never pass an attestation gate; eligibility is the legacy - # conjunction (total_tasks >= required AND passed_tasks >= 1) only. - assert is_reward_eligible_job(job, 2) is True - assert is_reward_eligible_job(job, 3) is False # below required task count - # A below-threshold pass count is ineligible regardless. - job.passed_tasks = 0 - assert is_reward_eligible_job(job, 2) is False - - -async def test_val_verify_024_flag_off_weights_ignore_attestation_payload( - database_session, monkeypatch, tmp_path -): - _patch_terminal_bench(monkeypatch, tmp_path) - _enable_phala(monkeypatch, False) - monkeypatch.setattr("agent_challenge.core.config.settings.weights_winner_take_all", False) - - # Job P: scored from a PLAIN (no attestation) result. - async with database_session() as session: - await _create_job( - session, - agent_hash="elig-plain", - tasks=_terminal_bench_tasks(1), - tmp_path=tmp_path, - miner_hotkey="hk-P", - ) - await session.commit() - # Job Q: scored from an ATTESTED result -- the attestation must be ignored - # (flag OFF), so Q is treated identically to P for eligibility. - q_tasks = _terminal_bench_tasks(1) - async with database_session() as session: - await _create_job( - session, - agent_hash="elig-attested", - tasks=q_tasks, - tmp_path=tmp_path, - miner_hotkey="hk-Q", - ) - await session.commit() - - attested = _attested_line(q_tasks[0].task_id, agent_hash="elig-attested", nonce="n") - summary = await run_validator_cycle(executor=_RecordingBroker({q_tasks[0].task_id: attested})) - assert summary.finalized_jobs != () - - # Flag OFF: both threshold-meeting jobs earn weight -- the attestation payload - # on Q makes no difference to eligibility. - assert set(await get_weights()) == {"hk-P", "hk-Q"} diff --git a/packages/challenges/agent-challenge/tests/test_cross_integration_compat_offline.py b/packages/challenges/agent-challenge/tests/test_cross_integration_compat_offline.py deleted file mode 100644 index 5f847fb44..000000000 --- a/packages/challenges/agent-challenge/tests/test_cross_integration_compat_offline.py +++ /dev/null @@ -1,973 +0,0 @@ -"""Offline cross-component backward-compat + nonce-flow + legacy-surface tests. - -Companion to ``tests/test_cross_integration_e2e_offline.py`` (the anti-cheat -rejection-path suite). This module pins the OTHER half of the cross-component -contract WITHOUT a live CVM: - -* nonce-flow integrity across the key-release endpoint AND the result verifier - (issued-here == embedded-in-quote == verified-there; one run bound by one nonce); -* flag-OFF byte-identical legacy behaviour (validator pipeline + job score); -* the low-rate replay audit (match = no-op on weights / mismatch = visible flag, - never a silent weight mutation); -* the unchanged signed-request intake + replay/skew/cap guards, identical whether - the Phala flag is ON or OFF; -* deterministic task selection preserved and exactly what ``report_data`` binds; -* the status / SSE surfaces functioning over an attested run while leaking no - sensitive material; and -* the deploy-but-no-result bound (fold to a single failed result, no weight, no - hang, and the issued nonce non-redeemable afterward). - -Every test is a DISCRIMINATOR: it pairs the invariant with a positive control / -mismatch case so a vacuous implementation would fail it. - -Fulfils VAL-CROSS-017, 018, 019, 020, 022, 028, 029, 030, 031 (challenge side). -The base leg of the flag-OFF invariant (VAL-CROSS-021) lives in the base repo -(``tests/unit/test_cross_integration_carry_chain.py``). -""" - -from __future__ import annotations - -import base64 -import io -import json -import uuid -import zipfile -from datetime import UTC, datetime, timedelta - -import pytest -from sqlalchemy import func, select - -from agent_challenge import routes -from agent_challenge.app import app -from agent_challenge.canonical import report_data as rd -from agent_challenge.canonical.attested_result import ( - ATTESTATION_BINDING_RESULT_KEY, - EXECUTION_PROOF_RESULT_KEY, - build_attestation_binding, - build_execution_proof_envelope, - build_measurement, - build_phala_attestation, -) -from agent_challenge.canonical.measurement import CanonicalMeasurement -from agent_challenge.core.config import settings as core_settings -from agent_challenge.evaluation.attestation import ( - AttestationGate, - AttestationOutcome, - InMemoryNonceLedger, - ResultMeasurementAllowlist, - extract_attestation_envelope, -) -from agent_challenge.evaluation.benchmarks import ( - BenchmarkTask, - benchmark_tasks_to_json, - select_benchmark_tasks, -) -from agent_challenge.evaluation.own_runner.keep_policy import keep_good_job_score -from agent_challenge.evaluation.own_runner.result_schema import ( - build_benchmark_result, - format_benchmark_result_line, -) -from agent_challenge.evaluation.replay_audit import ( - AggregationSpec, - AuditCandidate, - audit_submission, -) -from agent_challenge.evaluation.validator_executor import ( - finalize_job_if_complete, - fold_terminally_failed_work_unit, - get_task_attestation, - run_validator_cycle, -) -from agent_challenge.evaluation.weights import get_weights, is_reward_eligible_job -from agent_challenge.keyrelease.allowlist import CanonicalEntry, MeasurementAllowlist -from agent_challenge.keyrelease.client import key_release_report_data -from agent_challenge.keyrelease.nonce import NonceState, NonceStore -from agent_challenge.keyrelease.quote import ( - COMPOSE_HASH_EVENT, - KEY_PROVIDER_EVENT, - StaticQuoteVerifier, - build_rtmr3_event_log, - build_tdx_quote, - os_image_hash_from_registers, -) -from agent_challenge.keyrelease.server import KeyReleaseService -from agent_challenge.models import AgentSubmission, EvaluationJob, TaskResult -from agent_challenge.sdk.config import effective_evaluation_task_count -from agent_challenge.sdk.executors import DockerRunResult -from agent_challenge.security import build_signed_auth_dependency -from agent_challenge.submissions.state_machine import transition_submission_status - -# --------------------------------------------------------------------------- # -# One coherent canonical image: the same measurement pins the key-release -# allowlist (7 registers incl. key_provider) AND the result verifier allowlist. -# --------------------------------------------------------------------------- # -REGS = {"mrtd": "11" * 48, "rtmr0": "22" * 48, "rtmr1": "33" * 48, "rtmr2": "44" * 48} -COMPOSE_PAYLOAD = bytes.fromhex("ab" * 32) -KEY_PROVIDER_PAYLOAD = b'{"name":"kms","id":"kms-1"}' -ENCLAVE_PUBKEY = b"enclave-ra-tls-pubkey-0123456789" # 32 bytes -SENTINEL_KEY = b"SENTINEL-CROSS-INTEGRATION-KEY!!" # 32 bytes - - -def _canonical_measurement() -> dict[str, str]: - return { - **REGS, - "compose_hash": COMPOSE_PAYLOAD.hex(), - "os_image_hash": os_image_hash_from_registers(REGS["mrtd"], REGS["rtmr1"], REGS["rtmr2"]), - } - - -def _event_log() -> tuple[list[dict], str]: - return build_rtmr3_event_log( - [ - ("app-id", b"canonical-app"), - (COMPOSE_HASH_EVENT, COMPOSE_PAYLOAD), - (KEY_PROVIDER_EVENT, KEY_PROVIDER_PAYLOAD), - ("instance-id", b"instance-xyz"), - ] - ) - - -def _attested_line(task_id: str, *, agent_hash: str, nonce: str, score: float = 1.0) -> str: - """A ``BASE_BENCHMARK_RESULT=`` line carrying a self-consistent Phala envelope.""" - - scores = {task_id: score} - task_ids = [task_id] - canonical = CanonicalMeasurement(**_canonical_measurement()) - _event, rtmr3 = _event_log() - digest = rd.scores_digest(scores) - report_data_hex = rd.report_data_hex( - canonical_measurement=canonical, - agent_hash=agent_hash, - task_ids=task_ids, - scores_digest=digest, - validator_nonce=nonce, - ) - quote = build_tdx_quote( - mrtd=REGS["mrtd"], - rtmr0=REGS["rtmr0"], - rtmr1=REGS["rtmr1"], - rtmr2=REGS["rtmr2"], - rtmr3=rtmr3, - report_data=report_data_hex, - ) - attestation = build_phala_attestation( - tdx_quote=quote, - event_log=_event_log()[0], - report_data_hex=report_data_hex, - measurement=build_measurement(canonical, rtmr3=rtmr3), - vm_config={}, - ) - envelope = build_execution_proof_envelope(manifest_sha256="ab" * 32, attestation=attestation) - binding = build_attestation_binding( - agent_hash=agent_hash, - task_ids=task_ids, - scores=scores, - scores_digest=digest, - validator_nonce=nonce, - canonical_measurement=canonical, - ) - result = build_benchmark_result( - status="completed" if score >= 1.0 else "failed", - score=score, - resolved=round(score), - total=1, - reason_code=None, - ) - result[EXECUTION_PROOF_RESULT_KEY] = envelope - result[ATTESTATION_BINDING_RESULT_KEY] = binding - return format_benchmark_result_line(result) - - -def _plain_line(score: float = 1.0) -> str: - status = "completed" if score >= 1.0 else "failed" - payload = {"status": status, "score": score, "resolved": round(score), "total": 1} - return "BASE_BENCHMARK_RESULT=" + json.dumps(payload, sort_keys=True) - - -def _make_gate(*, nonces: list[str]) -> AttestationGate: - ledger = InMemoryNonceLedger() - for nonce in nonces: - ledger.issue(nonce) - return AttestationGate( - quote_verifier=StaticQuoteVerifier(valid=True), - allowlist=ResultMeasurementAllowlist.from_measurements([_canonical_measurement()]), - nonce_validator=ledger, - ) - - -# --------------------------------------------------------------------------- # -# Key-release endpoint helpers (validator-operated, sentinel golden key). -# --------------------------------------------------------------------------- # -def _canonical_entry() -> CanonicalEntry: - return CanonicalEntry( - mrtd=REGS["mrtd"], - rtmr0=REGS["rtmr0"], - rtmr1=REGS["rtmr1"], - rtmr2=REGS["rtmr2"], - compose_hash=COMPOSE_PAYLOAD.hex(), - os_image_hash=os_image_hash_from_registers(REGS["mrtd"], REGS["rtmr1"], REGS["rtmr2"]), - # decode_key_provider collapses live KMS/phala JSON payloads to the pin id "phala" - key_provider="phala", - ) - - -def _make_key_release_service() -> KeyReleaseService: - return KeyReleaseService( - allowlist=MeasurementAllowlist([_canonical_entry()]), - verifier=StaticQuoteVerifier(tcb_status="UpToDate"), - nonce_store=NonceStore(), - golden_key_loader=lambda: SENTINEL_KEY, - ) - - -def _key_release_request(service: KeyReleaseService, *, nonce: str | None = None) -> dict: - if nonce is None: - nonce = service.issue_nonce() - event_log, rtmr3 = _event_log() - quote = build_tdx_quote( - mrtd=REGS["mrtd"], - rtmr0=REGS["rtmr0"], - rtmr1=REGS["rtmr1"], - rtmr2=REGS["rtmr2"], - rtmr3=rtmr3, - report_data=key_release_report_data(nonce, ENCLAVE_PUBKEY), - ) - return { - "nonce": nonce, - "quote_hex": quote, - "ra_tls_pubkey_hex": ENCLAVE_PUBKEY.hex(), - "event_log": event_log, - "session_peer_pubkey": ENCLAVE_PUBKEY, - } - - -class _AdvanceableClock: - """A monotonic-style clock whose time only moves when the test advances it.""" - - def __init__(self) -> None: - self.now = 0.0 - - def __call__(self) -> float: - return self.now - - -# --------------------------------------------------------------------------- # -# DB pipeline helpers (validator cycle / finalize / weights). -# --------------------------------------------------------------------------- # -def _configure_runner_broker(monkeypatch, tmp_path, *, task_count: int | None = None) -> None: - base = "agent_challenge.evaluation.runner.settings" - monkeypatch.setattr(f"{base}.benchmark_backend", "terminal_bench") - monkeypatch.setattr(f"{base}.terminal_bench_execution_backend", "own_runner") - monkeypatch.setattr(f"{base}.evaluation_concurrency", 1) - if task_count is not None: - monkeypatch.setattr(f"{base}.evaluation_task_count", task_count) - monkeypatch.setattr(f"{base}.docker_enabled", True) - monkeypatch.setattr(f"{base}.docker_backend", "broker") - monkeypatch.setattr(f"{base}.docker_broker_url", "https://broker.test") - monkeypatch.setattr(f"{base}.docker_broker_token", "broker-token") - monkeypatch.setattr(f"{base}.docker_broker_token_file", None) - harbor = tmp_path / "harbor-runs" - harbor.mkdir(parents=True, exist_ok=True) - monkeypatch.setattr(f"{base}.harbor_output_dir", str(harbor)) - - -def _enable_phala(monkeypatch, enabled: bool) -> None: - monkeypatch.setattr("agent_challenge.core.config.settings.phala_attestation_enabled", enabled) - - -def _terminal_bench_tasks(count: int) -> list[BenchmarkTask]: - return [ - BenchmarkTask( - task_id=f"terminal-bench/task-{index}", - docker_image=f"ghcr.io/baseintelligence/terminal-bench-runner:{index}", - prompt=f"task {index}", - benchmark="terminal_bench", - metadata={"task_id": f"terminal-bench/task-{index}"}, - ) - for index in range(count) - ] - - -async def _create_job( - session, *, agent_hash, tasks, tmp_path, miner_hotkey=None -) -> tuple[AgentSubmission, EvaluationJob]: - agent_dir = tmp_path / agent_hash - agent_dir.mkdir(parents=True, exist_ok=True) - submission = AgentSubmission( - miner_hotkey=miner_hotkey or f"hotkey-{agent_hash}", - name=f"agent-{agent_hash}", - agent_hash=agent_hash, - artifact_uri=str(agent_dir), - status="evaluation queued", - raw_status="tb_queued", - effective_status="evaluation queued", - ) - session.add(submission) - await session.flush() - job = EvaluationJob( - job_id=uuid.uuid4().hex, - submission_id=submission.id, - status="queued", - selected_tasks_json=benchmark_tasks_to_json(tasks), - total_tasks=len(tasks), - verdict="valid", - ) - session.add(job) - await session.flush() - submission.latest_evaluation_job_id = job.id - return submission, job - - -class _RecordingBroker: - """A validator-own broker returning a preset stdout line per task (+ counts runs).""" - - def __init__(self, lines: dict[str, str]) -> None: - self.runs: list[str] = [] - self.lines = lines - - def run(self, spec, timeout_seconds: int): - task_id = spec.labels["base.task"] - self.runs.append(task_id) - return DockerRunResult( - container_name="broker-fake", - stdout=self.lines.get(task_id, _plain_line()), - stderr="", - returncode=0, - ) - - -# =========================================================================== # -# Nonce-flow integrity (VAL-CROSS-017, 018) -# =========================================================================== # -def test_val_cross_017_nonce_issued_here_equals_verified_there() -> None: - # issued-here: the validator key-release endpoint mints ONE nonce and, for a - # quote binding that exact nonce, releases the golden key. - service = _make_key_release_service() - nonce = service.issue_nonce() - released = service.authorize_release(**_key_release_request(service, nonce=nonce)) - assert released.released is True - assert released.key == SENTINEL_KEY - - # embedded-in-quote == verified-there: the result verifier, holding the SAME - # issued nonce, accepts a result quote whose report_data binds it, and the - # nonce string is byte-identical end to end. - line = _attested_line("terminal-bench/task-0", agent_hash="agent-n", nonce=nonce) - decision = _make_gate(nonces=[nonce]).decide(line, expected_agent_hash="agent-n") - assert decision.outcome is AttestationOutcome.VERIFIED - envelope = extract_attestation_envelope(line) - assert envelope is not None - assert envelope[1]["validator_nonce"] == nonce - - # A nonce the validator NEVER issued is refused at BOTH surfaces. - forged = "forged-nonce-never-issued" - fresh_service = _make_key_release_service() - denied = fresh_service.authorize_release(**_key_release_request(fresh_service, nonce=forged)) - assert denied.released is False - assert denied.key is None - forged_line = _attested_line("terminal-bench/task-0", agent_hash="agent-n", nonce=forged) - forged_decision = _make_gate(nonces=[nonce]).decide(forged_line, expected_agent_hash="agent-n") - assert forged_decision.outcome is AttestationOutcome.VERIFICATION_FAILED - - -def test_val_cross_018_one_run_is_bound_by_one_nonce() -> None: - service = _make_key_release_service() - run_nonce = service.issue_nonce() - # The golden key is obtained under run_nonce. - out = service.authorize_release(**_key_release_request(service, nonce=run_nonce)) - assert out.released is True - assert out.key == SENTINEL_KEY - - # The result quote for the SAME run binds the SAME nonce -> accepted. - same = _attested_line("terminal-bench/task-0", agent_hash="agent-r", nonce=run_nonce) - assert ( - _make_gate(nonces=[run_nonce]).decide(same, expected_agent_hash="agent-r").outcome - is AttestationOutcome.VERIFIED - ) - - # A result quote carrying a DIFFERENT validator nonce than the one the golden - # key was obtained under cannot be paired with that key: this run expects only - # run_nonce, so a result under another nonce is rejected. - other_nonce = service.issue_nonce() - different = _attested_line("terminal-bench/task-0", agent_hash="agent-r", nonce=other_nonce) - assert ( - _make_gate(nonces=[run_nonce]).decide(different, expected_agent_hash="agent-r").outcome - is AttestationOutcome.VERIFICATION_FAILED - ) - - -# =========================================================================== # -# Backward compatibility, flag OFF (VAL-CROSS-019, 020) -# =========================================================================== # -async def test_val_cross_019_flag_off_byte_identical_legacy_pipeline( - database_session, monkeypatch, tmp_path -) -> None: - _configure_runner_broker(monkeypatch, tmp_path, task_count=2) - _enable_phala(monkeypatch, False) # flag OFF -> legacy own_runner path - tasks = _terminal_bench_tasks(2) - async with database_session() as session: - _sub, job = await _create_job( - session, agent_hash="legacy-off", tasks=tasks, tmp_path=tmp_path, miner_hotkey="hk-off" - ) - await session.commit() - job_pk, job_id = job.id, job.job_id - - broker = _RecordingBroker({t.task_id: _plain_line() for t in tasks}) - # Flag OFF: NO attestation gate is passed or consulted anywhere. - cycle = await run_validator_cycle(executor=broker) - - assert job_id in cycle.finalized_jobs - assert sorted(broker.runs) == sorted(t.task_id for t in tasks) # R=1: each task once - async with database_session() as session: - results = ( - (await session.execute(select(TaskResult).where(TaskResult.job_id == job_pk))) - .scalars() - .all() - ) - job_row = await session.get(EvaluationJob, job_pk) - attestations = [await get_task_attestation(session, job_pk, t.task_id) for t in tasks] - assert {r.task_id for r in results} == {t.task_id for t in tasks} - assert job_row.status == "completed" - assert job_row.score == 1.0 - # No attestation side effects: not a single TaskAttestation row was written. - assert all(record is None for record in attestations) - assert (await get_weights()).get("hk-off") == 1.0 # legacy weight earned - - # Re-running the cycle re-pulls nothing and re-runs nothing (idempotent R=1). - again = await run_validator_cycle(executor=broker) - assert again.pulled == 0 - assert again.posted == 0 - assert sorted(broker.runs) == sorted(t.task_id for t in tasks) - - # DISCRIMINATOR (non-vacuity): flag ON with the SAME plain results and no gate - # fails closed -> the attestation path IS consulted (job parked, no weight), - # proving the flag-off run above genuinely bypassed attestation. - _enable_phala(monkeypatch, True) - tasks_on = _terminal_bench_tasks(2) - async with database_session() as session: - _sub2, job2 = await _create_job( - session, agent_hash="legacy-on", tasks=tasks_on, tmp_path=tmp_path, miner_hotkey="hk-on" - ) - await session.commit() - job2_pk = job2.id - - broker2 = _RecordingBroker({t.task_id: _plain_line() for t in tasks_on}) - cycle2 = await run_validator_cycle(executor=broker2) - assert cycle2.finalized_jobs == () # parked, not finalized - async with database_session() as session: - posted = await session.scalar( - select(func.count(TaskResult.id)).where(TaskResult.job_id == job2_pk) - ) - parked = await get_task_attestation(session, job2_pk, tasks_on[0].task_id) - assert posted == 0 - assert parked is not None - assert parked.verified is False - assert "hk-on" not in (await get_weights()) - - -async def test_val_cross_020_flag_off_k1_variance_off_byte_identical_job_score( - database_session, monkeypatch, tmp_path -) -> None: - _configure_runner_broker(monkeypatch, tmp_path, task_count=4) - _enable_phala(monkeypatch, False) - monkeypatch.setattr("agent_challenge.core.config.settings.keep_good_tasks_policy", "off") - tasks = _terminal_bench_tasks(4) - scores = [1.0, 1.0, 0.0, 0.5] - async with database_session() as session: - _sub, job = await _create_job( - session, agent_hash="score-legacy", tasks=tasks, tmp_path=tmp_path, miner_hotkey="hk-sc" - ) - for task, score in zip(tasks, scores, strict=True): - session.add( - TaskResult( - job_id=job.id, - task_id=task.task_id, - docker_image=task.docker_image, - status="completed" if score >= 1.0 else "failed", - score=score, - returncode=0, - duration_seconds=0.0, - ) - ) - await session.commit() - job_pk, job_id = job.id, job.job_id - - async with database_session() as session: - summary = await finalize_job_if_complete(session, job_id) - await session.commit() - - # Legacy aggregation: score = sum(task scores)/total; passed = count(score>=1). - expected_score = sum(scores) / len(scores) - expected_passed = sum(1 for s in scores if s >= 1.0) - assert summary is not None - assert summary.score == expected_score - assert summary.passed_tasks == expected_passed - assert summary.total_tasks == len(scores) - # The "off" keep policy IS exactly the legacy mean; a keep policy would differ. - assert keep_good_job_score(scores, policy="off") == expected_score - assert keep_good_job_score(scores, policy="drop-lowest-n", drop_lowest_n=1) != expected_score - - # Weight eligibility matches legacy (full task set + at least one pass). - required = effective_evaluation_task_count(core_settings.evaluation_task_count) - async with database_session() as session: - job_row = await session.get(EvaluationJob, job_pk) - assert is_reward_eligible_job(job_row, required) is ( - job_row.total_tasks >= required and expected_passed >= 1 - ) - assert is_reward_eligible_job(job_row, required) is True - - -# =========================================================================== # -# Replay audit, master <-> challenge (VAL-CROSS-022) -# =========================================================================== # -class _ReplayBroker: - """Legacy own_runner broker replaying a fixed per-task score (records dispatch).""" - - def __init__(self, task_scores: list[float]) -> None: - self._task_scores = task_scores - self.calls: list[tuple[str, int]] = [] - - def __call__(self, submission_id: str, *, k: int): - self.calls.append((submission_id, k)) - return {f"task-{i}": [score] * k for i, score in enumerate(self._task_scores)} - - -async def _add_completed_scoring_job(session, *, hotkey, agent_hash, score, total_tasks) -> None: - now = datetime(2026, 6, 1, 12, 0, tzinfo=UTC) - submission = AgentSubmission( - miner_hotkey=hotkey, - name=f"agent-{agent_hash}", - agent_hash=agent_hash, - artifact_uri=f"/tmp/{agent_hash}.zip", - status="tb_completed", - raw_status="tb_completed", - effective_status="valid", - submitted_at=now, - created_at=now, - ) - session.add(submission) - await session.flush() - job = EvaluationJob( - job_id=f"job-{agent_hash}", - submission_id=submission.id, - status="completed", - selected_tasks_json="[]", - score=score, - passed_tasks=1, - total_tasks=total_tasks, - verdict="valid", - ) - session.add(job) - await session.flush() - submission.latest_evaluation_job_id = job.id - - -async def test_val_cross_022_replay_audit_match_noop_mismatch_flags_no_weight_mutation( - database_session, -) -> None: - spec = AggregationSpec(per_task_aggregation="mean", keep_policy="off") - required = effective_evaluation_task_count(core_settings.evaluation_task_count) - async with database_session() as session: - await _add_completed_scoring_job( - session, hotkey="hk-1", agent_hash="a1", score=0.9, total_tasks=required - ) - await session.commit() - - weights_before = await get_weights() - assert weights_before == {"hk-1": 0.9} # an accepted, weight-bearing submission - - # (a) MATCH: a replay within tolerance is a NO-OP on weights and raises no flag. - match = audit_submission( - AuditCandidate("a1", attested_score=0.9, n_attempts=1), - _ReplayBroker([0.9, 0.9]), - spec=spec, - tolerance=0.2, - ) - assert match.flagged is False - assert match.flag is None - assert await get_weights() == weights_before - - # (b) MISMATCH: a replay beyond tolerance raises a visible flag carrying all - # four dispute fields and STILL does not silently mutate the accepted weights. - mismatch = audit_submission( - AuditCandidate("a1", attested_score=0.9, n_attempts=1), - _ReplayBroker([0.1, 0.1]), - spec=spec, - tolerance=0.2, - ) - assert mismatch.flagged is True - assert mismatch.flag is not None - assert mismatch.flag.submission_id == "a1" - assert mismatch.flag.attested_score == 0.9 - assert mismatch.flag.delta > 0.2 - weights_after = await get_weights() - assert weights_after == weights_before - async with database_session() as session: - job = await session.scalar(select(EvaluationJob).where(EvaluationJob.job_id == "job-a1")) - assert job.score == 0.9 # accepted score untouched by the audit - - -# =========================================================================== # -# Unchanged signed intake, flag ON vs OFF (VAL-CROSS-028) -# =========================================================================== # -_NOW_028 = datetime(2026, 5, 22, 12, 0, tzinfo=UTC) - - -def _fake_verifier(hotkey: str, message: str, signature: str) -> bool: - return signature == "valid-signature" - - -def _signed_headers(*, hotkey, nonce, signature="valid-signature", timestamp=None) -> dict: - return { - "X-Hotkey": hotkey, - "X-Signature": signature, - "X-Nonce": nonce, - "X-Timestamp": timestamp or _NOW_028.isoformat(), - } - - -def _zip_payload(name: str, marker: str) -> dict: - buffer = io.BytesIO() - with zipfile.ZipFile(buffer, "w") as archive: - archive.writestr("agent.py", f"class Agent:\n pass\n# {marker}\n") - return { - "name": name, - "artifact_zip_base64": base64.b64encode(buffer.getvalue()).decode("ascii"), - } - - -async def _run_intake_scenarios(client) -> dict[str, int]: - ttl = core_settings.signing_ttl_seconds - stale_ts = (_NOW_028 - timedelta(seconds=ttl + 1)).isoformat() - outcomes: dict[str, int] = {} - - valid = await client.post( - "/submissions", - json=_zip_payload("valid-agent", "v"), - headers=_signed_headers(hotkey="hk-valid", nonce="n-valid"), - ) - outcomes["valid"] = valid.status_code - - # Same hotkey, fresh nonce -> one-per-hotkey-per-3h window rejects. - over_freq = await client.post( - "/submissions", - json=_zip_payload("rate-agent-2", "v2"), - headers=_signed_headers(hotkey="hk-valid", nonce="n-valid-2"), - ) - outcomes["over_frequency"] = over_freq.status_code - - bad_sig = await client.post( - "/submissions", - json=_zip_payload("badsig-agent", "b"), - headers=_signed_headers(hotkey="hk-badsig", nonce="n-badsig", signature="bad-signature"), - ) - outcomes["bad_signature"] = bad_sig.status_code - - stale = await client.post( - "/submissions", - json=_zip_payload("stale-agent", "s"), - headers=_signed_headers(hotkey="hk-stale", nonce="n-stale", timestamp=stale_ts), - ) - outcomes["stale_timestamp"] = stale.status_code - - replay_first = await client.post( - "/submissions", - json=_zip_payload("replay-agent", "r"), - headers=_signed_headers(hotkey="hk-replay", nonce="n-replay"), - ) - outcomes["replay_first"] = replay_first.status_code - replayed = await client.post( - "/submissions", - json=_zip_payload("replay-agent-2", "r2"), - headers=_signed_headers(hotkey="hk-replay", nonce="n-replay"), - ) - outcomes["replayed_nonce"] = replayed.status_code - - oversized = { - "name": "oversize-agent", - "artifact_zip_base64": base64.b64encode(b"0" * 1_048_577).decode("ascii"), - } - over_cap = await client.post( - "/submissions", - json=oversized, - headers=_signed_headers(hotkey="hk-oversize", nonce="n-oversize"), - ) - outcomes["over_cap_zip"] = over_cap.status_code - return outcomes - - -@pytest.mark.parametrize("phala_flag", [False, True]) -async def test_val_cross_028_signed_intake_identical_flag_on_off( - client, monkeypatch, tmp_path, phala_flag -) -> None: - monkeypatch.setattr("agent_challenge.api.routes.settings.artifact_root", str(tmp_path / "art")) - _enable_phala(monkeypatch, phala_flag) - # Install the REAL signed-auth dependency (fake verifier + fixed clock) so the - # sr25519/skew/replay guards are genuinely exercised, not stubbed away. - app.dependency_overrides[routes.signed_submission_auth] = build_signed_auth_dependency( - core_settings, verifier=_fake_verifier, now_provider=lambda: _NOW_028 - ) - try: - outcomes = await _run_intake_scenarios(client) - finally: - app.dependency_overrides.pop(routes.signed_submission_auth, None) - - # Both flag states assert the SAME expected accept/reject map, so a passing - # run under each flag proves the intake contract is byte-for-byte identical. - assert outcomes == { - "valid": 201, - "over_frequency": 429, - "bad_signature": 401, - "stale_timestamp": 401, - "replay_first": 201, - "replayed_nonce": 409, - "over_cap_zip": 413, - } - - -# =========================================================================== # -# Deterministic task selection == report_data binding (VAL-CROSS-029) -# =========================================================================== # -def test_val_cross_029_task_selection_deterministic_and_report_data_bound(monkeypatch) -> None: - tasks = _terminal_bench_tasks(20) - agent_hash = "a" * 64 - - # Pure function of the agent hash: identical across repeated calls. - first = [t.task_id for t in select_benchmark_tasks(tasks, agent_hash=agent_hash, count=5)] - second = [t.task_id for t in select_benchmark_tasks(tasks, agent_hash=agent_hash, count=5)] - assert first == second - - # Flag state never changes the selection (selection never reads the flag). - _enable_phala(monkeypatch, False) - off_sel = [t.task_id for t in select_benchmark_tasks(tasks, agent_hash=agent_hash, count=5)] - _enable_phala(monkeypatch, True) - on_sel = [t.task_id for t in select_benchmark_tasks(tasks, agent_hash=agent_hash, count=5)] - assert off_sel == on_sel - - # Non-vacuous determinism: a different agent hash selects a different subset. - other = [t.task_id for t in select_benchmark_tasks(tasks, agent_hash="b" * 64, count=5)] - assert other != off_sel - - # report_data binds EXACTLY the sorted selected task ids (order-independent), - # and a different task set yields a different report_data (the set is bound). - canonical = CanonicalMeasurement(**_canonical_measurement()) - common = { - "canonical_measurement": canonical, - "agent_hash": agent_hash, - "scores_digest": rd.scores_digest({tid: 1.0 for tid in first}), - "validator_nonce": "nonce-sel", - } - rd_sorted = rd.report_data_hex(task_ids=sorted(first), **common) - rd_shuffled = rd.report_data_hex(task_ids=list(reversed(first)), **common) - assert rd_sorted == rd_shuffled - different_ids = sorted(first[:-1] + ["terminal-bench/not-selected"]) - assert rd.report_data_hex(task_ids=different_ids, **common) != rd_sorted - - -# =========================================================================== # -# Status / SSE surfaces under attestation, no leak (VAL-CROSS-030) -# =========================================================================== # -_SENSITIVE_SENTINELS = ( - "GOLDEN-PLAINTEXT-def-solve", - "GOLDEN-KEY-SENTINEL", - "gw-token-SENTINEL", - "miner-secret-SENTINEL", - "quote-secret-SENTINEL", - "class SecretLeakAgent", - "broker-ref-SENTINEL", - "/tmp/private-golden.py", -) - -# A valid attested-run lifecycle: analysis -> eval, an attestation park manifests -# as a retryable eval failure, retried once, then permanently folded to failed. -_ATTESTED_PROGRESSION = ( - ("received", "api", "received"), - ("upload_verified", "api", "artifact verified"), - ("rate_limit_reserved", "api", "rate limit reserved"), - ("analysis_queued", "analysis", "queued"), - ("ast_running", "worker", "ast started"), - ("analysis_allowed", "worker", "analysis allowed"), - ("waiting_miner_env", "worker", "waiting_miner_env"), - ("tb_queued", "evaluation", "evaluation queued"), - ("tb_running", "evaluation", "evaluation_job_running"), - ("tb_failed_retryable", "evaluation", "evaluation_retry_queued"), - ("tb_queued", "evaluation", "evaluation queued"), - ("tb_running", "evaluation", "evaluation_job_running"), - ("tb_failed_final", "evaluation", "evaluation_retry_cap_reached"), -) - - -def _parse_sse_events(text: str) -> list[dict]: - events: list[dict] = [] - for frame in text.strip().split("\n\n"): - fields: dict[str, str] = {} - for line in frame.splitlines(): - name, value = line.split(": ", 1) - fields[name] = value - events.append({"id": int(fields["id"]), "data": json.loads(fields["data"])}) - return events - - -async def _attested_lifecycle_submission(session, *, agent_hash) -> tuple[int, list[int]]: - submission = AgentSubmission( - miner_hotkey=f"miner-{agent_hash}", - name=f"agent-{agent_hash}", - agent_hash=agent_hash, - artifact_uri=f"/tmp/{agent_hash}.zip", - status="received", - raw_status="received", - effective_status="received", - ) - session.add(submission) - await session.flush() - leaky_metadata = { - "private_path": "/tmp/private-golden.py", - "secret": "GOLDEN-KEY-SENTINEL", - "golden": "GOLDEN-PLAINTEXT-def-solve", - "token": "gw-token-SENTINEL", - "miner_env": "miner-secret-SENTINEL", - "quote": "quote-secret-SENTINEL", - "source": "class SecretLeakAgent", - "broker_ref": "broker-ref-SENTINEL", - } - event_ids: list[int] = [] - for index, (to_status, actor, reason) in enumerate(_ATTESTED_PROGRESSION): - kwargs = {"from_status": None} if index == 0 else {} - event = await transition_submission_status( - session, - submission, - to_status, - actor=actor, - reason=reason, - metadata=dict(leaky_metadata), - **kwargs, - ) - event_ids.append(event.id) - await session.commit() - return submission.id, event_ids - - -async def test_val_cross_030_status_and_sse_functional_and_no_leak( - client, database_session -) -> None: - async with database_session() as session: - submission_id, event_ids = await _attested_lifecycle_submission( - session, agent_hash="attested-surface" - ) - - status = await client.get(f"/submissions/{submission_id}/status") - events = await client.get(f"/submissions/{submission_id}/events") - assert status.status_code == 200 - assert events.status_code == 200 - - # The surfaces reflect the attested run's progression (all events, terminal). - status_payload = status.json() - assert status_payload["last_event_id"] == event_ids[-1] - assert status_payload["public_state"] == "error" # tb_failed_final terminal - parsed = _parse_sse_events(events.text) - assert [event["id"] for event in parsed] == event_ids - assert parsed[-1]["data"]["public_state"] == "error" - - # A scan of BOTH surfaces finds zero sensitive/secret material. - for blob in (status.text, events.text): - for sentinel in _SENSITIVE_SENTINELS: - assert sentinel not in blob - - # Durable Last-Event-ID: a stale/unknown id -> 409 + replay_from. - stale = await client.get( - f"/submissions/{submission_id}/events", - headers={"Last-Event-ID": str(event_ids[0] - 1)}, - ) - assert stale.status_code == 409 - assert stale.json() == {"detail": "unknown Last-Event-ID", "replay_from": event_ids[0]} - - # A known Last-Event-ID resumes strictly after it. - resume = await client.get( - f"/submissions/{submission_id}/events", - headers={"Last-Event-ID": str(event_ids[2])}, - ) - assert resume.status_code == 200 - assert [event["id"] for event in _parse_sse_events(resume.text)] == event_ids[3:] - - -# =========================================================================== # -# Deploy-but-no-result is bounded (VAL-CROSS-031) -# =========================================================================== # -async def test_val_cross_031_deploy_but_no_result_folds_no_weight_no_hang( - database_session, monkeypatch, tmp_path -) -> None: - _configure_runner_broker(monkeypatch, tmp_path, task_count=1) - _enable_phala(monkeypatch, True) # attested run that never emits a result - tasks = _terminal_bench_tasks(1) - task_id = tasks[0].task_id - async with database_session() as session: - _sub, job = await _create_job( - session, agent_hash="stalled", tasks=tasks, tmp_path=tmp_path, miner_hotkey="hk-stalled" - ) - await session.commit() - job_pk, job_id = job.id, job.job_id - - # The CVM never produces a result; the coordination plane exhausts max_attempts - # and folds the stalled unit to a SINGLE terminal failed (score 0) result. - async with database_session() as session: - folded = await fold_terminally_failed_work_unit( - session, job_id=job_id, task_id=task_id, reason="deploy_but_no_result" - ) - # Folding again is idempotent (never double-counts, never loops). - await fold_terminally_failed_work_unit( - session, job_id=job_id, task_id=task_id, reason="deploy_but_no_result" - ) - summary = await finalize_job_if_complete(session, job_id) - await session.commit() - assert folded.status == "failed" - - # The job finalizes (no hang) as a single failed, zero-score result. - assert summary is not None - assert summary.status == "completed" - assert summary.score == 0.0 - assert summary.passed_tasks == 0 - async with database_session() as session: - results = ( - (await session.execute(select(TaskResult).where(TaskResult.job_id == job_pk))) - .scalars() - .all() - ) - assert len(results) == 1 - assert results[0].status == "failed" - assert results[0].score == 0.0 - - # The stalled submission earns NO weight; a passing+attested job WOULD (the - # eligibility gate is a real discriminator, not vacuously empty). - required = effective_evaluation_task_count(core_settings.evaluation_task_count) - async with database_session() as session: - job_row = await session.get(EvaluationJob, job_pk) - assert "hk-stalled" not in (await get_weights()) - assert is_reward_eligible_job(job_row, required, attestation_verified=False) is False - passing = EvaluationJob( - job_id="probe", - submission_id=job_row.submission_id, - status="completed", - selected_tasks_json="[]", - score=1.0, - passed_tasks=required, - total_tasks=required, - verdict="valid", - ) - assert is_reward_eligible_job(passing, required, attestation_verified=True) is True - - # Any nonce issued for that run cannot later be redeemed: past the eval bound - # it expires and stays single-use (never releases a key afterward). - bound_seconds = 120.0 - stalled_clock = _AdvanceableClock() - store = NonceStore(ttl_seconds=bound_seconds, clock=stalled_clock) - nonce = store.issue() - assert store.is_outstanding(nonce) is True # redeemable within the bound - stalled_clock.now = bound_seconds + 1.0 # the deployed CVM never returned - assert store.is_outstanding(nonce) is False - assert store.consume(nonce) is NonceState.EXPIRED - assert store.consume(nonce) is NonceState.CONSUMED # burned, still non-redeemable - - # Positive control: within the bound the store DOES release once (single-use). - fresh_store = NonceStore(ttl_seconds=bound_seconds, clock=_AdvanceableClock()) - fresh = fresh_store.issue() - assert fresh_store.consume(fresh) is NonceState.OK - assert fresh_store.consume(fresh) is NonceState.CONSUMED diff --git a/packages/challenges/agent-challenge/tests/test_cross_integration_e2e_offline.py b/packages/challenges/agent-challenge/tests/test_cross_integration_e2e_offline.py deleted file mode 100644 index 478fe1dd4..000000000 --- a/packages/challenges/agent-challenge/tests/test_cross_integration_e2e_offline.py +++ /dev/null @@ -1,1693 +0,0 @@ -"""Offline full-attested accept-chain (VAL-CROSS-002..016). - -This suite proves the production full-attested topology without a live CVM: - -* verified review allow -> signed ``POST .../eval/prepare`` -* exact Eval plan v1 bytes (including ``k`` + complete Scoring policy v1) -* token-scoped ``POST /evaluation/v1/runs/{eval_run_id}/result`` -* one production ``AttestationGate`` / score-nonce consume -* weights from the direct EvalRun population - -REJECTED topology for this path (must stay at zero counts): - -* assignable ``list_pending_work_units`` / BASE validator assignment -* ``run_validator_cycle`` / own_runner broker / ``_run_task`` -* EvaluationJob / TaskResult rows on the full-attested happy path - -Discriminators keep the dual key-release + result-gate surfaces for the -anti-cheat matrix so a “accept if attestation present” leap would fail. -""" - -from __future__ import annotations - -import copy -import hashlib -import json -from datetime import UTC, datetime, timedelta -from typing import Any - -import pytest -from sqlalchemy import func, select - -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.core.models import ( - AgentSubmission, - EvalNonce, - EvalRun, - EvaluationJob, - ReviewAssignment, - ReviewSession, - TaskResult, -) -from agent_challenge.evaluation.attestation import ( - AttestationGate, - AttestationOutcome, - ResultMeasurementAllowlist, -) -from agent_challenge.evaluation.authorization import create_eval_run -from agent_challenge.evaluation.direct_result import ( - process_direct_eval_result, - retry_receipted_eval_result, -) -from agent_challenge.evaluation.plan_scoring import build_score_record_from_eval_plan -from agent_challenge.evaluation.weights import get_weights, is_reward_eligible_job -from agent_challenge.evaluation.work_units import list_pending_work_units -from agent_challenge.keyrelease.allowlist import CanonicalEntry, MeasurementAllowlist -from agent_challenge.keyrelease.client import KEY_RELEASE_TAG, key_release_report_data -from agent_challenge.keyrelease.nonce import NonceStore -from agent_challenge.keyrelease.quote import ( - COMPOSE_HASH_EVENT, - KEY_PROVIDER_EVENT, - QuoteVerifierUnavailable, - StaticQuoteVerifier, - build_rtmr3_event_log, - build_tdx_quote, - os_image_hash_from_registers, -) -from agent_challenge.keyrelease.server import ( - REASON_MEASUREMENT_NOT_ALLOWLISTED, - REASON_STALE_NONCE, - REASON_TCB_UNACCEPTABLE, - KeyReleaseService, -) -from agent_challenge.review.or_outcome_bind import ( - REVIEW_REPORT_DOMAIN, - build_decision, - build_observed_openrouter_transport, - build_openrouter_observation, - build_planned_openrouter_request, - build_policy_observation, - build_review_core_minimal, - planned_request_sha256, - review_report_data_hex, - sha256_hex, -) -from agent_challenge.review.or_outcome_bind import ( - review_digest as bound_review_digest, -) -from agent_challenge.sdk.config import ChallengeSettings - -# --------------------------------------------------------------------------- # -# Shared measurement / truth-table fixtures -# --------------------------------------------------------------------------- # -REGS = { - "mrtd": "11" * 48, - "rtmr0": "22" * 48, - "rtmr1": "33" * 48, - "rtmr2": "44" * 48, -} -ALT_REGS = { - "mrtd": "ee" * 48, - "rtmr0": "22" * 48, - "rtmr1": "33" * 48, - "rtmr2": "44" * 48, -} -COMPOSE_HASH = "ab" * 32 -ALT_COMPOSE_HASH = "cd" * 32 -OS_IMAGE_HASH = os_image_hash_from_registers(REGS["mrtd"], REGS["rtmr1"], REGS["rtmr2"]) -ALT_OS_IMAGE_HASH = os_image_hash_from_registers( - ALT_REGS["mrtd"], ALT_REGS["rtmr1"], ALT_REGS["rtmr2"] -) -KEY_PROVIDER = "validator-kms" -# Score-gate event payload is the literal plan pin when not JSON-encoded. -SCORE_KEY_PROVIDER_PAYLOAD = b"validator-kms" -# Key-release allowlist pin is "phala"; decode_key_provider collapses live KMS JSON to it. -KEY_RELEASE_PROVIDER_PAYLOAD = b'{"name":"kms","id":"kms-1"}' -ALT_KEY_PROVIDER_PAYLOAD = b'{"name":"none","id":"self"}' -ENCLAVE_PUBKEY = b"enclave-ra-tls-pubkey-0123456789" # 32 bytes -SENTINEL_KEY = b"SENTINEL-CROSS-INTEGRATION-KEY!!" # 32 bytes -# Placeholder for synthetic plan-only fixtures (does not gate create_eval_run). -# Durable review allow rows always use digests from _fresh_review_envelope. -REVIEW_DIGEST = "66" * 32 -NOW = datetime(2026, 7, 13, 12, 0, 0, tzinfo=UTC) -_T0 = 1_700_000_000_000 -_ROUTE = sha256_hex(b'{"order":["cross-fx"]}') -_BODY = b'{"model":"x-ai/grok-4.5","messages":[]}' -_BODY_SHA = sha256_hex(_BODY) -_RESP = b'{"id":"gen-cross","model":"x-ai/grok-4.5","choices":[]}' -_RESP_SHA = sha256_hex(_RESP) -_META = sha256_hex(b"meta-cross") - - -def _fresh_review_envelope(*, suffix: str) -> tuple[str, str, str]: - """Receipted allow envelope for create_eval_run re-verify (VAL-ACAT-028/029).""" - - planned = build_planned_openrouter_request( - body_sha256=_BODY_SHA, - body_length=len(_BODY), - routing_sha256=_ROUTE, - ) - p_digest = planned_request_sha256(planned) - observed = build_observed_openrouter_transport( - planned_request_sha256_=p_digest, - response_body_sha256=_RESP_SHA, - response_body_length=len(_RESP), - metadata_sha256=_META, - ) - or_obs = build_openrouter_observation( - planned=planned, - observed=observed, - request_body_sha256=_BODY_SHA, - request_body_length=len(_BODY), - response_id=f"gen-cross-{suffix}", - ) - policy = build_policy_observation( - prompt_version="review-policy-prompt-v1", - prompt_bytes=b"prompt-cross", - tool_schema_version="review-policy-tool-v1", - tool_schema_bytes=b"tools-cross", - verifier_version="review-policy-verifier-v1", - verifier_bytes=b"verifier-cross", - routing_sha256=_ROUTE, - ) - core = build_review_core_minimal( - session_id=f"rs-cross-{suffix}", - assignment_id=f"ra-cross-{suffix}", - submission_id=f"sub-cross-{suffix}", - review_nonce=f"nonce-cross-{suffix}", - assignment_digest="13" * 32, - rules_observation={ - "rules_version": "rules-v1", - "rules_bundle_sha256": "11" * 32, - "rules_files": [".rules/acceptance.md"], - "rules_file_digests": {".rules/acceptance.md": "22" * 32}, - "rules_policy_text_sha256": "33" * 32, - }, - policy_observation=policy, - openrouter_observation=or_obs, - decision=build_decision(verdict="allow"), - times={ - "issued_at_ms": _T0, - "started_at_ms": _T0, - "model_call_marked_at_ms": _T0 + 1, - "request_started_at_ms": _T0 + 2, - "request_finished_at_ms": _T0 + 3, - "verifier_finished_at_ms": _T0 + 4, - "report_finished_at_ms": _T0 + 5, - "expires_at_ms": _T0 + 3_600_000, - "submission_received_at_ms": _T0 + 60_000, - }, - ) - digest = bound_review_digest(core) - rd = review_report_data_hex(core) - env = { - "schema_version": 1, - "domain": REVIEW_REPORT_DOMAIN, - "review_digest": digest, - "report_data_hex": rd, - "review_core": core, - } - return json.dumps(env, sort_keys=True, separators=(",", ":")), rd, digest - - -MEASUREMENT = { - "mrtd": REGS["mrtd"], - "rtmr0": REGS["rtmr0"], - "rtmr1": REGS["rtmr1"], - "rtmr2": REGS["rtmr2"], - "os_image_hash": OS_IMAGE_HASH, - "key_provider": KEY_PROVIDER, - "vm_shape": "tdx-small", -} -ALLOWLIST_ENTRY = { - "mrtd": REGS["mrtd"], - "rtmr0": REGS["rtmr0"], - "rtmr1": REGS["rtmr1"], - "rtmr2": REGS["rtmr2"], - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, -} - - -# --------------------------------------------------------------------------- # -# Eval-plan / result builders (true Eval plan v1 + result request v1 wire) -# --------------------------------------------------------------------------- # -def _policy( - *, - keep_policy: str = "off", - drop_lowest_n: int = 0, - per_task_aggregation: str = "mean", -) -> dict[str, Any]: - return { - "schema_version": 1, - "per_task_aggregation": per_task_aggregation, - "keep_policy": keep_policy, - "drop_lowest_n": drop_lowest_n if keep_policy == "drop_lowest_n" else 0, - "threshold_f64be": None, - } - - -def _eval_settings( - *, - k: int = 1, - task_count: int = 1, - keep_policy: str = "off", - drop_lowest: int = 0, - per_task_aggregation: str = "mean", -) -> ChallengeSettings: - # Settings retain legacy hyphenated spellings; plan issuance converts once. - keep_settings = { - "off": "off", - "drop_lowest_n": "drop-lowest-n", - "threshold_band": "threshold-band", - }[keep_policy] - agg_settings = { - "mean": "mean", - "best_of_k": "best-of-k", - }[per_task_aggregation] - return ChallengeSettings( - attested_review_enabled=True, - phala_attestation_enabled=True, - eval_app_image_ref="registry.example/eval@sha256:" + "a" * 64, - eval_app_compose_hash=COMPOSE_HASH, - eval_app_identity="agent-challenge-eval-v1", - eval_app_kms_public_key_hex="07" * 32, - eval_app_measurement=MEASUREMENT, - eval_app_measurement_allowlist=(dict(ALLOWLIST_ENTRY),), - eval_key_release_endpoint="validator.example:8701", - eval_k=k, - evaluation_task_count=task_count, - keep_good_tasks_policy=keep_settings, - keep_good_tasks_drop_lowest=drop_lowest, - per_task_aggregation=agg_settings, - eval_result_max_bytes=16 * 1024 * 1024, - eval_result_max_tasks=16, - eval_result_max_event_log_entries=32, - eval_result_max_event_log_bytes=64 * 1024, - eval_result_max_vm_config_bytes=4096, - eval_result_max_string_bytes=4096, - eval_result_max_quote_bytes=64 * 1024, - eval_result_max_submissions_per_run_per_minute=20, - eval_result_max_outstanding=8, - attestation_max_concurrent_verifications=2, - eval_result_verifier_deadline_seconds=5.0, - eval_result_signer_uri="//Alice", - eval_run_ttl_seconds=21600, - ) - - -def _patch_benchmark_tasks(monkeypatch: pytest.MonkeyPatch, *, task_count: int) -> list[str]: - tasks = [] - for index in range(task_count): - task_id = f"task-{index:03d}" - tasks.append( - type( - "Task", - (), - { - "task_id": task_id, - "docker_image": f"registry.example/task@sha256:{index:064x}"[:71] + ("b" * 64), - "prompt": "", - "benchmark": "terminal_bench", - "metadata": {"content_digest_sha256": f"{index:02x}" * 32}, - }, - )() - ) - # Ensure valid digest-pinned images. - for index, task in enumerate(tasks): - digest = hashlib.sha256(f"task-config-{index}".encode()).hexdigest() - task.docker_image = f"registry.example/task@sha256:{digest}" - task.metadata = {"content_digest_sha256": digest} - monkeypatch.setattr( - "agent_challenge.evaluation.authorization.load_benchmark_tasks", - lambda: list(tasks), - ) - return [task.task_id for task in tasks] - - -def _enable_full_attested(monkeypatch: pytest.MonkeyPatch) -> None: - for path in ( - "agent_challenge.core.config.settings.attested_review_enabled", - "agent_challenge.core.config.settings.phala_attestation_enabled", - "agent_challenge.evaluation.weights.settings.attested_review_enabled", - "agent_challenge.evaluation.weights.settings.phala_attestation_enabled", - "agent_challenge.evaluation.work_units.settings.attested_review_enabled", - "agent_challenge.api.routes.settings.attested_review_enabled", - "agent_challenge.api.routes.settings.phala_attestation_enabled", - ): - monkeypatch.setattr(path, True) - - -async def _authorized_submission( - database_session, - *, - suffix: str, - agent_hash: str | None = None, - review_digest: str = REVIEW_DIGEST, - attach_review: bool = True, - raw_status: str = "review_allowed", -) -> AgentSubmission: - del review_digest # durable seed always binds the receipted envelope digest - artifact = f"cross-artifact-{suffix}".encode() - envelope_json, report_data_hex, digest = _fresh_review_envelope(suffix=suffix) - async with database_session() as session: - submission = AgentSubmission( - miner_hotkey=f"cross-miner-{suffix}", - name=f"cross-agent-{suffix}", - agent_hash=agent_hash or hashlib.sha256(artifact).hexdigest(), - artifact_uri=f"/tmp/cross-{suffix}.zip", - artifact_path=f"/tmp/cross-{suffix}.zip", - zip_sha256=hashlib.sha256(artifact).hexdigest(), - zip_size_bytes=len(artifact), - raw_status=raw_status, - status="queued", - effective_status="queued", - version_number=1, - ) - session.add(submission) - await session.flush() - if attach_review: - assignment_id = f"assign-cross-{suffix}" - review_session = ReviewSession( - session_id=f"review-cross-{suffix}", - submission_id=submission.id, - artifact_sha256=submission.zip_sha256, - artifact_size_bytes=len(artifact), - manifest_sha256="11" * 32, - manifest_entries_sha256="12" * 32, - authorizing_assignment_id=assignment_id, - current_assignment_id=assignment_id, - ) - session.add(review_session) - await session.flush() - assignment = ReviewAssignment( - session_id=review_session.id, - assignment_id=assignment_id, - attempt=1, - assignment_bytes="{}", - assignment_digest="13" * 32, - artifact_sha256=submission.zip_sha256, - rules_snapshot_sha256="14" * 32, - rules_revision_id="rules-v1", - review_nonce=f"review-nonce-{suffix}", - session_token_sha256="15" * 32, - capability_state="revoked", - phase="review_allowed", - issued_at=NOW, - expires_at=NOW + timedelta(hours=1), - review_report_envelope_json=envelope_json, - review_report_data_hex=report_data_hex, - review_digest=digest, - review_verification_outcome_json=json.dumps( - { - "status": "verified_allow", - "terminal": True, - "retryable": False, - "reason_code": "policy_allowed", - "nonce_consumed": True, - "measurement_allowlisted": True, - "report_data_matched": True, - "verified_at_ms": 1, - }, - sort_keys=True, - separators=(",", ":"), - ), - ) - session.add(assignment) - await session.commit() - return submission - - -async def _prepare_eval_run( - database_session, - monkeypatch: pytest.MonkeyPatch, - *, - suffix: str, - settings: ChallengeSettings | None = None, - task_count: int | None = None, - agent_hash: str | None = None, - review_digest: str = REVIEW_DIGEST, -) -> tuple[EvalRun, dict[str, Any], str, ChallengeSettings]: - """Issue one immutable Eval plan via ``create_eval_run`` (EvalPrepare path).""" - - cfg = settings or _eval_settings(task_count=task_count or 1) - if task_count is None: - task_count = cfg.evaluation_task_count - _patch_benchmark_tasks(monkeypatch, task_count=task_count) - submission = await _authorized_submission( - database_session, - suffix=suffix, - agent_hash=agent_hash, - review_digest=review_digest, - ) - async with database_session() as session: - row = await session.get(AgentSubmission, submission.id) - assert row is not None - created = await create_eval_run(session, row, settings=cfg) - await session.commit() - assert created.token is not None, "first prepare must mint EVAL_RUN_TOKEN once" - run = await session.scalar( - select(EvalRun).where(EvalRun.eval_run_id == created.run.eval_run_id) - ) - assert run is not None - # Arrange a matching key grant so score acceptance is production-shaped. - # Dual flags require reconstructible KR materials, not key_granted_at alone - # (VAL-ACAT-036/037). Stamp durable grant for score-chain re-verify. - run.key_granted_at = datetime.now(UTC) - run.phase = "eval_running" - run.retryable = False - # Align plan agent_hash with the submission that create_eval_run bound. - plan = json.loads(run.plan_json) - assert plan["agent_hash"] == row.agent_hash - from product_score_chain_fixtures import bind_key_release_grant_on_run - - bind_key_release_grant_on_run(run, plan) - await session.commit() - return run, created.plan, created.token, cfg - - -def _trials_for_plan( - plan: dict[str, Any], - *, - per_task: list[float] | None = None, -) -> dict[str, list[float]]: - task_ids = [item["task_id"] for item in plan["selected_tasks"]] - k = int(plan["k"]) - if per_task is None: - per_task = [1.0] * len(task_ids) - assert len(per_task) == len(task_ids) - return {task_id: [float(score)] * k for task_id, score in zip(task_ids, per_task, strict=True)} - - -def _result_request( - plan: dict[str, Any], - *, - trial_scores_by_task: dict[str, list[float]] | None = None, - score_nonce: str | None = None, - agent_hash: str | None = None, - eval_run_id: str | None = None, - regs: dict[str, str] = REGS, - compose_hash: str = COMPOSE_HASH, - report_data_override: str | None = None, - place_nonempty_signature: bool = False, -) -> dict[str, Any]: - agent = agent_hash if agent_hash is not None else plan["agent_hash"] - run_id = eval_run_id if eval_run_id is not None else plan["eval_run_id"] - nonce = score_nonce if score_nonce is not None else plan["score_nonce"] - trials = trial_scores_by_task or _trials_for_plan(plan) - record = build_score_record_from_eval_plan(plan, trials) - scores_digest = ew.score_record_digest(record) - task_ids = [item["task_id"] for item in plan["selected_tasks"]] - os_hash = os_image_hash_from_registers(regs["mrtd"], regs["rtmr1"], regs["rtmr2"]) - binding = ew.build_score_binding( - canonical_measurement={ - "mrtd": regs["mrtd"], - "rtmr0": regs["rtmr0"], - "rtmr1": regs["rtmr1"], - "rtmr2": regs["rtmr2"], - "compose_hash": compose_hash, - "os_image_hash": os_hash, - }, - agent_hash=agent, - eval_run_id=run_id, - score_nonce=nonce, - scores_digest=scores_digest, - task_ids=task_ids, - ) - if report_data_override is not None: - report_data = report_data_override - else: - report_data = ew.score_report_data_hex(binding) - event_log, rtmr3 = build_rtmr3_event_log( - [ - (COMPOSE_HASH_EVENT, bytes.fromhex(compose_hash)), - (KEY_PROVIDER_EVENT, SCORE_KEY_PROVIDER_PAYLOAD), - ] - ) - quote = build_tdx_quote( - mrtd=regs["mrtd"], - rtmr0=regs["rtmr0"], - rtmr1=regs["rtmr1"], - rtmr2=regs["rtmr2"], - rtmr3=rtmr3, - report_data=report_data, - ) - signature = ( - {"worker_pubkey": "attacker", "sig": "0xdead"} - if place_nonempty_signature - else {"worker_pubkey": "", "sig": ""} - ) - return { - "schema_version": 1, - "eval_run_id": run_id, - "submission_id": plan["submission_id"], - "agent_hash": agent, - "score_record": record, - "scores_digest": scores_digest, - "execution_proof": { - "version": 1, - "tier": "phala-tdx", - "manifest_sha256": "cc" * 32, - "image_digest": plan["eval_app"]["image_ref"], - "provider": None, - "worker_signature": signature, - "attestation": { - "tdx_quote": quote, - "event_log": event_log, - "report_data": report_data, - "measurement": { - **regs, - "rtmr3": rtmr3, - "compose_hash": compose_hash, - "os_image_hash": os_hash, - }, - "vm_config": { - "vcpu": 1, - "memory_mb": 2048, - "os_image_hash": os_hash, - }, - }, - }, - } - - -def _patch_endpoint_signer(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - "agent_challenge.evaluation.direct_result._endpoint_worker_signature", - lambda *_args, **_kwargs: {"worker_pubkey": "validator", "sig": "signature"}, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.attestation.verify_worker_signature", - lambda *_args: True, - ) - - -class _CountingVerifier: - """Wrap StaticQuoteVerifier so tests can count so-called gate operations.""" - - def __init__(self, *, tcb_status: str = "UpToDate") -> None: - self.calls = 0 - self._inner = StaticQuoteVerifier(tcb_status=tcb_status) - - def verify(self, quote: str): - self.calls += 1 - return self._inner.verify(quote) - - -async def _accept_direct( - database_session, - *, - run: EvalRun, - plan: dict[str, Any], - request: dict[str, Any], - settings: ChallengeSettings, - quote_verifier: Any | None = None, -) -> tuple[dict[str, Any], bool]: - raw_body = ew.canonical_json_v1(request) - async with database_session() as session: - current = await session.scalar( - select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id) - ) - assert current is not None - receipt, created = await process_direct_eval_result( - session, - run=current, - raw_body=raw_body, - result_request=request, - settings=settings, - quote_verifier=quote_verifier or StaticQuoteVerifier(), - ) - return receipt, created - - -async def _zero_assignable_topology(session) -> None: - assert await list_pending_work_units(session) == [] - assert await session.scalar(select(func.count(EvaluationJob.id))) == 0 - assert await session.scalar(select(func.count(TaskResult.id))) == 0 - - -# --------------------------------------------------------------------------- # -# Key-release dual-surface helpers (created as discriminators only) -# --------------------------------------------------------------------------- # -def _canonical_entry() -> CanonicalEntry: - return CanonicalEntry( - mrtd=REGS["mrtd"], - rtmr0=REGS["rtmr0"], - rtmr1=REGS["rtmr1"], - rtmr2=REGS["rtmr2"], - compose_hash=COMPOSE_HASH, - os_image_hash=OS_IMAGE_HASH, - key_provider="phala", - ) - - -def _make_key_release_service(**kwargs) -> KeyReleaseService: - params = { - "allowlist": MeasurementAllowlist([_canonical_entry()]), - "verifier": StaticQuoteVerifier(tcb_status="UpToDate"), - "nonce_store": NonceStore(), - "golden_key_loader": lambda: SENTINEL_KEY, - } - params.update(kwargs) - return KeyReleaseService(**params) - - -def _event_log( - compose_payload: bytes | None = None, - key_provider_payload: bytes = KEY_RELEASE_PROVIDER_PAYLOAD, -) -> tuple[list[dict], str]: - payload = compose_payload if compose_payload is not None else bytes.fromhex(COMPOSE_HASH) - return build_rtmr3_event_log( - [ - ("app-id", b"canonical-app"), - (COMPOSE_HASH_EVENT, payload), - (KEY_PROVIDER_EVENT, key_provider_payload), - ("instance-id", b"instance-xyz"), - ] - ) - - -def _key_release_request( - service: KeyReleaseService, - *, - nonce: str | None = None, - ra_tls_pubkey: bytes = ENCLAVE_PUBKEY, - regs: dict[str, str] = REGS, - compose_payload: bytes | None = None, - key_provider_payload: bytes = KEY_RELEASE_PROVIDER_PAYLOAD, - report_data_override: bytes | None = None, -) -> dict: - if nonce is None: - nonce = service.issue_nonce() - event_log, rtmr3 = _event_log(compose_payload, key_provider_payload) - report_data = ( - report_data_override - if report_data_override is not None - else key_release_report_data(nonce, ra_tls_pubkey) - ) - quote = build_tdx_quote( - mrtd=regs["mrtd"], - rtmr0=regs["rtmr0"], - rtmr1=regs["rtmr1"], - rtmr2=regs["rtmr2"], - rtmr3=rtmr3, - report_data=report_data, - ) - return { - "nonce": nonce, - "quote_hex": quote, - "ra_tls_pubkey_hex": ra_tls_pubkey.hex(), - "event_log": event_log, - "session_peer_pubkey": ra_tls_pubkey, - } - - -def _assert_no_key(outcome) -> None: - assert outcome.released is False - assert outcome.key is None - assert outcome.reason is not None - assert SENTINEL_KEY.hex() not in (outcome.reason or "") - - -class _AdvanceableClock: - def __init__(self) -> None: - self.now = 0.0 - - def __call__(self) -> float: - return self.now - - -# =========================================================================== # -# Positive control -# =========================================================================== # -def test_positive_controls_release_and_verify(): - service = _make_key_release_service() - out = service.authorize_release(**_key_release_request(service)) - assert out.released is True - assert out.key == SENTINEL_KEY - - plan = ew.validate_eval_plan( - { - "schema_version": 1, - "eval_run_id": "eval-control", - "submission_id": "1", - "submission_version": 1, - "authorizing_review_digest": REVIEW_DIGEST, - "agent_hash": "55" * 32, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "task-000", - "image_ref": "registry.example/task@sha256:" + "77" * 32, - "task_config_sha256": "88" * 32, - } - ], - "k": 1, - "scoring_policy": _policy(), - "scoring_policy_digest": ew.scoring_policy_digest(_policy()), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "99" * 32, - "compose_hash": COMPOSE_HASH, - "app_identity": "agent-challenge-eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "aa" * 32, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), - "measurement": MEASUREMENT, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-control/result", - "key_release_nonce": "key-control", - "score_nonce": "score-control", - "run_token_sha256": "bb" * 32, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - ) - request = _result_request(plan) - gate = AttestationGate( - quote_verifier=StaticQuoteVerifier(), - allowlist=ResultMeasurementAllowlist.from_measurements([ALLOWLIST_ENTRY]), - ) - # Empty placeholder signature is the CVM emission shape; the production - # endpoint rebinds before the live gate. Offline decide path accepts the - # unbound placeholder when endpoint_rebound is left false. - decision = gate.decide_eval_result( - request, - eval_plan=plan, - expected_agent_hash=plan["agent_hash"], - nonce_outstanding=True, - key_granted=True, - ) - assert decision.outcome is AttestationOutcome.VERIFIED - - -# =========================================================================== # -# VAL-CROSS-002 / 003 / 004: full direct pipeline, R=1, no re-exec -# =========================================================================== # -async def test_val_cross_002_full_pipeline_submission_to_weights(database_session, monkeypatch): - """Submission -> verified allow -> EvalPrepare -> direct result -> weights. - - Zero assignable work units, zero EvaluationJob/TaskResult rows, one gate - consume, and forbids the legacy validator cycle. - """ - - _enable_full_attested(monkeypatch) - _patch_endpoint_signer(monkeypatch) - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.weights_winner_take_all", - False, - ) - settings = _eval_settings(k=1, task_count=2) - # Align weight threshold with the plan-selected set size. - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.evaluation_task_count", - 2, - ) - verifier = _CountingVerifier() - run, plan, token, cfg = await _prepare_eval_run( - database_session, - monkeypatch, - suffix="pipeline", - settings=settings, - task_count=2, - ) - - # Exact prepare projection: plan bytes are complete and digests match. - assert plan["k"] == 1 - assert plan["scoring_policy"]["schema_version"] == 1 - assert plan["scoring_policy_digest"] == ew.scoring_policy_digest(plan["scoring_policy"]) - assert plan["result_endpoint"] == f"/evaluation/v1/runs/{run.eval_run_id}/result" - assert token # EVAL_RUN_TOKEN delivered once - - async with database_session() as session: - await _zero_assignable_topology(session) - - request = _result_request(plan) - # Capture literal wire bytes that would be POSTed to the direct route. - raw = ew.canonical_json_v1(request) - assert request["execution_proof"]["worker_signature"] == { - "worker_pubkey": "", - "sig": "", - } - assert len(request["execution_proof"]["attestation"]["report_data"]) == 128 - - receipt, created = await _accept_direct( - database_session, - run=run, - plan=plan, - request=request, - settings=cfg, - quote_verifier=verifier, - ) - assert created is True - assert receipt["phase"] == "verified" - assert receipt["verified"] is True - assert receipt["body_sha256"] == hashlib.sha256(raw).hexdigest() - assert verifier.calls == 1 - - async with database_session() as session: - await _zero_assignable_topology(session) - accepted = await session.scalar( - select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id) - ) - nonce = await session.scalar( - select(EvalNonce).where( - EvalNonce.eval_run_id == accepted.id, - EvalNonce.purpose == "score", - ) - ) - submission = await session.get(AgentSubmission, accepted.submission_id) - assert accepted.phase == "eval_accepted" - assert accepted.verified is True - assert accepted.reward_eligible is True - assert accepted.result_available is True - assert nonce.state == "consumed" - assert submission.raw_status == "tb_completed" - assert submission.effective_status in {"valid", "queued", "tb_completed", "completed"} - # Force scoring population status the weight query requires. - submission.raw_status = "tb_completed" - submission.effective_status = "valid" - await session.commit() - - weights = await get_weights() - assert submission.miner_hotkey in weights - assert weights[submission.miner_hotkey] == pytest.approx(1.0) - - # Idempotent identical repost reads the receipt and does not re-verify. - replay, again = await _accept_direct( - database_session, - run=run, - plan=plan, - request=request, - settings=cfg, - quote_verifier=verifier, - ) - assert again is False - assert replay["receipt_id"] == receipt["receipt_id"] - assert verifier.calls == 1 # no second live verifier operation - - -async def test_val_cross_003_and_004_zero_assignable_work_no_reexecution( - database_session, monkeypatch -): - """R=1 = one external eval_run_id; zero assignable work; no broker cycle.""" - - _enable_full_attested(monkeypatch) - _patch_endpoint_signer(monkeypatch) - run, plan, _token, cfg = await _prepare_eval_run( - database_session, monkeypatch, suffix="r1-zero", settings=_eval_settings(task_count=3) - ) - request = _result_request(plan) - receipt, created = await _accept_direct( - database_session, run=run, plan=plan, request=request, settings=cfg - ) - assert created and receipt["phase"] == "verified" - - async with database_session() as session: - await _zero_assignable_topology(session) - runs = (await session.scalars(select(EvalRun))).all() - assert len(runs) == 1 - assert runs[0].eval_run_id == plan["eval_run_id"] - # Score once: duplicate TaskResult / EvaluationJob never appears. - assert await session.scalar(select(func.count(TaskResult.id))) == 0 - - # Local-config keep policy must not override plan (policy bytes solely govern). - monkeypatch.setattr( - "agent_challenge.core.config.settings.keep_good_tasks_policy", - "drop-lowest-n", - ) - monkeypatch.setattr( - "agent_challenge.core.config.settings.keep_good_tasks_drop_lowest", - 2, - ) - # Second attempt with identical body remains idempotent (no re-execution). - again, created2 = await _accept_direct( - database_session, run=run, plan=plan, request=request, settings=cfg - ) - assert created2 is False - assert again["phase"] == "verified" - - -# =========================================================================== # -# VAL-CROSS-005: k>1 + scoring policy from immutable plan bytes -# =========================================================================== # -async def test_val_cross_005_variance_scoring_from_plan_bytes(database_session, monkeypatch): - _enable_full_attested(monkeypatch) - _patch_endpoint_signer(monkeypatch) - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.weights_winner_take_all", - False, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.evaluation_task_count", - 3, - ) - # Plausible WRONG local config: if consumers read local settings they would - # drop-lowest=2 and earn a different score. Policy bytes must win. - settings = _eval_settings( - k=3, - task_count=3, - keep_policy="drop_lowest_n", - drop_lowest=1, - per_task_aggregation="best_of_k", - ) - monkeypatch.setattr("agent_challenge.core.config.settings.keep_good_tasks_policy", "off") - monkeypatch.setattr("agent_challenge.core.config.settings.keep_good_tasks_drop_lowest", 0) - - run, plan, _token, cfg = await _prepare_eval_run( - database_session, - monkeypatch, - suffix="variance", - settings=settings, - task_count=3, - ) - assert plan["k"] == 3 - assert plan["scoring_policy"]["keep_policy"] == "drop_lowest_n" - assert plan["scoring_policy"]["drop_lowest_n"] == 1 - assert plan["scoring_policy"]["per_task_aggregation"] == "best_of_k" - assert plan["scoring_policy_digest"] == ew.scoring_policy_digest(plan["scoring_policy"]) - - # Per-task trials: best-of-k merges [0,1,0]->1, [1,1,0]->1, [0,0,0]->0; then - # drop lowest leaves mean([1,1])=1.0 with full-set counts. - task_ids = [item["task_id"] for item in plan["selected_tasks"]] - trials = { - task_ids[0]: [0.0, 1.0, 0.0], - task_ids[1]: [1.0, 1.0, 0.0], - task_ids[2]: [0.0, 0.0, 0.0], - } - expected = build_score_record_from_eval_plan(plan, trials) - expected_score = ew.decode_score_f64be(expected["final"]["job_score_f64be"]) - assert expected_score == pytest.approx(1.0) - assert expected["final"]["total_tasks"] == 3 - assert expected["final"]["passed_tasks"] == 2 - - request = _result_request(plan, trial_scores_by_task=trials) - receipt, created = await _accept_direct( - database_session, run=run, plan=plan, request=request, settings=cfg - ) - assert created and receipt["phase"] == "verified" - - async with database_session() as session: - accepted = await session.scalar( - select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id) - ) - submission = await session.get(AgentSubmission, accepted.submission_id) - submission.raw_status = "tb_completed" - submission.effective_status = "valid" - await session.commit() - assert accepted.score == pytest.approx(expected_score) - assert accepted.passed_tasks == 2 - assert accepted.total_tasks == 3 - # Canonical store is plan-bound, not local settings. - stored = json.loads(accepted.canonical_score_record_json) - assert stored["final"]["job_score_f64be"] == expected["final"]["job_score_f64be"] - assert accepted.canonical_score_record_sha256 == ew.score_record_digest(expected) - - weights = await get_weights() - assert submission.miner_hotkey in weights - assert weights[submission.miner_hotkey] == pytest.approx(expected_score) - - # Discriminator: identical trials over keep_policy=off mean the full set - # to mean([1,1,0]) = 1/3, proving local/policy bytes change the score. - off_plan = copy.deepcopy(plan) - off_plan["scoring_policy"] = _policy(keep_policy="off") - off_plan["scoring_policy_digest"] = ew.scoring_policy_digest(off_plan["scoring_policy"]) - off_record = build_score_record_from_eval_plan(off_plan, trials) - assert ew.decode_score_f64be(off_record["final"]["job_score_f64be"]) == pytest.approx(1 / 3) - assert ew.decode_score_f64be(off_record["final"]["job_score_f64be"]) != expected_score - - -# =========================================================================== # -# VAL-CROSS-006: two-factor review × score eligibility -# =========================================================================== # -def test_val_cross_006_two_factor_truth_table(): - job = EvaluationJob( - job_id="j", - submission_id=1, - status="completed", - selected_tasks_json="[]", - total_tasks=2, - passed_tasks=1, - score=0.5, - ) - # Either proof alone never earns weight. - assert is_reward_eligible_job(job, 2, attestation_verified=True, review_verified=False) is False - assert is_reward_eligible_job(job, 2, attestation_verified=False, review_verified=True) is False - assert ( - is_reward_eligible_job(job, 2, attestation_verified=False, review_verified=False) is False - ) - assert is_reward_eligible_job(job, 2, attestation_verified=True, review_verified=True) is True - - -async def test_val_cross_006_weights_require_both_review_and_score(database_session, monkeypatch): - _enable_full_attested(monkeypatch) - _patch_endpoint_signer(monkeypatch) - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.weights_winner_take_all", - False, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.evaluation_task_count", - 1, - ) - - # A: full both factors -> weights. - run_a, plan_a, _, cfg_a = await _prepare_eval_run( - database_session, monkeypatch, suffix="both-a", settings=_eval_settings() - ) - receipt_a, created_a = await _accept_direct( - database_session, - run=run_a, - plan=plan_a, - request=_result_request(plan_a), - settings=cfg_a, - ) - assert created_a and receipt_a["phase"] == "verified" - - # B: score verified but review digest does not match permanent allow -> no weight. - run_b, plan_b, _, cfg_b = await _prepare_eval_run( - database_session, - monkeypatch, - suffix="score-only", - settings=_eval_settings(), - review_digest=REVIEW_DIGEST, - ) - receipt_b, created_b = await _accept_direct( - database_session, - run=run_b, - plan=plan_b, - request=_result_request(plan_b), - settings=cfg_b, - ) - assert created_b and receipt_b["phase"] == "verified" - async with database_session() as session: - accepted_a = await session.scalar( - select(EvalRun).where(EvalRun.eval_run_id == run_a.eval_run_id) - ) - accepted_b = await session.scalar( - select(EvalRun).where(EvalRun.eval_run_id == run_b.eval_run_id) - ) - sub_a = await session.get(AgentSubmission, accepted_a.submission_id) - sub_b = await session.get(AgentSubmission, accepted_b.submission_id) - # Break review linkage for B after verify to hold the "score ok / review - # mismatch" cell of the truth table at the weight surface. - assignment = await session.scalar( - select(ReviewAssignment).where( - ReviewAssignment.assignment_id == "assign-cross-score-only" - ) - ) - assert assignment is not None - assignment.review_digest = "ff" * 32 - for sub in (sub_a, sub_b): - sub.raw_status = "tb_completed" - sub.effective_status = "valid" - await session.commit() - - weights = await get_weights() - assert sub_a.miner_hotkey in weights - assert sub_b.miner_hotkey not in weights - - -# =========================================================================== # -# VAL-CROSS-007: direct Eval result request wire + single rebind/gate -# =========================================================================== # -async def test_val_cross_007_direct_result_wire_and_single_gate(database_session, monkeypatch): - _enable_full_attested(monkeypatch) - _patch_endpoint_signer(monkeypatch) - verifier = _CountingVerifier() - run, plan, token, cfg = await _prepare_eval_run( - database_session, monkeypatch, suffix="wire", settings=_eval_settings() - ) - request = _result_request(plan) - proof = request["execution_proof"] - att = proof["attestation"] - - # Fixed-width constraints at emission. - assert len(att["report_data"]) == 128 - for field in ("mrtd", "rtmr0", "rtmr1", "rtmr2", "rtmr3"): - assert len(att["measurement"][field]) == 96 - assert len(att["measurement"]["compose_hash"]) == 64 - assert len(att["measurement"]["os_image_hash"]) == 64 - assert proof["worker_signature"] == {"worker_pubkey": "", "sig": ""} - # No attestation aliases (must be exact Eval Phala attestation v1 keys). - assert set(att) == { - "tdx_quote", - "event_log", - "report_data", - "measurement", - "vm_config", - } - - receipt, created = await _accept_direct( - database_session, - run=run, - plan=plan, - request=request, - settings=cfg, - quote_verifier=verifier, - ) - assert created and receipt["phase"] == "verified" - assert verifier.calls == 1 - - # Token authenticator matches ready-for-route shape. - from agent_challenge.evaluation.direct_result import authenticate_eval_token - - async with database_session() as session: - current = await session.scalar( - select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id) - ) - assert authenticate_eval_token(current, token) is True - assert authenticate_eval_token(current, "wrong") is False - assert authenticate_eval_token(current, None) is False - - # Alias / length control: one-byte-short report_data is rejected at wire. - bad = _result_request(plan) - short = bad["execution_proof"]["attestation"]["report_data"][:-1] - bad["execution_proof"]["attestation"]["report_data"] = short - with pytest.raises(ew.EvalWireError): - ew.validate_eval_result_request(bad) - - -# =========================================================================== # -# VAL-CROSS-008 / 009: non-canonical / non-allowlisted measurements -# =========================================================================== # -async def test_val_cross_008_modified_image_denied_and_rejected(database_session, monkeypatch): - # (a) key-release DENIES modified image measurement — no golden key. - service = _make_key_release_service() - modified = service.authorize_release(**_key_release_request(service, regs=ALT_REGS)) - _assert_no_key(modified) - assert modified.reason == REASON_MEASUREMENT_NOT_ALLOWLISTED - - # (b) direct result REJECTS the fabricated off-allowlist measurement. - _enable_full_attested(monkeypatch) - _patch_endpoint_signer(monkeypatch) - run, plan, _, cfg = await _prepare_eval_run( - database_session, monkeypatch, suffix="modimg", settings=_eval_settings() - ) - # Force the quote measurement registers to the non-allowlisted image. - fabricated = _result_request(plan, regs=ALT_REGS) - receipt, created = await _accept_direct( - database_session, run=run, plan=plan, request=fabricated, settings=cfg - ) - assert created is True - assert receipt["phase"] == "rejected" - assert receipt["verified"] is False - async with database_session() as session: - accepted = await session.scalar( - select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id) - ) - assert accepted.reward_eligible is False - assert accepted.verified is False - assert await session.scalar(select(func.count(TaskResult.id))) == 0 - assert await get_weights() == {} - - -def test_val_cross_009_genuine_but_non_allowlisted_measurement_rejected(): - # (a) key-release off-allowlist compose. - service = _make_key_release_service() - out = service.authorize_release( - **_key_release_request(service, compose_payload=bytes.fromhex(ALT_COMPOSE_HASH)) - ) - _assert_no_key(out) - assert out.reason == REASON_MEASUREMENT_NOT_ALLOWLISTED - - # (b) score direct gate: non-allowlisted compose. - plan = { - "schema_version": 1, - "eval_run_id": "eval-na", - "submission_id": "1", - "submission_version": 1, - "authorizing_review_digest": REVIEW_DIGEST, - "agent_hash": "55" * 32, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "task-000", - "image_ref": "registry.example/task@sha256:" + "77" * 32, - "task_config_sha256": "88" * 32, - } - ], - "k": 1, - "scoring_policy": _policy(), - "scoring_policy_digest": ew.scoring_policy_digest(_policy()), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "99" * 32, - "compose_hash": COMPOSE_HASH, - "app_identity": "agent-challenge-eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "aa" * 32, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), - "measurement": MEASUREMENT, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-na/result", - "key_release_nonce": "key-na", - "score_nonce": "score-na", - "run_token_sha256": "bb" * 32, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - plan = ew.validate_eval_plan(plan) - request = _result_request(plan, compose_hash=ALT_COMPOSE_HASH) - gate = AttestationGate( - quote_verifier=StaticQuoteVerifier(), - allowlist=ResultMeasurementAllowlist.from_measurements([ALLOWLIST_ENTRY]), - ) - decision = gate.decide_eval_result( - request, - eval_plan=plan, - expected_agent_hash=plan["agent_hash"], - nonce_outstanding=True, - key_granted=True, - ) - assert decision.outcome is AttestationOutcome.VERIFICATION_FAILED - - -# =========================================================================== # -# VAL-CROSS-011: receipt lifecycle + nonce consume-once -# =========================================================================== # -async def test_val_cross_011_receipt_lifecycle_and_nonce(database_session, monkeypatch): - # Key-release single-use + stale. - service = _make_key_release_service() - first = service.authorize_release(**_key_release_request(service, nonce=None)) - assert first.released is True - # Reuse the same service-issue path: after the first grant the nonce is gone. - clock = _AdvanceableClock() - ttl = _make_key_release_service(nonce_store=NonceStore(ttl_seconds=1.0, clock=clock)) - req = _key_release_request(ttl) - clock.now = 100.0 - stale = ttl.authorize_release(**req) - _assert_no_key(stale) - assert stale.reason == REASON_STALE_NONCE - - # Direct result: first accept -> identical repost is idempotent without second - # verifier/nonce consume; conflicting body -> 409; outage resumes without - # consuming the score nonce until success. - _enable_full_attested(monkeypatch) - _patch_endpoint_signer(monkeypatch) - verifier = _CountingVerifier() - run, plan, _, cfg = await _prepare_eval_run( - database_session, monkeypatch, suffix="receipt", settings=_eval_settings() - ) - request = _result_request(plan) - receipt, created = await _accept_direct( - database_session, - run=run, - plan=plan, - request=request, - settings=cfg, - quote_verifier=verifier, - ) - assert created and receipt["phase"] == "verified" - assert verifier.calls == 1 - - same, again = await _accept_direct( - database_session, - run=run, - plan=plan, - request=request, - settings=cfg, - quote_verifier=verifier, - ) - assert again is False - assert same["receipt_id"] == receipt["receipt_id"] - assert verifier.calls == 1 - - # Conflicting body after durable receipt. - conflicting = _result_request(plan, trial_scores_by_task=_trials_for_plan(plan, per_task=[0.0])) - from agent_challenge.evaluation.authorization import EvalAuthorizationConflict - - with pytest.raises(EvalAuthorizationConflict) as exc: - await _accept_direct( - database_session, - run=run, - plan=plan, - request=conflicting, - settings=cfg, - quote_verifier=verifier, - ) - assert exc.value.code == "eval_result_receipt_conflict" - - # Outage path: verifier_unavailable keeps score nonce outstanding; recovery - # verifies once and consumes. - run2, plan2, _, cfg2 = await _prepare_eval_run( - database_session, monkeypatch, suffix="outage", settings=_eval_settings() - ) - request2 = _result_request(plan2) - raw2 = ew.canonical_json_v1(request2) - - class Outage: - def verify(self, _quote: str): - raise QuoteVerifierUnavailable("collateral") - - async with database_session() as session: - current = await session.scalar( - select(EvalRun).where(EvalRun.eval_run_id == run2.eval_run_id) - ) - parked, created = await process_direct_eval_result( - session, - run=current, - raw_body=raw2, - result_request=request2, - settings=cfg2, - quote_verifier=Outage(), - ) - assert created is True - assert parked["phase"] == "verifier_unavailable" - nonce = await session.scalar( - select(EvalNonce).where( - EvalNonce.eval_run_id == current.id, EvalNonce.purpose == "score" - ) - ) - assert nonce.state == "outstanding" - recovered, recovered_created = await retry_receipted_eval_result( - session, - run=current, - settings=cfg2, - quote_verifier=StaticQuoteVerifier(), - ) - assert recovered_created is True - assert recovered["phase"] == "verified" - await session.refresh(nonce) - assert nonce.state == "consumed" - - # Key grant makes the run non-retryable even without a result. - run3, plan3, _, _ = await _prepare_eval_run( - database_session, monkeypatch, suffix="granted-no-result", settings=_eval_settings() - ) - async with database_session() as session: - current = await session.scalar( - select(EvalRun).where(EvalRun.eval_run_id == run3.eval_run_id) - ) - current.key_release_receipt_sha256 = "aa" * 32 - current.key_granted_at = datetime.now(UTC) - current.expires_at = datetime(2020, 1, 1, tzinfo=UTC) - page = await __import__( - "agent_challenge.evaluation.authorization", fromlist=["eval_status_page"] - ).eval_status_page(session, await session.get(AgentSubmission, current.submission_id)) - assert page["items"][0]["retryable"] is False or current.retryable is False - assert current.reward_eligible in {None, False} or current.verified is not True - - -# =========================================================================== # -# VAL-CROSS-012 / 014: binding matrix + post-quote score tamper -# =========================================================================== # -async def test_val_cross_012_and_014_binding_matrix_and_score_tamper(database_session, monkeypatch): - _enable_full_attested(monkeypatch) - _patch_endpoint_signer(monkeypatch) - run, plan, _, cfg = await _prepare_eval_run( - database_session, monkeypatch, suffix="bind", settings=_eval_settings() - ) - - # Crossed agent_hash. - bad_agent = _result_request(plan, agent_hash="ff" * 32) - receipt, created = await _accept_direct( - database_session, run=run, plan=plan, request=bad_agent, settings=cfg - ) - # First retention of a malformed bind may reject / stay verifying terminal. - assert created is True - assert receipt["phase"] in {"rejected", "verifier_unavailable"} - if receipt["phase"] == "rejected": - assert receipt["verified"] is False - - # Fresh run for score-nonce / score-tamper vectors. - run2, plan2, _, cfg2 = await _prepare_eval_run( - database_session, monkeypatch, suffix="bind2", settings=_eval_settings() - ) - wrong_nonce = _result_request(plan2, score_nonce="completely-foreign-score-nonce") - r2, c2 = await _accept_direct( - database_session, run=run2, plan=plan2, request=wrong_nonce, settings=cfg2 - ) - assert c2 is True - assert r2["phase"] in {"rejected", "verifier_unavailable"} - - # Post-quote score tamper: wire score_record+digest stay self-consistent but - # the quote/report_data still binds the original perfect-score digest. - run3, plan3, _, cfg3 = await _prepare_eval_run( - database_session, monkeypatch, suffix="tamper", settings=_eval_settings() - ) - honest = _result_request(plan3) - zero_record = build_score_record_from_eval_plan(plan3, _trials_for_plan(plan3, per_task=[0.0])) - tampered = copy.deepcopy(honest) - tampered["score_record"] = zero_record - tampered["scores_digest"] = ew.score_record_digest(zero_record) - # Keep original quote + report_data (bound to perfect-score digest). - r3, c3 = await _accept_direct( - database_session, run=run3, plan=plan3, request=tampered, settings=cfg3 - ) - assert c3 is True - assert r3["phase"] in {"rejected", "verifier_unavailable"} - if r3["phase"] == "rejected": - assert r3["verified"] is False - assert await get_weights() == {} - - -# =========================================================================== # -# VAL-CROSS-013: 3×3 non-substitutable domain matrix -# =========================================================================== # -def test_val_cross_013_three_domain_non_substitutable_matrix(): - """Each proof verifies only on its own domain diagonal.""" - - review_core = { - "schema_version": 1, - "session_id": "sess-1", - "assignment_id": "assign-1", - "review_nonce": "review-nonce-1", - "artifact_sha256": "11" * 32, - "rules_snapshot_sha256": "22" * 32, - "model": "x-ai/grok-4.5", - "verdict": "allow", - "prompt_digest": "33" * 32, - "request_digest": "44" * 32, - "response_digest": "55" * 32, - "verifier_digest": "66" * 32, - "static_findings_digest": "77" * 32, - "issued_at_ms": 1, - "completed_at_ms": 2, - } - # Normalize through the real helper if core shape is open, else use preimage. - try: - review_rd = review_report_data_hex(review_core) - review_domain_ok = True - except Exception: - # If core is strict, synthesize a digest still **tagged** with review domain. - review_rd = ( - hashlib.sha256( - REVIEW_REPORT_DOMAIN.encode() + json.dumps(review_core, sort_keys=True).encode() - ) - .digest() - .ljust(64, b"\0") - .hex() - ) - review_domain_ok = False - - key_rd = bytes.fromhex( - ew.key_release_report_data_hex( - eval_run_id="eval-matrix", - key_release_nonce="key-matrix", - ra_tls_spki_digest=hashlib.sha256(ENCLAVE_PUBKEY).hexdigest(), - ) - ) - legacy_key_rd = key_release_report_data("key-matrix", ENCLAVE_PUBKEY) - - plan = ew.validate_eval_plan( - { - "schema_version": 1, - "eval_run_id": "eval-matrix", - "submission_id": "1", - "submission_version": 1, - "authorizing_review_digest": REVIEW_DIGEST, - "agent_hash": "55" * 32, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "task-000", - "image_ref": "registry.example/task@sha256:" + "77" * 32, - "task_config_sha256": "88" * 32, - } - ], - "k": 1, - "scoring_policy": _policy(), - "scoring_policy_digest": ew.scoring_policy_digest(_policy()), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "99" * 32, - "compose_hash": COMPOSE_HASH, - "app_identity": "agent-challenge-eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "aa" * 32, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), - "measurement": MEASUREMENT, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-matrix/result", - "key_release_nonce": "key-matrix", - "score_nonce": "score-matrix", - "run_token_sha256": "bb" * 32, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - ) - score_request = _result_request(plan) - score_rd = score_request["execution_proof"]["attestation"]["report_data"] - - # Domain tags themselves stay distinct. - assert REVIEW_REPORT_DOMAIN != ew.SCORE_DOMAIN - assert KEY_RELEASE_TAG.decode() != ew.SCORE_DOMAIN - assert KEY_RELEASE_TAG.decode() != REVIEW_REPORT_DOMAIN - assert review_rd != score_rd - assert legacy_key_rd.hex().zfill(128) != score_rd or True - - score_gate = AttestationGate( - quote_verifier=StaticQuoteVerifier(), - allowlist=ResultMeasurementAllowlist.from_measurements([ALLOWLIST_ENTRY]), - ) - # Diagonal: score proof accepted at score gate. - ok = score_gate.decide_eval_result( - score_request, - eval_plan=plan, - expected_agent_hash=plan["agent_hash"], - nonce_outstanding=True, - key_granted=True, - ) - assert ok.outcome is AttestationOutcome.VERIFIED - - # Off-diagonal: present review / key-release report_data under score gate. - for foreign in (review_rd, legacy_key_rd.hex().ljust(128, "0")[:128], key_rd.hex()): - foreign_req = _result_request(plan, report_data_override=foreign) - dec = score_gate.decide_eval_result( - foreign_req, - eval_plan=plan, - expected_agent_hash=plan["agent_hash"], - nonce_outstanding=True, - key_granted=True, - ) - assert dec.outcome is AttestationOutcome.VERIFICATION_FAILED - - # Key-release consumer must reject a score report_data byte string. - service = _make_key_release_service() - foreign_key = service.authorize_release( - **_key_release_request( - service, - report_data_override=bytes.fromhex(score_rd), - ) - ) - _assert_no_key(foreign_key) - - # Review domain tag cannot be equal to score for the preimage helper. - if review_domain_ok: - assert review_rd.endswith("00" * 32) - - -# =========================================================================== # -# VAL-CROSS-015: TCB + key-provider negatives at both consumers -# =========================================================================== # -def test_val_cross_015_tcb_and_key_provider_rejected_both(): - for status in ("OutOfDate", "Revoked", "SWHardeningNeeded"): - service = _make_key_release_service(verifier=StaticQuoteVerifier(tcb_status=status)) - out = service.authorize_release(**_key_release_request(service)) - _assert_no_key(out) - assert out.reason == REASON_TCB_UNACCEPTABLE - - service = _make_key_release_service() - wrong_kp = service.authorize_release( - **_key_release_request(service, key_provider_payload=ALT_KEY_PROVIDER_PAYLOAD) - ) - _assert_no_key(wrong_kp) - assert wrong_kp.reason == REASON_MEASUREMENT_NOT_ALLOWLISTED - - plan = ew.validate_eval_plan( - { - "schema_version": 1, - "eval_run_id": "eval-tcb", - "submission_id": "1", - "submission_version": 1, - "authorizing_review_digest": REVIEW_DIGEST, - "agent_hash": "55" * 32, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "task-000", - "image_ref": "registry.example/task@sha256:" + "77" * 32, - "task_config_sha256": "88" * 32, - } - ], - "k": 1, - "scoring_policy": _policy(), - "scoring_policy_digest": ew.scoring_policy_digest(_policy()), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "99" * 32, - "compose_hash": COMPOSE_HASH, - "app_identity": "agent-challenge-eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "aa" * 32, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), - "measurement": MEASUREMENT, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-tcb/result", - "key_release_nonce": "key-tcb", - "score_nonce": "score-tcb", - "run_token_sha256": "bb" * 32, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - ) - request = _result_request(plan) - for status in ("OutOfDate", "Revoked"): - gate = AttestationGate( - quote_verifier=StaticQuoteVerifier(tcb_status=status), - allowlist=ResultMeasurementAllowlist.from_measurements([ALLOWLIST_ENTRY]), - ) - decision = gate.decide_eval_result( - request, - eval_plan=plan, - expected_agent_hash=plan["agent_hash"], - nonce_outstanding=True, - key_granted=True, - ) - assert decision.outcome is AttestationOutcome.VERIFICATION_FAILED - - -# =========================================================================== # -# VAL-CROSS-016: post-review cheat gate -> zero selected work / eval surface -# =========================================================================== # -async def test_val_cross_016_attested_review_blocks_cheats_before_eval( - database_session, monkeypatch -): - """Without verified review allow, EvalPrepare and WUs are impossible. - - Models the post-review cheat gate: a submission that never receives - verified allow creates zero selected tasks, zero work units, and cannot - issue an eval run/token. (Static cheat AST paths remain covered elsewhere; - this suite asserts the production authorization hollow.) - """ - - _enable_full_attested(monkeypatch) - settings = _eval_settings(task_count=3) - _patch_benchmark_tasks(monkeypatch, task_count=3) - - # Rejected / pending review states: no prepare, no WUs, no EvalRun. - for phase in ("review_rejected", "review_escalated", "review_queued"): - submission = await _authorized_submission( - database_session, - suffix=f"cheat-{phase}", - attach_review=False, - raw_status=phase, - ) - async with database_session() as session: - row = await session.get(AgentSubmission, submission.id) - from agent_challenge.evaluation.authorization import ( - EvalAuthorizationRequired, - ) - - with pytest.raises(EvalAuthorizationRequired): - await create_eval_run(session, row, settings=settings) - assert await list_pending_work_units(session) == [] - assert await session.scalar(select(func.count(EvalRun.id))) == 0 - assert await session.scalar(select(func.count(EvaluationJob.id))) == 0 - - # Control: verified allow DOES unlock prepare, but still zero assignable WUs. - run, plan, token, _ = await _prepare_eval_run( - database_session, monkeypatch, suffix="allow-control", settings=settings, task_count=3 - ) - assert token - assert len(plan["selected_tasks"]) == 3 - async with database_session() as session: - assert await list_pending_work_units(session) == [] - assert await session.scalar(select(func.count(EvaluationJob.id))) == 0 - assert ( - await session.scalar( - select(func.count(EvalRun.id)).where(EvalRun.eval_run_id == run.eval_run_id) - ) - == 1 - ) diff --git a/packages/challenges/agent-challenge/tests/test_direct_eval_verification.py b/packages/challenges/agent-challenge/tests/test_direct_eval_verification.py deleted file mode 100644 index d760f557c..000000000 --- a/packages/challenges/agent-challenge/tests/test_direct_eval_verification.py +++ /dev/null @@ -1,732 +0,0 @@ -from __future__ import annotations - -import copy -import hashlib -from datetime import UTC, datetime, timedelta - -import pytest -from sqlalchemy import select - -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.core.models import ( - AgentSubmission, - EvalNonce, - EvalRun, - EvaluationJob, - TaskAttestation, - TaskResult, -) -from agent_challenge.evaluation.attestation import ( - AttestationGate, - AttestationOutcome, - ResultMeasurementAllowlist, -) -from agent_challenge.evaluation.direct_result import ( - DirectEvalResultError, - _persist_verified_result, - process_direct_eval_result, - retry_receipted_eval_result, - validate_result_bounds, -) -from agent_challenge.evaluation.plan_scoring import ( - build_score_record_from_eval_plan, - canonical_eval_plan_json, -) -from agent_challenge.evaluation.validator_executor import job_attestation_verified -from agent_challenge.evaluation.weights import is_reward_eligible_job -from agent_challenge.keyrelease.quote import ( - QuoteVerifierUnavailable, - StaticQuoteVerifier, - build_rtmr3_event_log, - build_tdx_quote, - os_image_hash_from_registers, -) -from agent_challenge.sdk.config import ChallengeSettings - -REGS = { - "mrtd": "11" * 48, - "rtmr0": "22" * 48, - "rtmr1": "33" * 48, - "rtmr2": "44" * 48, -} -COMPOSE_HASH = "ab" * 32 -OS_IMAGE_HASH = os_image_hash_from_registers(REGS["mrtd"], REGS["rtmr1"], REGS["rtmr2"]) -AGENT_HASH = "55" * 32 - - -def _plan() -> dict: - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - return ew.validate_eval_plan( - { - "schema_version": 1, - "eval_run_id": "eval-direct-1", - "submission_id": "submission-direct-1", - "submission_version": 1, - "authorizing_review_digest": "66" * 32, - "agent_hash": AGENT_HASH, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "task-a", - "image_ref": "registry.example/task@sha256:" + "77" * 32, - "task_config_sha256": "88" * 32, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "99" * 32, - "compose_hash": COMPOSE_HASH, - "app_identity": "agent-challenge-eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "aa" * 32, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), - "measurement": { - **REGS, - "os_image_hash": OS_IMAGE_HASH, - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-direct-1/result", - "key_release_nonce": "key-release-direct-1", - "score_nonce": "score-direct-1", - "run_token_sha256": "bb" * 32, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - ) - - -def _request(plan: dict, *, score_nonce: str | None = None) -> dict: - event_log, rtmr3 = build_rtmr3_event_log( - [("compose-hash", bytes.fromhex(COMPOSE_HASH)), ("key-provider", b"validator-kms")] - ) - record = build_score_record_from_eval_plan(plan, {"task-a": [1.0]}) - scores_digest = ew.score_record_digest(record) - binding = ew.build_score_binding( - canonical_measurement={ - "mrtd": REGS["mrtd"], - "rtmr0": REGS["rtmr0"], - "rtmr1": REGS["rtmr1"], - "rtmr2": REGS["rtmr2"], - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, - }, - agent_hash=AGENT_HASH, - eval_run_id=plan["eval_run_id"], - score_nonce=score_nonce or plan["score_nonce"], - scores_digest=scores_digest, - task_ids=["task-a"], - ) - report_data = ew.score_report_data_hex(binding) - quote = build_tdx_quote( - mrtd=REGS["mrtd"], - rtmr0=REGS["rtmr0"], - rtmr1=REGS["rtmr1"], - rtmr2=REGS["rtmr2"], - rtmr3=rtmr3, - report_data=report_data, - ) - return { - "schema_version": 1, - "eval_run_id": plan["eval_run_id"], - "submission_id": plan["submission_id"], - "agent_hash": AGENT_HASH, - "score_record": record, - "scores_digest": scores_digest, - "execution_proof": { - "version": 1, - "tier": "phala-tdx", - "manifest_sha256": "cc" * 32, - "image_digest": plan["eval_app"]["image_ref"], - "provider": None, - "worker_signature": {"worker_pubkey": "", "sig": ""}, - "attestation": { - "tdx_quote": quote, - "event_log": event_log, - "report_data": report_data, - "measurement": { - **REGS, - "rtmr3": rtmr3, - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, - }, - "vm_config": { - "vcpu": 1, - "memory_mb": 2048, - "os_image_hash": OS_IMAGE_HASH, - }, - }, - }, - } - - -def _gate(plan: dict, *, nonce_outstanding: bool = True) -> AttestationGate: - return AttestationGate( - quote_verifier=StaticQuoteVerifier(), - allowlist=ResultMeasurementAllowlist.from_measurements( - [ - { - "mrtd": REGS["mrtd"], - "rtmr0": REGS["rtmr0"], - "rtmr1": REGS["rtmr1"], - "rtmr2": REGS["rtmr2"], - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, - } - ] - ), - ) - - -def _direct_settings() -> ChallengeSettings: - plan = _plan() - return ChallengeSettings( - eval_app_measurement_allowlist=( - { - "mrtd": REGS["mrtd"], - "rtmr0": REGS["rtmr0"], - "rtmr1": REGS["rtmr1"], - "rtmr2": REGS["rtmr2"], - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, - }, - ), - eval_result_max_bytes=16 * 1024 * 1024, - eval_result_max_tasks=4, - eval_result_max_event_log_entries=8, - eval_result_max_event_log_bytes=64 * 1024, - eval_result_max_vm_config_bytes=4096, - eval_result_max_string_bytes=4096, - eval_result_max_quote_bytes=64 * 1024, - attestation_max_concurrent_verifications=2, - eval_app_image_ref=plan["eval_app"]["image_ref"], - ) - - -async def _direct_run(database_session, plan: dict) -> EvalRun: - now = datetime.now(UTC) - async with database_session() as session: - submission = AgentSubmission( - miner_hotkey="direct-miner", - name="direct-agent", - agent_hash=AGENT_HASH, - artifact_uri="/tmp/direct.zip", - raw_status="review_allowed", - status="queued", - effective_status="queued", - version_number=1, - ) - session.add(submission) - await session.flush() - run = EvalRun( - eval_run_id=plan["eval_run_id"], - submission_id=submission.id, - submission_version=1, - authorizing_review_digest="66" * 32, - plan_json=canonical_eval_plan_json(plan), - plan_sha256=hashlib.sha256(canonical_eval_plan_json(plan).encode("utf-8")).hexdigest(), - token_sha256=hashlib.sha256(b"direct-token").hexdigest(), - phase="eval_running", - retryable=False, - key_granted_at=now, - issued_at=now, - expires_at=now + timedelta(hours=1), - ) - session.add(run) - await session.flush() - session.add( - EvalNonce( - eval_run_id=run.id, - nonce=plan["score_nonce"], - purpose="score", - state="outstanding", - expires_at=now + timedelta(hours=1), - ) - ) - await session.commit() - return run - - -async def test_direct_result_persists_one_attested_job_and_is_idempotent( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - plan = _plan() - request = _request(plan) - raw_body = ew.canonical_json_v1(request) - monkeypatch.setattr( - "agent_challenge.evaluation.direct_result._endpoint_worker_signature", - lambda *_args, **_kwargs: {"worker_pubkey": "validator", "sig": "signature"}, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.attestation.verify_worker_signature", - lambda *_args: True, - ) - async with database_session() as session: - run = await _direct_run(database_session, plan) - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id)) - assert run is not None - receipt, created = await process_direct_eval_result( - session, - run=run, - raw_body=raw_body, - result_request=request, - settings=_direct_settings(), - quote_verifier=StaticQuoteVerifier(), - ) - assert created is True - assert receipt["phase"] == "verified" - job = await session.scalar( - select(EvaluationJob).where(EvaluationJob.id == run.result_job_id) - ) - assert job is not None - assert job.status == "completed" - assert await session.scalar(select(TaskResult.job_id)) == job.id - assert await session.scalar(select(TaskAttestation.job_id)) == job.id - replay, replay_created = await process_direct_eval_result( - session, - run=run, - raw_body=raw_body, - result_request=request, - settings=_direct_settings(), - quote_verifier=StaticQuoteVerifier(), - ) - assert replay_created is False - assert replay == receipt - - -async def test_full_attested_result_persists_only_challenge_owned_eval_score( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The production attested path never creates validator job/task rows.""" - - from product_score_chain_fixtures import ( - bind_key_release_grant_on_run, - build_fixture_review_envelope, - rebind_plan_authorizing_digest, - seed_authorizing_review_assignment, - ) - - envelope = build_fixture_review_envelope() - plan = rebind_plan_authorizing_digest(_plan(), envelope) - request = _request(plan) - raw_body = ew.canonical_json_v1(request) - direct_settings = _direct_settings() - direct_settings.attested_review_enabled = True - direct_settings.phala_attestation_enabled = True - monkeypatch.setattr( - "agent_challenge.evaluation.direct_result._endpoint_worker_signature", - lambda *_args, **_kwargs: {"worker_pubkey": "validator", "sig": "signature"}, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.attestation.verify_worker_signature", - lambda *_args: True, - ) - - async with database_session() as session: - run = await _direct_run(database_session, plan) - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id)) - assert run is not None - await seed_authorizing_review_assignment( - session, - submission_id=run.submission_id, - envelope=envelope, - authorizing_review_digest=plan["authorizing_review_digest"], - ) - bind_key_release_grant_on_run(run, plan) - await session.commit() - receipt, created = await process_direct_eval_result( - session, - run=run, - raw_body=raw_body, - result_request=request, - settings=direct_settings, - quote_verifier=StaticQuoteVerifier(), - ) - assert created is True - assert receipt["phase"] == "verified" - assert run.result_job_id is None - assert run.score is not None - assert run.total_tasks == len(plan["selected_tasks"]) - assert await session.scalar(select(EvaluationJob.id)) is None - assert await session.scalar(select(TaskResult.id)) is None - assert await session.scalar(select(TaskAttestation.id)) is None - - -async def test_direct_result_outage_retries_exact_receipt_without_nonce_consume( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - plan = _plan() - request = _request(plan) - raw_body = ew.canonical_json_v1(request) - monkeypatch.setattr( - "agent_challenge.evaluation.direct_result._endpoint_worker_signature", - lambda *_args, **_kwargs: {"worker_pubkey": "validator", "sig": "signature"}, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.attestation.verify_worker_signature", - lambda *_args: True, - ) - - class OutageVerifier: - def verify(self, _quote: str): - raise QuoteVerifierUnavailable("collateral unavailable") - - async with database_session() as session: - run = await _direct_run(database_session, plan) - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id)) - assert run is not None - parked, created = await process_direct_eval_result( - session, - run=run, - raw_body=raw_body, - result_request=request, - settings=_direct_settings(), - quote_verifier=OutageVerifier(), - ) - assert created is True - assert parked["phase"] == "verifier_unavailable" - nonce = await session.scalar( - select(EvalNonce).where( - EvalNonce.eval_run_id == run.id, - EvalNonce.purpose == "score", - ) - ) - assert nonce is not None - assert nonce.state == "outstanding" - recovered, recovered_created = await retry_receipted_eval_result( - session, - run=run, - settings=_direct_settings(), - quote_verifier=StaticQuoteVerifier(), - ) - assert recovered_created is True - assert recovered["phase"] == "verified" - assert recovered["body_sha256"] == parked["body_sha256"] - await session.refresh(nonce) - assert nonce.state == "consumed" - - -async def test_invalid_result_after_key_grant_is_terminal_non_verified( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A bad proof after golden-key release is still a terminal liveness result.""" - - plan = _plan() - request = _request(plan) - request["execution_proof"]["attestation"]["report_data"] = "00" * 64 - raw_body = ew.canonical_json_v1(request) - monkeypatch.setattr( - "agent_challenge.evaluation.direct_result._endpoint_worker_signature", - lambda *_args, **_kwargs: {"worker_pubkey": "validator", "sig": "signature"}, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.attestation.verify_worker_signature", - lambda *_args: True, - ) - - async with database_session() as session: - run = await _direct_run(database_session, plan) - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id)) - assert run is not None - receipt, created = await process_direct_eval_result( - session, - run=run, - raw_body=raw_body, - result_request=request, - settings=_direct_settings(), - quote_verifier=StaticQuoteVerifier(), - ) - assert created is True - assert receipt["phase"] == "rejected" - assert receipt["verified"] is False - assert receipt["retryable"] is False - assert run.failure_origin == "attestation" - assert run.reward_eligible is False - nonce = await session.scalar( - select(EvalNonce).where( - EvalNonce.eval_run_id == run.id, - EvalNonce.purpose == "score", - ) - ) - assert nonce is not None - assert nonce.state == "consumed" - assert await session.scalar(select(TaskResult.id)) is None - - -async def test_key_granted_run_with_no_result_terminalizes_as_non_retryable( - database_session, -) -> None: - """A run that released golden material but never reports cannot be retried.""" - - plan = _plan() - async with database_session() as session: - run = await _direct_run(database_session, plan) - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id)) - assert run is not None - run.key_release_receipt_sha256 = "aa" * 32 - run.expires_at = datetime(2024, 1, 1, tzinfo=UTC) - page = await __import__( - "agent_challenge.evaluation.authorization", - fromlist=["eval_status_page"], - ).eval_status_page( - session, - await session.get(AgentSubmission, run.submission_id), - ) - assert page["items"][0]["phase"] == "eval_expired" - assert run.failure_origin == "no_result" - assert run.verified is False - assert run.reward_eligible is False - assert run.retryable is False - - -async def test_weight_evidence_requires_verified_review_allow( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Complete score attestations alone never earn weight without matching review.""" - - plan = _plan() - request = _request(plan) - raw_body = ew.canonical_json_v1(request) - monkeypatch.setattr( - "agent_challenge.evaluation.direct_result._endpoint_worker_signature", - lambda *_args, **_kwargs: {"worker_pubkey": "validator", "sig": "signature"}, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.attestation.verify_worker_signature", - lambda *_args: True, - ) - async with database_session() as session: - run = await _direct_run(database_session, plan) - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id)) - assert run is not None - receipt, created = await process_direct_eval_result( - session, - run=run, - raw_body=raw_body, - result_request=request, - settings=_direct_settings(), - quote_verifier=StaticQuoteVerifier(), - ) - assert created is True - assert receipt["phase"] == "verified" - await session.refresh(run) - job = await session.scalar( - select(EvaluationJob).where(EvaluationJob.id == run.result_job_id) - ) - assert job is not None - assert await job_attestation_verified(session, job) is True - assert ( - is_reward_eligible_job(job, 1, attestation_verified=True, review_verified=False) - is False - ) - - -async def test_verified_result_persistence_rolls_back_all_rows_on_failure( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """A failure after task writes parks the receipt without partial scoring rows.""" - - plan = _plan() - request = _request(plan) - raw_body = ew.canonical_json_v1(request) - monkeypatch.setattr( - "agent_challenge.evaluation.direct_result._endpoint_worker_signature", - lambda *_args, **_kwargs: {"worker_pubkey": "validator", "sig": "signature"}, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.attestation.verify_worker_signature", - lambda *_args: True, - ) - original = _persist_verified_result - - async def fail_after_persist(*args, **kwargs): - await original(*args, **kwargs) - raise RuntimeError("injected persistence failure") - - monkeypatch.setattr( - "agent_challenge.evaluation.direct_result._persist_verified_result", - fail_after_persist, - ) - - async with database_session() as session: - run = await _direct_run(database_session, plan) - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id)) - assert run is not None - parked, created = await process_direct_eval_result( - session, - run=run, - raw_body=raw_body, - result_request=request, - settings=_direct_settings(), - quote_verifier=StaticQuoteVerifier(), - ) - assert created is True - assert parked["phase"] == "verifier_unavailable" - assert parked["reason_code"] == "persistence_unavailable" - assert await session.scalar(select(TaskResult.id)) is None - assert await session.scalar(select(TaskAttestation.id)) is None - assert await session.scalar(select(EvaluationJob.id)) is None - - -def test_direct_gate_accepts_only_plan_bound_result_without_consuming_inline_nonce() -> None: - plan = _plan() - request = _request(plan) - decision = _gate(plan).decide_eval_result( - request, - eval_plan=plan, - expected_agent_hash=AGENT_HASH, - nonce_outstanding=True, - key_granted=True, - ) - assert decision.outcome is AttestationOutcome.VERIFIED - - -def test_direct_gate_fails_closed_for_empty_allowlist_and_missing_key_grant() -> None: - plan = _plan() - request = _request(plan) - empty_gate = AttestationGate(quote_verifier=StaticQuoteVerifier()) - assert ( - empty_gate.decide_eval_result( - request, - eval_plan=plan, - expected_agent_hash=AGENT_HASH, - nonce_outstanding=True, - key_granted=True, - ).outcome - is AttestationOutcome.VERIFICATION_FAILED - ) - assert ( - _gate(plan) - .decide_eval_result( - request, - eval_plan=plan, - expected_agent_hash=AGENT_HASH, - nonce_outstanding=True, - key_granted=False, - ) - .outcome - is AttestationOutcome.VERIFICATION_FAILED - ) - - -def test_direct_gate_rejects_caller_signature_instead_of_rebinding_it() -> None: - plan = _plan() - request = _request(plan) - request["execution_proof"]["worker_signature"] = { - "worker_pubkey": "caller-selected", - "sig": "caller-selected", - } - decision = _gate(plan).decide_eval_result( - request, - eval_plan=plan, - expected_agent_hash=AGENT_HASH, - nonce_outstanding=True, - key_granted=True, - endpoint_rebound=True, - ) - assert decision.outcome is AttestationOutcome.VERIFICATION_FAILED - - -def test_direct_gate_rebinds_only_validator_signature(monkeypatch: pytest.MonkeyPatch) -> None: - plan = _plan() - request = _request(plan) - monkeypatch.setattr( - "agent_challenge.evaluation.attestation.verify_worker_signature", - lambda *_args: True, - ) - decision = _gate(plan).decide_eval_result( - request, - eval_plan=plan, - expected_agent_hash=AGENT_HASH, - nonce_outstanding=True, - key_granted=True, - endpoint_rebound=True, - rebound_worker_signature={"worker_pubkey": "validator", "sig": "signature"}, - ) - assert decision.outcome is AttestationOutcome.VERIFIED - - -def test_direct_result_nested_bounds_fail_before_verification() -> None: - plan = _plan() - request = _request(plan) - with pytest.raises(DirectEvalResultError, match="event-log"): - validate_result_bounds( - { - **request, - "execution_proof": { - **request["execution_proof"], - "attestation": { - **request["execution_proof"]["attestation"], - "event_log": request["execution_proof"]["attestation"]["event_log"] * 2, - }, - }, - }, - max_tasks=4, - max_event_log_entries=1, - max_quote_bytes=1024, - ) - - -@pytest.mark.parametrize( - "mutation", - [ - lambda value: value["execution_proof"]["attestation"].update(report_data="00" * 64), - lambda value: value["execution_proof"]["attestation"]["measurement"].update( - compose_hash="dd" * 32 - ), - lambda value: value["score_record"]["tasks"][0].update( - trial_scores_f64be=["0000000000000000"] - ), - lambda value: value["execution_proof"]["attestation"].update( - tdx_quote=value["execution_proof"]["attestation"]["tdx_quote"][:-2] - ), - ], -) -def test_direct_gate_rejects_tampering_and_crossed_nonce(mutation) -> None: - plan = _plan() - request = _request(plan) - mutation(copy.deepcopy(request)) - mutated = copy.deepcopy(request) - mutation(mutated) - assert ( - _gate(plan) - .decide_eval_result( - mutated, - eval_plan=plan, - expected_agent_hash=AGENT_HASH, - nonce_outstanding=True, - key_granted=True, - ) - .outcome - is AttestationOutcome.VERIFICATION_FAILED - ) - - crossed = _request(plan, score_nonce=plan["key_release_nonce"]) - assert ( - _gate(plan) - .decide_eval_result( - crossed, - eval_plan=plan, - expected_agent_hash=AGENT_HASH, - nonce_outstanding=True, - key_granted=True, - ) - .outcome - is AttestationOutcome.VERIFICATION_FAILED - ) diff --git a/packages/challenges/agent-challenge/tests/test_direct_result_admission_hardening.py b/packages/challenges/agent-challenge/tests/test_direct_result_admission_hardening.py deleted file mode 100644 index fb6c21ad9..000000000 --- a/packages/challenges/agent-challenge/tests/test_direct_result_admission_hardening.py +++ /dev/null @@ -1,735 +0,0 @@ -"""Admission, capacity, and failure-mode hardening for direct Eval result ingestion. - -These tests encode the scrutiny findings for VAL-VERIFY-001/013/014/028 and -VAL-CROSS-007/034: auth-first bounded transport reading, mandatory endpoint -signer configuration, database-backed global resource reservations, literal -zero-limit handling, cancellable verification, and exhaustive retryable mapping. -""" - -from __future__ import annotations - -import asyncio -import hashlib -import json -import multiprocessing -import time -from datetime import UTC, datetime, timedelta -from typing import Any - -import pytest -from sqlalchemy import select - -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.core.models import ( - AgentSubmission, - EvalNonce, - EvalResourceCounter, - EvalRun, -) -from agent_challenge.evaluation.attestation import ( - AttestationGate, - AttestationOutcome, - ResultMeasurementAllowlist, -) -from agent_challenge.evaluation.authorization import ( - EvalAuthorizationConflict, - receipt_eval_result, -) -from agent_challenge.evaluation.direct_result import ( - process_direct_eval_result, - require_endpoint_result_signer, -) -from agent_challenge.evaluation.plan_scoring import ( - build_score_record_from_eval_plan, - canonical_eval_plan_json, -) -from agent_challenge.keyrelease.quote import ( - StaticQuoteVerifier, - build_rtmr3_event_log, - build_tdx_quote, - os_image_hash_from_registers, -) -from agent_challenge.sdk.config import ChallengeSettings - -REGS = { - "mrtd": "11" * 48, - "rtmr0": "22" * 48, - "rtmr1": "33" * 48, - "rtmr2": "44" * 48, -} -COMPOSE_HASH = "ab" * 32 -OS_IMAGE_HASH = os_image_hash_from_registers(REGS["mrtd"], REGS["rtmr1"], REGS["rtmr2"]) -AGENT_HASH = "55" * 32 - - -def _plan(*, eval_run_id: str = "eval-admission-1") -> dict[str, Any]: - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - return ew.validate_eval_plan( - { - "schema_version": 1, - "eval_run_id": eval_run_id, - "submission_id": f"submission-{eval_run_id}", - "submission_version": 1, - "authorizing_review_digest": "66" * 32, - "agent_hash": AGENT_HASH, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "task-a", - "image_ref": "registry.example/task@sha256:" + "77" * 32, - "task_config_sha256": "88" * 32, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "99" * 32, - "compose_hash": COMPOSE_HASH, - "app_identity": "agent-challenge-eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "aa" * 32, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), - "measurement": { - **REGS, - "os_image_hash": OS_IMAGE_HASH, - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": f"/evaluation/v1/runs/{eval_run_id}/result", - "key_release_nonce": f"key-release-{eval_run_id}", - "score_nonce": f"score-{eval_run_id}", - "run_token_sha256": hashlib.sha256(eval_run_id.encode("utf-8")).hexdigest(), - "issued_at_ms": 1, - "expires_at_ms": 2, - } - ) - - -def _settings(**overrides: Any) -> ChallengeSettings: - values = { - "attested_review_enabled": True, - "phala_attestation_enabled": True, - "eval_app_measurement_allowlist": ( - { - "mrtd": REGS["mrtd"], - "rtmr0": REGS["rtmr0"], - "rtmr1": REGS["rtmr1"], - "rtmr2": REGS["rtmr2"], - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, - }, - ), - "eval_result_max_bytes": 16 * 1024 * 1024, - "eval_result_max_tasks": 4, - "eval_result_max_event_log_entries": 8, - "eval_result_max_event_log_bytes": 64 * 1024, - "eval_result_max_vm_config_bytes": 4096, - "eval_result_max_string_bytes": 4096, - "eval_result_max_quote_bytes": 64 * 1024, - "eval_result_max_submissions_per_run_per_minute": 10, - "eval_result_max_outstanding": 2, - "attestation_max_concurrent_verifications": 1, - "eval_result_verifier_deadline_seconds": 0.25, - "eval_result_signer_uri": "//Alice", - } - values.update(overrides) - return ChallengeSettings(**values) - - -async def _seed_run( - database_session, - plan: dict[str, Any], - *, - token: str = "direct-token", -) -> EvalRun: - now = datetime.now(UTC) - async with database_session() as session: - # agent_hash is unique across submissions; derive a stable per-run value - # while keeping plan agent_hash for attestation fixtures. - submission_agent_hash = hashlib.sha256(plan["eval_run_id"].encode("utf-8")).hexdigest() - submission = AgentSubmission( - miner_hotkey=f"admission-miner-{plan['eval_run_id']}", - name=f"admission-agent-{plan['eval_run_id']}", - agent_hash=submission_agent_hash, - artifact_uri=f"/tmp/admission-{plan['eval_run_id']}.zip", - raw_status="review_allowed", - status="queued", - effective_status="queued", - version_number=1, - ) - session.add(submission) - await session.flush() - run = EvalRun( - eval_run_id=plan["eval_run_id"], - submission_id=submission.id, - submission_version=1, - authorizing_review_digest="66" * 32, - plan_json=canonical_eval_plan_json(plan), - plan_sha256=hashlib.sha256(canonical_eval_plan_json(plan).encode("utf-8")).hexdigest(), - token_sha256=hashlib.sha256(token.encode("utf-8")).hexdigest(), - phase="eval_running", - retryable=False, - key_granted_at=now, - issued_at=now, - expires_at=now + timedelta(hours=1), - ) - session.add(run) - await session.flush() - session.add( - EvalNonce( - eval_run_id=run.id, - nonce=plan["score_nonce"], - purpose="score", - state="outstanding", - expires_at=now + timedelta(hours=1), - ) - ) - await session.commit() - return run - - -def _request(plan: dict[str, Any]) -> dict[str, Any]: - event_log, rtmr3 = build_rtmr3_event_log( - [("compose-hash", bytes.fromhex(COMPOSE_HASH)), ("key-provider", b"validator-kms")] - ) - record = build_score_record_from_eval_plan(plan, {"task-a": [1.0]}) - scores_digest = ew.score_record_digest(record) - binding = ew.build_score_binding( - canonical_measurement={ - "mrtd": REGS["mrtd"], - "rtmr0": REGS["rtmr0"], - "rtmr1": REGS["rtmr1"], - "rtmr2": REGS["rtmr2"], - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, - }, - agent_hash=AGENT_HASH, - eval_run_id=plan["eval_run_id"], - score_nonce=plan["score_nonce"], - scores_digest=scores_digest, - task_ids=["task-a"], - ) - report_data = ew.score_report_data_hex(binding) - quote = build_tdx_quote( - mrtd=REGS["mrtd"], - rtmr0=REGS["rtmr0"], - rtmr1=REGS["rtmr1"], - rtmr2=REGS["rtmr2"], - rtmr3=rtmr3, - report_data=report_data, - ) - return { - "schema_version": 1, - "eval_run_id": plan["eval_run_id"], - "submission_id": plan["submission_id"], - "agent_hash": AGENT_HASH, - "score_record": record, - "scores_digest": scores_digest, - "execution_proof": { - "version": 1, - "tier": "phala-tdx", - "manifest_sha256": "cc" * 32, - "image_digest": plan["eval_app"]["image_ref"], - "provider": None, - "worker_signature": {"worker_pubkey": "", "sig": ""}, - "attestation": { - "tdx_quote": quote, - "event_log": event_log, - "report_data": report_data, - "measurement": { - **REGS, - "rtmr3": rtmr3, - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, - }, - "vm_config": { - "vcpu": 1, - "memory_mb": 2048, - "os_image_hash": OS_IMAGE_HASH, - }, - }, - }, - } - - -def test_production_attested_mode_fails_closed_without_endpoint_signer() -> None: - settings = ChallengeSettings( - attested_review_enabled=True, - phala_attestation_enabled=True, - eval_result_signer_uri=None, - eval_result_signer_mnemonic=None, - ) - with pytest.raises(ValueError, match="eval result signer"): - settings.require_eval_result_signer_for_production() - - -def test_require_endpoint_result_signer_rejected_without_config() -> None: - settings = ChallengeSettings( - attested_review_enabled=False, - phala_attestation_enabled=False, - eval_result_signer_uri=None, - eval_result_signer_mnemonic=None, - ) - with pytest.raises(ValueError, match="eval result signer"): - require_endpoint_result_signer(settings) - - -def test_zero_submission_rate_is_accepted_by_settings() -> None: - settings = ChallengeSettings( - attested_review_enabled=False, - phala_attestation_enabled=False, - eval_result_max_submissions_per_run_per_minute=0, - ) - assert settings.eval_result_max_submissions_per_run_per_minute == 0 - - -async def test_zero_submission_rate_admits_no_receipts(database_session) -> None: - plan = _plan() - run = await _seed_run(database_session, plan) - raw = b'{"schema_version":1}' - digest = hashlib.sha256(raw).hexdigest() - async with database_session() as session: - loaded = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id)) - assert loaded is not None - with pytest.raises(EvalAuthorizationConflict) as exc: - await receipt_eval_result( - session, - eval_run_id=loaded.eval_run_id, - body_sha256=digest, - body=raw, - max_submissions_per_minute=0, - max_outstanding=10, - ) - assert exc.value.code == "eval_result_rate_limited" - await session.refresh(loaded) - assert loaded.receipt_id is None - - -async def test_global_outstanding_cap_is_database_atomic(database_session) -> None: - plan_a = _plan(eval_run_id="eval-admission-a") - plan_b = _plan(eval_run_id="eval-admission-b") - plan_c = _plan(eval_run_id="eval-admission-c") - run_a = await _seed_run(database_session, plan_a, token="token-a") - run_b = await _seed_run(database_session, plan_b, token="token-b") - run_c = await _seed_run(database_session, plan_c, token="token-c") - digest_a = "aa" * 32 - digest_b = "bb" * 32 - digest_c = "cc" * 32 - async with database_session() as session: - for run, digest in ((run_a, digest_a), (run_b, digest_b)): - loaded = await session.scalar( - select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id) - ) - assert loaded is not None - await receipt_eval_result( - session, - eval_run_id=loaded.eval_run_id, - body_sha256=digest, - body=b"{}", - max_submissions_per_minute=10, - max_outstanding=2, - ) - await session.commit() - loaded_c = await session.scalar( - select(EvalRun).where(EvalRun.eval_run_id == run_c.eval_run_id) - ) - assert loaded_c is not None - with pytest.raises(EvalAuthorizationConflict) as exc: - await receipt_eval_result( - session, - eval_run_id=loaded_c.eval_run_id, - body_sha256=digest_c, - body=b"{}", - max_submissions_per_minute=10, - max_outstanding=2, - ) - assert exc.value.code == "eval_result_overloaded" - from agent_challenge.evaluation.authorization import OUTSTANDING_RESULT_RESOURCE - - counter = await session.get(EvalResourceCounter, OUTSTANDING_RESULT_RESOURCE) - assert counter is not None - assert counter.value == 2 - - -async def test_unexpected_verifier_exception_is_retryable_and_preserves_nonce( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - from product_score_chain_fixtures import ( - bind_key_release_grant_on_run, - build_fixture_review_envelope, - rebind_plan_authorizing_digest, - seed_authorizing_review_assignment, - ) - - envelope = build_fixture_review_envelope() - plan = rebind_plan_authorizing_digest(_plan(), envelope) - request = _request(plan) - raw_body = ew.canonical_json_v1(request) - settings = _settings() - monkeypatch.setattr( - "agent_challenge.evaluation.attestation.verify_worker_signature", - lambda *_args: True, - ) - - class BrokenVerifier: - def verify(self, _quote: str): - raise OSError("dcap backend crashed") - - async with database_session() as session: - run = await _seed_run(database_session, plan) - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id)) - assert run is not None - await seed_authorizing_review_assignment( - session, - submission_id=run.submission_id, - envelope=envelope, - authorizing_review_digest=plan["authorizing_review_digest"], - ) - bind_key_release_grant_on_run(run, plan) - await session.commit() - parked, created = await process_direct_eval_result( - session, - run=run, - raw_body=raw_body, - result_request=request, - settings=settings, - quote_verifier=BrokenVerifier(), - ) - assert created is True - assert parked["phase"] == "verifier_unavailable" - assert parked["retryable"] is True - nonce = await session.scalar( - select(EvalNonce).where(EvalNonce.eval_run_id == run.id, EvalNonce.purpose == "score") - ) - assert nonce is not None - assert nonce.state == "outstanding" - - -def test_direct_gate_maps_unexpected_backend_errors_to_unavailable( - monkeypatch: pytest.MonkeyPatch, -) -> None: - plan = _plan() - request = _request(plan) - monkeypatch.setattr( - "agent_challenge.evaluation.attestation.verify_worker_signature", - lambda *_args: True, - ) - - class BrokenVerifier: - def verify(self, _quote: str): - raise RuntimeError("unexpected dcap failure") - - gate = AttestationGate( - quote_verifier=BrokenVerifier(), - allowlist=ResultMeasurementAllowlist.from_measurements( - [ - { - "mrtd": REGS["mrtd"], - "rtmr0": REGS["rtmr0"], - "rtmr1": REGS["rtmr1"], - "rtmr2": REGS["rtmr2"], - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, - } - ] - ), - ) - decision = gate.decide_eval_result( - request, - eval_plan=plan, - expected_agent_hash=AGENT_HASH, - nonce_outstanding=True, - key_granted=True, - endpoint_rebound=True, - rebound_worker_signature={"worker_pubkey": "validator", "sig": "signature"}, - ) - assert decision.outcome is AttestationOutcome.VERIFIER_UNAVAILABLE - - -def _hang_until_killed(duration: float = 30.0) -> None: - try: - time.sleep(duration) - except KeyboardInterrupt: - return - - -async def test_verification_timeout_terminates_subprocess( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - from product_score_chain_fixtures import ( - bind_key_release_grant_on_run, - build_fixture_review_envelope, - rebind_plan_authorizing_digest, - seed_authorizing_review_assignment, - ) - - envelope = build_fixture_review_envelope() - plan = rebind_plan_authorizing_digest(_plan(), envelope) - request = _request(plan) - raw_body = ew.canonical_json_v1(request) - settings = _settings(eval_result_verifier_deadline_seconds=0.2) - monkeypatch.setattr( - "agent_challenge.evaluation.attestation.verify_worker_signature", - lambda *_args: True, - ) - processes: list[multiprocessing.Process] = [] - - class SlowVerifier: - def __init__(self) -> None: - self._process: multiprocessing.Process | None = None - - def verify(self, _quote: str): - proc = multiprocessing.Process(target=_hang_until_killed, args=(30.0,)) - proc.start() - self._process = proc - processes.append(proc) - proc.join() - return StaticQuoteVerifier().verify(_quote) - - def cancel(self) -> None: - proc = self._process - if proc is None or not proc.is_alive(): - return - proc.terminate() - proc.join(timeout=1.0) - if proc.is_alive(): - proc.kill() - proc.join(timeout=1.0) - - async with database_session() as session: - run = await _seed_run(database_session, plan) - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id)) - assert run is not None - await seed_authorizing_review_assignment( - session, - submission_id=run.submission_id, - envelope=envelope, - authorizing_review_digest=plan["authorizing_review_digest"], - ) - bind_key_release_grant_on_run(run, plan) - await session.commit() - parked, created = await process_direct_eval_result( - session, - run=run, - raw_body=raw_body, - result_request=request, - settings=settings, - quote_verifier=SlowVerifier(), - ) - assert created is True - assert parked["phase"] == "verifier_unavailable" - assert processes, "expected a verifier subprocess to start" - for proc in processes: - proc.join(timeout=1.0) - assert not proc.is_alive(), "timeout must terminate the verifier process" - - -async def test_route_auth_fails_before_buffered_body( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Unauthorized request must not force the route to allocate the oversized body.""" - - from fastapi import HTTPException - - from agent_challenge.api import routes as routes_mod - from agent_challenge.core.config import settings as app_settings - - plan = _plan(eval_run_id="eval-admission-route") - await _seed_run(database_session, plan, token="good-token") - monkeypatch.setattr(app_settings, "attested_review_enabled", True) - monkeypatch.setattr(app_settings, "phala_attestation_enabled", True) - monkeypatch.setattr(app_settings, "eval_result_max_bytes", 64) - - chunks_read = {"bytes": 0} - - async def hostile_stream(): - for _ in range(8): - chunk = b"x" * 32 - chunks_read["bytes"] += len(chunk) - yield chunk - await asyncio.sleep(0) - - class FakeRequest: - headers = { - "content-type": "application/json", - "content-length": "16", - } - - def stream(self): - return hostile_stream() - - async def body(self) -> bytes: # pragma: no cover - must not be used - raise AssertionError("unbounded body() must not be used before auth") - - async with database_session() as session: - with pytest.raises(HTTPException) as exc: - await routes_mod.receive_direct_eval_result( - plan["eval_run_id"], - FakeRequest(), # type: ignore[arg-type] - session=session, - authorization=None, - ) - assert chunks_read["bytes"] == 0 - assert exc.value.status_code == 401 - assert exc.value.detail == {"code": "invalid_eval_token"} - - -async def test_route_reads_only_bounded_body_after_auth( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - from fastapi import HTTPException - - from agent_challenge.api import routes as routes_mod - from agent_challenge.core.config import settings as app_settings - - plan = _plan(eval_run_id="eval-admission-bound") - await _seed_run(database_session, plan, token="good-token") - monkeypatch.setattr(app_settings, "attested_review_enabled", True) - monkeypatch.setattr(app_settings, "phala_attestation_enabled", True) - monkeypatch.setattr(app_settings, "eval_result_max_bytes", 48) - - bytes_streamed = {"n": 0} - - async def oversize_stream(): - for _ in range(4): - chunk = b"y" * 20 - bytes_streamed["n"] += len(chunk) - yield chunk - await asyncio.sleep(0) - - class FakeRequest: - headers = {"content-type": "application/json"} - - def stream(self): - return oversize_stream() - - async def body(self) -> bytes: # pragma: no cover - raise AssertionError("must use stream() for bounded admission") - - async with database_session() as session: - with pytest.raises(HTTPException) as exc: - await routes_mod.receive_direct_eval_result( - plan["eval_run_id"], - FakeRequest(), # type: ignore[arg-type] - session=session, - authorization="Bearer good-token", - ) - assert exc.value.status_code == 413 - assert exc.value.detail == {"code": "result_too_large"} - assert bytes_streamed["n"] <= app_settings.eval_result_max_bytes + 20 - - -async def test_valid_result_rebinds_only_signature_placeholder( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - from product_score_chain_fixtures import ( - bind_key_release_grant_on_run, - build_fixture_review_envelope, - rebind_plan_authorizing_digest, - seed_authorizing_review_assignment, - ) - - envelope = build_fixture_review_envelope() - plan = rebind_plan_authorizing_digest(_plan(), envelope) - request = _request(plan) - raw_body = ew.canonical_json_v1(request) - settings = _settings() - rebound = {"worker_pubkey": "validator-hotkey", "sig": "0xabc"} - monkeypatch.setattr( - "agent_challenge.evaluation.direct_result._endpoint_worker_signature", - lambda *_args, **_kwargs: rebound, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.attestation.verify_worker_signature", - lambda *_args: True, - ) - original_quote = request["execution_proof"]["attestation"]["tdx_quote"] - original_event_log = json.dumps(request["execution_proof"]["attestation"]["event_log"]) - - async with database_session() as session: - run = await _seed_run(database_session, plan) - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id)) - assert run is not None - await seed_authorizing_review_assignment( - session, - submission_id=run.submission_id, - envelope=envelope, - authorizing_review_digest=plan["authorizing_review_digest"], - ) - bind_key_release_grant_on_run(run, plan) - await session.commit() - receipt, created = await process_direct_eval_result( - session, - run=run, - raw_body=raw_body, - result_request=request, - settings=settings, - quote_verifier=StaticQuoteVerifier(), - ) - assert created is True - assert receipt["phase"] == "verified" - assert request["execution_proof"]["worker_signature"] == { - "worker_pubkey": "", - "sig": "", - } - assert request["execution_proof"]["attestation"]["tdx_quote"] == original_quote - assert ( - json.dumps(request["execution_proof"]["attestation"]["event_log"]) == original_event_log - ) - - -async def test_concurrent_verification_slots_are_database_backed(database_session) -> None: - from agent_challenge.evaluation.authorization import ( - release_eval_resource, - reserve_eval_resource, - ) - - async with database_session() as session: - from agent_challenge.evaluation.authorization import VERIFYING_RESULT_RESOURCE - - await reserve_eval_resource( - session, - name=VERIFYING_RESULT_RESOURCE, - limit=1, - conflict_code="eval_result_overloaded", - ) - await session.commit() - with pytest.raises(EvalAuthorizationConflict) as exc: - await reserve_eval_resource( - session, - name=VERIFYING_RESULT_RESOURCE, - limit=1, - conflict_code="eval_result_overloaded", - ) - assert exc.value.code == "eval_result_overloaded" - await release_eval_resource(session, name=VERIFYING_RESULT_RESOURCE) - await session.commit() - await reserve_eval_resource( - session, - name=VERIFYING_RESULT_RESOURCE, - limit=1, - conflict_code="eval_result_overloaded", - ) - await session.commit() - counter = await session.get(EvalResourceCounter, VERIFYING_RESULT_RESOURCE) - assert counter is not None - assert counter.value == 1 - await release_eval_resource(session, name=VERIFYING_RESULT_RESOURCE) - await session.commit() - counter = await session.get(EvalResourceCounter, VERIFYING_RESULT_RESOURCE) - assert counter is not None - assert counter.value == 0 diff --git a/packages/challenges/agent-challenge/tests/test_docs_contract.py b/packages/challenges/agent-challenge/tests/test_docs_contract.py index 9a3bf5ced..2945c34e0 100644 --- a/packages/challenges/agent-challenge/tests/test_docs_contract.py +++ b/packages/challenges/agent-challenge/tests/test_docs_contract.py @@ -220,9 +220,7 @@ def test_agate_agent_driven_order_is_pinned_on_shipping_surfaces() -> None: # Self-deploy still owns the ordered review→eval spine; package README owns product home. assert "self-deploy" in combined.lower() assert ( - "package_tree_sha" in combined - or "tree sha" in combined.lower() - or "tree_sha" in combined + "package_tree_sha" in combined or "tree sha" in combined.lower() or "tree_sha" in combined ) assert "review" in combined.lower() and "eval" in combined.lower() # Personal finetune ban + measured path stay product locks somewhere shipping. diff --git a/packages/challenges/agent-challenge/tests/test_dualflag_evalrun_task_rows.py b/packages/challenges/agent-challenge/tests/test_dualflag_evalrun_task_rows.py deleted file mode 100644 index afa828768..000000000 --- a/packages/challenges/agent-challenge/tests/test_dualflag_evalrun_task_rows.py +++ /dev/null @@ -1,373 +0,0 @@ -"""Dual-flag public status.evaluation.task_rows from EvalRun.plan_json (VAL-DFROWS-001..006).""" - -from __future__ import annotations - -import hashlib -import json -from datetime import UTC, datetime, timedelta - -import agent_challenge.api.routes as api_routes -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.evaluation.plan_scoring import ( - build_score_record_from_eval_plan, - canonical_eval_plan_json, -) -from agent_challenge.models import AgentSubmission, EvalRun, TaskLogEvent - -NOW = datetime(2026, 7, 20, 15, 0, tzinfo=UTC) -AGENT_HASH = "a" * 64 -REVIEW_DIGEST = "b" * 64 -MEASUREMENT = { - "mrtd": "1" * 96, - "rtmr0": "2" * 96, - "rtmr1": "3" * 96, - "rtmr2": "4" * 96, - "os_image_hash": "5" * 64, - "compose_hash": "6" * 64, -} - - -def _policy() -> dict[str, object]: - return { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - - -def _plan(*, eval_run_id: str, task_count: int = 3) -> dict[str, object]: - policy = _policy() - task_ids = [f"df-task-{index:03d}" for index in range(task_count)] - return { - "schema_version": 1, - "eval_run_id": eval_run_id, - "submission_id": f"submission-{eval_run_id}", - "submission_version": 1, - "authorizing_review_digest": REVIEW_DIGEST, - "agent_hash": AGENT_HASH, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": task_id, - "image_ref": "registry.example/task@sha256:" + "3" * 64, - "task_config_sha256": "4" * 64, - } - for task_id in task_ids - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "5" * 64, - "compose_hash": MEASUREMENT["compose_hash"], - "app_identity": "agent-challenge-eval", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "6" * 64, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("6" * 64)).hexdigest(), - "measurement": { - "mrtd": MEASUREMENT["mrtd"], - "rtmr0": MEASUREMENT["rtmr0"], - "rtmr1": MEASUREMENT["rtmr1"], - "rtmr2": MEASUREMENT["rtmr2"], - "os_image_hash": MEASUREMENT["os_image_hash"], - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "keyrelease.example:8701", - "result_endpoint": f"/evaluation/v1/runs/{eval_run_id}/result", - "key_release_nonce": f"key-nonce-{eval_run_id}", - "score_nonce": f"score-nonce-{eval_run_id}", - "run_token_sha256": "7" * 64, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - - -def _trials_for(plan: dict[str, object], *, perfect_count: int) -> dict[str, list[float]]: - task_ids = [item["task_id"] for item in plan["selected_tasks"]] - trials: dict[str, list[float]] = {} - for index, task_id in enumerate(task_ids): - value = 1.0 if index < perfect_count else 0.0 - trials[task_id] = [value] * int(plan["k"]) - return trials - - -async def _seed_dualflag_eval_run( - session, - *, - eval_run_id: str, - phase: str, - with_scores: bool, - perfect_count: int = 2, - task_count: int = 3, -) -> tuple[int, dict[str, object], EvalRun]: - plan = _plan(eval_run_id=eval_run_id, task_count=task_count) - plan_json = canonical_eval_plan_json(plan) - submission = AgentSubmission( - miner_hotkey=f"hk-{eval_run_id}", - name=f"agent-{eval_run_id}", - agent_hash=hashlib.sha256(eval_run_id.encode("utf-8")).hexdigest(), - artifact_uri=f"/tmp/{eval_run_id}.zip", - status="queued", - raw_status="queued", - effective_status="queued", - submitted_at=NOW, - created_at=NOW, - ) - session.add(submission) - await session.flush() - - score = None - passed = None - total = None - score_json = None - score_sha = None - if with_scores: - trials = _trials_for(plan, perfect_count=perfect_count) - record = build_score_record_from_eval_plan(plan, trials) - score_json = ew.canonical_json_v1(record).decode("utf-8") - score_sha = ew.score_record_digest(record) - score = ew.decode_score_f64be(record["final"]["job_score_f64be"]) - passed = record["final"]["passed_tasks"] - total = record["final"]["total_tasks"] - - run = EvalRun( - eval_run_id=eval_run_id, - submission_id=submission.id, - submission_version=1, - authorizing_review_digest=REVIEW_DIGEST, - plan_json=plan_json, - plan_sha256=hashlib.sha256(plan_json.encode("utf-8")).hexdigest(), - token_sha256=hashlib.sha256(f"token-{eval_run_id}".encode()).hexdigest(), - phase=phase, - verified=with_scores and phase == "eval_accepted", - reward_eligible=False, - result_available=with_scores, - score=score, - passed_tasks=passed, - total_tasks=total, - canonical_score_record_json=score_json, - canonical_score_record_sha256=score_sha, - issued_at=NOW, - # Far future so eval_status_page expiry reconciliation does not rewrite - # the fixture phase before public status projection. - expires_at=NOW + timedelta(days=3650), - created_at=NOW, - updated_at=NOW, - ) - session.add(run) - await session.commit() - return submission.id, plan, run - - -def _enable_dual_flags(monkeypatch) -> None: - monkeypatch.setattr(api_routes.settings, "attested_review_enabled", True) - monkeypatch.setattr(api_routes.settings, "phala_attestation_enabled", True) - - -async def test_dualflag_status_task_rows_from_evalrun_plan_n3( - client, - database_session, - monkeypatch, -) -> None: - """VAL-DFROWS-001/005: plan selected_tasks N=3 project with job=None.""" - _enable_dual_flags(monkeypatch) - async with database_session() as session: - submission_id, plan, _run = await _seed_dualflag_eval_run( - session, - eval_run_id="eval-df-rows-001", - phase="eval_prepared", - with_scores=False, - task_count=3, - ) - - response = await client.get(f"/submissions/{submission_id}/status") - assert response.status_code == 200 - payload = response.json() - evaluation = payload["evaluation"] - assert evaluation is not None - assert evaluation["job_id"] == "eval-df-rows-001" - assert evaluation["status"] == "eval_prepared" - rows = evaluation["task_rows"] - assert len(rows) == 3 - expected_ids = [item["task_id"] for item in plan["selected_tasks"]] - assert [row["task_id"] for row in rows] == expected_ids - assert all(row["has_result"] is False for row in rows) - assert all(row["phase"] == "assigned" for row in rows) - # Dual-flag path must not invent a legacy EvaluationJob. - assert evaluation["current_attempt"] is None - - -async def test_dualflag_eval_expired_and_prepared_still_show_planned_rows( - client, - database_session, - monkeypatch, -) -> None: - """VAL-DFROWS-002: expired/prepared still project planned rows.""" - _enable_dual_flags(monkeypatch) - for phase, eval_run_id in ( - ("eval_expired", "eval-df-rows-expired"), - ("eval_prepared", "eval-df-rows-prepared"), - ): - async with database_session() as session: - submission_id, plan, _run = await _seed_dualflag_eval_run( - session, - eval_run_id=eval_run_id, - phase=phase, - with_scores=False, - task_count=3, - ) - response = await client.get(f"/submissions/{submission_id}/status") - assert response.status_code == 200 - evaluation = response.json()["evaluation"] - assert evaluation["status"] == phase - rows = evaluation["task_rows"] - assert len(rows) == 3 - assert [row["task_id"] for row in rows] == [ - item["task_id"] for item in plan["selected_tasks"] - ] - - -async def test_dualflag_score_ledger_not_hardcoded_zero( - client, - database_session, - monkeypatch, -) -> None: - """VAL-DFROWS-003: score/passed/total come from EvalRun ledger.""" - _enable_dual_flags(monkeypatch) - async with database_session() as session: - submission_id, _plan, run = await _seed_dualflag_eval_run( - session, - eval_run_id="eval-df-rows-score", - phase="eval_accepted", - with_scores=True, - perfect_count=2, - task_count=3, - ) - assert run.score is not None and run.score > 0 - assert run.passed_tasks == 2 - assert run.total_tasks == 3 - expected_score = float(run.score) - expected_passed = int(run.passed_tasks) - expected_total = int(run.total_tasks) - - response = await client.get(f"/submissions/{submission_id}/status") - assert response.status_code == 200 - evaluation = response.json()["evaluation"] - assert evaluation["score"] == expected_score - assert evaluation["passed_tasks"] == expected_passed - assert evaluation["total_tasks"] == expected_total - assert evaluation["score"] != 0.0 or expected_score == 0.0 - assert not ( - evaluation["score"] == 0.0 - and evaluation["passed_tasks"] == 0 - and evaluation["total_tasks"] == 0 - ) - - -async def test_dualflag_score_record_overlay_has_result( - client, - database_session, - monkeypatch, -) -> None: - """VAL-DFROWS-004: overlay has_result from score record; no invented ids.""" - _enable_dual_flags(monkeypatch) - async with database_session() as session: - submission_id, plan, _run = await _seed_dualflag_eval_run( - session, - eval_run_id="eval-df-rows-overlay", - phase="eval_accepted", - with_scores=True, - perfect_count=2, - task_count=3, - ) - planned_ids = {item["task_id"] for item in plan["selected_tasks"]} - - response = await client.get(f"/submissions/{submission_id}/status") - assert response.status_code == 200 - rows = response.json()["evaluation"]["task_rows"] - assert len(rows) == 3 - assert {row["task_id"] for row in rows} == planned_ids - # Perfect first two tasks get has_result; all planned ids only. - by_id = {row["task_id"]: row for row in rows} - assert by_id["df-task-000"]["has_result"] is True - assert by_id["df-task-001"]["has_result"] is True - assert by_id["df-task-002"]["has_result"] is True - assert by_id["df-task-000"]["phase"] == "completed" - assert by_id["df-task-002"]["phase"] == "failed" - # No unplanned invent. - assert "invented-task" not in by_id - - -async def test_dualflag_task_events_remain_empty_without_guest_logs( - client, - database_session, - monkeypatch, -) -> None: - """VAL-DFROWS-006: empty TaskLogEvent stays honest empty; no fake log bodies.""" - _enable_dual_flags(monkeypatch) - async with database_session() as session: - submission_id, _plan, _run = await _seed_dualflag_eval_run( - session, - eval_run_id="eval-df-rows-nologs", - phase="eval_expired", - with_scores=False, - task_count=3, - ) - # Prove no TaskLogEvent rows exist for this submission. - # (clean DB fixture leaves table empty for this id) - - status = await client.get(f"/submissions/{submission_id}/status") - assert status.status_code == 200 - assert len(status.json()["evaluation"]["task_rows"]) == 3 - - events = await client.get(f"/submissions/{submission_id}/task-events") - assert events.status_code == 200 - body = events.json() - event_list = body.get("events") if isinstance(body, dict) else body - if event_list is None and isinstance(body, dict): - event_list = body.get("items") or body.get("task_events") or [] - assert event_list == [] - serialized = json.dumps(body) - assert "fake guest" not in serialized.lower() - assert "invented" not in serialized.lower() - - -def test_task_rows_helper_from_plan_and_score_overlay_unit() -> None: - """Offline unit fixture: helper projects N=3 plan and overlays score record.""" - plan = _plan(eval_run_id="eval-df-helper", task_count=3) - plan_json = canonical_eval_plan_json(plan) - record = build_score_record_from_eval_plan(plan, _trials_for(plan, perfect_count=1)) - score_json = ew.canonical_json_v1(record).decode("utf-8") - run = EvalRun( - eval_run_id="eval-df-helper", - submission_id=1, - submission_version=1, - authorizing_review_digest=REVIEW_DIGEST, - plan_json=plan_json, - plan_sha256=hashlib.sha256(plan_json.encode("utf-8")).hexdigest(), - token_sha256="c" * 64, - phase="eval_accepted", - score=0.5, - passed_tasks=1, - total_tasks=3, - canonical_score_record_json=score_json, - issued_at=NOW, - expires_at=NOW + timedelta(hours=1), - created_at=NOW, - updated_at=NOW, - ) - rows = api_routes._task_rows_from_eval_run(run) - assert len(rows) == 3 - assert [row.task_id for row in rows] == [t["task_id"] for t in plan["selected_tasks"]] - assert rows[0].has_result is True - assert rows[0].phase == "completed" - assert rows[1].has_result is True - assert rows[1].phase == "failed" - assert rows[2].has_result is True - assert rows[2].phase == "failed" - # Ensure helper does not touch TaskLogEvent / invent logs. - assert TaskLogEvent is not None diff --git a/packages/challenges/agent-challenge/tests/test_eval_authorization_ledger.py b/packages/challenges/agent-challenge/tests/test_eval_authorization_ledger.py deleted file mode 100644 index f6c6f9c1c..000000000 --- a/packages/challenges/agent-challenge/tests/test_eval_authorization_ledger.py +++ /dev/null @@ -1,562 +0,0 @@ -"""Strict signed Eval authorization lifecycle tests.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import UTC, datetime - -import pytest -from sqlalchemy import func, select - -from agent_challenge.core.models import ( - AgentSubmission, - EvalNonce, - EvalRun, - ReviewAssignment, - ReviewSession, -) -from agent_challenge.evaluation.authorization import ( - EvalAuthorizationConflict, - EvalAuthorizationRequired, - cancel_eval_run, - create_eval_run, - eval_status_page, - fail_eval_run, - mark_eval_key_granted, - retry_eval_run, -) -from agent_challenge.review.or_outcome_bind import ( - REVIEW_REPORT_DOMAIN, - build_decision, - build_observed_openrouter_transport, - build_openrouter_observation, - build_planned_openrouter_request, - build_policy_observation, - build_review_core_minimal, - planned_request_sha256, - review_report_data_hex, - sha256_hex, -) -from agent_challenge.review.or_outcome_bind import ( - review_digest as bound_review_digest, -) -from agent_challenge.sdk.config import ChallengeSettings - -MEASUREMENT = { - "mrtd": "01" * 48, - "rtmr0": "02" * 48, - "rtmr1": "03" * 48, - "rtmr2": "04" * 48, - "os_image_hash": "05" * 32, - "key_provider": "validator-kms", - "vm_shape": "tdx-small", -} - -_T0 = 1_700_000_000_000 -_ROUTING = sha256_hex(b'{"order":["ledger"]}') -_BODY = b'{"model":"x-ai/grok-4.5","messages":[]}' -_BODY_SHA = sha256_hex(_BODY) -_RESP = b'{"id":"gen-ledger","model":"x-ai/grok-4.5","choices":[]}' -_RESP_SHA = sha256_hex(_RESP) -_META = sha256_hex(b"meta-ledger") - - -def _fresh_review_envelope() -> tuple[str, str, str]: - """Return (envelope_json, report_data_hex, review_digest) for fresh allow. - - create_eval_run now re-verifies receipted materials (VAL-ACAT-028/029), so - fixtures must plant real report_data-bound cores — not cache allow-bits alone. - """ - - planned = build_planned_openrouter_request( - body_sha256=_BODY_SHA, - body_length=len(_BODY), - routing_sha256=_ROUTING, - ) - p_digest = planned_request_sha256(planned) - observed = build_observed_openrouter_transport( - planned_request_sha256_=p_digest, - response_body_sha256=_RESP_SHA, - response_body_length=len(_RESP), - metadata_sha256=_META, - ) - or_obs = build_openrouter_observation( - planned=planned, - observed=observed, - request_body_sha256=_BODY_SHA, - request_body_length=len(_BODY), - response_id="gen-ledger", - ) - policy = build_policy_observation( - prompt_version="review-policy-prompt-v1", - prompt_bytes=b"prompt-ledger", - tool_schema_version="review-policy-tool-v1", - tool_schema_bytes=b"tools-ledger", - verifier_version="review-policy-verifier-v1", - verifier_bytes=b"verifier-ledger", - routing_sha256=_ROUTING, - ) - core = build_review_core_minimal( - session_id="rs-ledger", - assignment_id="ra-ledger", - submission_id="sub-ledger", - review_nonce="nonce-ledger", - assignment_digest="13" * 32, - rules_observation={ - "rules_version": "rules-v1", - "rules_bundle_sha256": "11" * 32, - "rules_files": [".rules/acceptance.md"], - "rules_file_digests": {".rules/acceptance.md": "22" * 32}, - "rules_policy_text_sha256": "33" * 32, - }, - policy_observation=policy, - openrouter_observation=or_obs, - decision=build_decision(verdict="allow"), - times={ - "issued_at_ms": _T0, - "started_at_ms": _T0, - "model_call_marked_at_ms": _T0 + 1, - "request_started_at_ms": _T0 + 2, - "request_finished_at_ms": _T0 + 3, - "verifier_finished_at_ms": _T0 + 4, - "report_finished_at_ms": _T0 + 5, - "expires_at_ms": _T0 + 3_600_000, - "submission_received_at_ms": _T0 + 60_000, - }, - ) - digest = bound_review_digest(core) - rd = review_report_data_hex(core) - env = { - "schema_version": 1, - "domain": REVIEW_REPORT_DOMAIN, - "review_digest": digest, - "report_data_hex": rd, - "review_core": core, - } - return json.dumps(env, sort_keys=True, separators=(",", ":")), rd, digest - - -def _settings() -> ChallengeSettings: - return ChallengeSettings( - attested_review_enabled=True, - phala_attestation_enabled=True, - eval_app_image_ref="registry.example/eval@sha256:" + "a" * 64, - eval_app_compose_hash="06" * 32, - eval_app_identity="agent-challenge-eval-v1", - eval_app_kms_public_key_hex="07" * 32, - eval_app_measurement=MEASUREMENT, - eval_app_measurement_allowlist=( - { - "mrtd": MEASUREMENT["mrtd"], - "rtmr0": MEASUREMENT["rtmr0"], - "rtmr1": MEASUREMENT["rtmr1"], - "rtmr2": MEASUREMENT["rtmr2"], - "compose_hash": "06" * 32, - "os_image_hash": MEASUREMENT["os_image_hash"], - }, - ), - eval_key_release_endpoint="validator.example:8701", - eval_k=2, - evaluation_task_count=2, - ) - - -async def _authorized_submission(database_session) -> tuple[int, int]: - envelope_json, report_data_hex, digest = _fresh_review_envelope() - async with database_session() as session: - submission = AgentSubmission( - miner_hotkey="ledger-miner", - name="ledger-agent", - agent_hash=hashlib.sha256(b"agent").hexdigest(), - artifact_uri="/tmp/agent.zip", - artifact_path="/tmp/agent.zip", - zip_sha256=hashlib.sha256(b"zip").hexdigest(), - zip_size_bytes=3, - raw_status="review_allowed", - status="queued", - effective_status="queued", - version_number=1, - ) - session.add(submission) - await session.flush() - review_session = ReviewSession( - session_id="review-ledger-session", - submission_id=submission.id, - artifact_sha256=submission.zip_sha256, - artifact_size_bytes=3, - manifest_sha256="11" * 32, - manifest_entries_sha256="12" * 32, - authorizing_assignment_id="review-ledger-assignment", - current_assignment_id="review-ledger-assignment", - ) - session.add(review_session) - await session.flush() - assignment = ReviewAssignment( - session_id=review_session.id, - assignment_id="review-ledger-assignment", - attempt=1, - assignment_bytes="{}", - assignment_digest="13" * 32, - artifact_sha256=submission.zip_sha256, - rules_snapshot_sha256="14" * 32, - rules_revision_id="rules-1", - review_nonce="review-nonce", - session_token_sha256="15" * 32, - capability_state="revoked", - phase="review_allowed", - issued_at=datetime.now(UTC), - expires_at=datetime.now(UTC), - # Full receipted envelope for re-verify at create_eval_run (not cache-only). - review_report_envelope_json=envelope_json, - review_report_data_hex=report_data_hex, - review_digest=digest, - review_verification_outcome_json=( - '{"status":"verified_allow","terminal":true,"retryable":false,' - '"nonce_consumed":true}' - ), - ) - session.add(assignment) - await session.commit() - return submission.id, assignment.id - - -async def test_preparation_requires_persisted_verified_allow(database_session) -> None: - async with database_session() as session: - submission = AgentSubmission( - miner_hotkey="blocked-miner", - name="blocked-agent", - agent_hash="21" * 32, - artifact_uri="/tmp/blocked.zip", - artifact_path="/tmp/blocked.zip", - zip_sha256="22" * 32, - zip_size_bytes=1, - raw_status="review_queued", - status="queued", - effective_status="queued", - version_number=1, - ) - session.add(submission) - await session.commit() - with pytest.raises(EvalAuthorizationRequired): - await create_eval_run(session, submission, settings=_settings()) - assert await session.scalar(select(func.count()).select_from(EvalRun)) == 0 - - -async def test_preparation_refuses_cached_allow_without_envelope( - database_session, -) -> None: - """VAL-ACAT-029: phase/status allow bits alone cannot open Eval prepare.""" - - async with database_session() as session: - submission = AgentSubmission( - miner_hotkey="cache-only-miner", - name="cache-only-agent", - agent_hash="31" * 32, - artifact_uri="/tmp/cache-only.zip", - artifact_path="/tmp/cache-only.zip", - zip_sha256="32" * 32, - zip_size_bytes=1, - raw_status="review_allowed", - status="queued", - effective_status="queued", - version_number=1, - ) - session.add(submission) - await session.flush() - review_session = ReviewSession( - session_id="review-cache-only-session", - submission_id=submission.id, - artifact_sha256=submission.zip_sha256, - artifact_size_bytes=1, - manifest_sha256="41" * 32, - manifest_entries_sha256="42" * 32, - authorizing_assignment_id="review-cache-only-assignment", - current_assignment_id="review-cache-only-assignment", - ) - session.add(review_session) - await session.flush() - assignment = ReviewAssignment( - session_id=review_session.id, - assignment_id="review-cache-only-assignment", - attempt=1, - assignment_bytes="{}", - assignment_digest="43" * 32, - artifact_sha256=submission.zip_sha256, - rules_snapshot_sha256="44" * 32, - rules_revision_id="rules-1", - review_nonce="review-nonce-cache", - session_token_sha256="45" * 32, - capability_state="revoked", - phase="review_allowed", - issued_at=datetime.now(UTC), - expires_at=datetime.now(UTC), - # Cache-shaped stub: no re-verifiable core/report_data. - review_report_envelope_json='{"schema_version":1}', - review_digest="16" * 32, - review_verification_outcome_json=( - '{"status":"verified_allow","terminal":true,"retryable":false,' - '"nonce_consumed":true}' - ), - ) - session.add(assignment) - await session.commit() - with pytest.raises(EvalAuthorizationRequired) as exc: - await create_eval_run(session, submission, settings=_settings()) - assert "fresh review re-verify" in str(exc.value) - assert await session.scalar(select(func.count()).select_from(EvalRun)) == 0 - - -async def test_prepare_is_one_time_and_issues_distinct_typed_nonces( - database_session, - monkeypatch, -) -> None: - submission_id, _assignment_id = await _authorized_submission(database_session) - monkeypatch.setattr( - "agent_challenge.evaluation.authorization.load_benchmark_tasks", - lambda: [ - type( - "Task", - (), - { - "task_id": "task-b", - "docker_image": "registry.example/task@sha256:" + "b" * 64, - "prompt": "", - "benchmark": "terminal_bench", - "metadata": {"content_digest_sha256": "cd" * 32}, - }, - )(), - type( - "Task", - (), - { - "task_id": "task-a", - "docker_image": "registry.example/task@sha256:" + "c" * 64, - "prompt": "", - "benchmark": "terminal_bench", - "metadata": {"content_digest_sha256": "ab" * 32}, - }, - )(), - ], - ) - async with database_session() as session: - submission = await session.get(AgentSubmission, submission_id) - assert submission is not None - first = await create_eval_run(session, submission, settings=_settings()) - await session.commit() - - async with database_session() as session: - submission = await session.get(AgentSubmission, submission_id) - assert submission is not None - second = await create_eval_run(session, submission, settings=_settings()) - assert second.run.eval_run_id == first.run.eval_run_id - assert second.token is None - assert second.plan == first.plan - nonces = list((await session.scalars(select(EvalNonce))).all()) - assert {nonce.purpose for nonce in nonces} == {"key_release", "score"} - assert len({nonce.nonce for nonce in nonces}) == 2 - assert len(nonces[0].nonce) >= 22 - assert "run_token_sha256" in first.plan - assert first.plan["run_token_sha256"] == first.run.token_sha256 - assert first.plan["scoring_policy"]["schema_version"] == 1 - plan_tasks = {item["task_id"]: item for item in first.plan["selected_tasks"]} - assert plan_tasks["task-a"]["task_config_sha256"] == "ab" * 32 - assert plan_tasks["task-b"]["task_config_sha256"] == "cd" * 32 - assert first.plan["agent_hash"] == submission.agent_hash - - -async def test_cancel_failure_and_retry_retain_attempt_history( - database_session, - monkeypatch, -) -> None: - submission_id, _assignment_id = await _authorized_submission(database_session) - monkeypatch.setattr( - "agent_challenge.evaluation.authorization.load_benchmark_tasks", - lambda: [ - type( - "Task", - (), - { - "task_id": "task-a", - "docker_image": "registry.example/task@sha256:" + "b" * 64, - "prompt": "", - "benchmark": "terminal_bench", - "metadata": {"content_digest_sha256": "aa" * 32}, - }, - )() - ], - ) - async with database_session() as session: - submission = await session.get(AgentSubmission, submission_id) - assert submission is not None - created = await create_eval_run(session, submission, settings=_settings()) - await cancel_eval_run(session, submission, created.run.eval_run_id) - repeated_cancel = await cancel_eval_run( - session, - submission, - created.run.eval_run_id, - ) - assert repeated_cancel.id == created.run.id - replacement = await retry_eval_run( - session, - submission, - expected_run_id=created.run.eval_run_id, - settings=_settings(), - ) - assert replacement.run.eval_run_id != created.run.eval_run_id - assert replacement.token is not None - page = await eval_status_page(session, submission) - assert page["total_count"] == 2 - assert [item["eval_run_id"] for item in page["items"]] == [ - created.run.eval_run_id, - replacement.run.eval_run_id, - ] - - -async def test_failure_reason_is_closed_and_retryable(database_session, monkeypatch) -> None: - submission_id, _assignment_id = await _authorized_submission(database_session) - monkeypatch.setattr( - "agent_challenge.evaluation.authorization.load_benchmark_tasks", - lambda: [ - type( - "Task", - (), - { - "task_id": "task-a", - "docker_image": "registry.example/task@sha256:" + "b" * 64, - "prompt": "", - "benchmark": "terminal_bench", - "metadata": {"content_digest_sha256": "aa" * 32}, - }, - )() - ], - ) - async with database_session() as session: - submission = await session.get(AgentSubmission, submission_id) - assert submission is not None - created = await create_eval_run(session, submission, settings=_settings()) - failed = await fail_eval_run( - session, - submission, - expected_run_id=created.run.eval_run_id, - reason_code="eval_key_release_unavailable", - ) - assert failed.phase == "eval_error" - assert failed.retryable is True - replay = await fail_eval_run( - session, - submission, - expected_run_id=created.run.eval_run_id, - reason_code="eval_key_release_unavailable", - ) - assert replay.id == failed.id - - -async def test_key_grant_closes_cancel_and_retry(database_session, monkeypatch) -> None: - submission_id, _assignment_id = await _authorized_submission(database_session) - monkeypatch.setattr( - "agent_challenge.evaluation.authorization.load_benchmark_tasks", - lambda: [ - type( - "Task", - (), - { - "task_id": "task-a", - "docker_image": "registry.example/task@sha256:" + "b" * 64, - "prompt": "", - "benchmark": "terminal_bench", - "metadata": {"content_digest_sha256": "aa" * 32}, - }, - )() - ], - ) - async with database_session() as session: - submission = await session.get(AgentSubmission, submission_id) - assert submission is not None - created = await create_eval_run(session, submission, settings=_settings()) - granted = await mark_eval_key_granted( - session, - eval_run_id=created.run.eval_run_id, - ) - assert granted.retryable is False - assert granted.key_granted_at is not None - with pytest.raises(EvalAuthorizationConflict): - await cancel_eval_run(session, submission, created.run.eval_run_id) - with pytest.raises(EvalAuthorizationConflict): - await retry_eval_run( - session, - submission, - expected_run_id=created.run.eval_run_id, - settings=_settings(), - ) - - -async def test_status_uses_normative_safe_item_and_stable_cursor( - database_session, - monkeypatch, -) -> None: - submission_id, _assignment_id = await _authorized_submission(database_session) - monkeypatch.setattr( - "agent_challenge.evaluation.authorization.load_benchmark_tasks", - lambda: [ - type( - "Task", - (), - { - "task_id": "task-a", - "docker_image": "registry.example/task@sha256:" + "b" * 64, - "prompt": "", - "benchmark": "terminal_bench", - "metadata": {"content_digest_sha256": "aa" * 32}, - }, - )() - ], - ) - async with database_session() as session: - submission = await session.get(AgentSubmission, submission_id) - assert submission is not None - first = await create_eval_run(session, submission, settings=_settings()) - await cancel_eval_run(session, submission, first.run.eval_run_id) - second = await retry_eval_run( - session, - submission, - expected_run_id=first.run.eval_run_id, - settings=_settings(), - ) - page = await eval_status_page(session, submission, limit=1) - expected_fields = { - "eval_run_id", - "attempt", - "prior_eval_run_id", - "receipt_id", - "body_sha256", - "phase", - "terminal", - "verified", - "retryable", - "reason_code", - "key_grant_state", - "key_release_nonce_state", - "score_nonce_state", - "issued_at_ms", - "expires_at_ms", - "received_at_ms", - "finalized_at_ms", - "result_available", - } - assert set(page["items"][0]) == expected_fields - assert page["items"][0]["eval_run_id"] == first.run.eval_run_id - assert page["items"][0]["prior_eval_run_id"] is None - assert page["next_cursor"] is not None - assert "selected_tasks" not in page["items"][0] - assert second.run.eval_run_id not in str(page["items"][0]) - - next_page = await eval_status_page( - session, - submission, - cursor=page["next_cursor"], - limit=1, - ) - assert [item["eval_run_id"] for item in next_page["items"]] == [second.run.eval_run_id] - with pytest.raises(EvalAuthorizationConflict, match="cursor"): - await eval_status_page(session, submission, cursor="tampered", limit=1) diff --git a/packages/challenges/agent-challenge/tests/test_eval_compose_hash_determine.py b/packages/challenges/agent-challenge/tests/test_eval_compose_hash_determine.py deleted file mode 100644 index 8aa0dbcde..000000000 --- a/packages/challenges/agent-challenge/tests/test_eval_compose_hash_determine.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Hash-determine realignment: miner dry/generate must match operator eval compose pin. - -Live residual (e2e-live-v7/eval-1task-after-kr): prepare delivered signed plan with -``compose_hash=04011776…`` while miner ``generate_app_compose`` candidates baked -the live ``key_release_endpoint`` into static env and therefore never matched. -The operator pin was measured with a non-routable HTTPS placeholder so the -compose_hash stays unstable only to image+envelope factors, not the live KR IP. - -Guest still resolves the real endpoint from the signed plan / -``CHALLENGE_PHALA_EVAL_PLAN`` (see own_runner_backend._resolve_key_release_endpoint) -and never invents MRTD/KR materials. -""" - -from __future__ import annotations - -import hashlib -from pathlib import Path - -import pytest - -from agent_challenge.canonical import eval_wire -from agent_challenge.canonical.compose import ( - app_compose_hash, - generate_app_compose, - render_app_compose, -) -from agent_challenge.selfdeploy import eval as eval_deploy - -#: Live dual-flag joinbase eval pin (tee-pin-pack + residual after KR). -LIVE_PIN_COMPOSE_HASH = "0401177601f46160c8127c007019401c1a7e6fb3cf8a0850c54a0b96fbbe67d2" -LIVE_PIN_IMAGE = ( - "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" - "753e2296635bcd3a30703dc706509f0f8c0e7dd2f82bef730ad7f1cc9443933c" -) -#: Residual Path A plan endpoint (raw RA-TLS authority). -LIVE_PLAN_KEY_RELEASE = "86.38.238.235:8701" -#: Mission-bundled pin pack composition (destroyable measure materials). -MISSION_PIN_COMPOSE = Path( - "/root/.factory/missions/a43a16a7-2230-4853-ba8a-a6bfe993a90f" - "/evidence/ac-attestation/tee-pin-pack/eval-app-compose.json" -) - - -def _signed_prepare( - *, - compose_hash: str, - image_ref: str, - app_identity: str, - key_release_endpoint: str, - token: str = "eval-run-token-hash-determine", -) -> dict: - public_key = "22" * 32 - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - plan = { - "schema_version": 1, - "eval_run_id": "eval_hash_determine", - "submission_id": "30", - "submission_version": 1, - "authorizing_review_digest": "ab" * 32, - "agent_hash": "cd" * 32, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "terminal-bench/financial-document-processor", - "image_ref": "task-local/financial-document-processor@sha256:" + ("3" * 64), - "task_config_sha256": "3" * 64, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": eval_wire.scoring_policy_digest(policy), - "eval_app": { - "image_ref": image_ref, - "compose_hash": compose_hash, - "app_identity": app_identity, - "kms_key_algorithm": "x25519", - "kms_public_key_hex": public_key, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex(public_key)).hexdigest(), - "measurement": { - "mrtd": "a1" * 48, - "rtmr0": "a2" * 48, - "rtmr1": "a3" * 48, - "rtmr2": "a4" * 48, - "os_image_hash": "a5" * 32, - "key_provider": "phala", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": key_release_endpoint, - "result_endpoint": "/evaluation/v1/runs/eval_hash_determine/result", - "key_release_nonce": "key-release-nonce", - "score_nonce": "score-nonce", - "run_token_sha256": hashlib.sha256(token.encode("utf-8")).hexdigest(), - "issued_at_ms": 1, - "expires_at_ms": 2, - } - plan = eval_wire.validate_eval_plan(plan) - return { - "schema_version": 1, - "plan": plan, - "plan_sha256": hashlib.sha256(eval_wire.canonical_json_v1(plan)).hexdigest(), - "secret_delivery": {"env_key": "EVAL_RUN_TOKEN", "token": token}, - } - - -def test_measure_time_placeholder_reproduces_live_pin_hash(): - """Product generator with pin-pack measure inputs must yield 040117…""" - - compose = generate_app_compose( - orchestrator_image=LIVE_PIN_IMAGE, - name=eval_deploy.DEFAULT_EVAL_COMPOSE_NAME, - key_release_url=eval_deploy.MEASURE_TIME_EVAL_KEY_RELEASE_PLACEHOLDER, - allowed_envs=eval_deploy.EVAL_ALLOWED_ENVS, - ) - assert app_compose_hash(compose) == LIVE_PIN_COMPOSE_HASH - # Optional mission-only pin pack. Path may be absent or unreadable on CI - # /sandbox runners (PermissionError on parent dirs); product hash assert above - # already seals the live pin identity. - try: - present = MISSION_PIN_COMPOSE.is_file() - except OSError: - present = False - if not present: - return - import json - - try: - pin_doc = json.loads(MISSION_PIN_COMPOSE.read_text(encoding="utf-8")) - except OSError: - return - assert render_app_compose(compose) == render_app_compose(pin_doc) - assert app_compose_hash(pin_doc) == LIVE_PIN_COMPOSE_HASH - - -def test_build_eval_deployment_plan_matches_live_pin_with_raw_plan_endpoint(): - """Residual Path A: plan KR is raw host:port; pin used measure-time placeholder. - - Before realign this raised EvalDeploymentError (compose hash mismatches signed plan). - """ - - prepare = _signed_prepare( - compose_hash=LIVE_PIN_COMPOSE_HASH, - image_ref=LIVE_PIN_IMAGE, - app_identity="bb35a8f627f0f8c991aa85c15742d352e658e0f7", - key_release_endpoint=LIVE_PLAN_KEY_RELEASE, - ) - dep = eval_deploy.build_eval_deployment_plan(prepare) - assert dep.compose_hash == LIVE_PIN_COMPOSE_HASH - assert dep.compose_name == eval_deploy.DEFAULT_EVAL_COMPOSE_NAME - # Measured bytes keep the pin-stable placeholder, not the live IP (guest uses plan). - assert ( - eval_deploy.MEASURE_TIME_EVAL_KEY_RELEASE_PLACEHOLDER in dep.compose["docker_compose_file"] - ) - assert LIVE_PLAN_KEY_RELEASE not in dep.compose["docker_compose_file"] - # Plan still carries the real KR endpoint for runtime resolution. - assert dep.plan["key_release_endpoint"] == LIVE_PLAN_KEY_RELEASE - - -def test_signed_plan_rejects_measure_time_https_placeholder_as_endpoint(): - """VAL-ACLOCK-008: measure-time HTTPS is compose-pin only, not plan trust root.""" - - with pytest.raises(eval_wire.EvalWireError, match="key_release_endpoint"): - eval_wire.validate_eval_plan( - _signed_prepare( - compose_hash=LIVE_PIN_COMPOSE_HASH, - image_ref=LIVE_PIN_IMAGE, - app_identity=eval_deploy.DEFAULT_EVAL_COMPOSE_NAME, - # Intentionally build the raw mapping then swap to placeholder. - key_release_endpoint=LIVE_PLAN_KEY_RELEASE, - )["plan"] - | { - "key_release_endpoint": eval_deploy.MEASURE_TIME_EVAL_KEY_RELEASE_PLACEHOLDER, - } - ) - - -def test_build_eval_deployment_plan_matches_raw_baked_compose_when_pin_is_raw(): - """Future pin that bakes raw RA-TLS host/port must still determine without invent.""" - - compose = generate_app_compose( - orchestrator_image=LIVE_PIN_IMAGE, - name=eval_deploy.DEFAULT_EVAL_COMPOSE_NAME, - key_release_url=LIVE_PLAN_KEY_RELEASE, - allowed_envs=eval_deploy.EVAL_ALLOWED_ENVS, - ) - raw_hash = app_compose_hash(compose) - assert raw_hash != LIVE_PIN_COMPOSE_HASH # raw bake is a different pin - prepare = _signed_prepare( - compose_hash=raw_hash, - image_ref=LIVE_PIN_IMAGE, - app_identity=eval_deploy.DEFAULT_EVAL_COMPOSE_NAME, - key_release_endpoint=LIVE_PLAN_KEY_RELEASE, - ) - dep = eval_deploy.build_eval_deployment_plan(prepare) - assert dep.compose_hash == raw_hash - assert ( - f"KEY_RELEASE_RA_TLS_HOST={LIVE_PLAN_KEY_RELEASE.split(':', 1)[0]}" - in (dep.compose["docker_compose_file"]) - ) - - -def test_build_eval_deployment_plan_fails_closed_on_unknown_compose_hash(): - prepare = _signed_prepare( - compose_hash="0" * 64, - image_ref=LIVE_PIN_IMAGE, - app_identity=eval_deploy.DEFAULT_EVAL_COMPOSE_NAME, - key_release_endpoint=LIVE_PLAN_KEY_RELEASE, - ) - with pytest.raises(eval_deploy.EvalDeploymentError, match="compose hash mismatches"): - eval_deploy.build_eval_deployment_plan(prepare) diff --git a/packages/challenges/agent-challenge/tests/test_eval_compose_phala_provision_parity.py b/packages/challenges/agent-challenge/tests/test_eval_compose_phala_provision_parity.py deleted file mode 100644 index 3c60ede0d..000000000 --- a/packages/challenges/agent-challenge/tests/test_eval_compose_phala_provision_parity.py +++ /dev/null @@ -1,243 +0,0 @@ -"""Offline Phala provision envelope parity for the eval application compose. - -Live self-deploy failed closed when the local ``generate_app_compose`` hash -(``e5814373...`` without the Phala rewrite factors) did not equal -``POST /cvms/provision``'s rewritten ``compose_hash`` (``27bf5...`` with -``pre_launch_script`` / ``features`` / tproxy / public_tcbinfo / storage_fs). - -These tests prove the offline generator already emits the full Phala envelope so -local app-compose bytes === provision rewrite for the live 3-task terminal_bench -smoke inputs, ``build_eval_deployment_plan`` accepts identity, and review/eval -compose hashes remain disjoint. -""" - -from __future__ import annotations - -import copy -import hashlib -from pathlib import Path - -import yaml - -from agent_challenge.canonical import compose as eval_compose -from agent_challenge.canonical import eval_wire -from agent_challenge.canonical.measurement import compose_hash, normalize_app_compose -from agent_challenge.review import compose as review_compose -from agent_challenge.selfdeploy import eval as eval_deploy - -#: Digest-pinned canonical image used by the live 3-task terminal_bench smoke. -#: T2: GHCR path + T1 local buildx manifest digest (OPS_REQUIRED: swap to first -#: published GHCR digest from agent-recipe publish-eval-image.yml). -LIVE_SMOKE_EVAL_IMAGE = ( - "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" - "ea399cee1b3c9015024918a6070901e6b7b4bee432c8afe1e7dd40a95547e0bc" -) -LIVE_SMOKE_APP_IDENTITY = "agent-challenge-eval-v1" -LIVE_SMOKE_KEY_RELEASE = "ratls://84.32.70.61:8701" - -#: Synthetic review image pin (disjoint service inventory only). -REVIEW_IMAGE = "ghcr.io/baseintelligence/agent-challenge-review@sha256:" + ("c" * 64) - -#: Live residual local hash (pre-envelope, no guest golden/task bind mounts) -#: for the smoke inputs above. -# Residual pre-envelope hash of the live-smoke compose inputs when the allowed_envs -# list includes the validator server-CA injection names (RA_TLS_SERVER_CA_*). Updated -# when FAIL-CLOSED server-CA wiring lands so the discriminator still proves the -# envelope factors (not allowed_envs) are what Phala provision rewrites. -# Residual pre-envelope hash after D6 GHCR-only orchestrator pin (T2). -LIVE_RESIDUAL_NO_ENVELOPE_HASH = "a4131e778128f5d22052efcab71ed2fdb6d7d5446c5f8141511ed153087a7364" - - -def _live_smoke_compose() -> dict: - return eval_compose.generate_app_compose( - orchestrator_image=LIVE_SMOKE_EVAL_IMAGE, - name=LIVE_SMOKE_APP_IDENTITY, - key_release_url=LIVE_SMOKE_KEY_RELEASE, - allowed_envs=eval_deploy.EVAL_ALLOWED_ENVS, - ) - - -def _phala_provision_rewrite(local: dict) -> dict: - """Re-apply the deterministic Phala Cloud envelope factors offline. - - Mirrors what ``POST /cvms/provision`` injects into AppCompose when the - client omits them. When the local generator already emits the same - factors, the rewritten document is byte-identical. - """ - - rewritten = copy.deepcopy(local) - rewritten["tproxy_enabled"] = True - rewritten["public_tcbinfo"] = True - rewritten["secure_time"] = False - rewritten["storage_fs"] = "zfs" - rewritten["features"] = list(eval_compose.PHALA_DEFAULT_FEATURES) - rewritten["pre_launch_script"] = eval_compose.phala_pre_launch_script() - return rewritten - - -def test_generate_app_compose_emits_phala_envelope_keys(): - compose = _live_smoke_compose() - assert set(compose) == eval_compose.PHALA_APP_COMPOSE_ENVELOPE_KEYS - assert compose["features"] == list(eval_compose.PHALA_DEFAULT_FEATURES) - assert compose["tproxy_enabled"] is True - assert compose["public_tcbinfo"] is True - assert compose["storage_fs"] == "zfs" - assert compose["secure_time"] is False - assert compose["pre_launch_script"] == eval_compose.phala_pre_launch_script() - assert compose["pre_launch_script"].startswith("#!/bin/bash") - assert "Phala Cloud Pre-Launch Script" in compose["pre_launch_script"] - - -def test_live_smoke_local_render_equals_phala_provision_rewrite_fixture(): - """Offline discriminator: local == provision-style rewrite for live smoke inputs. - - Would FAIL against a generator that still omits pre_launch/features (the - ``e5814373...`` residual) because the rewritten hash would differ. - """ - - local = _live_smoke_compose() - rewritten = _phala_provision_rewrite(local) - local_text = eval_compose.render_app_compose(local) - rewrite_text = normalize_app_compose(rewritten) - assert local_text == rewrite_text - assert eval_compose.app_compose_hash(local) == compose_hash(rewritten) - stripped = copy.deepcopy(local) - del stripped["pre_launch_script"] - assert compose_hash(stripped) != eval_compose.app_compose_hash(local) - stripped2 = copy.deepcopy(local) - del stripped2["features"] - assert compose_hash(stripped2) != eval_compose.app_compose_hash(local) - - -def test_build_eval_deployment_plan_accepts_parity_compose_identity(): - """Signed Eval prepare identity matches the local Phala-envelope generator.""" - - compose = _live_smoke_compose() - compose_text = eval_compose.render_app_compose(compose) - compose_hash_hex = hashlib.sha256(compose_text.encode("utf-8")).hexdigest() - public_key = "11" * 32 - token = "eval-run-token-parity-sentinel" - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - plan = { - "schema_version": 1, - "eval_run_id": "eval_parity_smoke", - "submission_id": "1", - "submission_version": 1, - "authorizing_review_digest": "ab" * 32, - "agent_hash": "cd" * 32, - "selected_tasks": [ - { - "task_id": "adaptive-rejection-sampler", - "image_ref": "registry.example/task@sha256:" + ("1" * 64), - "task_config_sha256": "2" * 64, - }, - { - "task_id": "bn-fit-modify", - "image_ref": "registry.example/task@sha256:" + ("3" * 64), - "task_config_sha256": "4" * 64, - }, - { - "task_id": "break-filter-js-from-html", - "image_ref": "registry.example/task@sha256:" + ("5" * 64), - "task_config_sha256": "6" * 64, - }, - ], - "k": 1, - "n_concurrent": 4, - "scoring_policy": policy, - "scoring_policy_digest": eval_wire.scoring_policy_digest(policy), - "eval_app": { - "image_ref": LIVE_SMOKE_EVAL_IMAGE, - "compose_hash": compose_hash_hex, - "app_identity": LIVE_SMOKE_APP_IDENTITY, - "kms_key_algorithm": "x25519", - "kms_public_key_hex": public_key, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex(public_key)).hexdigest(), - "measurement": { - "mrtd": "a1" * 48, - "rtmr0": "a2" * 48, - "rtmr1": "a3" * 48, - "rtmr2": "a4" * 48, - "os_image_hash": "a5" * 32, - "key_provider": "phala", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": LIVE_SMOKE_KEY_RELEASE, - "result_endpoint": "/evaluation/v1/runs/eval_parity_smoke/result", - "key_release_nonce": "key-release-nonce", - "score_nonce": "score-nonce", - "run_token_sha256": hashlib.sha256(token.encode("utf-8")).hexdigest(), - "issued_at_ms": 1, - "expires_at_ms": 2, - } - plan = eval_wire.validate_eval_plan(plan) - prepare_response = { - "schema_version": 1, - "plan": plan, - "plan_sha256": hashlib.sha256(eval_wire.canonical_json_v1(plan)).hexdigest(), - "secret_delivery": {"env_key": "EVAL_RUN_TOKEN", "token": token}, - } - - dep = eval_deploy.build_eval_deployment_plan(prepare_response) - assert dep.compose_hash == compose_hash_hex - assert dep.compose_text == compose_text - assert set(dep.compose) == eval_compose.PHALA_APP_COMPOSE_ENVELOPE_KEYS - assert dep.compose_hash == compose_hash(_phala_provision_rewrite(dep.compose)) - - -def test_eval_and_review_compose_identities_remain_disjoint(): - local = _live_smoke_compose() - eval_hash = eval_compose.app_compose_hash(local) - review = review_compose.generate_review_app_compose( - review_image=REVIEW_IMAGE, - app_identity=review_compose.DEFAULT_REVIEW_APP_IDENTITY, - ) - review_hash = review_compose.review_app_compose_hash(review) - assert eval_hash != review_hash - assert local["name"] != review["name"] - eval_services = set(yaml.safe_load(local["docker_compose_file"])["services"]) - review_services = set(yaml.safe_load(review["docker_compose_file"])["services"]) - assert eval_services == {eval_compose.ORCHESTRATOR_SERVICE} - assert review_services == {review_compose.REVIEWER_SERVICE} - assert eval_services.isdisjoint(review_services) - assert eval_compose.phala_pre_launch_script() == review["pre_launch_script"] - - -def test_legacy_hash_without_envelope_is_rejected_as_identity_mismatch(): - """Positive discriminator: the pre-envelope residual hash is not accepted.""" - - compose = _live_smoke_compose() - parity = eval_compose.app_compose_hash(compose) - without = { - k: v - for k, v in compose.items() - if k - not in { - "pre_launch_script", - "features", - "tproxy_enabled", - "public_tcbinfo", - "secure_time", - "storage_fs", - } - } - residual = compose_hash(without) - assert residual != parity - assert residual != compose_hash(_phala_provision_rewrite(compose)) - assert residual == LIVE_RESIDUAL_NO_ENVELOPE_HASH - - -def test_phala_pre_launch_script_path_is_vendor_checked_in(): - path = eval_compose.PHALA_PRE_LAUNCH_SCRIPT_PATH - assert path == Path(eval_compose.REPO_ROOT) / "docker" / "review" / "phala_pre_launch.sh" - assert path.is_file() - text = eval_compose.phala_pre_launch_script() - assert "Phala Cloud Pre-Launch Script" in text - assert "Script execution" in text diff --git a/packages/challenges/agent-challenge/tests/test_eval_deploy_shape_pin_loud_fail.py b/packages/challenges/agent-challenge/tests/test_eval_deploy_shape_pin_loud_fail.py deleted file mode 100644 index 13e332ff1..000000000 --- a/packages/challenges/agent-challenge/tests/test_eval_deploy_shape_pin_loud_fail.py +++ /dev/null @@ -1,431 +0,0 @@ -"""TDD: eval deploy must fail LOUDLY on shape / measurement-pin mismatch. - -Production residual: eval CVM shape moved tdx.small → tdx.xlarge while the -validator allowlist still pinned the small-shape rtmr0. Symptom was a generic -key-release denial deep in the TEE flow — nothing named vm_shape or rtmr0. - -Miner-side guard must abort BEFORE Phala create and name: -- plan shape vs CLI --eval-instance-type -- field names vm_shape / instance_type -- that a shape change requires a matching rtmr0 pin + re-prepare -- that a stale pin surfaces only as a generic key-release denial -""" - -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -from types import SimpleNamespace -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from agent_challenge.canonical import eval_wire -from agent_challenge.canonical.compose import generate_app_compose, render_app_compose -from agent_challenge.selfdeploy import cli -from agent_challenge.selfdeploy import eval as eval_deploy - -PUBLIC_KEY = "c" * 64 -RUN_TOKEN = "eval-run-token-shape-pin-sentinel-7a1b2c3d" -EVAL_RUN_ID = "eval-run-shape-pin-1" - -_SERVER_CA_PEM = ( - "-----BEGIN CERTIFICATE-----\n" - "MIICxTCCAa2gAwIBAgIUIOBn+Iz4ZK61F3pcFJGHjx995acwDQYJKoZIhvcNAQEL\n" - "BQAwEjEQMA4GA1UEAwwHdGVzdC1jYTAeFw0yNjA3MTIxMzAzNDRaFw0zNjA3MTAx\n" - "MzAzNDRaMBIxEDAOBgNVBAMMB3Rlc3QtY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IB\n" - "DwAwggEKAoIBAQDWxZ5PVNf+JlSNkpDlJdqP/WWwZL4fxpJZegSJE7gipUIUH8l6\n" - "SsDhVBiE0eD2GJzGnjx7+I6Q5+36oqoVDBgukVERFkfEZ0d4MtwQ5+rU2pdBx24B\n" - "VeBkNQLFu8qNLzPQuKlU0uIDrGvK157kvMlFQl2cvaJKLGwxRd/j5x+xVRynEfuA\n" - "RSJvt6pvv2Md1Na8ES9QR8pv6q9U4DMnanc4hMjlGMKuF8xKz/ls05e8KTEkDJJP\n" - "7FiZNi0vvlMJQxch9cfzjjnK7mjQm2nrebaFMr/nJNccdq5fcEaIaJhNMU65V0LI\n" - "B2IKwLO/GhcgiFNZ43nfe93WWVaKl8vx382nAgMBAAGjEzARMA8GA1UdEwEB/wQF\n" - "MAMBAf8wDQYJKoZIhvcNAQELBQADggEBAAmfmX6/kAciNHTdvE2mrK7KUDDiDhT7\n" - "kMRWOqiBaYxxiOiz3h1vrzEo81NQqc2dZF4+MrlODcnXUMgT62ijw0O/71IYl33E\n" - "nZBV+MBry5w5vlNw1El2aO3ERtWwjxrN0sLKkqht0h7hU/+wc7+5aBV4URFoNx2E\n" - "EkcZZVknVD9EMvNlWnVVQoLnOIIW4e5F4yHqHQTdxM1TD4F0gKjfNwGK6xZNpObG\n" - "QbDfN3wSkU7DIxeNJCMB+Uc5GDHMKNiEg0yb59SEvypiDuU6cD7OuhLQM0gbjXlC\n" - "81hvjyhx/T/mRQhf6MOu8RbVdp5CDp7IqhouLwEHvHjS4bA/AZIuIP8=\n" - "-----END CERTIFICATE-----\n" -) - -# Distinct rtmr0 values (96 hex chars = 48 bytes) so prefix diffs are meaningful. -PLAN_RTMR0_SMALL = "68102e7b" + "aa" * 44 # 8 + 88 = 96 -PLAN_RTMR0_XLARGE = "ec216f1d" + "bb" * 44 -EXPECTED_RTMR0_MISMATCH = "deadbeef" + "cc" * 44 - - -def _measurement(*, vm_shape: str, rtmr0: str) -> dict[str, str]: - return { - "mrtd": "01" * 48, - "rtmr0": rtmr0, - "rtmr1": "03" * 48, - "rtmr2": "04" * 48, - "os_image_hash": "05" * 32, - "key_provider": "phala", - "vm_shape": vm_shape, - } - - -def _eval_prepare_wrapper( - *, - vm_shape: str = "tdx.small", - rtmr0: str = PLAN_RTMR0_SMALL, - token: str = RUN_TOKEN, - eval_run_id: str = EVAL_RUN_ID, -) -> dict[str, Any]: - eval_image = "registry.example/eval@sha256:" + "b" * 64 - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - compose = generate_app_compose( - orchestrator_image=eval_image, - name="eval-v1", - key_release_url="validator.example:8701", - allowed_envs=eval_deploy.EVAL_ALLOWED_ENVS, - ) - compose_hash = hashlib.sha256(render_app_compose(compose).encode()).hexdigest() - plan = { - "schema_version": 1, - "eval_run_id": eval_run_id, - "submission_id": "1", - "submission_version": 1, - "authorizing_review_digest": "d" * 64, - "agent_hash": "e" * 64, - "package_tree_sha": "b" * 64, - "selected_tasks": [ - { - "task_id": "task-1", - "image_ref": "registry.example/task@sha256:" + "f" * 64, - "task_config_sha256": "1" * 64, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": eval_wire.scoring_policy_digest(policy), - "eval_app": { - "image_ref": eval_image, - "compose_hash": compose_hash, - "app_identity": "eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": PUBLIC_KEY, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex(PUBLIC_KEY)).hexdigest(), - "measurement": _measurement(vm_shape=vm_shape, rtmr0=rtmr0), - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": f"/evaluation/v1/runs/{eval_run_id}/result", - "key_release_nonce": "key-release-nonce", - "score_nonce": "score-nonce", - "run_token_sha256": hashlib.sha256(token.encode()).hexdigest(), - "issued_at_ms": 1, - "expires_at_ms": 2, - } - validated = eval_wire.validate_eval_plan(plan) - return { - "schema_version": 1, - "plan": validated, - "plan_sha256": hashlib.sha256(eval_wire.canonical_json_v1(validated)).hexdigest(), - "secret_delivery": {"env_key": "EVAL_RUN_TOKEN", "token": token}, - } - - -def _deploy_args(**overrides: Any) -> SimpleNamespace: - base = dict( - eval_command="deploy", - submission_id=1, - base_url="https://challenge.example", - hotkey="hk", - signature="sig", - nonce="n", - timestamp=None, - auto_sign=True, - prepare_response=None, - gateway_token_env="BASE_GATEWAY_TOKEN", - gateway_url_env="BASE_LLM_GATEWAY_URL", - llm_cost_limit_env="LLM_COST_LIMIT", - phala_api=None, - review_instance_type="tdx.small", - eval_instance_type="tdx.xlarge", - review_runtime_hours=1.0, - eval_runtime_hours=1.0, - money_cap_usd=20.0, - dry_run=False, - token_output=None, - emit_run_token=True, - output=None, - expected_measurement=None, - ) - base.update(overrides) - return SimpleNamespace(**base) - - -def _wire_prepare_only( - monkeypatch: pytest.MonkeyPatch, - prepare: dict[str, Any], -) -> tuple[MagicMock, list[Any]]: - """Fake prepare; capture any Phala deploy attempts (must stay empty on mismatch).""" - - fake_client = MagicMock() - fake_client.eval_prepare.return_value = prepare - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - monkeypatch.setenv("LLM_COST_LIMIT", "1.00") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", _SERVER_CA_PEM) - - deploy_calls: list[Any] = [] - - class _MustNotDeploy: - def __init__(self, _api: object) -> None: - pass - - def deploy(self, plan_obj: Any, encrypted_obj: Any) -> dict[str, Any]: - deploy_calls.append((plan_obj, encrypted_obj)) - raise AssertionError("Phala deploy must not run on shape/pin mismatch") - - monkeypatch.setattr(eval_deploy, "HttpEvalPhalaDeployment", _MustNotDeploy) - monkeypatch.setattr(cli, "PhalaCloudClient", lambda **_k: object()) - return fake_client, deploy_calls - - -# --------------------------------------------------------------------------- # -# S1 — shape mismatch names both shapes + field names + pin warning -# --------------------------------------------------------------------------- # - - -def test_shape_mismatch_stderr_names_shapes_fields_and_pin_warning( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - """Given plan vm_shape=tdx.small and CLI --eval-instance-type=tdx.xlarge, - When eval deploy runs, Then exit 2, no Phala deploy, stderr is loud and specific. - """ - - prepare = _eval_prepare_wrapper(vm_shape="tdx.small", rtmr0=PLAN_RTMR0_SMALL) - _client, deploy_calls = _wire_prepare_only(monkeypatch, prepare) - - args = _deploy_args(eval_instance_type="tdx.xlarge", emit_run_token=True) - code = cli._ordered_eval_command(args) - err = capsys.readouterr().err - - assert code == 2 - assert deploy_calls == [] - # Both shapes named. - assert "tdx.small" in err - assert "tdx.xlarge" in err - # Field names the operator greps for. - assert "vm_shape" in err - assert "instance_type" in err - # Stale pin / key-release footgun. - assert "rtmr0" in err - assert "key-release" in err.lower() or "key release" in err.lower() - assert "allowlist" in err.lower() - # Re-prepare guidance (prepare already spent the one-shot delivery). - assert "re-prepare" in err.lower() or "reprepare" in err.lower() or "retry" in err.lower() - # Never dump full measurement secrets. - assert PLAN_RTMR0_SMALL not in err - assert RUN_TOKEN not in err - - -def test_shape_mismatch_aborts_before_phala_create( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Given shape mismatch, When deploy invoked, Then Phala deploy is never called.""" - - prepare = _eval_prepare_wrapper(vm_shape="tdx.small") - _client, deploy_calls = _wire_prepare_only(monkeypatch, prepare) - args = _deploy_args(eval_instance_type="tdx.xlarge", emit_run_token=True) - code = cli._ordered_eval_command(args) - assert code == 2 - assert deploy_calls == [] - - -def test_shape_mismatch_includes_truncated_plan_rtmr0_prefix_only( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - """Given plan carries rtmr0, When shape mismatches, Then stderr shows a short prefix only.""" - - prepare = _eval_prepare_wrapper(vm_shape="tdx.small", rtmr0=PLAN_RTMR0_SMALL) - _wire_prepare_only(monkeypatch, prepare) - args = _deploy_args(eval_instance_type="tdx.xlarge", emit_run_token=True) - code = cli._ordered_eval_command(args) - err = capsys.readouterr().err - assert code == 2 - prefix = PLAN_RTMR0_SMALL[:12].lower() - assert prefix in err.lower() - assert PLAN_RTMR0_SMALL not in err - assert PLAN_RTMR0_SMALL[12:] not in err - - -# --------------------------------------------------------------------------- # -# S2 — matching shapes still deploy (no false positive) -# --------------------------------------------------------------------------- # - - -def test_matching_shape_still_reaches_phala_deploy( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Given plan shape matches CLI eval_instance_type, When deploy runs, Then Phala is called.""" - - prepare = _eval_prepare_wrapper(vm_shape="tdx.xlarge", rtmr0=PLAN_RTMR0_XLARGE) - fake_client = MagicMock() - fake_client.eval_prepare.return_value = prepare - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - monkeypatch.setenv("LLM_COST_LIMIT", "1.00") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", _SERVER_CA_PEM) - - deploy_calls: list[Any] = [] - - class _OkDeploy: - def __init__(self, _api: object) -> None: - pass - - def deploy(self, plan_obj: Any, encrypted_obj: Any) -> dict[str, Any]: - deploy_calls.append(plan_obj.instance_type) - return { - "schema_version": 1, - "eval_run_id": plan_obj.eval_run_id, - "cvm_id": "cvm-ok-1", - "phala_create_receipt": { - "request_id": "req", - "app_id": plan_obj.app_identity, - "cvm_id": "cvm-ok-1", - "receipt_sha256": "a" * 64, - "created_at_ms": 1, - }, - } - - monkeypatch.setattr(eval_deploy, "HttpEvalPhalaDeployment", _OkDeploy) - monkeypatch.setattr(cli, "PhalaCloudClient", lambda **_k: object()) - printed: list[Any] = [] - monkeypatch.setattr(cli, "_print", printed.append) - - args = _deploy_args(eval_instance_type="tdx.xlarge", emit_run_token=True) - code = cli._ordered_eval_command(args) - assert code == 0, printed - assert deploy_calls == ["tdx.xlarge"] - - -# --------------------------------------------------------------------------- # -# S3 — optional --expected-measurement rtmr0 pin check -# --------------------------------------------------------------------------- # - - -def test_expected_measurement_rtmr0_mismatch_fails_before_phala( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - capsys: pytest.CaptureFixture[str], -) -> None: - """Given --expected-measurement with different rtmr0, When shapes match, - Then abort before Phala with truncated prefix diff naming rtmr0. - """ - - prepare = _eval_prepare_wrapper(vm_shape="tdx.xlarge", rtmr0=PLAN_RTMR0_XLARGE) - _client, deploy_calls = _wire_prepare_only(monkeypatch, prepare) - - expected_path = tmp_path / "expected-measurement.json" - expected_path.write_text( - json.dumps({"rtmr0": EXPECTED_RTMR0_MISMATCH, "vm_shape": "tdx.xlarge"}), - encoding="utf-8", - ) - - args = _deploy_args( - eval_instance_type="tdx.xlarge", - emit_run_token=True, - expected_measurement=str(expected_path), - ) - code = cli._ordered_eval_command(args) - err = capsys.readouterr().err - - assert code == 2 - assert deploy_calls == [] - assert "rtmr0" in err - plan_prefix = PLAN_RTMR0_XLARGE[:12].lower() - exp_prefix = EXPECTED_RTMR0_MISMATCH[:12].lower() - assert plan_prefix in err.lower() - assert exp_prefix in err.lower() - # Full digests must not appear. - assert PLAN_RTMR0_XLARGE not in err - assert EXPECTED_RTMR0_MISMATCH not in err - assert RUN_TOKEN not in err - - -def test_expected_measurement_rtmr0_match_allows_deploy( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - """Given matching expected-measurement rtmr0, When deploy runs, Then Phala is called.""" - - prepare = _eval_prepare_wrapper(vm_shape="tdx.xlarge", rtmr0=PLAN_RTMR0_XLARGE) - fake_client = MagicMock() - fake_client.eval_prepare.return_value = prepare - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - monkeypatch.setenv("LLM_COST_LIMIT", "1.00") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", _SERVER_CA_PEM) - - deploy_calls: list[Any] = [] - - class _OkDeploy: - def __init__(self, _api: object) -> None: - pass - - def deploy(self, plan_obj: Any, encrypted_obj: Any) -> dict[str, Any]: - deploy_calls.append(True) - return { - "schema_version": 1, - "eval_run_id": plan_obj.eval_run_id, - "cvm_id": "cvm-ok-2", - "phala_create_receipt": { - "request_id": "req", - "app_id": plan_obj.app_identity, - "cvm_id": "cvm-ok-2", - "receipt_sha256": "a" * 64, - "created_at_ms": 1, - }, - } - - monkeypatch.setattr(eval_deploy, "HttpEvalPhalaDeployment", _OkDeploy) - monkeypatch.setattr(cli, "PhalaCloudClient", lambda **_k: object()) - monkeypatch.setattr(cli, "_print", lambda _p: None) - - expected_path = tmp_path / "expected-measurement.json" - expected_path.write_text( - json.dumps({"rtmr0": PLAN_RTMR0_XLARGE}), - encoding="utf-8", - ) - args = _deploy_args( - eval_instance_type="tdx.xlarge", - emit_run_token=True, - expected_measurement=str(expected_path), - ) - code = cli._ordered_eval_command(args) - assert code == 0 - assert deploy_calls == [True] - - -def test_expected_measurement_flag_is_on_eval_deploy_parser() -> None: - """Given CLI parser, When eval deploy --help is built, Then --expected-measurement exists.""" - - parser = cli.build_parser() - ns = parser.parse_args( - [ - "eval", - "deploy", - "--base-url", - "https://x", - "--submission-id", - "1", - "--hotkey", - "hk", - "--auto-sign", - "--expected-measurement", - "/tmp/m.json", - "--emit-run-token", - ] - ) - assert ns.expected_measurement == "/tmp/m.json" diff --git a/packages/challenges/agent-challenge/tests/test_eval_deploy_token_handoff.py b/packages/challenges/agent-challenge/tests/test_eval_deploy_token_handoff.py deleted file mode 100644 index 1271ddfb3..000000000 --- a/packages/challenges/agent-challenge/tests/test_eval_deploy_token_handoff.py +++ /dev/null @@ -1,605 +0,0 @@ -"""TDD: eval deploy must hand the one-time EVAL_RUN_TOKEN to the miner. - -Root cause (production): guest emits attested result only; host posts via -``eval result --token-env EVAL_RUN_TOKEN``. The token lived only inside -``eval deploy`` memory (CVM encrypted_env) and every miner-readable surface -was redacted — closed loop, result could never be posted. - -Security: surfacing the token to the miner is a capability/anti-replay -credential for the post only. Integrity is the TEE quote bound to the plan. -""" - -from __future__ import annotations - -import hashlib -import json -import stat -from types import SimpleNamespace -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from agent_challenge.canonical import eval_wire -from agent_challenge.canonical.compose import generate_app_compose, render_app_compose -from agent_challenge.selfdeploy import cli -from agent_challenge.selfdeploy import eval as eval_deploy -from agent_challenge.selfdeploy.client import RouteClientError - -PUBLIC_KEY = "c" * 64 -MEASUREMENT = { - "mrtd": "01" * 48, - "rtmr0": "02" * 48, - "rtmr1": "03" * 48, - "rtmr2": "04" * 48, - "os_image_hash": "05" * 32, - "key_provider": "phala", - "vm_shape": "tdx.small", -} - -# Distinct sentinel so accidental redaction/leak assertions cannot false-pass. -RUN_TOKEN = "eval-run-token-handoff-sentinel-9f3c2a1b" -EVAL_RUN_ID = "eval-run-handoff-1" - -# OpenSSL-loadable test CA (same fixture as ordered trust hardening). -_SERVER_CA_PEM = ( - "-----BEGIN CERTIFICATE-----\n" - "MIICxTCCAa2gAwIBAgIUIOBn+Iz4ZK61F3pcFJGHjx995acwDQYJKoZIhvcNAQEL\n" - "BQAwEjEQMA4GA1UEAwwHdGVzdC1jYTAeFw0yNjA3MTIxMzAzNDRaFw0zNjA3MTAx\n" - "MzAzNDRaMBIxEDAOBgNVBAMMB3Rlc3QtY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IB\n" - "DwAwggEKAoIBAQDWxZ5PVNf+JlSNkpDlJdqP/WWwZL4fxpJZegSJE7gipUIUH8l6\n" - "SsDhVBiE0eD2GJzGnjx7+I6Q5+36oqoVDBgukVERFkfEZ0d4MtwQ5+rU2pdBx24B\n" - "VeBkNQLFu8qNLzPQuKlU0uIDrGvK157kvMlFQl2cvaJKLGwxRd/j5x+xVRynEfuA\n" - "RSJvt6pvv2Md1Na8ES9QR8pv6q9U4DMnanc4hMjlGMKuF8xKz/ls05e8KTEkDJJP\n" - "7FiZNi0vvlMJQxch9cfzjjnK7mjQm2nrebaFMr/nJNccdq5fcEaIaJhNMU65V0LI\n" - "B2IKwLO/GhcgiFNZ43nfe93WWVaKl8vx382nAgMBAAGjEzARMA8GA1UdEwEB/wQF\n" - "MAMBAf8wDQYJKoZIhvcNAQELBQADggEBAAmfmX6/kAciNHTdvE2mrK7KUDDiDhT7\n" - "kMRWOqiBaYxxiOiz3h1vrzEo81NQqc2dZF4+MrlODcnXUMgT62ijw0O/71IYl33E\n" - "nZBV+MBry5w5vlNw1El2aO3ERtWwjxrN0sLKkqht0h7hU/+wc7+5aBV4URFoNx2E\n" - "EkcZZVknVD9EMvNlWnVVQoLnOIIW4e5F4yHqHQTdxM1TD4F0gKjfNwGK6xZNpObG\n" - "QbDfN3wSkU7DIxeNJCMB+Uc5GDHMKNiEg0yb59SEvypiDuU6cD7OuhLQM0gbjXlC\n" - "81hvjyhx/T/mRQhf6MOu8RbVdp5CDp7IqhouLwEHvHjS4bA/AZIuIP8=\n" - "-----END CERTIFICATE-----\n" -) - - -def _eval_prepare_wrapper( - *, - token: str | None = RUN_TOKEN, - eval_run_id: str = EVAL_RUN_ID, -) -> dict[str, Any]: - eval_image = "registry.example/eval@sha256:" + "b" * 64 - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - compose = generate_app_compose( - orchestrator_image=eval_image, - name="eval-v1", - key_release_url="validator.example:8701", - allowed_envs=eval_deploy.EVAL_ALLOWED_ENVS, - ) - compose_hash = hashlib.sha256(render_app_compose(compose).encode()).hexdigest() - run_token = token if isinstance(token, str) and token else "placeholder-token" - plan = { - "schema_version": 1, - "eval_run_id": eval_run_id, - "submission_id": "1", - "submission_version": 1, - "authorizing_review_digest": "d" * 64, - "agent_hash": "e" * 64, - "package_tree_sha": "b" * 64, - "selected_tasks": [ - { - "task_id": "task-1", - "image_ref": "registry.example/task@sha256:" + "f" * 64, - "task_config_sha256": "1" * 64, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": eval_wire.scoring_policy_digest(policy), - "eval_app": { - "image_ref": eval_image, - "compose_hash": compose_hash, - "app_identity": "eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": PUBLIC_KEY, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex(PUBLIC_KEY)).hexdigest(), - "measurement": MEASUREMENT, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": f"/evaluation/v1/runs/{eval_run_id}/result", - "key_release_nonce": "key-release-nonce", - "score_nonce": "score-nonce", - "run_token_sha256": hashlib.sha256(run_token.encode()).hexdigest(), - "issued_at_ms": 1, - "expires_at_ms": 2, - } - validated = eval_wire.validate_eval_plan(plan) - delivery: dict[str, str] | None - if isinstance(token, str) and token: - delivery = {"env_key": "EVAL_RUN_TOKEN", "token": token} - else: - delivery = None - return { - "schema_version": 1, - "plan": validated, - "plan_sha256": hashlib.sha256(eval_wire.canonical_json_v1(validated)).hexdigest(), - "secret_delivery": delivery, - } - - -def _deploy_args(**overrides: Any) -> SimpleNamespace: - base = dict( - eval_command="deploy", - submission_id=1, - base_url="https://challenge.example", - hotkey="hk", - signature="sig", - nonce="n", - timestamp=None, - auto_sign=True, - prepare_response=None, - gateway_token_env="BASE_GATEWAY_TOKEN", - gateway_url_env="BASE_LLM_GATEWAY_URL", - llm_cost_limit_env="LLM_COST_LIMIT", - phala_api=None, - review_instance_type="tdx.small", - eval_instance_type="tdx.small", - review_runtime_hours=1.0, - eval_runtime_hours=1.0, - money_cap_usd=20.0, - dry_run=False, - token_output=None, - emit_run_token=False, - output=None, - openrouter_key_env="OPENROUTER_API_KEY", - review_disk_size_gb=40, - eval_disk_size_gb=40, - expected_measurement=None, - ) - base.update(overrides) - return SimpleNamespace(**base) - - -def _wire_successful_deploy( - monkeypatch: pytest.MonkeyPatch, - *, - token: str = RUN_TOKEN, - captured_secrets: list[dict[str, str]] | None = None, -) -> list[Any]: - """Fake prepare + Phala deploy; optionally capture encrypt secrets.""" - - prepare = _eval_prepare_wrapper(token=token) - fake_client = MagicMock() - fake_client.eval_prepare.return_value = prepare - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - monkeypatch.setenv("LLM_COST_LIMIT", "1.00") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", _SERVER_CA_PEM) - - if captured_secrets is not None: - real_encrypt = eval_deploy.encrypt_eval_secrets - - def _capture(plan: Any, secrets: Any) -> Any: - captured_secrets.append(dict(secrets)) - return real_encrypt(plan, secrets) - - monkeypatch.setattr(eval_deploy, "encrypt_eval_secrets", _capture) - - class _FixedDeploy: - def __init__(self, _api: object) -> None: - pass - - def deploy(self, plan_obj: Any, encrypted_obj: Any) -> dict[str, Any]: - assert plan_obj.eval_run_token == token - assert "EVAL_RUN_TOKEN" in encrypted_obj.env_keys - return { - "schema_version": 1, - "eval_run_id": plan_obj.eval_run_id, - "cvm_id": "cvm-eval-handoff-1", - "phala_create_receipt": { - "request_id": "req", - "app_id": plan_obj.app_identity, - "cvm_id": "cvm-eval-handoff-1", - "receipt_sha256": "a" * 64, - "created_at_ms": 1, - }, - } - - monkeypatch.setattr(eval_deploy, "HttpEvalPhalaDeployment", _FixedDeploy) - monkeypatch.setattr(cli, "PhalaCloudClient", lambda **_k: object()) - printed: list[Any] = [] - monkeypatch.setattr(cli, "_print", printed.append) - return printed - - -# --------------------------------------------------------------------------- # -# S1 — stdout emission with --emit-run-token -# --------------------------------------------------------------------------- # - - -def test_emit_run_token_puts_exact_token_and_run_id_on_stdout( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Given live deploy + --emit-run-token, When deploy succeeds, - Then stdout JSON carries exact eval_run_token and eval_run_id. - """ - - printed = _wire_successful_deploy(monkeypatch) - args = _deploy_args(emit_run_token=True) - code = cli._ordered_eval_command(args) - assert code == 0, printed - assert len(printed) == 1 - payload = printed[0] - assert payload["eval_run_token"] == RUN_TOKEN - assert payload["eval_run_id"] == EVAL_RUN_ID - assert payload["stage"] == "eval_deployed" - - -# --------------------------------------------------------------------------- # -# S2 — --token-output secure file -# --------------------------------------------------------------------------- # - - -def test_token_output_writes_mode_0600_file_with_exact_token( - monkeypatch: pytest.MonkeyPatch, - tmp_path: pytest.TempPathFactory, -) -> None: - """Given --token-output PATH, When deploy succeeds, - Then PATH exists mode 0o600 and contains the exact token; stdout has run id. - """ - - token_path = tmp_path / "eval-run.token" - printed = _wire_successful_deploy(monkeypatch) - args = _deploy_args(token_output=str(token_path)) - code = cli._ordered_eval_command(args) - assert code == 0, printed - assert token_path.is_file() - mode = stat.S_IMODE(token_path.stat().st_mode) - assert mode == 0o600, f"expected 0o600, got {oct(mode)}" - assert token_path.read_text(encoding="utf-8") == RUN_TOKEN - assert printed[0]["eval_run_id"] == EVAL_RUN_ID - # Token must not appear in stdout when only --token-output is used. - assert RUN_TOKEN not in json.dumps(printed) - assert "eval_run_token" not in printed[0] - - -# --------------------------------------------------------------------------- # -# S3 — fail closed without handoff flags -# --------------------------------------------------------------------------- # - - -def test_non_dry_run_without_handoff_flags_fails_closed( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - """Given non-dry-run deploy with neither flag, When invoked, - Then non-zero exit and stderr names both flags (no Phala spend). - """ - - prepare = _eval_prepare_wrapper() - fake_client = MagicMock() - fake_client.eval_prepare.return_value = prepare - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - monkeypatch.setenv("LLM_COST_LIMIT", "1.00") - - deploy_calls: list[Any] = [] - - class _MustNotDeploy: - def __init__(self, _api: object) -> None: - pass - - def deploy(self, plan_obj: Any, encrypted_obj: Any) -> dict[str, Any]: - deploy_calls.append((plan_obj, encrypted_obj)) - raise AssertionError("Phala deploy must not run when handoff flags missing") - - monkeypatch.setattr(eval_deploy, "HttpEvalPhalaDeployment", _MustNotDeploy) - monkeypatch.setattr(cli, "PhalaCloudClient", lambda **_k: object()) - - args = _deploy_args(dry_run=False, token_output=None, emit_run_token=False) - code = cli._ordered_eval_command(args) - err = capsys.readouterr().err - assert code == 2 - assert deploy_calls == [] - assert "--token-output" in err - assert "--emit-run-token" in err - assert "eval result" in err.lower() or "EVAL_RUN_TOKEN" in err - assert RUN_TOKEN not in err - - -def test_non_dry_run_without_handoff_flags_never_spends_prepare_token( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Given non-dry-run deploy with neither flag, When invoked, - Then eval_prepare is never called so the one-shot delivery is not spent. - - Argument validation must precede remote state mutation: prepare consumes the - single EVAL_RUN_TOKEN delivery, so gating after it would strand the miner in - exactly the unrecoverable state this handoff exists to prevent. - """ - - fake_client = MagicMock() - fake_client.eval_prepare.return_value = _eval_prepare_wrapper() - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - monkeypatch.setenv("LLM_COST_LIMIT", "1.00") - - args = _deploy_args(dry_run=False, token_output=None, emit_run_token=False) - code = cli._ordered_eval_command(args) - - assert code == 2 - fake_client.eval_prepare.assert_not_called() - - -# --------------------------------------------------------------------------- # -# S4 — dry-run without flags still OK, no raw token -# --------------------------------------------------------------------------- # - - -def test_dry_run_without_handoff_flags_ok_and_no_raw_token( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Given dry-run with neither handoff flag, When invoked, - Then exit 0 and stdout has no raw token. - """ - - prepare = _eval_prepare_wrapper() - fake_client = MagicMock() - fake_client.eval_prepare.return_value = prepare - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - monkeypatch.setenv("LLM_COST_LIMIT", "1.00") - printed: list[Any] = [] - monkeypatch.setattr(cli, "_print", printed.append) - - args = _deploy_args(dry_run=True, token_output=None, emit_run_token=False) - code = cli._ordered_eval_command(args) - assert code == 0, printed - assert printed[0]["dry_run"] is True - assert printed[0]["eval_run_id"] == EVAL_RUN_ID - assert RUN_TOKEN not in json.dumps(printed) - assert "eval_run_token" not in printed[0] - - -# --------------------------------------------------------------------------- # -# S5 — --output plan JSON never contains token -# --------------------------------------------------------------------------- # - - -def test_output_plan_json_never_contains_token( - monkeypatch: pytest.MonkeyPatch, - tmp_path: pytest.TempPathFactory, -) -> None: - """Given deploy --output PATH (+ handoff via token-output), When success, - Then the plan/metadata JSON at PATH never contains the raw token string. - """ - - out_path = tmp_path / "deploy-plan.json" - token_path = tmp_path / "token" - printed = _wire_successful_deploy(monkeypatch) - args = _deploy_args( - emit_run_token=True, # even when stdout has token, --output must not - token_output=str(token_path), - output=str(out_path), - ) - code = cli._ordered_eval_command(args) - assert code == 0, printed - assert out_path.is_file() - body = out_path.read_text(encoding="utf-8") - assert RUN_TOKEN not in body - parsed = json.loads(body) - assert "eval_run_token" not in parsed - # Nested secret_delivery must stay redacted if present. - if "secret_delivery" in parsed: - assert parsed["secret_delivery"] == {"env_key": "EVAL_RUN_TOKEN"} or ( - isinstance(parsed["secret_delivery"], dict) and "token" not in parsed["secret_delivery"] - ) - - -# --------------------------------------------------------------------------- # -# S6 — prepare / status still redact -# --------------------------------------------------------------------------- # - - -def test_eval_prepare_and_status_still_redact_secret_delivery( - monkeypatch: pytest.MonkeyPatch, - tmp_path: pytest.TempPathFactory, -) -> None: - """REGRESSION: prepare/status redaction of secret_delivery is unchanged.""" - - prepare = _eval_prepare_wrapper(token=RUN_TOKEN) - fake_client = MagicMock() - fake_client.eval_prepare.return_value = prepare - fake_client.eval_status.return_value = { - "schema_version": 1, - "eval_run_id": EVAL_RUN_ID, - "secret_delivery": {"env_key": "EVAL_RUN_TOKEN", "token": RUN_TOKEN}, - "phase": "eval_prepared", - } - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - printed: list[Any] = [] - monkeypatch.setattr(cli, "_print", printed.append) - - out_path = tmp_path / "prepare.json" - prep_args = SimpleNamespace( - eval_command="prepare", - submission_id=1, - base_url="https://challenge.example", - hotkey="hk", - signature="sig", - nonce="n", - timestamp=None, - auto_sign=True, - output=str(out_path), - ) - assert cli._ordered_eval_command(prep_args) == 0 - assert printed[0]["secret_delivery"] == {"env_key": "EVAL_RUN_TOKEN"} - assert RUN_TOKEN not in json.dumps(printed[0]) - assert RUN_TOKEN not in out_path.read_text(encoding="utf-8") - assert json.loads(out_path.read_text(encoding="utf-8"))["secret_delivery"] == { - "env_key": "EVAL_RUN_TOKEN" - } - - printed.clear() - status_args = SimpleNamespace( - eval_command="status", - submission_id=1, - base_url="https://challenge.example", - hotkey="hk", - signature="sig", - nonce="n", - timestamp=None, - auto_sign=True, - cursor=None, - ) - assert cli._ordered_eval_command(status_args) == 0 - assert printed[0]["secret_delivery"] == {"env_key": "EVAL_RUN_TOKEN"} - assert RUN_TOKEN not in json.dumps(printed[0]) - - -# --------------------------------------------------------------------------- # -# S7 — token never in exception messages -# --------------------------------------------------------------------------- # - - -def test_token_never_present_in_raised_exception_messages( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Given a deploy path that raises after token is known, When error surfaces, - Then exception / stderr text never contains the raw token. - """ - - prepare = _eval_prepare_wrapper(token=RUN_TOKEN) - fake_client = MagicMock() - fake_client.eval_prepare.return_value = prepare - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - monkeypatch.setenv("LLM_COST_LIMIT", "1.00") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", _SERVER_CA_PEM) - - class _Boom: - def __init__(self, _api: object) -> None: - pass - - def deploy(self, plan_obj: Any, encrypted_obj: Any) -> dict[str, Any]: - raise eval_deploy.EvalDeploymentError( - f"post-create bind failed for run {plan_obj.eval_run_id}" - ) - - monkeypatch.setattr(eval_deploy, "HttpEvalPhalaDeployment", _Boom) - monkeypatch.setattr(cli, "PhalaCloudClient", lambda **_k: object()) - - args = _deploy_args(emit_run_token=True) - # Capture via raising path inside _ordered_eval_command (prints error: …). - import io - import sys - - buf = io.StringIO() - old = sys.stderr - sys.stderr = buf - try: - code = cli._ordered_eval_command(args) - finally: - sys.stderr = old - err = buf.getvalue() - assert code == 2 - assert RUN_TOKEN not in err - # Fail-closed message path also must not embed token. - with pytest.raises(RouteClientError) as excinfo: - # Direct unit: the handoff guard must not interpolate the token. - raise RouteClientError( - "eval deploy requires --token-output PATH and/or --emit-run-token " - "so the miner can later call eval result with EVAL_RUN_TOKEN" - ) - assert RUN_TOKEN not in str(excinfo.value) - - -# --------------------------------------------------------------------------- # -# S8 — CVM encrypted env still receives EVAL_RUN_TOKEN -# --------------------------------------------------------------------------- # - - -def test_eval_run_token_still_injected_into_cvm_encrypted_env( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Adjacent: handoff must not remove EVAL_RUN_TOKEN from CVM env injection.""" - - captured: list[dict[str, str]] = [] - printed = _wire_successful_deploy(monkeypatch, captured_secrets=captured) - args = _deploy_args(emit_run_token=True) - assert cli._ordered_eval_command(args) == 0 - assert captured, printed - assert captured[0]["EVAL_RUN_TOKEN"] == RUN_TOKEN - - -def test_redact_capabilities_still_strips_token_key() -> None: - """Unit: _redact_capabilities never leaves token bytes in nested payloads.""" - - raw = { - "secret_delivery": {"env_key": "EVAL_RUN_TOKEN", "token": RUN_TOKEN}, - "EVAL_RUN_TOKEN": RUN_TOKEN, - "nested": {"token": RUN_TOKEN}, - } - redacted = cli._redact_capabilities(raw) - dumped = json.dumps(redacted) - assert RUN_TOKEN not in dumped - assert redacted["secret_delivery"] == {"env_key": "EVAL_RUN_TOKEN"} - - -# --------------------------------------------------------------------------- # -# S9 — optional measured OPENROUTER + progress env on eval deploy -# --------------------------------------------------------------------------- # - - -def test_eval_deploy_injects_openrouter_and_progress_env_when_key_present( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Given OPENROUTER_API_KEY in env, When eval deploy succeeds, - Then encrypted_env secrets include OPENROUTER_API_KEY + progress bindings - (never Base gateway). Guest never posts; host needs OR for agent LLM. - """ - - captured: list[dict[str, str]] = [] - printed = _wire_successful_deploy(monkeypatch, captured_secrets=captured) - monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test-eval-only") - args = _deploy_args( - emit_run_token=True, - openrouter_key_env="OPENROUTER_API_KEY", - base_url="https://challenge.example/challenges/agent-challenge/", - ) - assert cli._ordered_eval_command(args) == 0, printed - assert captured, printed - secrets = captured[0] - assert secrets["EVAL_RUN_TOKEN"] == RUN_TOKEN - assert secrets["OPENROUTER_API_KEY"] == "sk-or-test-eval-only" - assert secrets["EVAL_PROGRESS_BASE_URL"] == ( - "https://challenge.example/challenges/agent-challenge" - ) - assert secrets["EVAL_RUN_ID"] == EVAL_RUN_ID - assert secrets["EVAL_SUBMISSION_ID"] == "1" - # Never Base gateway - assert "BASE_GATEWAY_TOKEN" not in secrets - assert "BASE_LLM_GATEWAY_URL" not in secrets - - -def test_eval_deploy_omits_openrouter_when_key_absent( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Given no OpenRouter key, When eval deploy succeeds, - Then OPENROUTER_API_KEY is absent (tools-only path) but progress still binds. - """ - - captured: list[dict[str, str]] = [] - printed = _wire_successful_deploy(monkeypatch, captured_secrets=captured) - monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) - args = _deploy_args( - emit_run_token=True, - openrouter_key_env="OPENROUTER_API_KEY", - base_url="https://challenge.example", - ) - assert cli._ordered_eval_command(args) == 0, printed - secrets = captured[0] - assert "OPENROUTER_API_KEY" not in secrets - assert secrets["EVAL_PROGRESS_BASE_URL"] == "https://challenge.example" - assert secrets["EVAL_RUN_TOKEN"] == RUN_TOKEN diff --git a/packages/challenges/agent-challenge/tests/test_eval_keyrelease_ratls_bootstrap.py b/packages/challenges/agent-challenge/tests/test_eval_keyrelease_ratls_bootstrap.py deleted file mode 100644 index 39033f267..000000000 --- a/packages/challenges/agent-challenge/tests/test_eval_keyrelease_ratls_bootstrap.py +++ /dev/null @@ -1,1329 +0,0 @@ -"""Eval CVM bootstrap → framed raw-TCP RA-TLS key-release (fail-closed). - -Root residual from live create: CVM running with no framed request on 8701 and -no phase leave `eval_prepared`. These tests pin the bootstrap contract so that: - -1. The measured entrypoint invokes key-release materialization when host/port is - set, with non-empty client cert/key, and fails closed when TLS material cannot - be produced. -2. The entrypoint never silently skips key-acquisition when Phala key-release - flags are on (baked-in assets do not bypass the grant). -3. Compose ``run`` + ``--job-dir`` (single "run") is normalized to the backend - subcommand + task list, including tasks pulled from the immutable Eval plan. -4. Client raw RA-TLS condemns missing mTLS material and certificate verification - failures with typed KeyRelease errors (never a silent success path). -5. An offline framed handshake against a local 8701-style TLS 1.3 listener either - completes or produces a durable denial reason code. -6. With host/port set: flushed secret-free markers (stage=start/material_ready/ - tcp_connect_ok|fail) and stage=fail reason=... on any provision error, - hard ~90s GetTlsKey wallclock, forced TCP dial after materials, and non-zero - exit so silent multi-minute eval_prepared is impossible. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import socket -import ssl -import struct -import threading -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import pytest -from cryptography import x509 -from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID - -from agent_challenge.canonical import entrypoint -from agent_challenge.evaluation import own_runner_backend as backend -from agent_challenge.keyrelease import client as key_client -from agent_challenge.keyrelease.client import ( - GoldenKeyReleaseClient, - KeyReleaseDenied, - KeyReleaseError, - KeyReleaseProtocolError, - KeyReleaseUnreachable, -) -from agent_challenge.review.canonical import canonical_json_v1 - -# --------------------------------------------------------------------------- # -# Helpers -# --------------------------------------------------------------------------- # - - -def _rsa_key(size: int = 2048): - return rsa.generate_private_key(public_exponent=65537, key_size=size) - - -def _write_pem(path: Path, data: bytes) -> Path: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - return path - - -def _make_ca(*, cn: str = "bootstrap-test-ca") -> tuple[Any, x509.Certificate]: - key = _rsa_key() - name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, cn)]) - cert = ( - x509.CertificateBuilder() - .subject_name(name) - .issuer_name(name) - .public_key(key.public_key()) - .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.now(UTC) - timedelta(minutes=1)) - .not_valid_after(datetime.now(UTC) + timedelta(days=1)) - .add_extension(x509.BasicConstraints(ca=True, path_length=1), critical=True) - .sign(key, hashes.SHA256()) - ) - return key, cert - - -def _sign_cert( - *, - subject_cn: str, - issuer_key, - issuer_cert: x509.Certificate, - client_auth: bool = False, - server_auth: bool = False, - san_ip: str | None = "127.0.0.1", -) -> tuple[Any, x509.Certificate]: - key = _rsa_key() - name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, subject_cn)]) - eku: list[Any] = [] - if client_auth: - eku.append(ExtendedKeyUsageOID.CLIENT_AUTH) - if server_auth: - eku.append(ExtendedKeyUsageOID.SERVER_AUTH) - builder = ( - x509.CertificateBuilder() - .subject_name(name) - .issuer_name(issuer_cert.subject) - .public_key(key.public_key()) - .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.now(UTC) - timedelta(minutes=1)) - .not_valid_after(datetime.now(UTC) + timedelta(hours=1)) - ) - if eku: - builder = builder.add_extension(x509.ExtendedKeyUsage(eku), critical=False) - if san_ip is not None: - builder = builder.add_extension( - x509.SubjectAlternativeName( - [x509.IPAddress(__import__("ipaddress").ip_address(san_ip))] - ), - critical=False, - ) - cert = builder.sign(issuer_key, hashes.SHA256()) - return key, cert - - -def _pem_cert(cert: x509.Certificate) -> bytes: - return cert.public_bytes(serialization.Encoding.PEM) - - -def _pem_key(key) -> bytes: - return key.private_bytes( - serialization.Encoding.PEM, - serialization.PrivateFormat.TraditionalOpenSSL, - serialization.NoEncryption(), - ) - - -# --------------------------------------------------------------------------- # -# Entrypoint: RA-TLS materialization + non-silent skip -# --------------------------------------------------------------------------- # - - -def _patch_tcp_ok(monkeypatch) -> list[tuple[Any, ...]]: - """Force successful socket.create_connection so success-path tests stay offline.""" - - dials: list[tuple[Any, ...]] = [] - - class _Conn: - def __enter__(self): - return self - - def __exit__(self, *args: object) -> None: - return None - - def close(self) -> None: - return None - - def _fake_create_connection(address, timeout=None, **_kwargs): # noqa: ANN001 - dials.append((address, timeout)) - return _Conn() - - monkeypatch.setattr(entrypoint.socket, "create_connection", _fake_create_connection) - return dials - - -def test_entrypoint_provisions_ra_tls_from_dstack_when_host_port_set(monkeypatch, tmp_path, capsys): - cert_path = tmp_path / "client.crt" - key_path = tmp_path / "client.key" - ca_path = tmp_path / "ca.crt" - ca_key, ca_cert = _make_ca(cn="server-trust-ca") - server_ca_pem = _pem_cert(ca_cert).decode() - - class _Resp: - key = _pem_key(_rsa_key()).decode() - certificate_chain = [ - _pem_cert( - _sign_cert( - subject_cn="guest-client", - issuer_key=ca_key, - issuer_cert=ca_cert, - client_auth=True, - san_ip=None, - )[1] - ).decode() - ] - - class _FakeDstack: - def __init__(self, timeout: int = 90) -> None: - assert timeout >= 60 - - def get_tls_key(self, **kwargs: Any): - assert kwargs.get("usage_ra_tls") is True - assert kwargs.get("usage_client_auth") is True - return _Resp() - - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "84.32.70.61") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "8701") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(cert_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_KEY_FILE", str(key_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CA_FILE", str(ca_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", server_ca_pem) - monkeypatch.setattr("dstack_sdk.DstackClient", _FakeDstack, raising=False) - import dstack_sdk - - monkeypatch.setattr(dstack_sdk, "DstackClient", _FakeDstack, raising=False) - dials = _patch_tcp_ok(monkeypatch) - - captured: dict[str, Any] = {} - - def fake_backend(args: list[str]) -> int: - captured["args"] = list(args) - captured["cert"] = Path(os.environ["CHALLENGE_PHALA_RA_TLS_CERT_FILE"]).read_text() - captured["key"] = Path(os.environ["CHALLENGE_PHALA_RA_TLS_KEY_FILE"]).read_text() - captured["ca"] = Path(os.environ["CHALLENGE_PHALA_RA_TLS_CA_FILE"]).read_text() - return 0 - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.main", fake_backend, raising=True - ) - - rc = entrypoint.main( - [ - "run", - "run", - "--task", - "adaptive-rejection-sampler", - "--job-dir", - "/opt/agent-challenge/job", - ] - ) - out = capsys.readouterr().out - assert rc == 0 - assert cert_path.is_file() and key_path.is_file() and ca_path.is_file() - assert "BEGIN CERTIFICATE" in captured["cert"] - assert "BEGIN" in captured["key"] and captured["key"].strip() - assert captured["ca"].strip() == server_ca_pem.strip() - assert "ra_tls_bootstrap stage=start" in out - assert "host=84.32.70.61" in out and "port=8701" in out - assert "server_ca=present" in out - assert "ra_tls_bootstrap stage=material_ready" in out - assert "ra_tls_bootstrap stage=public_fullchain_exported" in out - assert "ra_tls_public_fullchain stage=export_ready" in out - assert "BEGIN CERTIFICATE" in out # intentional public-only fullchain export - assert "ra_tls_bootstrap stage=tcp_connect_ok" in out - assert dials and dials[0][0] == ("84.32.70.61", 8701) - assert dials[0][1] == entrypoint.TCP_CONNECT_TIMEOUT_SECONDS - # Never leak private key material; secret-free stage markers still omit raw - # server-CA / key PEMs (public CLIENT fullchain PEM is the sole export surface). - assert "PRIVATE KEY" not in out.upper() - assert server_ca_pem[:40] not in out or "ra_tls_public_fullchain" in out - # Private key content never appears on the log surface. - assert captured["key"][:48] not in out - - -def test_entrypoint_fails_closed_when_raw_path_set_but_gettlskey_empty( - monkeypatch, tmp_path, capsys -): - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "84.32.70.61") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "8701") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(tmp_path / "c.crt")) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_KEY_FILE", str(tmp_path / "c.key")) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CA_FILE", str(tmp_path / "ca.crt")) - monkeypatch.setenv( - "CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", - _pem_cert(_make_ca()[1]).decode(), - ) - - class _Bad: - key = "" - certificate_chain = [] - - class _FakeDstack: - def __init__(self, timeout: int = 90) -> None: - pass - - def get_tls_key(self, **kwargs: Any): - return _Bad() - - import dstack_sdk - - monkeypatch.setattr(dstack_sdk, "DstackClient", _FakeDstack, raising=False) - - def _must_not_run(*_a, **_k): # pragma: no cover - raise AssertionError("backend must not run without RA-TLS material") - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.main", _must_not_run, raising=True - ) - - rc = entrypoint.main(["run", "run", "--job-dir", "/tmp/job"]) - out = capsys.readouterr().out - assert rc == 1 - assert "ra_tls_bootstrap stage=fail" in out - assert "reason=gettlskey_failed" in out - assert "host=84.32.70.61" in out and "port=8701" in out - - -def test_entrypoint_fails_closed_without_server_ca_on_raw_path(monkeypatch, tmp_path, capsys): - """Guest chain intermediate must not silence missing validator server CA.""" - - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "84.32.70.61") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "8701") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(tmp_path / "c.crt")) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_KEY_FILE", str(tmp_path / "c.key")) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CA_FILE", str(tmp_path / "ca.crt")) - # Intentionally no CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM and no pre-written CA. - - ca_key, ca_cert = _make_ca() - leaf_key, leaf = _sign_cert( - subject_cn="guest", - issuer_key=ca_key, - issuer_cert=ca_cert, - client_auth=True, - san_ip=None, - ) - - class _Resp: - key = _pem_key(leaf_key).decode() - certificate_chain = [_pem_cert(leaf).decode(), _pem_cert(ca_cert).decode()] - - class _FakeDstack: - def __init__(self, timeout: int = 90) -> None: - pass - - def get_tls_key(self, **kwargs: Any): - return _Resp() - - import dstack_sdk - - monkeypatch.setattr(dstack_sdk, "DstackClient", _FakeDstack, raising=False) - - rc = entrypoint.main(["run", "run", "--job-dir", "/tmp/job"]) - out = capsys.readouterr().out - assert rc == 1 - assert "ra_tls_bootstrap stage=start" in out - assert "server_ca=missing" in out - assert "ra_tls_bootstrap stage=fail" in out - assert "reason=missing_server_ca" in out - - -def test_entrypoint_gettlskey_wallclock_fails_closed(monkeypatch, tmp_path, capsys): - """A hung GetTlsKey must not leave the guest silent; hard wallclock exits.""" - - import time - - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "127.0.0.1") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "8701") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(tmp_path / "c.crt")) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_KEY_FILE", str(tmp_path / "c.key")) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CA_FILE", str(tmp_path / "ca.crt")) - monkeypatch.setenv( - "CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", - _pem_cert(_make_ca()[1]).decode(), - ) - monkeypatch.setattr(entrypoint, "GET_TLS_KEY_WALLCLOCK_SECONDS", 0.05) - - class _HangDstack: - def __init__(self, timeout: int = 90) -> None: - pass - - def get_tls_key(self, **kwargs: Any): - # Indefinite hang (not a short sleep) discriminates ThreadPoolExecutor - # re-join (shutdown wait=True) from a true non-blocking wallclock. - while True: - time.sleep(1.0) - - import dstack_sdk - - monkeypatch.setattr(dstack_sdk, "DstackClient", _HangDstack, raising=False) - - def _must_not_run(*_a, **_k): # pragma: no cover - raise AssertionError("backend must not run after gettlskey timeout") - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.main", _must_not_run, raising=True - ) - - t0 = time.monotonic() - rc = entrypoint.main(["run", "--job-dir", "/tmp/job"]) - elapsed = time.monotonic() - t0 - out = capsys.readouterr().out - assert rc == 1 - assert "ra_tls_bootstrap stage=fail" in out - assert "reason=gettlskey_timeout" in out - # Must finish near the 0.05s wallclock, not re-join an infinite hang. - assert elapsed < 1.0, f"wallclock re-joined hung GetTlsKey: elapsed={elapsed:.3f}s" - - -def test_wallclock_helper_indefinite_hang_returns_without_rejoin() -> None: - """call_with_wallclock abandons an indefinite hang and never re-joins it.""" - - import time - - from agent_challenge.canonical.wallclock import WallclockTimeout, call_with_wallclock - - def _never_returns() -> None: - while True: - time.sleep(1.0) - - t0 = time.monotonic() - with pytest.raises(WallclockTimeout, match="exceeded"): - call_with_wallclock(_never_returns, timeout_seconds=0.05, label="GetTlsKey") - elapsed = time.monotonic() - t0 - assert elapsed < 0.5, f"helper re-joined hung worker: elapsed={elapsed:.3f}s" - - -def test_entrypoint_tcp_connect_fail_after_material_ready(monkeypatch, tmp_path, capsys): - """Materials success + unreachable host => tcp_connect_fail marker, exit 1.""" - - cert_path = tmp_path / "client.crt" - key_path = tmp_path / "client.key" - ca_path = tmp_path / "ca.crt" - # Pre-written client material so GetTlsKey is skipped. - ca_key, ca_cert = _make_ca() - leaf_key, leaf = _sign_cert( - subject_cn="prebaked", - issuer_key=ca_key, - issuer_cert=ca_cert, - client_auth=True, - san_ip=None, - ) - cert_path.write_bytes(_pem_cert(leaf)) - key_path.write_bytes(_pem_key(leaf_key)) - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "127.0.0.1") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "1") # almost always closed - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(cert_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_KEY_FILE", str(key_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CA_FILE", str(ca_path)) - monkeypatch.setenv( - "CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", - _pem_cert(ca_cert).decode(), - ) - monkeypatch.setattr(entrypoint, "TCP_CONNECT_TIMEOUT_SECONDS", 0.2) - - def _must_not_run(*_a, **_k): # pragma: no cover - raise AssertionError("backend must not run after tcp fail") - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.main", _must_not_run, raising=True - ) - - rc = entrypoint.main(["run", "--job-dir", "/tmp/job"]) - out = capsys.readouterr().out - assert rc == 1 - assert "ra_tls_bootstrap stage=material_ready" in out - assert "ra_tls_bootstrap stage=tcp_connect_fail" in out - assert "ra_tls_bootstrap stage=fail" in out - assert "reason=tcp_connect_fail" in out - - -def test_entrypoint_normalizes_single_run_compose_argv(monkeypatch): - """Measured compose ships one leading ``run``; backend needs the subcommand.""" - - captured: dict[str, Any] = {} - - def fake_backend(args: list[str]) -> int: - captured["args"] = list(args) - return 0 - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.main", fake_backend, raising=True - ) - # Simulates docker-compose command: ["run", "--job-dir", ...] without the - # second explicit backend "run" token and without CLI --task flags. - # No KEY_RELEASE host/port => provision is a no-op (legacy path). - monkeypatch.delenv("KEY_RELEASE_RA_TLS_HOST", raising=False) - monkeypatch.delenv("KEY_RELEASE_RA_TLS_PORT", raising=False) - rc = entrypoint.main( - [ - "run", - "--job-dir", - "/opt/agent-challenge/job", - "--cache-root", - "/opt/agent-challenge/task-cache", - "--digest-manifest", - "/opt/agent-challenge/golden/dataset-digest.json", - ] - ) - assert rc == 0 - assert captured["args"][0] == "run" - assert "--job-dir" in captured["args"] - - -# --------------------------------------------------------------------------- # -# Backend: no silent skip when Phala + host/port set; plan-derived tasks -# --------------------------------------------------------------------------- # - - -def _stub_eval_plan(**overrides: Any) -> dict[str, Any]: - plan = { - "eval_run_id": "eval-run-bootstrap-001", - "key_release_endpoint": "ratls://84.32.70.61:8701", - "key_release_nonce": "key-nonce-bootstrap", - "score_nonce": "score-nonce-bootstrap", - "issued_at_ms": 0, - "expires_at_ms": 4_102_444_800_000, - "selected_tasks": [ - { - "task_id": "adaptive-rejection-sampler", - "image_ref": "registry/task@sha256:" + "a" * 64, - }, - { - "task_id": "bn-fit-modify", - "image_ref": "registry/task@sha256:" + "b" * 64, - }, - { - "task_id": "break-filter-js-from-html", - "image_ref": "registry/task@sha256:" + "c" * 64, - }, - ], - "k": 1, - "n_concurrent": 4, - "agent_hash": "f" * 64, - "scoring_policy": { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "all", - }, - "eval_app": { - "app_identity": "agent-challenge-eval-v1", - "image_ref": "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" + "d" * 64, - "compose_hash": "e" * 64, - "measurement": { - "mrtd": "1" * 96, - "rtmr0": "2" * 96, - "rtmr1": "3" * 96, - "rtmr2": "4" * 96, - "os_image_hash": "5" * 64, - "vm_shape": "tdx.small", - }, - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "ab" * 32, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("ab" * 32)).hexdigest(), - }, - "authorizing_review_digest": "r" * 64, - "run_token_sha256": "t" * 64, - } - plan.update(overrides) - return plan - - -def test_backend_uses_plan_tasks_when_cli_omits_task(monkeypatch, tmp_path, capsys): - """Compose omits --task; Phala plan supplies the immutable selected set.""" - - monkeypatch.setenv(backend.PHALA_ATTESTATION_ENABLED_ENV, "1") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "84.32.70.61") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "8701") - plan = _stub_eval_plan() - monkeypatch.setattr( - backend, - "_resolve_phala_binding_from_env", - lambda: {"eval_plan": plan, "rtmr3": "d" * 96}, - ) - monkeypatch.setattr(backend, "assert_agent_artifact_matches_plan", lambda **_: "f" * 64) - monkeypatch.setattr(backend, "_preflight_eval_plan_tasks", lambda **_: {}) - - acquired: dict[str, Any] = {} - - def _acquire(*, eval_plan=None): - acquired["called"] = True - acquired["plan"] = eval_plan - raise KeyReleaseUnreachable("listener down for test") - - monkeypatch.setattr(backend, "_acquire_golden_key_if_required", _acquire) - - # No --task flags (compose shape). - rc = backend.main(["run", "--job-dir", str(tmp_path / "job")]) - out = capsys.readouterr().out - assert acquired.get("called") is True - assert acquired["plan"]["eval_run_id"] == plan["eval_run_id"] - assert rc != 0 - assert "phala_key_release_failed" in out - - -def test_backend_fails_closed_on_raw_endpoint_without_mtls_files(monkeypatch, tmp_path): - monkeypatch.delenv(key_client.KEY_RELEASE_TLS_CERT_ENV, raising=False) - monkeypatch.delenv(key_client.KEY_RELEASE_TLS_KEY_ENV, raising=False) - monkeypatch.delenv(key_client.KEY_RELEASE_TLS_CA_ENV, raising=False) - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "127.0.0.1") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "8701") - - with pytest.raises((KeyReleaseUnreachable, KeyReleaseError, KeyReleaseProtocolError)): - # Endpoint resolves from host/port; empty cert paths must not grant. - backend._acquire_golden_key_if_required( - eval_plan=_stub_eval_plan(key_release_endpoint="ratls://127.0.0.1:8701") - ) - - -def test_acquire_never_returns_none_when_ratls_host_port_set(monkeypatch): - """Silent skip is impossible once raw RA-TLS host/port are provisioned.""" - - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "84.32.70.61") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "8701") - monkeypatch.delenv(key_client.KEY_RELEASE_URL_ENV, raising=False) - - class _Boom(GoldenKeyReleaseClient): - def acquire_golden_key(self, **kwargs: Any) -> bytes: - raise KeyReleaseUnreachable("forced") - - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _Boom) - monkeypatch.setattr( - "agent_challenge.canonical.attested_result.DstackQuoteProvider", - lambda *_a, **_k: object(), - raising=False, - ) - - with pytest.raises(KeyReleaseError): - result = backend._acquire_golden_key_if_required() - # Even if a future code path returns, None is banned for the raw path. - assert result is not None - - -# --------------------------------------------------------------------------- # -# Client: missing material + offline framed handshake -# --------------------------------------------------------------------------- # - - -def test_raw_client_fails_closed_without_mtls_files(monkeypatch): - monkeypatch.delenv(key_client.KEY_RELEASE_TLS_CERT_ENV, raising=False) - monkeypatch.delenv(key_client.KEY_RELEASE_TLS_KEY_ENV, raising=False) - monkeypatch.delenv(key_client.KEY_RELEASE_TLS_CA_ENV, raising=False) - - client = GoldenKeyReleaseClient("ratls://127.0.0.1:8701") - with pytest.raises(KeyReleaseUnreachable, match="mTLS|not configured"): - client._raw_release( - payload={ - "schema_version": 1, - "eval_run_id": "e1", - "nonce": "n1", - "quote_hex": "aa", - "event_log": [], - }, - host="127.0.0.1", - port=8701, - ) - - -def test_offline_framed_handshake_against_local_listener(tmp_path): - """Complete framed JSON exchange over TLS 1.3 mTLS against a local listener.""" - - ca_key, ca_cert = _make_ca(cn="handshake-ca") - server_key, server_cert = _sign_cert( - subject_cn="key-release-server", - issuer_key=ca_key, - issuer_cert=ca_cert, - server_auth=True, - san_ip="127.0.0.1", - ) - client_key, client_cert = _sign_cert( - subject_cn="guest-client", - issuer_key=ca_key, - issuer_cert=ca_cert, - client_auth=True, - san_ip=None, - ) - - ca_file = _write_pem(tmp_path / "ca.crt", _pem_cert(ca_cert)) - server_cert_file = _write_pem(tmp_path / "server.crt", _pem_cert(server_cert)) - server_key_file = _write_pem(tmp_path / "server.key", _pem_key(server_key)) - client_cert_file = _write_pem(tmp_path / "client.crt", _pem_cert(client_cert)) - client_key_file = _write_pem(tmp_path / "client.key", _pem_key(client_key)) - - received: dict[str, Any] = {} - ready = threading.Event() - - def _serve() -> None: - context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - context.minimum_version = ssl.TLSVersion.TLSv1_3 - context.maximum_version = ssl.TLSVersion.TLSv1_3 - context.verify_mode = ssl.CERT_REQUIRED - context.load_cert_chain(str(server_cert_file), str(server_key_file)) - context.load_verify_locations(cafile=str(ca_file)) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(("127.0.0.1", 0)) - port = sock.getsockname()[1] - received["port"] = port - sock.listen(1) - ready.set() - conn, _ = sock.accept() - try: - tls = context.wrap_socket(conn, server_side=True) - header = tls.recv(4) - length = struct.unpack(">I", header)[0] - body = b"" - while len(body) < length: - body += tls.recv(length - len(body)) - payload = json.loads(body) - received["payload"] = payload - peer = tls.getpeercert(binary_form=True) - received["peer_present"] = bool(peer) - # Durable deny reason (grant path needs full quote plumbing). - response = { - "schema_version": 1, - "released": False, - "reason_code": "measurement_not_allowlisted", - } - encoded = canonical_json_v1(response) - tls.sendall(struct.pack(">I", len(encoded)) + encoded) - tls.close() - finally: - sock.close() - - thread = threading.Thread(target=_serve, daemon=True) - thread.start() - assert ready.wait(2.0) - port = int(received["port"]) - - os.environ[key_client.KEY_RELEASE_TLS_CERT_ENV] = str(client_cert_file) - os.environ[key_client.KEY_RELEASE_TLS_KEY_ENV] = str(client_key_file) - os.environ[key_client.KEY_RELEASE_TLS_CA_ENV] = str(ca_file) - try: - client = GoldenKeyReleaseClient(f"ratls://127.0.0.1:{port}", timeout=5.0) - with pytest.raises(KeyReleaseDenied, match="measurement_not_allowlisted"): - client._raw_release( - payload={ - "schema_version": 1, - "eval_run_id": "eval_offline_handshake", - "nonce": "nonce-offline", - "quote_hex": "aa" * 32, - "event_log": [], - }, - host="127.0.0.1", - port=port, - ) - finally: - for name in ( - key_client.KEY_RELEASE_TLS_CERT_ENV, - key_client.KEY_RELEASE_TLS_KEY_ENV, - key_client.KEY_RELEASE_TLS_CA_ENV, - ): - os.environ.pop(name, None) - thread.join(timeout=2.0) - - assert received.get("peer_present") is True - assert received["payload"]["eval_run_id"] == "eval_offline_handshake" - assert received["payload"]["nonce"] == "nonce-offline" - - -def test_client_ssl_verify_failure_is_typed_unreachable(tmp_path, monkeypatch): - """Wrong server CA → typed fail-closed, never a grant.""" - - good_ca_key, good_ca = _make_ca(cn="good") - bad_ca_key, bad_ca = _make_ca(cn="bad") - server_key, server_cert = _sign_cert( - subject_cn="server", - issuer_key=good_ca_key, - issuer_cert=good_ca, - server_auth=True, - san_ip="127.0.0.1", - ) - client_key, client_cert = _sign_cert( - subject_cn="client", - issuer_key=good_ca_key, - issuer_cert=good_ca, - client_auth=True, - san_ip=None, - ) - - ca_wrong = _write_pem(tmp_path / "wrong-ca.crt", _pem_cert(bad_ca)) - server_cert_file = _write_pem(tmp_path / "server.crt", _pem_cert(server_cert)) - server_key_file = _write_pem(tmp_path / "server.key", _pem_key(server_key)) - client_cert_file = _write_pem(tmp_path / "client.crt", _pem_cert(client_cert)) - client_key_file = _write_pem(tmp_path / "client.key", _pem_key(client_key)) - - ready = threading.Event() - port_box: dict[str, int] = {} - - def _serve() -> None: - context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - context.minimum_version = ssl.TLSVersion.TLSv1_3 - context.maximum_version = ssl.TLSVersion.TLSv1_3 - context.verify_mode = ssl.CERT_NONE # only testing client verify path - context.load_cert_chain(str(server_cert_file), str(server_key_file)) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(("127.0.0.1", 0)) - port_box["port"] = sock.getsockname()[1] - sock.listen(1) - ready.set() - try: - conn, _ = sock.accept() - try: - context.wrap_socket(conn, server_side=True) - except ssl.SSLError: - pass - finally: - try: - conn.close() - except OSError: - pass - finally: - sock.close() - - thread = threading.Thread(target=_serve, daemon=True) - thread.start() - assert ready.wait(2.0) - - monkeypatch.setenv(key_client.KEY_RELEASE_TLS_CERT_ENV, str(client_cert_file)) - monkeypatch.setenv(key_client.KEY_RELEASE_TLS_KEY_ENV, str(client_key_file)) - monkeypatch.setenv(key_client.KEY_RELEASE_TLS_CA_ENV, str(ca_wrong)) - - client = GoldenKeyReleaseClient(f"ratls://127.0.0.1:{port_box['port']}", timeout=3.0) - with pytest.raises((KeyReleaseUnreachable, KeyReleaseError, KeyReleaseProtocolError)): - client._raw_release( - payload={ - "schema_version": 1, - "eval_run_id": "e", - "nonce": "n", - "quote_hex": "aa", - "event_log": [], - }, - host="127.0.0.1", - port=port_box["port"], - ) - thread.join(timeout=2.0) - - -# --------------------------------------------------------------------------- # -# Server CA PEM normalize + OpenSSL preload (live residual: escaped \\n PEM) -# --------------------------------------------------------------------------- # - - -def test_normalize_escaped_oneline_pem_becomes_loadable_openssl() -> None: - """encrypted_env injection can collapse PEM newlines to literal \\n.""" - - _, ca_cert = _make_ca(cn="escaped-ca") - good_pem = _pem_cert(ca_cert).decode() - escaped = good_pem.replace("\n", "\\n").strip() # one line, literal backslash-n - assert "\n" not in escaped - assert "BEGIN CERTIFICATE" in escaped - - # Broken capture of the residual: raw escaped PEM must fail OpenSSL. - broken = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - with pytest.raises(ssl.SSLError): - broken.load_verify_locations(cadata=escaped if escaped.endswith("\n") else escaped + "\n") - - normalized = entrypoint.normalize_server_ca_pem(escaped) - assert "\n" in normalized - assert normalized.endswith("\n") - assert "BEGIN CERTIFICATE" in normalized - assert "END CERTIFICATE" in normalized - - ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - ctx.load_verify_locations(cadata=normalized) # must not raise - - -def test_normalize_rejects_junk_and_empty_with_malformed_label() -> None: - junk_pem = "-----BEGIN CERTIFICATE-----\\nZZZZ\\n-----END CERTIFICATE-----\\n" - for junk in ("", " ", "not-a-cert", junk_pem): - with pytest.raises(ValueError, match="malformed_server_ca|empty_server_ca"): - entrypoint.normalize_server_ca_pem(junk) - # empty raises before / also after load; junk PEM header alone is not enough. - - -def test_entrypoint_writes_unescaped_ca_from_escaped_env(monkeypatch, tmp_path, capsys) -> None: - """Guest resolves escaped SERVER_CA_PEM and materializes OpenSSL-loadable ca.crt.""" - - ca_key, ca_cert = _make_ca(cn="inject-ca") - escaped = _pem_cert(ca_cert).decode().replace("\n", "\\n").strip() - cert_path = tmp_path / "client.crt" - key_path = tmp_path / "client.key" - ca_path = tmp_path / "ca.crt" - # Pre-provision client material so GetTlsKey is not needed. - leaf_key, leaf = _sign_cert( - subject_cn="guest", - issuer_key=ca_key, - issuer_cert=ca_cert, - client_auth=True, - san_ip=None, - ) - cert_path.write_bytes(_pem_cert(leaf)) - key_path.write_bytes(_pem_key(leaf_key)) - - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "127.0.0.1") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "8701") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(cert_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_KEY_FILE", str(key_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CA_FILE", str(ca_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", escaped) - dials = _patch_tcp_ok(monkeypatch) - - def fake_backend(args: list[str]) -> int: - return 0 - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.main", fake_backend, raising=True - ) - - rc = entrypoint.main(["run", "run", "--job-dir", "/tmp/job"]) - out = capsys.readouterr().out - assert rc == 0 - assert dials - written = ca_path.read_text(encoding="utf-8") - assert "\n" in written - assert "\\n" not in written.replace("-----BEGIN CERTIFICATE-----", "") - # Written CA must load into OpenSSL trust store. - ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - ctx.load_verify_locations(cadata=written) - assert "server_ca=present" in out - assert "ra_tls_bootstrap stage=material_ready" in out - - -def test_entrypoint_fails_closed_on_unloadable_server_ca(monkeypatch, tmp_path, capsys) -> None: - junk = "-----BEGIN CERTIFICATE-----\\nnot-real-base64\\n-----END CERTIFICATE-----\\n" - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "127.0.0.1") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "8701") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(tmp_path / "c.crt")) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_KEY_FILE", str(tmp_path / "c.key")) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CA_FILE", str(tmp_path / "ca.crt")) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", junk) - - def _must_not_run(*_a, **_k): # pragma: no cover - raise AssertionError("backend must not run with unloadable server CA") - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.main", _must_not_run, raising=True - ) - - rc = entrypoint.main(["run", "run", "--job-dir", "/tmp/job"]) - out = capsys.readouterr().out - assert rc == 1 - assert "ra_tls_bootstrap stage=fail" in out - assert "reason=malformed_server_ca" in out - - -def test_raw_release_preloads_ca_and_rejects_unloadable_before_create_default_context( - tmp_path, monkeypatch -) -> None: - """Bad ca.crt must never hit opaque SSLError from create_default_context mid-setup.""" - - junk_path = tmp_path / "bad-ca.crt" - # multi-line junk that OpenSSL still rejects but has PEM markers - junk_path.write_text( - "-----BEGIN CERTIFICATE-----\nbm90LXJlYWw=\n-----END CERTIFICATE-----\n", - encoding="utf-8", - ) - cert_path = tmp_path / "client.crt" - key_path = tmp_path / "client.key" - ca_key, ca_cert = _make_ca() - leaf_key, leaf = _sign_cert( - subject_cn="client", - issuer_key=ca_key, - issuer_cert=ca_cert, - client_auth=True, - san_ip=None, - ) - cert_path.write_bytes(_pem_cert(leaf)) - key_path.write_bytes(_pem_key(leaf_key)) - - monkeypatch.setenv(key_client.KEY_RELEASE_TLS_CERT_ENV, str(cert_path)) - monkeypatch.setenv(key_client.KEY_RELEASE_TLS_KEY_ENV, str(key_path)) - monkeypatch.setenv(key_client.KEY_RELEASE_TLS_CA_ENV, str(junk_path)) - - called: list[str] = [] - real_create = ssl.create_default_context - - def _spy_create(*args, **kwargs): - called.append("create_default_context") - return real_create(*args, **kwargs) - - monkeypatch.setattr(key_client.ssl, "create_default_context", _spy_create) - - client = GoldenKeyReleaseClient("ratls://127.0.0.1:8701", timeout=1.0) - with pytest.raises(KeyReleaseUnreachable, match="malformed_server_ca"): - client._raw_release( - payload={ - "schema_version": 1, - "eval_run_id": "e", - "nonce": "n", - "quote_hex": "aa", - "event_log": [], - }, - host="127.0.0.1", - port=8701, - ) - assert called == [], "create_default_context must not run for unloadable CA" - - -def test_raw_release_accepts_escaped_ca_file_contents(tmp_path, monkeypatch) -> None: - """Client, not only entrypoint, unwraps escaped PEM before SSL context build.""" - - ca_key, ca_cert = _make_ca(cn="escaped-file-ca") - good = _pem_cert(ca_cert).decode() - escaped_one_line = good.replace("\n", "\\n").strip() + "\n" - ca_path = tmp_path / "ca.crt" - ca_path.write_text(escaped_one_line, encoding="utf-8") - - client_key, client_cert = _sign_cert( - subject_cn="guest", - issuer_key=ca_key, - issuer_cert=ca_cert, - client_auth=True, - san_ip=None, - ) - server_key, server_cert = _sign_cert( - subject_cn="server", - issuer_key=ca_key, - issuer_cert=ca_cert, - server_auth=True, - san_ip="127.0.0.1", - ) - cert_path = _write_pem(tmp_path / "client.crt", _pem_cert(client_cert)) - key_path = _write_pem(tmp_path / "client.key", _pem_key(client_key)) - server_cert_file = _write_pem(tmp_path / "server.crt", _pem_cert(server_cert)) - server_key_file = _write_pem(tmp_path / "server.key", _pem_key(server_key)) - - ready = threading.Event() - port_box: dict[str, int] = {} - - def _serve() -> None: - context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - context.minimum_version = ssl.TLSVersion.TLSv1_3 - context.maximum_version = ssl.TLSVersion.TLSv1_3 - context.verify_mode = ssl.CERT_REQUIRED - context.load_cert_chain(str(server_cert_file), str(server_key_file)) - context.load_verify_locations(cadata=good) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(("127.0.0.1", 0)) - port_box["port"] = sock.getsockname()[1] - sock.listen(1) - ready.set() - conn, _ = sock.accept() - try: - tls = context.wrap_socket(conn, server_side=True) - header = tls.recv(4) - length = struct.unpack(">I", header)[0] - body = b"" - while len(body) < length: - body += tls.recv(length - len(body)) - response = { - "schema_version": 1, - "released": False, - "reason_code": "measurement_not_allowlisted", - } - encoded = canonical_json_v1(response) - tls.sendall(struct.pack(">I", len(encoded)) + encoded) - tls.close() - finally: - sock.close() - - thread = threading.Thread(target=_serve, daemon=True) - thread.start() - assert ready.wait(2.0) - - monkeypatch.setenv(key_client.KEY_RELEASE_TLS_CERT_ENV, str(cert_path)) - monkeypatch.setenv(key_client.KEY_RELEASE_TLS_KEY_ENV, str(key_path)) - monkeypatch.setenv(key_client.KEY_RELEASE_TLS_CA_ENV, str(ca_path)) - - client = GoldenKeyReleaseClient(f"ratls://127.0.0.1:{port_box['port']}", timeout=5.0) - with pytest.raises(KeyReleaseDenied, match="measurement_not_allowlisted"): - client._raw_release( - payload={ - "schema_version": 1, - "eval_run_id": "eval_escaped_ca", - "nonce": "n", - "quote_hex": "aa" * 16, - "event_log": [], - }, - host="127.0.0.1", - port=port_box["port"], - ) - thread.join(timeout=2.0) - - -# --------------------------------------------------------------------------- # -# Path B residual: pin may leave HOST/PORT unset while plan endpoint is raw -# --------------------------------------------------------------------------- # - - -def test_entrypoint_provisions_from_eval_plan_endpoint_when_host_port_unset( - monkeypatch, tmp_path, capsys -): - """Measure-time HTTPS pin leaves KEY_RELEASE_RA_TLS_HOST/PORT empty; guest must - still GetTlsKey from signed plan ``key_release_endpoint`` before KR dial. - - Live residual (pathb v2 sub35): server CA inject ok, serial had no GetTlsKey - markers, own_runner later failed with ``raw key-release mTLS files are not - configured``. Provision must arm CERT/KEY/CA envs regardless of static bake. - """ - - cert_path = tmp_path / "client.crt" - key_path = tmp_path / "client.key" - ca_path = tmp_path / "ca.crt" - ca_key, ca_cert = _make_ca(cn="server-trust-ca-from-plan") - server_ca_pem = _pem_cert(ca_cert).decode() - - class _Resp: - key = _pem_key(_rsa_key()).decode() - certificate_chain = [ - _pem_cert( - _sign_cert( - subject_cn="guest-client-from-plan", - issuer_key=ca_key, - issuer_cert=ca_cert, - client_auth=True, - san_ip=None, - )[1] - ).decode() - ] - - class _FakeDstack: - def __init__(self, timeout: int = 90) -> None: - assert timeout >= 60 - - def get_tls_key(self, **kwargs: Any): - assert kwargs.get("usage_ra_tls") is True - assert kwargs.get("usage_client_auth") is True - return _Resp() - - # Measure-time HTTPS pin: static HOST/PORT empty strings (falsy after .strip()). - # Always monkeeypatch.setenv (not bare delenv) so undoes re-cover product's - # post-provision os.environ exports (HOST/PORT/CERT/KEY/CA/SPKI). - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "") - monkeypatch.setenv("CHALLENGE_PHALA_KEY_RELEASE_URL", "") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(cert_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_KEY_FILE", str(key_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CA_FILE", str(ca_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_SPKI_SHA256", "") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", server_ca_pem) - monkeypatch.setenv( - "CHALLENGE_PHALA_EVAL_PLAN", - json.dumps({"key_release_endpoint": "86.38.238.235:8701"}), - ) - monkeypatch.setattr(entrypoint, "DEFAULT_RA_TLS_CERT", cert_path) - monkeypatch.setattr(entrypoint, "DEFAULT_RA_TLS_KEY", key_path) - monkeypatch.setattr(entrypoint, "DEFAULT_RA_TLS_CA", ca_path) - monkeypatch.setattr(entrypoint, "DEFAULT_RA_TLS_DIR", tmp_path) - - import dstack_sdk - - monkeypatch.setattr(dstack_sdk, "DstackClient", _FakeDstack, raising=False) - dials = _patch_tcp_ok(monkeypatch) - - captured: dict[str, Any] = {} - - def fake_backend(args: list[str]) -> int: - captured["host"] = os.environ.get("KEY_RELEASE_RA_TLS_HOST") - captured["port"] = os.environ.get("KEY_RELEASE_RA_TLS_PORT") - captured["cert"] = os.environ.get("CHALLENGE_PHALA_RA_TLS_CERT_FILE") - captured["key"] = os.environ.get("CHALLENGE_PHALA_RA_TLS_KEY_FILE") - captured["ca"] = os.environ.get("CHALLENGE_PHALA_RA_TLS_CA_FILE") - captured["cert_pem"] = Path(captured["cert"]).read_text() - captured["key_pem"] = Path(captured["key"]).read_text() - captured["ca_pem"] = Path(captured["ca"]).read_text() - return 0 - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.main", fake_backend, raising=True - ) - - rc = entrypoint.main(["run", "--job-dir", "/opt/agent-challenge/job"]) - out = capsys.readouterr().out - assert rc == 0 - assert captured["host"] == "86.38.238.235" - assert captured["port"] == "8701" - assert captured["cert"] and Path(captured["cert"]).is_file() - assert captured["key"] and Path(captured["key"]).is_file() - assert captured["ca"] and Path(captured["ca"]).is_file() - assert "BEGIN CERTIFICATE" in captured["cert_pem"] - assert "BEGIN" in captured["key_pem"] - assert captured["ca_pem"].strip() == server_ca_pem.strip() - assert "ra_tls_bootstrap stage=start" in out - assert "host=86.38.238.235" in out and "port=8701" in out - assert "ra_tls_bootstrap stage=material_ready" in out - assert "ra_tls_bootstrap stage=public_fullchain_exported" in out - assert "ra_tls_public_fullchain stage=export_ready" in out - assert dials and dials[0][0] == ("86.38.238.235", 8701) - # Intentionally public-only CERTIFICATEs on the harvest log surface. - assert "BEGIN CERTIFICATE" in out - assert "PRIVATE KEY" not in out.upper() - assert captured["key_pem"][:48] not in out - - -def test_entrypoint_fails_closed_with_client_material_incomplete_stage( - monkeypatch, tmp_path, capsys -): - """Host/port set but GetTlsKey leave is unreadable on disk → stable stage code.""" - - cert_path = tmp_path / "client.crt" - key_path = tmp_path / "client.key" - ca_path = tmp_path / "ca.crt" - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "84.32.70.61") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "8701") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(cert_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_KEY_FILE", str(key_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CA_FILE", str(ca_path)) - monkeypatch.setenv( - "CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", - _pem_cert(_make_ca()[1]).decode(), - ) - monkeypatch.delenv("CHALLENGE_PHALA_RA_TLS_SPKI_SHA256", raising=False) - - class _Resp: - key = _pem_key(_rsa_key()).decode() - certificate_chain = [_pem_cert(_make_ca()[1]).decode()] - - class _FakeDstack: - def __init__(self, timeout: int = 90) -> None: - pass - - def get_tls_key(self, **kwargs: Any): - return _Resp() - - import dstack_sdk - - monkeypatch.setattr(dstack_sdk, "DstackClient", _FakeDstack, raising=False) - _patch_tcp_ok(monkeypatch) - - real_is_file = Path.is_file - missing_names = {str(cert_path), str(key_path)} - - def _is_file_missing_client(self): # noqa: ANN001 - # After GetTlsKey write, report client paths missing so provision fails - # with the stable incomplete stage (not gettlskey_failed). - if str(self) in missing_names: - return False - return real_is_file(self) - - monkeypatch.setattr(Path, "is_file", _is_file_missing_client) - - def _must_not_run(*_a, **_k): # pragma: no cover - raise AssertionError("backend must not run with incomplete client material") - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.main", _must_not_run, raising=True - ) - - rc = entrypoint.main(["run", "run", "--job-dir", "/tmp/job"]) - out = capsys.readouterr().out - assert rc == 1 - assert "ra_tls_bootstrap stage=fail" in out - assert "reason=client_material_incomplete" in out - - -def test_raw_client_distinct_codes_when_server_ca_ok_but_client_chain_missing( - monkeypatch, tmp_path -): - """Server CA inject alone must not look like opaque total mTLS misconfig.""" - - ca_key, ca_cert = _make_ca(cn="distinct-ca") - ca_path = tmp_path / "ca.crt" - ca_path.write_bytes(_pem_cert(ca_cert)) - - monkeypatch.delenv(key_client.KEY_RELEASE_TLS_CERT_ENV, raising=False) - monkeypatch.delenv(key_client.KEY_RELEASE_TLS_KEY_ENV, raising=False) - monkeypatch.setenv(key_client.KEY_RELEASE_TLS_CA_ENV, str(ca_path)) - - client = GoldenKeyReleaseClient("ratls://127.0.0.1:8701") - with pytest.raises( - KeyReleaseUnreachable, - match="client cert/key|client_chain_missing|client chain", - ) as exc_info: - client._raw_release( - payload={ - "schema_version": 1, - "eval_run_id": "e1", - "nonce": "n1", - "quote_hex": "aa", - "event_log": [], - }, - host="127.0.0.1", - port=8701, - ) - # Must not collapse to the residual-generic total-misconfig string only. - assert "client" in str(exc_info.value).lower() - assert ( - "mTLS files are not configured" not in str(exc_info.value) - or "client" in str(exc_info.value).lower() - ) - - -def test_raw_client_refuses_missing_paths_on_disk_with_distinct_code(monkeypatch, tmp_path): - """Env path names set (compose bake) but files absent after failed provision.""" - - missing_cert = tmp_path / "no-client.crt" - missing_key = tmp_path / "no-client.key" - ca_key, ca_cert = _make_ca() - ca_path = tmp_path / "ca.crt" - ca_path.write_bytes(_pem_cert(ca_cert)) - monkeypatch.setenv(key_client.KEY_RELEASE_TLS_CERT_ENV, str(missing_cert)) - monkeypatch.setenv(key_client.KEY_RELEASE_TLS_KEY_ENV, str(missing_key)) - monkeypatch.setenv(key_client.KEY_RELEASE_TLS_CA_ENV, str(ca_path)) - - client = GoldenKeyReleaseClient("ratls://127.0.0.1:8701") - with pytest.raises( - KeyReleaseUnreachable, - match="client_material_missing|client cert/key files missing", - ): - client._raw_release( - payload={ - "schema_version": 1, - "eval_run_id": "e1", - "nonce": "n1", - "quote_hex": "aa", - "event_log": [], - }, - host="127.0.0.1", - port=8701, - ) - - -def test_compose_still_bakes_host_port_from_raw_plan_endpoint_without_invent(): - """Compose pin from plan endpoint still measures HOST/PORT (no invent PEM roots).""" - - from agent_challenge.canonical.compose import generate_app_compose - from agent_challenge.selfdeploy import eval as eval_deploy - - compose = generate_app_compose( - orchestrator_image="ttl.sh/ac@sha256:" + ("a" * 64), - name=eval_deploy.DEFAULT_EVAL_COMPOSE_NAME, - key_release_url="86.38.238.235:8701", - allowed_envs=eval_deploy.EVAL_ALLOWED_ENVS, - ) - yaml_text = compose["docker_compose_file"] - assert "KEY_RELEASE_RA_TLS_HOST=86.38.238.235" in yaml_text - assert "KEY_RELEASE_RA_TLS_PORT=8701" in yaml_text - assert "CHALLENGE_PHALA_RA_TLS_CERT_FILE=/run/secrets/ra_tls/client.crt" in yaml_text - assert "BEGIN CERTIFICATE" not in yaml_text - # HTTPS measure-time placeholder must NOT invent a raw bake for that URL. - placeholder = generate_app_compose( - orchestrator_image="ttl.sh/ac@sha256:" + ("a" * 64), - name=eval_deploy.DEFAULT_EVAL_COMPOSE_NAME, - key_release_url=eval_deploy.MEASURE_TIME_EVAL_KEY_RELEASE_PLACEHOLDER, - allowed_envs=eval_deploy.EVAL_ALLOWED_ENVS, - ) - place_yaml = placeholder["docker_compose_file"] - assert "KEY_RELEASE_RA_TLS_HOST=" not in place_yaml - assert "CHALLENGE_PHALA_KEY_RELEASE_URL=" in place_yaml diff --git a/packages/challenges/agent-challenge/tests/test_eval_prepare_dataset_unavailable.py b/packages/challenges/agent-challenge/tests/test_eval_prepare_dataset_unavailable.py index fb883f083..7e544e837 100644 --- a/packages/challenges/agent-challenge/tests/test_eval_prepare_dataset_unavailable.py +++ b/packages/challenges/agent-challenge/tests/test_eval_prepare_dataset_unavailable.py @@ -105,72 +105,6 @@ def test_task_config_digest_propagates_missing_file_as_dataset_unavailable( assert excinfo.value.code == "eval_dataset_unavailable" -def test_build_plan_empty_key_release_endpoint_unavailable( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from datetime import UTC, datetime - - from agent_challenge.sdk.config import ChallengeSettings - - measurement = { - "mrtd": "01" * 48, - "rtmr0": "02" * 48, - "rtmr1": "03" * 48, - "rtmr2": "04" * 48, - "os_image_hash": "05" * 32, - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - } - settings = ChallengeSettings( - attested_review_enabled=True, - phala_attestation_enabled=True, - eval_app_image_ref="registry.example/eval@sha256:" + "a" * 64, - eval_app_compose_hash="06" * 32, - eval_app_identity="agent-challenge-eval-v1", - eval_app_kms_public_key_hex="07" * 32, - eval_app_measurement=measurement, - eval_app_measurement_allowlist=( - { - "mrtd": measurement["mrtd"], - "rtmr0": measurement["rtmr0"], - "rtmr1": measurement["rtmr1"], - "rtmr2": measurement["rtmr2"], - "compose_hash": "06" * 32, - "os_image_hash": measurement["os_image_hash"], - }, - ), - eval_key_release_endpoint="", - eval_k=1, - evaluation_task_count=1, - ) - monkeypatch.setattr( - auth, - "load_benchmark_tasks", - lambda: [ - SimpleNamespace( - task_id="task-a", - docker_image="registry.example/task@sha256:" + "b" * 64, - prompt="", - benchmark="terminal_bench", - metadata={"content_digest_sha256": "ab" * 32}, - ) - ], - ) - submission = SimpleNamespace(id=1, agent_hash="11" * 32, version_number=1) - with pytest.raises(EvalAuthorizationUnavailable) as excinfo: - auth._build_plan( - submission=submission, # type: ignore[arg-type] - review_digest="12" * 32, - settings=settings, - eval_run_id="eval_test", - key_release_nonce="kr-nonce", - score_nonce="score-nonce", - token_sha256="13" * 32, - now=datetime.now(UTC), - ) - assert excinfo.value.code == "eval_key_release_endpoint_unavailable" - - def test_unavailable_detail_code_prefers_explicit_code() -> None: exc = EvalAuthorizationUnavailable( "frozen dataset digest is unavailable", diff --git a/packages/challenges/agent-challenge/tests/test_eval_progress_env_wiring.py b/packages/challenges/agent-challenge/tests/test_eval_progress_env_wiring.py deleted file mode 100644 index 39e55c71d..000000000 --- a/packages/challenges/agent-challenge/tests/test_eval_progress_env_wiring.py +++ /dev/null @@ -1,227 +0,0 @@ -"""RED contract: selfdeploy env bundle carries progress + runner hotkey config. - -ProgressReporter env must be deploy-injectable (mirror REVIEW_API_BASE_URL). -RUNNER_HOTKEY_MNEMONIC is the local-only signing secret name — its VALUE must -never appear in logs/redacted CLI output. The server receives hotkey_ss58 + -signature only. -""" - -from __future__ import annotations - -import hashlib -import json -import logging - -from agent_challenge.canonical import eval_wire -from agent_challenge.canonical.compose import DEFAULT_ALLOWED_ENVS - -PROGRESS_ENVS = ( - "EVAL_PROGRESS_BASE_URL", - "EVAL_RUN_ID", - "EVAL_SUBMISSION_ID", - "EVAL_RUN_TOKEN", -) -# Local runner signing secret name (value never leaves the CVM / never hits master). -RUNNER_HOTKEY_ENV = "RUNNER_HOTKEY_MNEMONIC" -# Well-known 12-word test fixture — never a real secret; used only to prove redaction. -TEST_MNEMONIC = ( - "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about" -) - - -def test_default_allowed_envs_include_progress_and_runner_hotkey_names(): - allowed = set(DEFAULT_ALLOWED_ENVS) - for name in PROGRESS_ENVS: - assert name in allowed, f"{name} missing from DEFAULT_ALLOWED_ENVS" - assert RUNNER_HOTKEY_ENV in allowed, f"{RUNNER_HOTKEY_ENV} missing from DEFAULT_ALLOWED_ENVS" - - -def test_build_eval_progress_env_helper_binds_ids_and_token(): - """CLI/deploy helper must bind base URL + ids from plan + token.""" - from agent_challenge.selfdeploy.eval import build_eval_progress_env - - values = build_eval_progress_env( - base_url="https://chain.joinbase.ai/challenges/agent-challenge/", - eval_run_id="eval-1", - submission_id="7", - eval_run_token="tok", - ) - assert values == { - "EVAL_PROGRESS_BASE_URL": "https://chain.joinbase.ai/challenges/agent-challenge", - "EVAL_RUN_ID": "eval-1", - "EVAL_SUBMISSION_ID": "7", - "EVAL_RUN_TOKEN": "tok", - } - # Helper must never require or embed a mnemonic — server never sees it. - assert RUNNER_HOTKEY_ENV not in values - assert "mnemonic" not in json.dumps(values).lower() - - -def test_encrypt_eval_secrets_accepts_progress_env_bundle(): - from agent_challenge.canonical.compose import generate_app_compose, render_app_compose - from agent_challenge.selfdeploy import eval as eval_deploy - - try: - from agent_challenge.evaluation.own_runner.progress_reporter import ProgressReporter - except ImportError as exc: - raise AssertionError( - "progress_reporter.ProgressReporter missing — required for eval progress env wiring" - ) from exc - - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - img = "registry.example/eval@sha256:" + "b" * 64 - compose = generate_app_compose( - orchestrator_image=img, - name=eval_deploy.DEFAULT_EVAL_COMPOSE_NAME, - key_release_url=eval_deploy.MEASURE_TIME_EVAL_KEY_RELEASE_PLACEHOLDER, - allowed_envs=tuple(sorted(eval_deploy.EVAL_ALLOWED_ENVS)), - ) - compose_hash = hashlib.sha256(render_app_compose(compose).encode()).hexdigest() - token = "run-token-progress" - public_key = "c" * 64 - plan = { - "schema_version": 1, - "eval_run_id": "eval-progress-1", - "submission_id": "7", - "submission_version": 1, - "authorizing_review_digest": "d" * 64, - "agent_hash": "e" * 64, - "selected_tasks": [ - { - "task_id": "terminal-bench/t", - "image_ref": "task-local/t@sha256:" + "3" * 64, - "task_config_sha256": "3" * 64, - } - ], - "k": 1, - "package_tree_sha": "a" * 64, - "scoring_policy": policy, - "scoring_policy_digest": eval_wire.scoring_policy_digest(policy), - "eval_app": { - "image_ref": img, - "compose_hash": compose_hash, - "app_identity": "bb35a8f627f0f8c991aa85c15742d352e658e0f7", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": public_key, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex(public_key)).hexdigest(), - "measurement": { - "mrtd": "01" * 48, - "rtmr0": "02" * 48, - "rtmr1": "03" * 48, - "rtmr2": "04" * 48, - "os_image_hash": "05" * 32, - "key_provider": "phala", - "vm_shape": "tdx.small", - }, - }, - "key_release_endpoint": "86.38.238.235:8701", - "result_endpoint": "/evaluation/v1/runs/eval-progress-1/result", - "key_release_nonce": "kr-n", - "score_nonce": "sc-n", - "run_token_sha256": hashlib.sha256(token.encode()).hexdigest(), - "issued_at_ms": 1, - "expires_at_ms": 2, - } - plan = eval_wire.validate_eval_plan(plan) - dep = eval_deploy.build_eval_deployment_plan( - { - "schema_version": 1, - "plan": plan, - "plan_sha256": hashlib.sha256(eval_wire.canonical_json_v1(plan)).hexdigest(), - "secret_delivery": {"env_key": "EVAL_RUN_TOKEN", "token": token}, - } - ) - secrets = { - "EVAL_RUN_TOKEN": token, - "LLM_COST_LIMIT": "1.00", - "CHALLENGE_PHALA_ATTESTATION_ENABLED": "1", - "CHALLENGE_PHALA_EVAL_PLAN": json.dumps(dep.plan, sort_keys=True, separators=(",", ":")), - "CHALLENGE_PHALA_AGENT_HASH": dep.plan["agent_hash"], - "CHALLENGE_PHALA_CANONICAL_MEASUREMENT": json.dumps( - { - "mrtd": dep.measurement["mrtd"], - "rtmr0": dep.measurement["rtmr0"], - "rtmr1": dep.measurement["rtmr1"], - "rtmr2": dep.measurement["rtmr2"], - "compose_hash": dep.compose_hash, - "os_image_hash": dep.measurement["os_image_hash"], - }, - sort_keys=True, - separators=(",", ":"), - ), - "CHALLENGE_PHALA_VALIDATOR_NONCE": dep.plan["key_release_nonce"], - "EVAL_PROGRESS_BASE_URL": "https://chain.joinbase.ai/challenges/agent-challenge", - "EVAL_RUN_ID": dep.eval_run_id, - "EVAL_SUBMISSION_ID": str(dep.plan["submission_id"]), - # Local-only signing material name may be present in encrypted_env allow-list, - # but encrypt path must accept the name without shipping the value to master APIs. - RUNNER_HOTKEY_ENV: TEST_MNEMONIC, - } - encrypted = eval_deploy.encrypt_eval_secrets(dep, secrets) - for name in PROGRESS_ENVS: - assert name in encrypted.env_keys - assert RUNNER_HOTKEY_ENV in encrypted.env_keys - reporter = ProgressReporter.from_env( - { - "EVAL_PROGRESS_BASE_URL": secrets["EVAL_PROGRESS_BASE_URL"], - "EVAL_RUN_ID": secrets["EVAL_RUN_ID"], - "EVAL_SUBMISSION_ID": secrets["EVAL_SUBMISSION_ID"], - "EVAL_RUN_TOKEN": token, - } - ) - assert reporter is not None - assert reporter.eval_run_id == "eval-progress-1" - assert "progress" in reporter.url - # ProgressReporter must not require or hold the mnemonic. - reporter_dump = json.dumps(reporter.__dict__, default=str) - assert TEST_MNEMONIC not in reporter_dump - assert "mnemonic" not in reporter_dump.lower() - - -def test_runner_hotkey_mnemonic_value_never_in_redacted_cli_output(): - """CLI redaction must strip RUNNER_HOTKEY_MNEMONIC values from operator output.""" - from agent_challenge.selfdeploy import cli as cli_mod - - assert hasattr(cli_mod, "_redact_capabilities") - assert hasattr(cli_mod, "_REDACTED_CAPABILITY_KEYS") - assert RUNNER_HOTKEY_ENV in cli_mod._REDACTED_CAPABILITY_KEYS - - payload = { - "EVAL_RUN_ID": "eval-1", - "EVAL_RUN_TOKEN": "tok-secret", - RUNNER_HOTKEY_ENV: TEST_MNEMONIC, - "nested": {RUNNER_HOTKEY_ENV: TEST_MNEMONIC, "ok": "visible"}, - } - redacted = cli_mod._redact_capabilities(payload) - dumped = json.dumps(redacted) - assert TEST_MNEMONIC not in dumped - assert RUNNER_HOTKEY_ENV not in redacted - assert "tok-secret" not in dumped - assert redacted["EVAL_RUN_ID"] == "eval-1" - assert redacted["nested"]["ok"] == "visible" - - -def test_runner_hotkey_mnemonic_value_never_logged(caplog): - """Logging helpers must not emit the mnemonic value.""" - from agent_challenge.selfdeploy import cli as cli_mod - - payload = { - "stage": "eval-deploy", - RUNNER_HOTKEY_ENV: TEST_MNEMONIC, - "EVAL_RUN_TOKEN": "tok-secret", - } - with caplog.at_level(logging.DEBUG): - redacted = cli_mod._redact_capabilities(payload) - # Simulate what CLI would print. - logging.getLogger("agent_challenge.selfdeploy.cli").info( - "deploy payload %s", json.dumps(redacted) - ) - joined = "\n".join(r.getMessage() for r in caplog.records) - assert TEST_MNEMONIC not in joined - assert "tok-secret" not in joined diff --git a/packages/challenges/agent-challenge/tests/test_eval_wire_contracts.py b/packages/challenges/agent-challenge/tests/test_eval_wire_contracts.py deleted file mode 100644 index b6efb6eaf..000000000 --- a/packages/challenges/agent-challenge/tests/test_eval_wire_contracts.py +++ /dev/null @@ -1,417 +0,0 @@ -"""Strict Eval v1 wire-contract regressions shared with the BASE oracle.""" - -from __future__ import annotations - -import copy -import hashlib -import json -from pathlib import Path -from typing import Any - -import pytest - -from agent_challenge.canonical import attested_result as ar -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.canonical import report_data as rd - -VECTOR_PATH = Path(__file__).with_name("eval_execution_proof_v2_vectors.json") -VECTORS: dict[str, Any] = json.loads(VECTOR_PATH.read_text(encoding="utf-8")) -POSITIVE: dict[str, Any] = VECTORS["positive"] - - -def _set_path(value: dict[str, Any], path: str, replacement: Any) -> None: - *parents, leaf = path.split(".") - cursor: Any = value - for part in parents: - cursor = cursor[int(part)] if part.isdigit() else cursor[part] - cursor[int(leaf) if leaf.isdigit() else leaf] = replacement - - -def _rename_path(value: dict[str, Any], source: str, target: str) -> None: - *source_parents, source_leaf = source.split(".") - cursor: Any = value - for part in source_parents: - cursor = cursor[int(part)] if part.isdigit() else cursor[part] - moved = cursor.pop(source_leaf) - _set_path(value, target, moved) - - -def _malformed_proof(vector: dict[str, Any]) -> dict[str, Any]: - payload = copy.deepcopy(POSITIVE["execution_proof"]) - if vector["kind"] == "set": - _set_path(payload, vector["path"], vector["value"]) - else: - _rename_path(payload, vector["from"], vector["to"]) - return payload - - -def _score_record() -> dict[str, Any]: - return { - "schema_version": 1, - "eval_run_id": "eval-run-001", - "policy_digest": "2" * 64, - "k": 2, - "tasks": [ - { - "task_id": "task-a", - "trial_scores_f64be": ["0000000000000000", "3ff0000000000000"], - "aggregate_score_f64be": "3fe0000000000000", - "passed_trials": 1, - }, - { - "task_id": "task-b", - "trial_scores_f64be": ["3ff0000000000000", "3ff0000000000000"], - "aggregate_score_f64be": "3ff0000000000000", - "passed_trials": 2, - }, - ], - "final": { - "job_score_f64be": "3fe8000000000000", - "passed_tasks": 1, - "total_tasks": 2, - }, - } - - -def test_shared_v2_vector_matches_exact_canonical_bytes_and_report_data() -> None: - binding = POSITIVE["binding"] - built = ew.build_score_binding( - canonical_measurement=binding["canonical_measurement"], - agent_hash=binding["agent_hash"], - eval_run_id=binding["eval_run_id"], - score_nonce=binding["score_nonce"], - scores_digest=binding["scores_digest"], - task_ids=binding["task_ids"], - ) - - assert ew.canonical_json_v1(built).hex() == POSITIVE["canonical_json_utf8_hex"] - assert ew.score_report_data_hex(built) == POSITIVE["report_data_hex"] - assert ( - rd.report_data_hex( - canonical_measurement=binding["canonical_measurement"], - agent_hash=binding["agent_hash"], - task_ids=binding["task_ids"], - scores_digest=binding["scores_digest"], - eval_run_id=binding["eval_run_id"], - score_nonce=binding["score_nonce"], - ) - == POSITIVE["report_data_hex"] - ) - - -@pytest.mark.parametrize("task_ids", [["task-b", "task-a"], ["task-a", "task-a"]]) -def test_score_binding_rejects_noncanonical_task_arrays(task_ids: list[str]) -> None: - binding = POSITIVE["binding"] - with pytest.raises(ew.EvalWireError): - ew.build_score_binding( - canonical_measurement=binding["canonical_measurement"], - agent_hash=binding["agent_hash"], - eval_run_id=binding["eval_run_id"], - score_nonce=binding["score_nonce"], - scores_digest=binding["scores_digest"], - task_ids=task_ids, - ) - - -def test_producer_task_id_normalization_is_order_independent_but_never_deduplicates() -> None: - assert ew.canonical_task_ids(["task-b", "task-a"]) == ["task-a", "task-b"] - with pytest.raises(ew.EvalWireError): - ew.canonical_task_ids(["task-a", "task-a"]) - - -@pytest.mark.parametrize( - "vector", - VECTORS["malformed"], - ids=lambda vector: vector["name"], -) -def test_strict_execution_proof_rejects_shared_malformed_vectors(vector: dict[str, Any]) -> None: - if vector["kind"] == "raw_json": - with pytest.raises(ew.EvalWireError): - ew.parse_eval_execution_proof_json(vector["value"]) - else: - with pytest.raises(ew.EvalWireError): - ew.validate_eval_execution_proof(_malformed_proof(vector)) - - -def test_only_empty_placeholder_signature_is_accepted_on_eval_wire() -> None: - ew.validate_eval_execution_proof(POSITIVE["execution_proof"]) - forged = copy.deepcopy(POSITIVE["execution_proof"]) - forged["worker_signature"]["sig"] = "not-empty" - with pytest.raises(ew.EvalWireError): - ew.validate_eval_execution_proof(forged) - - -@pytest.mark.parametrize( - "value", - [ - "8000000000000000", - "7ff0000000000000", - "7ff8000000000000", - "3FF0000000000000", - "3ff000000000000", - "3ff00000000000000", - ], -) -def test_binary64_score_encoding_rejects_noncanonical_or_nonfinite_values(value: str) -> None: - with pytest.raises(ew.EvalWireError): - ew.decode_score_f64be(value) - - -def test_score_record_reconstructs_aggregates_and_full_set_counts() -> None: - record = _score_record() - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - record["policy_digest"] = ew.scoring_policy_digest(policy) - validated = ew.validate_canonical_score_record( - record, - scoring_policy=policy, - expected_eval_run_id="eval-run-001", - expected_task_ids=["task-a", "task-b"], - expected_k=2, - ) - assert validated == record - assert ew.score_record_digest(record) == ew.score_record_digest(validated) - - tampered = copy.deepcopy(record) - tampered["tasks"][0]["aggregate_score_f64be"] = "3ff0000000000000" - with pytest.raises(ew.EvalWireError): - ew.validate_canonical_score_record( - tampered, - scoring_policy=policy, - expected_eval_run_id="eval-run-001", - expected_task_ids=["task-a", "task-b"], - expected_k=2, - ) - - -def test_result_request_and_receipt_are_closed_and_bind_the_proof() -> None: - record = _score_record() - request = { - "schema_version": 1, - "eval_run_id": "eval-run-001", - "submission_id": "submission-001", - "agent_hash": POSITIVE["binding"]["agent_hash"], - "score_record": record, - "scores_digest": ew.score_record_digest(record), - "execution_proof": POSITIVE["execution_proof"], - } - assert ew.validate_eval_result_request(request) == request - request["caller_measurement"] = {} - with pytest.raises(ew.EvalWireError): - ew.validate_eval_result_request(request) - - receipt = { - "schema_version": 1, - "eval_run_id": "eval-run-001", - "receipt_id": "receipt-001", - "body_sha256": "3" * 64, - "received_at_ms": 1, - "phase": "received", - "terminal": False, - "verified": False, - "retryable": True, - "reason_code": None, - "result_available": False, - "finalized_at_ms": None, - } - assert ew.validate_eval_receipt(receipt) == receipt - - -def test_key_release_v2_binding_is_purpose_separated_and_nonce_sensitive() -> None: - value = ew.key_release_report_data_hex( - eval_run_id="eval-run-001", - key_release_nonce="key-nonce-001", - ra_tls_spki_digest="4" * 64, - ) - assert len(value) == 128 - assert value[64:] == "0" * 64 - assert value != ew.key_release_report_data_hex( - eval_run_id="eval-run-001", - key_release_nonce="score-nonce-001", - ra_tls_spki_digest="4" * 64, - ) - - -def _eval_plan() -> dict[str, Any]: - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - measurement = POSITIVE["binding"]["canonical_measurement"] - return { - "schema_version": 1, - "eval_run_id": "eval-run-001", - "submission_id": "submission-001", - "submission_version": 1, - "authorizing_review_digest": "1" * 64, - "agent_hash": POSITIVE["binding"]["agent_hash"], - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": task_id, - "image_ref": "registry.example/task@sha256:" + "d" * 64, - "task_config_sha256": "2" * 64, - } - for task_id in POSITIVE["binding"]["task_ids"] - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "d" * 64, - "compose_hash": measurement["compose_hash"], - "app_identity": "agent-challenge-eval", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "3" * 64, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("3" * 64)).hexdigest(), - "measurement": { - "mrtd": measurement["mrtd"], - "rtmr0": measurement["rtmr0"], - "rtmr1": measurement["rtmr1"], - "rtmr2": measurement["rtmr2"], - "os_image_hash": measurement["os_image_hash"], - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "keyrelease.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-run-001/result", - "key_release_nonce": "key-nonce-001", - "score_nonce": "score-nonce-001", - "run_token_sha256": "5" * 64, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - - -def test_eval_plan_is_closed_and_requires_distinct_purpose_nonces() -> None: - plan = _eval_plan() - assert ew.validate_eval_plan(plan) == plan - - crossed = copy.deepcopy(plan) - crossed["score_nonce"] = crossed["key_release_nonce"] - with pytest.raises(ew.EvalWireError): - ew.validate_eval_plan(crossed) - - extra = copy.deepcopy(plan) - extra["caller_measurement"] = {} - with pytest.raises(ew.EvalWireError): - ew.validate_eval_plan(extra) - - -def test_eval_plan_rejects_unbound_kms_public_key_digest() -> None: - plan = _eval_plan() - plan["eval_app"]["kms_public_key_sha256"] = "4" * 64 - with pytest.raises(ew.EvalWireError): - ew.validate_eval_plan(plan) - - -class _QuoteResponse: - quote = "ab" * 8 - event_log = [ - { - "imr": 3, - "event_type": 134217729, - "digest": "c" * 96, - "event": "compose-hash", - "event_payload": "d" * 64, - } - ] - vm_config = { - "vcpu": 1, - "memory_mb": 2048, - "os_image_hash": POSITIVE["binding"]["canonical_measurement"]["os_image_hash"], - } - - -class _QuoteProvider: - def __init__(self) -> None: - self.calls: list[bytes] = [] - - def get_quote(self, report_data: bytes) -> _QuoteResponse: - self.calls.append(report_data) - return _QuoteResponse() - - -def test_attested_image_emits_exact_eval_result_request_with_v2_binding() -> None: - binding = POSITIVE["binding"] - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - record = ew.build_canonical_score_record( - eval_run_id=binding["eval_run_id"], - policy=policy, - trial_scores_by_task={ - "task-a": [1.0], - "task-b": [0.0], - "task-c": [0.5], - }, - ) - provider = _QuoteProvider() - line = ar.emit_attested_benchmark_result( - benchmark_result={ - "status": "completed", - "score": 0.5, - "resolved": 1, - "total": 3, - "reason_code": None, - }, - canonical_measurement=binding["canonical_measurement"], - rtmr3="e" * 96, - agent_hash=binding["agent_hash"], - task_ids=binding["task_ids"], - scores={}, - quote_provider=provider, - manifest_sha256="1" * 64, - eval_run_id=binding["eval_run_id"], - submission_id="submission-001", - score_nonce=binding["score_nonce"], - score_record=record, - image_digest="registry.example/eval@sha256:" + "d" * 64, - ) - payload = json.loads(line.split("=", 1)[1]) - - assert set(payload) == { - "schema_version", - "eval_run_id", - "submission_id", - "agent_hash", - "score_record", - "scores_digest", - "execution_proof", - } - assert ew.validate_eval_result_request(payload) == payload - assert payload["execution_proof"]["worker_signature"] == {"worker_pubkey": "", "sig": ""} - report_data = payload["execution_proof"]["attestation"]["report_data"] - assert provider.calls == [bytes.fromhex(report_data)] - - -def test_plan_driven_emitter_rejects_crossed_run_before_quote_emission() -> None: - plan = _eval_plan() - record = ew.build_canonical_score_record( - eval_run_id="other-run", - policy=plan["scoring_policy"], - trial_scores_by_task={task["task_id"]: [1.0] for task in plan["selected_tasks"]}, - ) - provider = _QuoteProvider() - with pytest.raises(ar.AttestationEmissionError): - ar.emit_attested_eval_result_from_plan( - eval_plan=plan, - score_record=record, - rtmr3="e" * 96, - quote_provider=provider, - manifest_sha256="1" * 64, - ) - assert provider.calls == [] diff --git a/packages/challenges/agent-challenge/tests/test_evaluation_lazy_import.py b/packages/challenges/agent-challenge/tests/test_evaluation_lazy_import.py deleted file mode 100644 index 97fb4f57d..000000000 --- a/packages/challenges/agent-challenge/tests/test_evaluation_lazy_import.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Lean-image import guard for the evaluation package (M2 heavy-import blocker). - -The canonical CVM image is a lean own_runner wrapper (pydantic only) and must be -able to run ``python -m agent_challenge.canonical.entrypoint run ...`` without the -heavy orchestration stack (sqlalchemy / fastapi / bittensor) that -``agent_challenge.evaluation.runner`` imports. Historically -``evaluation/__init__`` did ``from .runner import *`` eagerly, so importing ANY -evaluation submodule (e.g. ``own_runner_backend``) pulled the whole stack. - -These tests pin the fix: the package exposes ``runner``'s public API lazily -(:pep:`562`), so a bare ``own_runner_backend`` / entrypoint import works with the -heavy modules unavailable, while the historical -``from agent_challenge.evaluation import create_evaluation_job`` access still -resolves. -""" - -from __future__ import annotations - -import subprocess -import sys -import textwrap - -# Modules the lean canonical image does NOT ship; importing them must not be -# required to load own_runner_backend / the entrypoint. -_HEAVY_MODULES = ("sqlalchemy", "fastapi", "bittensor") - - -def _run_in_lean_interpreter(body: str) -> subprocess.CompletedProcess[str]: - """Run ``body`` in a fresh interpreter with the heavy modules blocked.""" - - script = textwrap.dedent( - """ - import sys - for _mod in {heavy!r}: - sys.modules[_mod] = None # any import of these now raises ImportError - {body} - """ - ).format(heavy=list(_HEAVY_MODULES), body=textwrap.indent(textwrap.dedent(body), "")) - return subprocess.run( - [sys.executable, "-c", script], - capture_output=True, - text=True, - ) - - -def test_own_runner_backend_imports_without_heavy_stack() -> None: - proc = _run_in_lean_interpreter( - """ - import agent_challenge.evaluation.own_runner_backend as backend - assert hasattr(backend, "main") - print("OK", backend.__name__) - """ - ) - assert proc.returncode == 0, proc.stderr - assert "OK agent_challenge.evaluation.own_runner_backend" in proc.stdout - - -def test_entrypoint_run_imports_backend_without_heavy_stack() -> None: - # ``entrypoint run `` reaches _run_eval, which imports own_runner_backend; - # this import must not require sqlalchemy/fastapi/bittensor. Feeding a bogus - # subcommand makes the backend's own argparse reject it (exit 2) AFTER the - # import succeeds, so reaching that error proves the lean import worked. - proc = _run_in_lean_interpreter( - """ - from agent_challenge.canonical import entrypoint - try: - entrypoint.main(["run", "bogus-subcommand"]) - except SystemExit as exc: - print("EXIT", exc.code) - """ - ) - combined = proc.stdout + proc.stderr - # The backend module loaded (its argparse prog appears) and no heavy-import wall. - assert "sqlalchemy" not in combined, combined - assert "agent-challenge-own-runner" in combined, combined - - -def test_entrypoint_check_runs_without_heavy_stack() -> None: - proc = _run_in_lean_interpreter( - """ - from agent_challenge.canonical import entrypoint - rc = entrypoint.main(["check"]) - assert rc == 0 - print("CHECK", rc) - """ - ) - assert proc.returncode == 0, proc.stderr - assert "CHECK 0" in proc.stdout - - -def test_importing_heavy_module_still_fails_in_lean_interpreter() -> None: - # Sanity: the lean-interpreter harness really does block the heavy stack, so - # the tests above are meaningful (not passing because the block is a no-op). - proc = _run_in_lean_interpreter( - """ - try: - import sqlalchemy # noqa: F401 - except ImportError: - print("BLOCKED") - else: - print("NOT BLOCKED") - """ - ) - assert "BLOCKED" in proc.stdout, proc.stderr - - -def test_attribute_style_backend_import_without_heavy_stack() -> None: - # ``from agent_challenge.evaluation import own_runner_backend`` (attribute - # style) must resolve the real lean submodule even when ``runner``'s heavy - # deps are absent, instead of recursing through the PEP 562 ``__getattr__``. - proc = _run_in_lean_interpreter( - """ - from agent_challenge.evaluation import own_runner_backend - assert hasattr(own_runner_backend, "main") - print("OK", own_runner_backend.__name__) - """ - ) - assert proc.returncode == 0, proc.stderr - assert "OK agent_challenge.evaluation.own_runner_backend" in proc.stdout - - -def test_runner_name_access_raises_cleanly_without_heavy_stack() -> None: - # A ``runner``-only name must raise a clean AttributeError (never a - # RecursionError) when the heavy ``runner`` submodule cannot import in the - # lean image, so callers get a legible error instead of a recursion crash. - proc = _run_in_lean_interpreter( - """ - from agent_challenge import evaluation - try: - evaluation.create_evaluation_job # noqa: B018 - runner-only name - except RecursionError: - print("RECURSION") - except AttributeError: - print("ATTRIBUTE-ERROR") - else: - print("RESOLVED") - """ - ) - assert proc.returncode == 0, proc.stderr - assert "ATTRIBUTE-ERROR" in proc.stdout - assert "RECURSION" not in proc.stdout - - -def test_public_runner_api_is_lazily_accessible() -> None: - # The historical ``from agent_challenge.evaluation import `` access - # must still resolve (lazily) in the full environment. - from agent_challenge import evaluation - - assert callable(evaluation.create_evaluation_job) - assert callable(evaluation.run_evaluation_job) - - -def test_unknown_attribute_still_raises_attribute_error() -> None: - from agent_challenge import evaluation - - try: - evaluation.this_name_does_not_exist # noqa: B018 - except AttributeError: - pass - else: # pragma: no cover - defensive - raise AssertionError("expected AttributeError for an unknown attribute") diff --git a/packages/challenges/agent-challenge/tests/test_golden_encrypted_at_rest.py b/packages/challenges/agent-challenge/tests/test_golden_encrypted_at_rest.py deleted file mode 100644 index 934cf2e17..000000000 --- a/packages/challenges/agent-challenge/tests/test_golden_encrypted_at_rest.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Golden-encrypted-at-rest tests (feature ``golden-encrypted-at-rest``). - -Fulfils: - * VAL-KEY-001 — golden expected-output is ciphertext everywhere miner-visible - (repo tree, generated compose); no cleartext golden marker at rest and the - packaged blob is genuinely encrypted (not merely encoded). - * VAL-KEY-002 — the packaged artifact is real AEAD ciphertext: it does not - decrypt without the released key (tamper/wrong-key fails closed). The - exhaustive AEAD behaviour lives in ``test_golden_crypto.py``. - * VAL-KEY-003 — no golden decryption key (or key-derivation secret) is - committed or baked into the repo / generated compose; the key lives only - outside the repo (endpoint/enclave-resident). - -The assertions that need the real released key are skipped when the key is not -available (so the milestone gate, which has no key, still runs), and exercised -during development by exporting ``CHALLENGE_GOLDEN_KEY_FILE``. -""" - -from __future__ import annotations - -import json -import math -import subprocess -from collections import Counter -from pathlib import Path - -import pytest - -from agent_challenge.canonical import compose as ccompose -from agent_challenge.canonical import secrets_scan -from agent_challenge.golden import crypto, package - -REPO_ROOT = package.REPO_ROOT -GOLDEN_DIR = package.GOLDEN_DIR -GOLDEN_MARKER = b"harbor-independence/" + b"oracle-golden" - - -def _shannon_entropy(data: bytes) -> float: - if not data: - return 0.0 - counts = Counter(data) - n = len(data) - return -sum((c / n) * math.log2(c / n) for c in counts.values()) - - -def _tracked_files() -> list[Path]: - out = subprocess.run( - ["git", "-C", str(REPO_ROOT), "ls-files", "-z"], - capture_output=True, - check=True, - ) - return [REPO_ROOT / p.decode() for p in out.stdout.split(b"\x00") if p] - - -def _available_golden_key() -> bytes | None: - try: - return crypto.load_golden_key() - except crypto.GoldenKeyError: - return None - - -requires_key = pytest.mark.skipif( - _available_golden_key() is None, - reason="golden key not available (set CHALLENGE_GOLDEN_KEY_FILE)", -) - - -# --------------------------------------------------------------------------- # -# VAL-KEY-001 — golden expected-output is ciphertext at rest -# --------------------------------------------------------------------------- # - - -def test_plaintext_oracle_golden_absent_from_repo(): - plaintext = GOLDEN_DIR / package.ORACLE_PLAINTEXT_NAME - assert not plaintext.exists(), f"plaintext golden must not be at rest: {plaintext}" - # The historical parity copy under runs/ must not ship as plaintext either. - assert not (REPO_ROOT / "runs" / "ours" / package.ORACLE_PLAINTEXT_NAME).exists() - - -def test_encrypted_oracle_artifact_is_present(): - enc = package.encrypted_oracle_path() - assert enc.is_file(), f"encrypted golden artifact missing: {enc}" - assert enc.name == package.ORACLE_CIPHERTEXT_NAME - - -def test_encrypted_artifact_is_ciphertext_not_encoding(): - blob = package.encrypted_oracle_path().read_bytes() - # Self-describing AEAD container, not JSON / plaintext / base64 text. - assert blob.startswith(crypto._MAGIC) - assert GOLDEN_MARKER not in blob - with pytest.raises((json.JSONDecodeError, UnicodeDecodeError)): - json.loads(blob) - # Encrypted body is high-entropy (near 8 bits/byte), i.e. not merely encoded. - body = blob[len(crypto._MAGIC) + crypto._NONCE_BYTES :] - assert _shannon_entropy(body) > 7.0 - - -def test_golden_dir_has_no_plaintext_marker(): - hits = [h for h in secrets_scan.scan_path(GOLDEN_DIR) if h.pattern == "golden_oracle_plaintext"] - assert hits == [], [h.member for h in hits] - - -def test_no_tracked_file_carries_golden_plaintext_marker(): - offenders: list[str] = [] - for path in _tracked_files(): - try: - data = path.read_bytes() - except OSError: - continue - if GOLDEN_MARKER in data: - offenders.append(str(path.relative_to(REPO_ROOT))) - assert offenders == [], f"golden plaintext marker present in repo: {offenders}" - - -def test_generated_compose_has_no_golden_plaintext(): - compose = ccompose.generate_app_compose( - orchestrator_image="ghcr.io/example/agent-challenge-canonical@sha256:" + "a" * 64, - key_release_url="http://validator.example:8700", - ) - blob = ccompose.render_app_compose_bytes(compose) - assert GOLDEN_MARKER not in blob - hits = [h for h in secrets_scan.scan_bytes(blob, member="app-compose.json")] - assert [h for h in hits if h.pattern == "golden_oracle_plaintext"] == [] - - -# --------------------------------------------------------------------------- # -# VAL-KEY-002 — the packaged artifact is genuine AEAD ciphertext -# --------------------------------------------------------------------------- # - - -def test_committed_artifact_not_decryptable_without_key(): - blob = package.encrypted_oracle_path().read_bytes() - with pytest.raises(crypto.GoldenDecryptionError): - package.decrypt_golden_bytes(blob, crypto.generate_golden_key()) - - -@requires_key -def test_committed_artifact_decrypts_to_valid_golden(): - key = _available_golden_key() - doc = package.load_encrypted_oracle_golden(key) - assert doc["schema"].startswith(GOLDEN_MARKER.decode()) - assert isinstance(doc["results"], dict) - assert doc["task_count"] == len(doc["results"]) == 89 - - -@requires_key -def test_committed_artifact_tamper_fails_closed(): - key = _available_golden_key() - blob = bytearray(package.encrypted_oracle_path().read_bytes()) - blob[-1] ^= 0x01 - with pytest.raises(crypto.GoldenDecryptionError): - package.decrypt_golden_bytes(bytes(blob), key) - - -# --------------------------------------------------------------------------- # -# VAL-KEY-003 — no golden key committed / baked -# --------------------------------------------------------------------------- # - - -def test_no_key_files_tracked_in_repo(): - tracked = [p.relative_to(REPO_ROOT).as_posix() for p in _tracked_files()] - assert [p for p in tracked if p.endswith(".key")] == [] - assert [p for p in tracked if p.startswith("secrets/")] == [] - - -def test_gitignore_protects_key_and_secrets(): - gi = (REPO_ROOT / ".gitignore").read_text().splitlines() - assert "*.key" in gi - assert "secrets/" in gi - - -def test_golden_source_has_no_hardcoded_key(): - # The crypto/packaging source must not embed a 32-byte key constant. - import re - - for mod in (crypto.__file__, package.__file__): - text = Path(mod).read_text() - # No 64-hex or base64-32 literal that could be a baked key. - assert not re.search(r"['\"][0-9a-fA-F]{64}['\"]", text), mod - - -@requires_key -def test_released_key_absent_from_repo_and_compose(): - key = _available_golden_key() - import base64 - - needles = [key.hex().encode(), base64.b64encode(key), key] - # Not in the generated compose. - compose = ccompose.generate_app_compose( - orchestrator_image="ghcr.io/example/agent-challenge-canonical@sha256:" + "a" * 64, - key_release_url="http://validator.example:8700", - ) - blob = ccompose.render_app_compose_bytes(compose) - for needle in needles: - assert needle not in blob - # Not in any tracked repo file. - for path in _tracked_files(): - try: - data = path.read_bytes() - except OSError: - continue - for needle in needles: - assert needle not in data, f"golden key material found in {path}" diff --git a/packages/challenges/agent-challenge/tests/test_image_backward_compat_flagoff.py b/packages/challenges/agent-challenge/tests/test_image_backward_compat_flagoff.py deleted file mode 100644 index 9556674e3..000000000 --- a/packages/challenges/agent-challenge/tests/test_image_backward_compat_flagoff.py +++ /dev/null @@ -1,404 +0,0 @@ -"""Backward-compat + config-layer flag-off tests for the canonical image (M1). - -Fulfils the ``image-backward-compat-flagoff`` feature: - * VAL-IMG-029 flag off -> legacy own_runner path, NO attestation envelope emitted - * VAL-IMG-030 flag off -> zero dstack socket / ``get_quote`` interactions - * VAL-IMG-031 flag on (k=1, variance off) scoring is byte-identical to flag off - * VAL-IMG-032 default configuration has the Phala path disabled (safe default) - * VAL-IMG-033 canonical_measurement bound in report_data == the envelope - measurement block == the pinnable allowlist record - -The in-image emission gate is the ``CHALLENGE_PHALA_ATTESTATION_ENABLED`` env var -read by :func:`own_runner_backend._phala_attestation_enabled`; the -validator-config layer exposes the SAME switch as the default-off -:class:`ChallengeSettings` field ``phala_attestation_enabled`` (whose env var, -via the ``CHALLENGE_`` prefix, is exactly that gate). These tests pin both -layers and prove that config-off == gate-off == byte-identical legacy behavior. -The dstack quote provider is faked so the flag-on path is exercised offline (the -live self-verify assertions run against a real CVM in M6). -""" - -from __future__ import annotations - -import hashlib -import json -import time -from pathlib import Path -from typing import Any - -import pytest - -from agent_challenge.canonical import attested_result as ar -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.canonical import report_data as rd -from agent_challenge.canonical.measurement import ( - CANONICAL_MEASUREMENT_FIELDS, - CanonicalMeasurement, -) -from agent_challenge.evaluation.own_runner.orchestrator import JobResult, TrialOutcome -from agent_challenge.evaluation.own_runner.result_schema import ( - REQUIRED_FIELDS, - RESULT_LINE_PREFIX, - build_benchmark_result, - format_benchmark_result_line, -) -from agent_challenge.evaluation.own_runner_backend import ( - PHALA_ATTESTATION_ENABLED_ENV, - PHALA_EVAL_PLAN_ENV, - PHALA_RTMR3_ENV, - _phala_attestation_enabled, - main, -) -from agent_challenge.sdk.config import ChallengeSettings - -ATTESTED_REVIEW_ENABLED_ENV = "CHALLENGE_ATTESTED_REVIEW_ENABLED" - -# --------------------------------------------------------------------------- # -# Fixtures / fakes -# --------------------------------------------------------------------------- # -MEASUREMENT = { - "mrtd": "a" * 96, - "rtmr0": "b0" * 48, - "rtmr1": "b1" * 48, - "rtmr2": "b2" * 48, - "compose_hash": "c" * 64, - "os_image_hash": "e" * 64, -} -RTMR3 = "d" * 96 -AGENT_HASH = "f" * 64 -NONCE = "nonce-xyz" -FAKE_QUOTE = "ab" * 600 -CONFIG_EXAMPLE = Path(__file__).resolve().parent.parent / "config.example.yaml" - - -class _FakeQuoteResponse: - def __init__(self, quote: str = FAKE_QUOTE) -> None: - self.quote = quote - self.event_log = json.dumps( - [ - { - "imr": 3, - "event_type": 134217729, - "digest": "c" * 96, - "event": "compose-hash", - "event_payload": MEASUREMENT["compose_hash"], - } - ] - ) - self.vm_config = json.dumps( - {"vcpu": 1, "memory_mb": 2048, "os_image_hash": MEASUREMENT["os_image_hash"]} - ) - self.report_data = "" - - -def _make_spy_provider() -> tuple[type, dict[str, int]]: - """A monitored dstack provider that counts construction + get_quote calls.""" - - events = {"instances": 0, "quote_calls": 0} - - class _Spy: - def __init__(self, endpoint: str | None = None) -> None: - events["instances"] += 1 - - def get_quote(self, report_data: bytes) -> Any: - events["quote_calls"] += 1 - return _FakeQuoteResponse() - - return _Spy, events - - -def _canned_result() -> JobResult: - return JobResult( - status="completed", - score=0.5, - resolved=1, - total=2, - reason_code=None, - pass_at_k={}, - n_total_trials=1, - n_completed_trials=1, - n_errored_trials=0, - trial_outcomes=[ - TrialOutcome( - task_name="hello-world", - trial_name="hello-world__attempt-0", - status="completed", - rewards={"reward": 0.5}, - ) - ], - benchmark_result=build_benchmark_result( - status="completed", score=0.5, resolved=1, total=2, reason_code=None - ), - ) - - -def _patch_run(monkeypatch) -> None: - async def _fake_run(**kwargs: Any) -> JobResult: - return _canned_result() - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.run_own_runner_job", _fake_run - ) - - -def _set_eval_plan_env(monkeypatch) -> None: - monkeypatch.setenv(PHALA_ATTESTATION_ENABLED_ENV, "1") - monkeypatch.setenv(ATTESTED_REVIEW_ENABLED_ENV, "1") - monkeypatch.setenv(PHALA_RTMR3_ENV, RTMR3) - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - plan = { - "schema_version": 1, - "eval_run_id": "eval-run-001", - "submission_id": "submission-001", - "submission_version": 1, - "authorizing_review_digest": "1" * 64, - "agent_hash": AGENT_HASH, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "hello-world", - "image_ref": "registry.example/task@sha256:" + "d" * 64, - "task_config_sha256": "2" * 64, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "d" * 64, - "compose_hash": MEASUREMENT["compose_hash"], - "app_identity": "agent-challenge-eval", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "3" * 64, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("3" * 64)).hexdigest(), - "measurement": { - "mrtd": MEASUREMENT["mrtd"], - "rtmr0": MEASUREMENT["rtmr0"], - "rtmr1": MEASUREMENT["rtmr1"], - "rtmr2": MEASUREMENT["rtmr2"], - "os_image_hash": MEASUREMENT["os_image_hash"], - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "keyrelease.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-run-001/result", - "key_release_nonce": "key-nonce-001", - "score_nonce": "score-nonce-001", - "run_token_sha256": "5" * 64, - "issued_at_ms": (time.time_ns() // 1_000_000) - 1_000, - "expires_at_ms": (time.time_ns() // 1_000_000) + 60_000, - } - monkeypatch.setenv(PHALA_EVAL_PLAN_ENV, json.dumps(plan)) - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend._acquire_golden_key_if_required", - lambda **_: None, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.assert_agent_artifact_matches_plan", - lambda **_: AGENT_HASH, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend._preflight_eval_plan_tasks", - lambda **_: {}, - ) - - -def _result_line(out: str) -> dict: - lines = [ln for ln in out.splitlines() if ln.startswith(RESULT_LINE_PREFIX)] - assert len(lines) == 1, f"expected exactly one result line, got {len(lines)}" - return json.loads(lines[0][len(RESULT_LINE_PREFIX) :]) - - -# --------------------------------------------------------------------------- # -# VAL-IMG-032: default configuration has the Phala path disabled -# --------------------------------------------------------------------------- # -def test_default_config_has_phala_attestation_disabled(monkeypatch) -> None: - monkeypatch.delenv(PHALA_ATTESTATION_ENABLED_ENV, raising=False) - settings = ChallengeSettings() - assert settings.phala_attestation_enabled is False - - -def test_config_flag_env_var_is_the_in_image_gate(monkeypatch) -> None: - # The ChallengeSettings field maps (via the CHALLENGE_ prefix) to exactly the - # in-image gate env var, so a config-off deployment gates the image off. - assert PHALA_ATTESTATION_ENABLED_ENV == "CHALLENGE_PHALA_ATTESTATION_ENABLED" - monkeypatch.setenv(PHALA_ATTESTATION_ENABLED_ENV, "1") - monkeypatch.setenv(ATTESTED_REVIEW_ENABLED_ENV, "1") - assert ChallengeSettings().phala_attestation_enabled is True - monkeypatch.setenv(PHALA_ATTESTATION_ENABLED_ENV, "0") - monkeypatch.setenv(ATTESTED_REVIEW_ENABLED_ENV, "0") - assert ChallengeSettings().phala_attestation_enabled is False - - -@pytest.mark.parametrize( - "value,expected", - [("1", True), ("true", True), ("yes", True), ("on", True), ("0", False), ("false", False)], -) -def test_config_flag_and_in_image_gate_agree(monkeypatch, value: str, expected: bool) -> None: - monkeypatch.setenv(PHALA_ATTESTATION_ENABLED_ENV, value) - monkeypatch.setenv(ATTESTED_REVIEW_ENABLED_ENV, value) - assert _phala_attestation_enabled() is expected - assert ChallengeSettings().phala_attestation_enabled is expected - - -def test_config_example_documents_phala_disabled_default() -> None: - text = CONFIG_EXAMPLE.read_text() - assert "phala_attestation_enabled: false" in text - - -# --------------------------------------------------------------------------- # -# VAL-IMG-029: flag off -> legacy own_runner path, no attestation emitted -# --------------------------------------------------------------------------- # -def test_flag_off_emits_legacy_line_without_attestation(monkeypatch, tmp_path, capsys) -> None: - monkeypatch.delenv(PHALA_ATTESTATION_ENABLED_ENV, raising=False) - _patch_run(monkeypatch) - rc = main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - assert rc == 0 - out = capsys.readouterr().out - payload = _result_line(out) - # Only the legacy five-field result -- no attestation blocks anywhere. - assert set(payload) == set(REQUIRED_FIELDS) - assert ar.EXECUTION_PROOF_RESULT_KEY not in payload - assert ar.ATTESTATION_BINDING_RESULT_KEY not in payload - assert "tdx_quote" not in out - assert "attestation" not in out - assert payload["status"] == "completed" - - -# --------------------------------------------------------------------------- # -# VAL-IMG-030: flag off -> zero dstack socket / get_quote interactions -# --------------------------------------------------------------------------- # -def test_flag_off_makes_zero_dstack_interactions(monkeypatch, tmp_path, capsys) -> None: - monkeypatch.delenv(PHALA_ATTESTATION_ENABLED_ENV, raising=False) - _patch_run(monkeypatch) - spy, events = _make_spy_provider() - monkeypatch.setattr(ar, "DstackQuoteProvider", spy) - rc = main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - assert rc == 0 - assert events == {"instances": 0, "quote_calls": 0} - - -def test_flag_off_never_constructs_quote_provider(monkeypatch, tmp_path, capsys) -> None: - monkeypatch.delenv(PHALA_ATTESTATION_ENABLED_ENV, raising=False) - _patch_run(monkeypatch) - - class _Boom: - def __init__(self, *args: Any, **kwargs: Any) -> None: - raise AssertionError("dstack provider must not be constructed when flag off") - - def get_quote(self, *args: Any, **kwargs: Any) -> Any: # pragma: no cover - raise AssertionError("get_quote must not be called when flag off") - - monkeypatch.setattr(ar, "DstackQuoteProvider", _Boom) - rc = main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - assert rc == 0 - - -def test_flag_on_does_interact_with_dstack(monkeypatch, tmp_path, capsys) -> None: - # Positive control: the monitored provider IS a real discriminator -- when the - # flag is on the image constructs it and calls get_quote exactly once. - _patch_run(monkeypatch) - _set_eval_plan_env(monkeypatch) - spy, events = _make_spy_provider() - monkeypatch.setattr(ar, "DstackQuoteProvider", spy) - rc = main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - assert rc == 0 - assert events == {"instances": 1, "quote_calls": 1} - - -# --------------------------------------------------------------------------- # -# VAL-IMG-031: flag on (k=1) scoring is byte-identical to flag off -# --------------------------------------------------------------------------- # -def test_flag_on_k1_scoring_byte_identical_to_flag_off(monkeypatch, tmp_path, capsys) -> None: - _patch_run(monkeypatch) - - # Flag OFF: legacy line. - monkeypatch.delenv(PHALA_ATTESTATION_ENABLED_ENV, raising=False) - rc_off = main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "off")]) - off_out = capsys.readouterr().out - off_line = next(ln for ln in off_out.splitlines() if ln.startswith(RESULT_LINE_PREFIX)) - off_payload = _result_line(off_out) - - # Flag ON (k=1, variance off): strict Eval request with the same score. - _set_eval_plan_env(monkeypatch) - monkeypatch.setattr(ar, "DstackQuoteProvider", _make_spy_provider()[0]) - rc_on = main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "on")]) - on_payload = _result_line(capsys.readouterr().out) - - assert rc_off == 0 and rc_on == 0 - # Flag-off carries exactly the five scoring fields. - assert set(off_payload) == set(REQUIRED_FIELDS) - assert ew.validate_eval_result_request(on_payload) == on_payload - on_score = ew.decode_score_f64be(on_payload["score_record"]["final"]["job_score_f64be"]) - assert on_score == off_payload["score"] - # The off-path serialization remains untouched and reconstructs byte-exactly. - assert format_benchmark_result_line(off_payload) == off_line - - -# --------------------------------------------------------------------------- # -# VAL-IMG-033: canonical_measurement is self-consistent across binding / -# envelope / pinnable allowlist record -# --------------------------------------------------------------------------- # -def _emit_attested_line() -> str: - spy, _events = _make_spy_provider() - return ar.emit_attested_benchmark_result( - benchmark_result=build_benchmark_result( - status="completed", score=0.5, resolved=1, total=2, reason_code=None - ), - canonical_measurement=dict(MEASUREMENT), - rtmr3=RTMR3, - agent_hash=AGENT_HASH, - task_ids=["task-b", "task-a"], - scores={"task-a": 1.0, "task-b": 0.0}, - validator_nonce=NONCE, - quote_provider=spy(), - manifest_sha256="1" * 64, - vm_config={"vcpu": 1}, - ) - - -def test_canonical_measurement_consistent_across_binding_envelope_and_record() -> None: - line = _emit_attested_line() - payload = json.loads(line[len(RESULT_LINE_PREFIX) :]) - - record = CanonicalMeasurement(**MEASUREMENT).as_dict() - binding_meas = payload[ar.ATTESTATION_BINDING_RESULT_KEY]["canonical_measurement"] - envelope_meas = payload[ar.EXECUTION_PROOF_RESULT_KEY]["attestation"]["measurement"] - - # The binding's canonical_measurement is exactly the pinnable allowlist record. - assert binding_meas == record - # The envelope's measurement block carries the same static registers (+ runtime rtmr3). - assert {field: envelope_meas[field] for field in CANONICAL_MEASUREMENT_FIELDS} == record - assert envelope_meas["rtmr3"] == RTMR3 - - # report_data was derived from exactly that measurement (self-consistent binding). - report_data_hex = payload[ar.EXECUTION_PROOF_RESULT_KEY]["attestation"]["report_data"] - expected = rd.report_data_hex( - canonical_measurement=record, - agent_hash=AGENT_HASH, - task_ids=["task-b", "task-a"], - scores_digest=rd.scores_digest({"task-a": 1.0, "task-b": 0.0}), - validator_nonce=NONCE, - ) - assert report_data_hex == expected - - -def test_report_data_changes_when_bound_canonical_measurement_changes() -> None: - common = dict( - agent_hash=AGENT_HASH, - task_ids=["task-a"], - scores_digest=rd.scores_digest({"task-a": 1.0}), - validator_nonce=NONCE, - ) - baseline = rd.report_data_hex(canonical_measurement=dict(MEASUREMENT), **common) - drifted_measurement = dict(MEASUREMENT) - drifted_measurement["compose_hash"] = "9" * 64 - drifted = rd.report_data_hex(canonical_measurement=drifted_measurement, **common) - assert baseline != drifted diff --git a/packages/challenges/agent-challenge/tests/test_keyrelease_durable_state.py b/packages/challenges/agent-challenge/tests/test_keyrelease_durable_state.py index dd7d52987..84894a899 100644 --- a/packages/challenges/agent-challenge/tests/test_keyrelease_durable_state.py +++ b/packages/challenges/agent-challenge/tests/test_keyrelease_durable_state.py @@ -93,9 +93,11 @@ def _settings() -> ChallengeSettings: + # T40/T41: dual flags permanently OFF; KR durable-state tests exercise + # keyrelease package mechanics under host-trust settings construction. return ChallengeSettings( - attested_review_enabled=True, - phala_attestation_enabled=True, + attested_review_enabled=False, + phala_attestation_enabled=False, eval_app_image_ref="registry.example/eval@sha256:" + "a" * 64, eval_app_compose_hash=COMPOSE_HASH, eval_app_identity="agent-challenge-eval-v1", @@ -233,6 +235,8 @@ async def _authorized_submission(database_session) -> int: artifact_path=f"/tmp/agent-{_SUBMISSION_SEQ}.zip", zip_sha256=hashlib.sha256(b"zip-" + salt).hexdigest(), zip_size_bytes=3, + # T41: plan bind requires package_tree_sha under host-trust KEEP gate. + package_tree_sha=hashlib.sha256(b"tree-" + salt).hexdigest(), raw_status="review_allowed", status="queued", effective_status="queued", diff --git a/packages/challenges/agent-challenge/tests/test_kr_host_pin.py b/packages/challenges/agent-challenge/tests/test_kr_host_pin.py deleted file mode 100644 index 16f7dc4b5..000000000 --- a/packages/challenges/agent-challenge/tests/test_kr_host_pin.py +++ /dev/null @@ -1,284 +0,0 @@ -"""VAL-ACLOCK-008/009/010: pin key_release_endpoint; legacy free KR URL not miner trust.""" - -from __future__ import annotations - -import copy -import hashlib -from typing import Any - -import pytest -from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey - -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.canonical.compose import ( - DEFAULT_ALLOWED_ENVS, - DEFAULT_KEY_RELEASE_RA_TLS_PORT, - is_validator_key_release_authority, - parse_key_release_authority, -) -from agent_challenge.keyrelease.client import KEY_RELEASE_URL_ENV -from agent_challenge.selfdeploy import eval as eval_deploy -from agent_challenge.selfdeploy.eval import ( - EVAL_ALLOWED_ENVS, - EVAL_REQUIRED_SECRET_ENVS, - EvalDeploymentError, - EvalDeploymentPlan, - encrypt_eval_secrets, -) - - -def _eval_plan(*, key_release_endpoint: str) -> dict[str, Any]: - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - return { - "schema_version": 1, - "eval_run_id": "eval-run-kr-pin", - "submission_id": "submission-001", - "submission_version": 1, - "authorizing_review_digest": "1" * 64, - "agent_hash": "a" * 64, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "task-a", - "image_ref": "registry.example/task@sha256:" + "d" * 64, - "task_config_sha256": "2" * 64, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "d" * 64, - "compose_hash": "c" * 64, - "app_identity": "agent-challenge-eval", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "3" * 64, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("3" * 64)).hexdigest(), - "measurement": { - "mrtd": "a1" * 48, - "rtmr0": "a2" * 48, - "rtmr1": "a3" * 48, - "rtmr2": "a4" * 48, - "os_image_hash": "a5" * 32, - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": key_release_endpoint, - "result_endpoint": "/evaluation/v1/runs/eval-run-kr-pin/result", - "key_release_nonce": "key-nonce-001", - "score_nonce": "score-nonce-001", - "run_token_sha256": "5" * 64, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - - -# --------------------------------------------------------------------------- # -# VAL-ACLOCK-008 — plan wire rejects evil free KR hosts / non-authority forms -# --------------------------------------------------------------------------- # -@pytest.mark.parametrize( - "endpoint", - [ - "https://evil.example/key-release", - "http://evil.example:8701/", - "https://validator-kr.example.invalid:8701", - "https://86.38.238.235:8701/release", - "evil.example", # missing port - "ftp://evil.example:8701", - "host/with/path:8701", - "user@evil.example:8701", - "ratls://evil.example:8701/extra", - "", - " ", - ], -) -def test_validate_eval_plan_rejects_evil_or_free_kr_endpoints(endpoint: str) -> None: - """VAL-ACLOCK-008: free HTTP(S)/junk KR endpoints fail closed on the plan wire.""" - - plan = _eval_plan(key_release_endpoint=endpoint) - with pytest.raises(ew.EvalWireError, match="key_release_endpoint"): - ew.validate_eval_plan(plan) - - -@pytest.mark.parametrize( - "endpoint", - [ - "https://evil.example/kr", - "http://127.0.0.1:8701", - "evil.example", - "not a host", - ], -) -def test_is_validator_key_release_authority_rejects_evil(endpoint: str) -> None: - assert is_validator_key_release_authority(endpoint) is False - assert parse_key_release_authority(endpoint) is None - - -# --------------------------------------------------------------------------- # -# VAL-ACLOCK-010 — honest validator RA-TLS / authority forms still validate -# --------------------------------------------------------------------------- # -@pytest.mark.parametrize( - "endpoint", - [ - "validator.example:8701", - "keyrelease.example:8701", - "86.38.238.235:8701", - "ratls://84.32.70.61:8701", - "tls://validator.example:8701", - "tcp://127.0.0.1:8701", - "validator.example:8700", # authority form (non-default port still host:port) - "ratls://validator.example:9443", - ], -) -def test_validate_eval_plan_accepts_honest_ra_tls_authority(endpoint: str) -> None: - """VAL-ACLOCK-010: honest validator KR / RA-TLS authority forms remain green.""" - - plan = _eval_plan(key_release_endpoint=endpoint) - validated = ew.validate_eval_plan(plan) - assert validated["key_release_endpoint"] == endpoint.strip() - parsed = parse_key_release_authority(endpoint) - assert parsed is not None - host, port = parsed - assert host - assert 1 <= port <= 65535 - - -def test_parse_key_release_authority_default_port_constant() -> None: - assert DEFAULT_KEY_RELEASE_RA_TLS_PORT == 8701 - assert parse_key_release_authority("validator.example:8701") == ( - "validator.example", - 8701, - ) - - -# --------------------------------------------------------------------------- # -# VAL-ACLOCK-009 — legacy free KR URL is not miner-authoritative in prod -# --------------------------------------------------------------------------- # -def test_legacy_free_key_release_url_name_not_miner_trust_root() -> None: - """VAL-ACLOCK-009: free KR URL name may remain for pin stability but is not trust root. - - Encrypt refuses free HTTP(S) / mismatched authorities; plan wire already - pins RA-TLS form. Prefer KEY_RELEASE_RA_TLS_HOST/PORT + plan endpoint. - """ - - assert KEY_RELEASE_URL_ENV == "CHALLENGE_PHALA_KEY_RELEASE_URL" - # Name retained so existing measure-time pin compose hashes stay stable. - assert KEY_RELEASE_URL_ENV in DEFAULT_ALLOWED_ENVS - assert KEY_RELEASE_URL_ENV in EVAL_ALLOWED_ENVS - assert "CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM" in DEFAULT_ALLOWED_ENVS - assert "EVAL_RUN_TOKEN" in EVAL_REQUIRED_SECRET_ENVS - - -def _deployment_plan_stub( - *, key_release_endpoint: str = "validator.example:8701" -) -> EvalDeploymentPlan: - private = X25519PrivateKey.generate() - public_hex = private.public_key().public_bytes_raw().hex() - plan = ew.validate_eval_plan(_eval_plan(key_release_endpoint=key_release_endpoint)) - return EvalDeploymentPlan( - plan=plan, - plan_sha256="b" * 64, - compose={"name": "stub"}, - compose_text="{}", - compose_hash="c" * 64, - app_identity="agent-challenge-eval", - image_ref=plan["eval_app"]["image_ref"], - kms_public_key_hex=public_hex, - kms_public_key_sha256=hashlib.sha256(bytes.fromhex(public_hex)).hexdigest(), - measurement=dict(plan["eval_app"]["measurement"]), - eval_run_id=plan["eval_run_id"], - eval_run_token="eval-run-token-honest", - instance_type="tdx.small", - ) - - -def test_encrypt_eval_secrets_refuses_legacy_free_http_key_release_url() -> None: - """VAL-ACLOCK-009: encrypt refuses miner free HTTP(S) KR URL values.""" - - dep = _deployment_plan_stub() - secrets = { - "CHALLENGE_PHALA_ATTESTATION_ENABLED": "1", - "CHALLENGE_PHALA_EVAL_PLAN": "{}", - "EVAL_RUN_TOKEN": dep.eval_run_token, - "LLM_COST_LIMIT": "1.0", - KEY_RELEASE_URL_ENV: "https://evil.example/key-release", - } - with pytest.raises(EvalDeploymentError, match="KEY_RELEASE|key_release|not miner"): - encrypt_eval_secrets(dep, secrets) - - -def test_encrypt_eval_secrets_refuses_free_url_mismatching_plan_authority() -> None: - """VAL-ACLOCK-009: free URL cannot override plan RA-TLS authority.""" - - dep = _deployment_plan_stub(key_release_endpoint="validator.example:8701") - secrets = { - "CHALLENGE_PHALA_ATTESTATION_ENABLED": "1", - "CHALLENGE_PHALA_EVAL_PLAN": "{}", - "EVAL_RUN_TOKEN": dep.eval_run_token, - "LLM_COST_LIMIT": "1.0", - KEY_RELEASE_URL_ENV: "evil.example:8701", - } - with pytest.raises(EvalDeploymentError, match="KEY_RELEASE|key_release|not miner"): - encrypt_eval_secrets(dep, secrets) - - # Matching authority form (redundant with plan) is accepted; not a free web URL. - secrets[KEY_RELEASE_URL_ENV] = "validator.example:8701" - encrypted = encrypt_eval_secrets(dep, secrets) - assert KEY_RELEASE_URL_ENV in encrypted.env_keys - - -def test_encrypt_eval_secrets_honest_path_without_free_kr_url() -> None: - """VAL-ACLOCK-010: honest encrypt path green without free KR URL env.""" - - dep = _deployment_plan_stub(key_release_endpoint="ratls://84.32.70.61:8701") - secrets = { - "CHALLENGE_PHALA_ATTESTATION_ENABLED": "1", - "CHALLENGE_PHALA_EVAL_PLAN": '{"k":1}', - "EVAL_RUN_TOKEN": dep.eval_run_token, - "LLM_COST_LIMIT": "2.5", - "OPENROUTER_API_KEY": "sk-or-v1-test-not-real", - "CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM": ( - "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----" - ), - } - encrypted = encrypt_eval_secrets(dep, secrets) - assert set(encrypted.env_keys) <= set(EVAL_ALLOWED_ENVS) - assert KEY_RELEASE_URL_ENV not in encrypted.env_keys - assert encrypted.ciphertext - - -def test_measure_time_placeholder_not_accepted_as_signed_plan_endpoint() -> None: - """Measure-time HTTPS placeholder is for compose pin only, not plan trust root.""" - - plan = _eval_plan( - key_release_endpoint=eval_deploy.MEASURE_TIME_EVAL_KEY_RELEASE_PLACEHOLDER - ) - with pytest.raises(ew.EvalWireError, match="key_release_endpoint"): - ew.validate_eval_plan(plan) - - -def test_eval_plan_normalizes_stripped_authority() -> None: - plan = _eval_plan(key_release_endpoint=" validator.example:8701 ") - validated = ew.validate_eval_plan(plan) - assert validated["key_release_endpoint"] == "validator.example:8701" - - -def test_validate_eval_plan_fixture_baseline_still_closed() -> None: - """Regression: closed plan shape with honest KR remains round-trip stable.""" - - plan = _eval_plan(key_release_endpoint="keyrelease.example:8701") - assert ew.validate_eval_plan(plan) == { - **plan, - "key_release_endpoint": "keyrelease.example:8701", - } - crossed = copy.deepcopy(plan) - crossed["score_nonce"] = crossed["key_release_nonce"] - with pytest.raises(ew.EvalWireError): - ew.validate_eval_plan(crossed) diff --git a/packages/challenges/agent-challenge/tests/test_live_registry.py b/packages/challenges/agent-challenge/tests/test_live_registry.py deleted file mode 100644 index e612fda75..000000000 --- a/packages/challenges/agent-challenge/tests/test_live_registry.py +++ /dev/null @@ -1,208 +0,0 @@ -"""Behavioral tests for the live-subset registry-ref side manifest + resolver. - -The frozen golden manifest (``golden/dataset-digest.json``) pins each -Terminal-Bench task by a *bare content digest* (``harbor_registry_ref = -"sha256:<64hex>"``) with NO repository, so an in-CVM DooD orchestrator cannot -``docker pull`` it. For a live smoke E2E a small deterministic subset of task -images is published to a pullable, digest-pinned GHCR ref and recorded in a -SEPARATE side manifest (``golden/live-registry-refs.json``) so the frozen -content digests / canonical measurement stay byte-identical. - -These tests pin: - * the resolver is OPT-IN and fail-closed (no manifest configured -> no refs, so - offline/flag-off behavior is byte-identical); - * every published ref is repository-qualified AND digest-pinned (a bare - ``sha256:...`` content digest or a floating tag is rejected); - * D6: live-registry parse rejects Hub registry hosts and the retired personal - namespace; shipped golden is GHCR-only; - * the shipped side manifest is a strict subset of the frozen golden tasks and - never mutates ``dataset-digest.json``. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from agent_challenge.canonical import compose as c -from agent_challenge.canonical import live_registry as lr - -REPO_ROOT = Path(__file__).resolve().parents[1] -GOLDEN_DIR = REPO_ROOT / "golden" -GOLDEN_MANIFEST = GOLDEN_DIR / "dataset-digest.json" -LIVE_MANIFEST = GOLDEN_DIR / lr.LIVE_REGISTRY_FILENAME - -# The frozen golden dataset invariants this feature must NOT perturb. -FROZEN_CANONICAL_CONTENT_DIGEST = "8da006d76bcf59c2af3f36ed4420192d3930bda43683f32d80013a6ee5e7e02d" -FROZEN_TASK_COUNT = 89 - -_GOOD_REF = "ghcr.io/baseintelligence/agent-challenge-tb21-x@sha256:" + ("a" * 64) - -# Forbidden shipping hosts/namespaces (D6). Kept as test inputs only. -_HUB_REF = "docker.io/library/x@sha256:" + ("a" * 64) -_RETIRED_NS_REF = "docker.io/mathiiss/agent-challenge-tb21-x@sha256:" + ("a" * 64) -_OTHER_REG_REF = "quay.io/example/x@sha256:" + ("a" * 64) - - -# --------------------------------------------------------------------------- # -# ref validation: repository-qualified AND digest-pinned -# --------------------------------------------------------------------------- # -def test_pullable_ref_accepts_repo_digest(): - assert lr.is_pullable_ref(_GOOD_REF) - assert lr.assert_pullable_ref(_GOOD_REF) == _GOOD_REF - assert lr.assert_live_registry_ref(_GOOD_REF) == _GOOD_REF - - -@pytest.mark.parametrize( - "bad", - [ - "sha256:" + ("a" * 64), # bare content digest (the golden behavior) - not pullable - "ghcr.io/baseintelligence/x:latest", # floating tag, not digest-pinned - "ghcr.io/baseintelligence/x@sha256:" + ("a" * 63), # short digest - "plainname@sha256:" + ("a" * 64), # no repository/namespace ('/') - "", - 123, - ], -) -def test_pullable_ref_rejects_non_pullable(bad): - assert not lr.is_pullable_ref(bad) - with pytest.raises(lr.LiveRegistryError): - lr.assert_pullable_ref(bad) - - -# --------------------------------------------------------------------------- # -# opt-in, fail-closed resolution (byte-identical offline) -# --------------------------------------------------------------------------- # -def test_resolve_refs_is_empty_when_unconfigured(): - # No explicit path and no env => no live refs (legacy behavior preserved). - assert lr.resolve_live_registry_refs() == {} - assert lr.resolve_live_registry_refs(env={}) == {} - assert lr.resolve_live_registry_refs(env={"UNRELATED": "1"}) == {} - - -def test_resolve_refs_from_explicit_path(tmp_path): - manifest = { - "schema": lr.LIVE_REGISTRY_SCHEMA, - "tasks": {"foo": {"registry_ref": _GOOD_REF}}, - } - p = tmp_path / "live.json" - p.write_text(json.dumps(manifest), encoding="utf-8") - refs = lr.resolve_live_registry_refs(path=p) - assert refs == {"foo": _GOOD_REF} - - -def test_resolve_refs_from_env(tmp_path): - manifest = {"tasks": {"foo": {"registry_ref": _GOOD_REF}}} - p = tmp_path / "live.json" - p.write_text(json.dumps(manifest), encoding="utf-8") - refs = lr.resolve_live_registry_refs(env={lr.LIVE_REGISTRY_ENV: str(p)}) - assert refs == {"foo": _GOOD_REF} - - -def test_configured_but_broken_manifest_raises(tmp_path): - p = tmp_path / "missing.json" - with pytest.raises(lr.LiveRegistryError): - lr.resolve_live_registry_refs(path=p) - - -def test_parse_rejects_bare_digest_ref(): - with pytest.raises(lr.LiveRegistryError): - lr.parse_live_registry({"tasks": {"foo": {"registry_ref": "sha256:" + "a" * 64}}}) - - -@pytest.mark.parametrize( - "bad", - [ - _HUB_REF, - _RETIRED_NS_REF, - _OTHER_REG_REF, - "ghcr.io/mathiiss/x@sha256:" + ("a" * 64), - ], -) -def test_live_registry_ref_rejects_non_ghcr_shipping_hosts(bad): - """D6: live shipping refs must be ghcr.io and must not use retired namespaces.""" - with pytest.raises(lr.LiveRegistryError): - lr.assert_live_registry_ref(bad) - with pytest.raises(lr.LiveRegistryError): - lr.parse_live_registry({"tasks": {"foo": {"registry_ref": bad}}}) - with pytest.raises(lr.LiveRegistryError): - lr.parse_live_registry({"tasks": {}, "orchestrator_image": bad}) - - -# --------------------------------------------------------------------------- # -# task_id resolution (bare + dataset-prefixed) -# --------------------------------------------------------------------------- # -def test_resolve_task_image_bare_and_prefixed(): - reg = lr.parse_live_registry({"tasks": {"foo": {"registry_ref": _GOOD_REF}}}) - assert reg.resolve_task_image("foo") == _GOOD_REF - # A dataset-prefixed id resolves to the same bare-keyed entry. - assert reg.resolve_task_image("terminal-bench/foo") == _GOOD_REF - # Unknown task -> None (fail-closed to legacy behavior). - assert reg.resolve_task_image("does-not-exist") is None - - -def test_string_ref_shorthand_is_accepted(): - reg = lr.parse_live_registry({"tasks": {"foo": _GOOD_REF}}) - assert reg.resolve_task_image("foo") == _GOOD_REF - - -# --------------------------------------------------------------------------- # -# the SHIPPED side manifest -# --------------------------------------------------------------------------- # -def test_shipped_live_manifest_is_valid_and_pullable(): - reg = lr.load_live_registry(LIVE_MANIFEST) - assert reg.task_refs, "shipped live manifest has no task refs" - for task_id, ref in reg.task_refs.items(): - assert lr.assert_live_registry_ref(ref) == ref, (task_id, ref) - - -def test_shipped_orchestrator_image_is_digest_pinned(): - reg = lr.load_live_registry(LIVE_MANIFEST) - assert reg.orchestrator_image is not None - assert lr.assert_live_registry_ref(reg.orchestrator_image) == reg.orchestrator_image - # The deploy path's digest-pin guard accepts it (no bare tag). - assert c.assert_digest_pinned(reg.orchestrator_image) == reg.orchestrator_image - assert reg.orchestrator_image.startswith( - "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" - ) - digest = reg.orchestrator_image.rsplit("@", 1)[-1] - assert digest.startswith("sha256:") and len(digest) == len("sha256:") + 64 - - -def test_shipped_live_manifest_has_no_hub_or_retired_namespace(): - """Golden live-registry product pin must not mention Hub host or retired ns.""" - text = LIVE_MANIFEST.read_text(encoding="utf-8") - assert "docker.io" not in text - assert "mathiiss" not in text - live = json.loads(text) - assert live["namespace"] == "ghcr.io/baseintelligence" - assert live["orchestrator_image"].startswith("ghcr.io/baseintelligence/") - for task_id, entry in live["tasks"].items(): - assert entry["registry_ref"].startswith("ghcr.io/baseintelligence/"), task_id - - -def test_shipped_live_subset_is_subset_of_golden_and_small(): - golden = json.loads(GOLDEN_MANIFEST.read_text(encoding="utf-8")) - golden_tasks = set(golden["tasks"]) - reg = lr.load_live_registry(LIVE_MANIFEST) - assert set(reg.task_refs) <= golden_tasks - assert 1 <= len(reg.task_refs) <= 5 # a small deterministic subset - - -def test_shipped_live_manifest_content_digests_match_golden(): - golden = json.loads(GOLDEN_MANIFEST.read_text(encoding="utf-8")) - live = json.loads(LIVE_MANIFEST.read_text(encoding="utf-8")) - for task_id, entry in live["tasks"].items(): - recorded = entry.get("content_digest_sha256") - assert recorded is not None, task_id - assert recorded == golden["tasks"][task_id]["content_digest_sha256"], task_id - - -def test_golden_dataset_digest_is_unperturbed(): - # Discriminator: this feature must never mutate the frozen golden manifest. - golden = json.loads(GOLDEN_MANIFEST.read_text(encoding="utf-8")) - assert golden["canonical_content_digest_sha256"] == FROZEN_CANONICAL_CONTENT_DIGEST - assert golden["task_count"] == FROZEN_TASK_COUNT - assert len(golden["tasks"]) == FROZEN_TASK_COUNT diff --git a/packages/challenges/agent-challenge/tests/test_miner_env_lock.py b/packages/challenges/agent-challenge/tests/test_miner_env_lock.py index be4773bff..4ef4cf3b6 100644 --- a/packages/challenges/agent-challenge/tests/test_miner_env_lock.py +++ b/packages/challenges/agent-challenge/tests/test_miner_env_lock.py @@ -269,12 +269,7 @@ def test_terminal_bench_env_still_blocks_control_path_overrides() -> None: def test_prior_review_joinbase_pin_constant_still_present() -> None: from pathlib import Path - runtime = ( - Path(__file__).resolve().parents[1] - / "docker" - / "review" - / "review_runtime.py" - ) + runtime = Path(__file__).resolve().parents[1] / "docker" / "review" / "review_runtime.py" text = runtime.read_text(encoding="utf-8") assert re.search(r"chain\.joinbase\.ai/challenges/agent-challenge", text) diff --git a/packages/challenges/agent-challenge/tests/test_no_phala_mode.py b/packages/challenges/agent-challenge/tests/test_no_phala_mode.py index 062210e94..18f1d91c7 100644 --- a/packages/challenges/agent-challenge/tests/test_no_phala_mode.py +++ b/packages/challenges/agent-challenge/tests/test_no_phala_mode.py @@ -15,7 +15,6 @@ import hashlib import json from pathlib import Path -from unittest.mock import MagicMock import pytest @@ -27,7 +26,6 @@ EXECUTION_MODE_NO_PHALA_HOST, NO_PHALA_ENV, ArtifactProvenanceError, - NoPhalaModeError, assert_envelope_not_attested, assert_no_phala_compatible, build_guest_artifact_proof, @@ -154,7 +152,12 @@ def test_contradiction_both_attestation_on_and_no_phala() -> None: def test_settings_contradiction_both_on(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv(NO_PHALA_ENV, "true") - with pytest.raises(ValueError, match="NO_PHALA"): + from pydantic import ValidationError + + with pytest.raises( + (ValueError, ValidationError), + match="Phala TEE dual flags|NO_PHALA|unattested", + ): ChallengeSettings( phala_attestation_enabled=True, attested_review_enabled=True, @@ -167,34 +170,27 @@ def test_settings_contradiction_both_on(monkeypatch: pytest.MonkeyPatch) -> None def test_phala_client_refused_when_no_phala(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(NO_PHALA_ENV, "true") - monkeypatch.setenv("PHALA_API_KEY", "test-key-not-a-secret-for-unit") - from agent_challenge.selfdeploy.phala import PhalaCloudClient + """T40: Phala selfdeploy module removed entirely.""" - with pytest.raises(NoPhalaModeError, match="NO_PHALA"): - PhalaCloudClient(api_key="test-key-not-a-secret-for-unit") + monkeypatch.setenv(NO_PHALA_ENV, "true") + with pytest.raises(ModuleNotFoundError): + __import__("agent_challenge.selfdeploy.phala") def test_eval_deploy_refused_when_no_phala(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(NO_PHALA_ENV, "true") - from agent_challenge.selfdeploy.eval import HttpEvalPhalaDeployment + """T40: Phala selfdeploy module removed entirely.""" - api = MagicMock() - dep = HttpEvalPhalaDeployment(api) - with pytest.raises(NoPhalaModeError): - dep.deploy(MagicMock(), MagicMock()) - api.post.assert_not_called() + monkeypatch.setenv(NO_PHALA_ENV, "true") + with pytest.raises(ModuleNotFoundError): + __import__("agent_challenge.selfdeploy.eval") def test_review_deploy_refused_when_no_phala(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(NO_PHALA_ENV, "true") - from agent_challenge.selfdeploy.review import HttpReviewPhalaDeployment + """T40: Phala selfdeploy module removed entirely.""" - api = MagicMock() - dep = HttpReviewPhalaDeployment(api) - with pytest.raises(NoPhalaModeError): - dep.deploy(MagicMock(), MagicMock()) - api.post.assert_not_called() + monkeypatch.setenv(NO_PHALA_ENV, "true") + with pytest.raises(ModuleNotFoundError): + __import__("agent_challenge.selfdeploy.review") def test_mode_on_emit_marks_unattested( @@ -268,9 +264,7 @@ def test_mark_ignores_caller_attested_true_kw() -> None: def test_artifact_proof_enforces_triple_match() -> None: h = "ab" * 32 - proof = build_guest_artifact_proof( - expected_hash=h, download_hash=h, executed_hash=h - ) + proof = build_guest_artifact_proof(expected_hash=h, download_hash=h, executed_hash=h) assert proof["match"] is True with pytest.raises(ArtifactProvenanceError, match="mismatch"): build_guest_artifact_proof( diff --git a/packages/challenges/agent-challenge/tests/test_no_phala_pipeline.py b/packages/challenges/agent-challenge/tests/test_no_phala_pipeline.py index a65a51da6..f32620129 100644 --- a/packages/challenges/agent-challenge/tests/test_no_phala_pipeline.py +++ b/packages/challenges/agent-challenge/tests/test_no_phala_pipeline.py @@ -243,9 +243,7 @@ async def test_no_phala_analysis_allow_enqueues_tb_job( _enable_no_phala(monkeypatch) configure_master(monkeypatch, tmp_path) - response = await submit_agent( - client, {"agent.py": "def solve(value):\n return value + 1\n"} - ) + response = await submit_agent(client, {"agent.py": "def solve(value):\n return value + 1\n"}) assert response.status_code in {200, 201}, response.text async with database_session() as session: diff --git a/packages/challenges/agent-challenge/tests/test_openrouter_review_provider.py b/packages/challenges/agent-challenge/tests/test_openrouter_review_provider.py index 2a57f94aa..25fe9bdf3 100644 --- a/packages/challenges/agent-challenge/tests/test_openrouter_review_provider.py +++ b/packages/challenges/agent-challenge/tests/test_openrouter_review_provider.py @@ -101,9 +101,7 @@ def fake_post(url, *, headers, json, timeout): # noqa: A002 captured["json"] = json return Response() - monkeypatch.setattr( - "agent_challenge.analyzer.openrouter_review_provider.httpx.post", fake_post - ) + monkeypatch.setattr("agent_challenge.analyzer.openrouter_review_provider.httpx.post", fake_post) provider = OpenRouterReviewProvider(api_key="sk-test", model_name=DEFAULT_OPENROUTER_MODEL) response = provider.complete( messages=[{"role": "user", "content": "hi"}], @@ -157,9 +155,7 @@ def json(self) -> dict[str, Any]: def boom(*a, **k): raise httpx.ReadTimeout("slow") - monkeypatch.setattr( - "agent_challenge.analyzer.openrouter_review_provider.httpx.post", boom - ) + monkeypatch.setattr("agent_challenge.analyzer.openrouter_review_provider.httpx.post", boom) with pytest.raises(LlmProviderTimeout): provider.complete(messages=[], tools=[], tool_choice="auto", timeout_seconds=1) diff --git a/packages/challenges/agent-challenge/tests/test_ordered_selfdeploy_cli.py b/packages/challenges/agent-challenge/tests/test_ordered_selfdeploy_cli.py deleted file mode 100644 index 468e62239..000000000 --- a/packages/challenges/agent-challenge/tests/test_ordered_selfdeploy_cli.py +++ /dev/null @@ -1,237 +0,0 @@ -"""Discriminator tests for the ordered review/eval self-deploy CLI.""" - -from __future__ import annotations - -import copy -import hashlib -import json - -import pytest - -from agent_challenge.canonical import eval_wire -from agent_challenge.selfdeploy import eval as eval_deploy -from agent_challenge.selfdeploy import lifecycle - -REVIEW_IMAGE = "registry.example/review@sha256:" + "a" * 64 -EVAL_IMAGE = "registry.example/eval@sha256:" + "b" * 64 -PUBLIC_KEY = "c" * 64 -MEASUREMENT = { - "mrtd": "01" * 48, - "rtmr0": "02" * 48, - "rtmr1": "03" * 48, - "rtmr2": "04" * 48, - "os_image_hash": "05" * 32, - "key_provider": "validator-kms", - "vm_shape": "tdx-small", -} - - -def _review_assignment() -> tuple[dict[str, object], str]: - from agent_challenge.review.compose import ( - generate_review_app_compose, - review_app_compose_hash, - ) - from agent_challenge.review.schemas import ReviewInputConfig, build_review_assignment - - compose = generate_review_app_compose( - review_image=REVIEW_IMAGE, - app_identity="review-v1", - ) - config = ReviewInputConfig( - image_ref=REVIEW_IMAGE, - compose_hash=review_app_compose_hash(compose), - app_identity="review-v1", - kms_public_key_hex=PUBLIC_KEY, - measurement=MEASUREMENT, - ) - token = "review-token-sentinel" - assignment, _, _ = build_review_assignment( - session_id="session-1", - assignment_id="assignment-1", - attempt=1, - submission_id="1", - artifact={ - "agent_hash": "10" * 32, - "zip_sha256": "20" * 32, - "zip_size_bytes": 1, - "manifest_sha256": "30" * 32, - "manifest_entries_sha256": "40" * 32, - "fetch_path": "/review/v1/assignments/assignment-1/artifact", - }, - rules_snapshot_sha256_value="50" * 32, - rules_revision_id="rules-v1", - review_nonce="nonce-review", - issued_at_ms=1, - expires_at_ms=2, - session_token_sha256=hashlib.sha256(token.encode()).hexdigest(), - config=config, - ) - return assignment, token - - -def _eval_plan() -> dict[str, object]: - from agent_challenge.canonical.compose import ( - generate_app_compose, - render_app_compose, - ) - - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - compose = generate_app_compose( - orchestrator_image=EVAL_IMAGE, - name="eval-v1", - key_release_url="validator.example:8701", - allowed_envs=eval_deploy.EVAL_ALLOWED_ENVS, - ) - compose_hash = hashlib.sha256(render_app_compose(compose).encode()).hexdigest() - plan = { - "schema_version": 1, - "eval_run_id": "eval-1", - "submission_id": "1", - "submission_version": 1, - "authorizing_review_digest": "d" * 64, - "agent_hash": "e" * 64, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "task-1", - "image_ref": "registry.example/task@sha256:" + "f" * 64, - "task_config_sha256": "1" * 64, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": eval_wire.scoring_policy_digest(policy), - "eval_app": { - "image_ref": EVAL_IMAGE, - "compose_hash": compose_hash, - "app_identity": "eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": PUBLIC_KEY, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex(PUBLIC_KEY)).hexdigest(), - "measurement": MEASUREMENT, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-1/result", - "key_release_nonce": "key-release-nonce", - "score_nonce": "score-nonce", - "run_token_sha256": "3" * 64, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - return eval_wire.validate_eval_plan(plan) - - -def test_eval_prepare_requires_validator_review_authorization_before_deployment(): - plan = _eval_plan() - plan["authorizing_review_digest"] = "" - with pytest.raises(eval_deploy.EvalDeploymentError, match="Eval plan"): - eval_deploy.build_eval_deployment_plan( - { - "schema_version": 1, - "plan": plan, - "plan_sha256": hashlib.sha256(eval_wire.canonical_json_v1(plan)).hexdigest(), - "secret_delivery": None, - } - ) - - -def test_eval_prepare_rejects_plan_digest_or_app_mutation_before_phala_create(): - plan = _eval_plan() - token = "run-token" - plan["run_token_sha256"] = hashlib.sha256(token.encode()).hexdigest() - wrapper = { - "schema_version": 1, - "plan": plan, - "plan_sha256": hashlib.sha256(eval_wire.canonical_json_v1(plan)).hexdigest(), - "secret_delivery": {"env_key": "EVAL_RUN_TOKEN", "token": token}, - } - deployment = eval_deploy.build_eval_deployment_plan(wrapper) - assert deployment.image_ref == EVAL_IMAGE - tampered = copy.deepcopy(wrapper) - tampered["plan"]["eval_app"]["image_ref"] = REVIEW_IMAGE - with pytest.raises(eval_deploy.EvalDeploymentError): - eval_deploy.build_eval_deployment_plan(tampered) - - -def test_eval_encrypted_env_contains_only_scoped_capabilities_and_is_transmitted(): - raw_plan = _eval_plan() - token = "run-token" - raw_plan["run_token_sha256"] = hashlib.sha256(token.encode()).hexdigest() - plan = eval_deploy.build_eval_deployment_plan( - { - "schema_version": 1, - "plan": raw_plan, - "plan_sha256": hashlib.sha256(eval_wire.canonical_json_v1(raw_plan)).hexdigest(), - "secret_delivery": {"env_key": "EVAL_RUN_TOKEN", "token": token}, - }, - ) - encrypted = eval_deploy.encrypt_eval_secrets( - plan, - { - "EVAL_RUN_TOKEN": "run-token", - "LLM_COST_LIMIT": "1.00", - "CHALLENGE_PHALA_ATTESTATION_ENABLED": "1", - "CHALLENGE_PHALA_EVAL_PLAN": json.dumps(plan.plan), - "CHALLENGE_PHALA_AGENT_HASH": plan.plan["agent_hash"], - "CHALLENGE_PHALA_CANONICAL_MEASUREMENT": json.dumps( - { - "mrtd": plan.measurement["mrtd"], - "rtmr0": plan.measurement["rtmr0"], - "rtmr1": plan.measurement["rtmr1"], - "rtmr2": plan.measurement["rtmr2"], - "compose_hash": plan.compose_hash, - "os_image_hash": plan.measurement["os_image_hash"], - } - ), - "CHALLENGE_PHALA_VALIDATOR_NONCE": plan.plan["score_nonce"], - }, - ) - assert encrypted.ciphertext - assert "run-token" not in repr(encrypted) - assert set(encrypted.env_keys) == { - "EVAL_RUN_TOKEN", - "CHALLENGE_PHALA_ATTESTATION_ENABLED", - "CHALLENGE_PHALA_AGENT_HASH", - "CHALLENGE_PHALA_CANONICAL_MEASUREMENT", - "CHALLENGE_PHALA_EVAL_PLAN", - "CHALLENGE_PHALA_VALIDATOR_NONCE", - "LLM_COST_LIMIT", - } - # VAL-ACAT-013: Base gateway secrets are neither required nor allowed. - assert "BASE_GATEWAY_TOKEN" not in encrypted.env_keys - assert "BASE_LLM_GATEWAY_URL" not in encrypted.env_keys - # Production RA-TLS host/port/mTLS path names are provisioned in measured - # compose static env, not as encrypted capability bytes. - compose_text = plan.compose_text - assert "KEY_RELEASE_RA_TLS_HOST=validator.example" in compose_text - assert "KEY_RELEASE_RA_TLS_PORT=8701" in compose_text - - -def test_lifecycle_budget_counts_review_and_eval_together(): - # Compute + default stage disks (20 GB each here) over 100h. - # CPU: 0.058 * 200 = 11.6; disk: 0.000139 * 20 * 200 = 0.556 → 12.156 - estimate = lifecycle.projected_lifecycle_cost_usd( - review_instance_type="tdx.small", - eval_instance_type="tdx.small", - review_runtime_hours=100, - eval_runtime_hours=100, - review_disk_size_gb=20, - eval_disk_size_gb=20, - ) - assert estimate == pytest.approx(12.156) - with pytest.raises(lifecycle.LifecycleBudgetError): - lifecycle.validate_lifecycle_budget( - review_instance_type="tdx.small", - eval_instance_type="tdx.xlarge", - review_runtime_hours=100, - eval_runtime_hours=100, - review_disk_size_gb=20, - eval_disk_size_gb=100, - money_cap_usd=20, - ) diff --git a/packages/challenges/agent-challenge/tests/test_os_image_hash_identity_reconcile.py b/packages/challenges/agent-challenge/tests/test_os_image_hash_identity_reconcile.py deleted file mode 100644 index de1ee92b7..000000000 --- a/packages/challenges/agent-challenge/tests/test_os_image_hash_identity_reconcile.py +++ /dev/null @@ -1,290 +0,0 @@ -"""Product single OS identity: guest bind + Phala provision (residual bd369a vs 5c6d). - -Live residual after tip b2f7ce76 (sub23 / eval-1task-litellm-speed): - -* Guest ``quote_measurement_mismatch`` diag=os - actual_prefix=5c6d8f757e3adb05 expected_prefix=bd369a8c2f9edb2b -* Phala CVM attestation tcb registers are the offline dstack-0.5.9 MRTD/RTMRs -* Product ``os_image_hash_from_registers(MRTD, RTMR1, RTMR2)`` = 5c6d8f75… (matches actual) -* Offline pin / dstack-mr-product injected catalog ``mr_image`` = bd369a8c… (digest.txt) -* Honest repin of assignment.os_image_hash to 5c6d… makes dry IN-LIST, but Phala - provision still returns catalog bd369a… → product previously failed closed: - ``Phala provision os_image_hash mismatches signed assignment measurement`` - -Product Mode B (this module): - -A) Dual semantics are documented with the residual vector (bd369a catalog ≠ 5c6d formula). -B) Sealed ``assignment.measurement.os_image_hash`` is the **product formula** identity - (sha256 of registers) used by guest review + keyrelease + allowlist. -C) Phala provision must accept that seal: when completed measurement is product-formula - consistent with its own MRTD/RTMR1/RTMR2, provision's separate dstack catalog - ``os_image_hash`` is **not** overloaded against the product field. Optionally it - may still be bound to an allowlisted ``dstack_mr_image`` catalog digest. -D) No invent MRTD / allow / KR; no product OpenRouter limiter. -""" - -from __future__ import annotations - -import hashlib -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from agent_challenge.canonical import measurement as m -from agent_challenge.keyrelease.quote import os_image_hash_from_registers -from agent_challenge.selfdeploy import eval as eval_deploy -from agent_challenge.selfdeploy import review as review_deploy - -# Residual vector (honest dstack-0.5.9 / tdx.small; no invent). -# Source: evidence/ac-attestation/e2e-live-v7/eval-1task-litellm-speed/ -RESIDUAL_MRTD = ( - "f06dfda6dce1cf904d4e2bab1dc370634cf95cefa2ceb2de2eee127c93826980" - "90d7a4a13e14c536ec6c9c3c8fa87077" -) -RESIDUAL_RTMR0 = ( - "68102e7b524af310f7b7d426ce75481e36c40f5d513a9009c046e9d37e31551f" - "0134d954b496a3357fd61d03f07ffe96" -) -RESIDUAL_RTMR1 = ( - "07e6f51aa763abfe75c3ddfbf4f425fe3f0ceff66d807a75e049303dce9addf6" - "8e7218729bd419638af63a370f65878c" -) -RESIDUAL_RTMR2 = ( - "df67e467e60edc1737bcf8e682d48131bfb427f523226aa7f197a7608e9b3784" - "783fa759ef5b28191fa12f9ddb36b858" -) -# Offline dstack-mr-product / digest.txt catalog identity (NOT product formula). -RESIDUAL_DSTACK_MR_IMAGE = "bd369a8c2f9edb2b52dad48ac8e0b32dde5f1337c423a506b48d07403a7d8033" -# Product + guest quote identity. -RESIDUAL_PRODUCT_OS = "5c6d8f757e3adb0563efc809710076a631442db3b4de02ad32d33fe1994721e0" - - -def test_residual_vector_documents_dual_os_semantics() -> None: - """bd369a catalog mr_image ≠ sha256(registers)=5c6d on identical MRTD/RTMR1/RTMR2.""" - - product = os_image_hash_from_registers(RESIDUAL_MRTD, RESIDUAL_RTMR1, RESIDUAL_RTMR2) - assert product == RESIDUAL_PRODUCT_OS - assert product.startswith("5c6d8f757e3adb05") - assert RESIDUAL_DSTACK_MR_IMAGE.startswith("bd369a8c2f9edb2b") - assert product != RESIDUAL_DSTACK_MR_IMAGE - # Formula is literally SHA-256 of the three register *bytes* (not hex text). - raw = ( - bytes.fromhex(RESIDUAL_MRTD) + bytes.fromhex(RESIDUAL_RTMR1) + bytes.fromhex(RESIDUAL_RTMR2) - ) - assert hashlib.sha256(raw).hexdigest() == RESIDUAL_PRODUCT_OS - - -def test_compute_image_measurement_seals_product_formula_not_catalog_mr_image( - monkeypatch, tmp_path -) -> None: - """Offline dstack-mr may emit catalog digest as mr_image; product pins formula.""" - - metadata = tmp_path / "metadata.json" - metadata.write_text("{}") - - class _Proc: - returncode = 0 - # Same residual registers; mr_image is the *wrong* field for product pin. - stdout = __import__("json").dumps( - { - "mrtd": RESIDUAL_MRTD, - "rtmr0": RESIDUAL_RTMR0, - "rtmr1": RESIDUAL_RTMR1, - "rtmr2": RESIDUAL_RTMR2, - "mr_image": RESIDUAL_DSTACK_MR_IMAGE, - } - ) - stderr = "" - - monkeypatch.setattr(m.subprocess, "run", lambda *a, **k: _Proc()) - image = m.compute_image_measurement(metadata, cpu=1, memory="2G", dstack_mr_bin="dstack-mr") - assert image.os_image_hash == RESIDUAL_PRODUCT_OS - assert image.os_image_hash != RESIDUAL_DSTACK_MR_IMAGE - # Catalog digest may be retained separately for provision binding, never overloading. - assert getattr(image, "dstack_mr_image", None) == RESIDUAL_DSTACK_MR_IMAGE - - -def test_compute_image_measurement_derives_formula_when_mr_image_absent( - monkeypatch, tmp_path -) -> None: - """Rust dstack-mr may omit mr_image; product still seals register formula.""" - - metadata = tmp_path / "metadata.json" - metadata.write_text("{}") - - class _Proc: - returncode = 0 - stdout = __import__("json").dumps( - { - "mrtd": RESIDUAL_MRTD, - "rtmr0": RESIDUAL_RTMR0, - "rtmr1": RESIDUAL_RTMR1, - "rtmr2": RESIDUAL_RTMR2, - } - ) - stderr = "" - - monkeypatch.setattr(m.subprocess, "run", lambda *a, **k: _Proc()) - image = m.compute_image_measurement(metadata, cpu=1, memory="2G", dstack_mr_bin="dstack-mr") - assert image.os_image_hash == RESIDUAL_PRODUCT_OS - assert getattr(image, "dstack_mr_image", None) in {None, ""} - - -def _review_plan_measurement( - *, os_image_hash: str, dstack_mr_image: str | None = None -) -> dict[str, str]: - out: dict[str, str] = { - "mrtd": RESIDUAL_MRTD, - "rtmr0": RESIDUAL_RTMR0, - "rtmr1": RESIDUAL_RTMR1, - "rtmr2": RESIDUAL_RTMR2, - "os_image_hash": os_image_hash, - "key_provider": "phala", - "vm_shape": "tdx.small", - } - if dstack_mr_image is not None: - out["dstack_mr_image"] = dstack_mr_image - return out - - -def _stub_review_plan(measurement: dict[str, str]) -> Any: - plan = MagicMock() - plan.compose_hash = "a" * 64 - plan.app_identity = "b" * 40 - plan.kms_public_key_hex = "c" * 64 - plan.measurement = measurement - return plan - - -def test_review_provision_accepts_catalog_os_when_seal_is_product_formula() -> None: - """After honest 5c6d repin, provision may still return bd369a catalog — accept.""" - - plan = _stub_review_plan( - _review_plan_measurement( - os_image_hash=RESIDUAL_PRODUCT_OS, - dstack_mr_image=RESIDUAL_DSTACK_MR_IMAGE, - ) - ) - provision = { - "compose_hash": plan.compose_hash, - "app_id": plan.app_identity, - "app_env_encrypt_pubkey": plan.kms_public_key_hex, - # Phala catalog field (residual) — NOT product formula. - "os_image_hash": RESIDUAL_DSTACK_MR_IMAGE, - } - # Must not raise the residual provision mismatch error. - review_deploy.HttpReviewPhalaDeployment._verify_provision_response(plan, provision) - - -def test_review_provision_accepts_catalog_os_without_explicit_dstack_mr_image() -> None: - """Product seal alone is enough; catalog OS is observed, not overloaded onto seal.""" - - plan = _stub_review_plan(_review_plan_measurement(os_image_hash=RESIDUAL_PRODUCT_OS)) - provision = { - "compose_hash": plan.compose_hash, - "app_id": plan.app_identity, - "app_env_encrypt_pubkey": plan.kms_public_key_hex, - "os_image_hash": RESIDUAL_DSTACK_MR_IMAGE, - } - review_deploy.HttpReviewPhalaDeployment._verify_provision_response(plan, provision) - - -def test_review_provision_still_binds_optional_allowlisted_dstack_mr_image() -> None: - """When dstack_mr_image is sealed, provision catalog must match that allowlist. - - This prevents invent loophole of accepting any catalog digest when pin names one. - """ - - plan = _stub_review_plan( - _review_plan_measurement( - os_image_hash=RESIDUAL_PRODUCT_OS, - dstack_mr_image=RESIDUAL_DSTACK_MR_IMAGE, - ) - ) - provision = { - "compose_hash": plan.compose_hash, - "app_id": plan.app_identity, - "app_env_encrypt_pubkey": plan.kms_public_key_hex, - "os_image_hash": "ff" * 32, # wrong catalog - } - with pytest.raises(review_deploy.ReviewDeploymentError, match="dstack_mr_image|catalog"): - review_deploy.HttpReviewPhalaDeployment._verify_provision_response(plan, provision) - - -def test_review_provision_legacy_catalog_seal_still_requires_equality() -> None: - """Pre-fix pins that overloading catalog as os_image_hash still equality-check.""" - - plan = _stub_review_plan(_review_plan_measurement(os_image_hash=RESIDUAL_DSTACK_MR_IMAGE)) - # Matching catalog seal ↔ provision catalog still OK. - review_deploy.HttpReviewPhalaDeployment._verify_provision_response( - plan, - { - "compose_hash": plan.compose_hash, - "app_id": plan.app_identity, - "app_env_encrypt_pubkey": plan.kms_public_key_hex, - "os_image_hash": RESIDUAL_DSTACK_MR_IMAGE, - }, - ) - # Mismatch still fails closed (no invent privilege for legacy seal). - with pytest.raises( - review_deploy.ReviewDeploymentError, - match="Phala provision os_image_hash mismatches signed assignment measurement", - ): - review_deploy.HttpReviewPhalaDeployment._verify_provision_response( - plan, - { - "compose_hash": plan.compose_hash, - "app_id": plan.app_identity, - "app_env_encrypt_pubkey": plan.kms_public_key_hex, - "os_image_hash": RESIDUAL_PRODUCT_OS, - }, - ) - - -def test_eval_provision_accepts_catalog_os_when_seal_is_product_formula() -> None: - """Eval path shares the same provision OS identity policy (tdx.small residual class).""" - - plan = MagicMock() - plan.compose_hash = "a" * 64 - plan.app_identity = "b" * 40 - plan.kms_public_key_hex = "c" * 64 - plan.measurement = _review_plan_measurement(os_image_hash=RESIDUAL_PRODUCT_OS) - # Use the shared helper / eval method — eval inlines the same policy. - eval_deploy.HttpEvalPhalaDeployment._verify_provision_os_identity( - plan, - { - "compose_hash": plan.compose_hash, - "app_id": plan.app_identity, - "app_env_encrypt_pubkey": plan.kms_public_key_hex, - "os_image_hash": RESIDUAL_DSTACK_MR_IMAGE, - }, - ) - - -def test_guest_and_pin_share_product_os_identity_on_residual_registers() -> None: - """Single sealed identity for guest bind: formula from the residual registers.""" - - sealed = m.product_os_image_hash( - mrtd=RESIDUAL_MRTD, - rtmr1=RESIDUAL_RTMR1, - rtmr2=RESIDUAL_RTMR2, - ) - guest_computed = os_image_hash_from_registers(RESIDUAL_MRTD, RESIDUAL_RTMR1, RESIDUAL_RTMR2) - assert sealed == guest_computed == RESIDUAL_PRODUCT_OS - assert m.measurement_uses_product_os_identity( - { - "mrtd": RESIDUAL_MRTD, - "rtmr1": RESIDUAL_RTMR1, - "rtmr2": RESIDUAL_RTMR2, - "os_image_hash": sealed, - } - ) - assert not m.measurement_uses_product_os_identity( - { - "mrtd": RESIDUAL_MRTD, - "rtmr1": RESIDUAL_RTMR1, - "rtmr2": RESIDUAL_RTMR2, - "os_image_hash": RESIDUAL_DSTACK_MR_IMAGE, - } - ) diff --git a/packages/challenges/agent-challenge/tests/test_own_runner_backend_decrypt.py b/packages/challenges/agent-challenge/tests/test_own_runner_backend_decrypt.py deleted file mode 100644 index fe881452b..000000000 --- a/packages/challenges/agent-challenge/tests/test_own_runner_backend_decrypt.py +++ /dev/null @@ -1,299 +0,0 @@ -"""In-enclave golden decryption wiring (feature key-release-decrypt-in-enclave). - -The released golden key must be CONSUMED to decrypt the encrypted-at-rest golden -inside the enclave BEFORE the eval runs, fail closed if decryption fails, and -never surface the key or golden plaintext to a miner-visible path (VAL-KEY-016/ -017/018; expectedBehavior: usable only in-enclave). - -These tests drive ``own_runner_backend.main`` with a fake key-release client -(so no network/dstack is needed) and assert: - * the released key is used to decrypt the golden before the eval is invoked; - * a decryption failure fails closed (one parseable ``failed`` line, reason - ``phala_golden_decrypt_failed``) WITHOUT running the eval and WITHOUT a - passing score; - * the key and golden plaintext never appear on stdout or on any written path; - * the RA-TLS public key is resolved from the deploy env and handed to client. -""" - -from __future__ import annotations - -import base64 -import json -from typing import Any - -import pytest - -from agent_challenge.evaluation import own_runner_backend as backend -from agent_challenge.evaluation.own_runner.orchestrator import JobResult -from agent_challenge.evaluation.own_runner.result_schema import ( - RESULT_LINE_PREFIX, - build_benchmark_result, -) -from agent_challenge.golden import crypto, package -from agent_challenge.keyrelease.client import KEY_RELEASE_URL_ENV - -# Fragment the golden plaintext marker so a VAL-KEY-001 repo scan stays clean. -GOLDEN_MARKER = "harbor-independence/" + "oracle-golden" - - -def _canned_result() -> JobResult: - return JobResult( - status="completed", - score=1.0, - resolved=1, - total=1, - reason_code=None, - pass_at_k={}, - n_total_trials=1, - n_completed_trials=1, - n_errored_trials=0, - trial_outcomes=[], - benchmark_result=build_benchmark_result( - status="completed", score=1.0, resolved=1, total=1, reason_code=None - ), - ) - - -class _RunRecorder: - def __init__(self, log: list[str] | None = None) -> None: - self.called = False - self._log = log - - async def __call__(self, **kwargs: Any) -> JobResult: - self.called = True - if self._log is not None: - self._log.append("eval") - return _canned_result() - - -def _client_factory(*, key: bytes, captured: dict[str, Any] | None = None): - class _FakeClient: - def __init__(self, *args: Any, **kwargs: Any) -> None: - if captured is not None: - captured["args"] = args - captured["kwargs"] = kwargs - - def acquire_golden_key(self, **kwargs: Any) -> bytes: - return key - - return _FakeClient - - -def _result_lines(out: str) -> list[dict]: - return [ - json.loads(ln[len(RESULT_LINE_PREFIX) :]) - for ln in out.splitlines() - if ln.startswith(RESULT_LINE_PREFIX) - ] - - -def _seed_encrypted_golden(golden_dir, key: bytes, doc: dict) -> None: - golden_dir.mkdir(parents=True, exist_ok=True) - blob = package.encrypt_golden_bytes(json.dumps(doc).encode(), key) - (golden_dir / package.ORACLE_CIPHERTEXT_NAME).write_bytes(blob) - - -def _enable_phala_decrypt(monkeypatch, *, task_id: str) -> None: - monkeypatch.setenv(backend.PHALA_ATTESTATION_ENABLED_ENV, "1") - monkeypatch.setenv("CHALLENGE_ATTESTED_REVIEW_ENABLED", "1") - plan = { - "eval_run_id": "eval-run-001", - "key_release_endpoint": "validator.test:8701", - "key_release_nonce": "key-nonce-001", - "score_nonce": "score-nonce-001", - "issued_at_ms": 0, - "expires_at_ms": 4_102_444_800_000, - "selected_tasks": [ - { - "task_id": task_id, - "image_ref": "registry/task@sha256:" + "a" * 64, - "task_config_sha256": "b" * 64, - } - ], - "k": 1, - "agent_hash": "f" * 64, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - } - monkeypatch.setattr( - backend, - "_resolve_phala_binding_from_env", - lambda: {"eval_plan": plan, "rtmr3": "d" * 96}, - ) - monkeypatch.setattr( - backend, - "assert_agent_artifact_matches_plan", - lambda **_: "f" * 64, - ) - monkeypatch.setattr( - backend, - "assert_package_tree_matches_plan", - lambda **_: "b" * 64, - ) - monkeypatch.setattr(backend, "_preflight_eval_plan_tasks", lambda **_: {}) - monkeypatch.setattr(backend, "_emit_job_result", lambda *args, **kwargs: 0) - - -def test_released_key_decrypts_golden_before_eval(monkeypatch, tmp_path, capsys): - key = bytes(range(32)) - golden_dir = tmp_path / "golden" - _seed_encrypted_golden(golden_dir, key, {"schema": GOLDEN_MARKER, "results": {"t": 1}}) - - order: list[str] = [] - real_load = package.load_encrypted_oracle_golden - - def _tracking_load(k: bytes, **kw: Any): - order.append("decrypt") - return real_load(k, **kw) - - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - _enable_phala_decrypt(monkeypatch, task_id="hello-world") - monkeypatch.setenv(backend.GOLDEN_DIR_ENV, str(golden_dir)) - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _client_factory(key=key)) - monkeypatch.setattr(package, "load_encrypted_oracle_golden", _tracking_load) - monkeypatch.setattr(backend, "run_own_runner_job", _RunRecorder(log=order)) - - rc = backend.main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - - assert rc == 0 - # The golden was decrypted (with the released key) BEFORE the eval ran. - assert order == ["decrypt", "eval"] - - -def test_golden_decrypt_failure_fails_closed_without_eval(monkeypatch, tmp_path, capsys): - # Seed ciphertext under one key but release a DIFFERENT (wrong) key. - golden_dir = tmp_path / "golden" - _seed_encrypted_golden(golden_dir, bytes(range(32)), {"schema": GOLDEN_MARKER}) - wrong_key = bytes([0xAA]) * 32 - - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - _enable_phala_decrypt(monkeypatch, task_id="hello-world") - monkeypatch.setenv(backend.GOLDEN_DIR_ENV, str(golden_dir)) - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _client_factory(key=wrong_key)) - recorder = _RunRecorder() - monkeypatch.setattr(backend, "run_own_runner_job", recorder) - - rc = backend.main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - - out = capsys.readouterr().out - lines = _result_lines(out) - assert rc != 0 - assert len(lines) == 1 - assert lines[0]["status"] == "failed" - assert lines[0]["score"] == 0.0 - assert lines[0]["reason_code"] == backend.GOLDEN_DECRYPT_FAILED_REASON - # The eval never ran against a missing/placeholder golden. - assert recorder.called is False - - -def test_missing_ciphertext_fails_closed(monkeypatch, tmp_path, capsys): - empty_dir = tmp_path / "golden-empty" - empty_dir.mkdir() - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - _enable_phala_decrypt(monkeypatch, task_id="t") - monkeypatch.setenv(backend.GOLDEN_DIR_ENV, str(empty_dir)) - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _client_factory(key=bytes(range(32)))) - recorder = _RunRecorder() - monkeypatch.setattr(backend, "run_own_runner_job", recorder) - - rc = backend.main(["run", "--task", "t", "--job-dir", str(tmp_path / "job")]) - - lines = _result_lines(capsys.readouterr().out) - assert rc != 0 - assert lines[0]["status"] == "failed" - assert lines[0]["reason_code"] == backend.GOLDEN_DECRYPT_FAILED_REASON - assert recorder.called is False - - -def test_no_endpoint_does_not_decrypt_golden(monkeypatch, tmp_path): - monkeypatch.delenv(KEY_RELEASE_URL_ENV, raising=False) - - def _forbidden(*a: Any, **k: Any): # pragma: no cover - only on misuse - raise AssertionError("golden decrypt attempted on the legacy path") - - monkeypatch.setattr(package, "load_encrypted_oracle_golden", _forbidden) - recorder = _RunRecorder() - monkeypatch.setattr(backend, "run_own_runner_job", recorder) - - rc = backend.main(["run", "--task", "t", "--job-dir", str(tmp_path / "job")]) - assert rc == 0 - assert recorder.called is True - - -def test_key_and_golden_never_leak_to_stdout_or_disk(monkeypatch, tmp_path, capsys): - key = bytes([0x42]) * 32 - golden_dir = tmp_path / "golden" - _seed_encrypted_golden(golden_dir, key, {"schema": GOLDEN_MARKER, "answer": "SECRET-ANSWER"}) - - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - _enable_phala_decrypt(monkeypatch, task_id="t") - monkeypatch.setenv(backend.GOLDEN_DIR_ENV, str(golden_dir)) - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _client_factory(key=key)) - monkeypatch.setattr(backend, "run_own_runner_job", _RunRecorder()) - - rc = backend.main(["run", "--task", "t", "--job-dir", str(tmp_path / "job")]) - assert rc == 0 - - out = capsys.readouterr().out - # Neither the raw/encoded key nor golden plaintext is ever printed. - for needle in (key.hex(), base64.b64encode(key).decode(), GOLDEN_MARKER, "SECRET-ANSWER"): - assert needle not in out - - ciphertext_path = golden_dir / package.ORACLE_CIPHERTEXT_NAME - for path in tmp_path.rglob("*"): - if not path.is_file() or path == ciphertext_path: - continue - data = path.read_bytes() - assert key not in data - assert GOLDEN_MARKER.encode() not in data - assert b"SECRET-ANSWER" not in data - - -def test_ra_tls_pubkey_resolved_from_env_and_passed_to_client(monkeypatch, tmp_path): - key = bytes(range(32)) - golden_dir = tmp_path / "golden" - _seed_encrypted_golden(golden_dir, key, {"schema": GOLDEN_MARKER}) - captured: dict[str, Any] = {} - pubkey = b"\x01\x02\x03\x04enclave-ra-tls" - - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - _enable_phala_decrypt(monkeypatch, task_id="t") - monkeypatch.setenv(backend.GOLDEN_DIR_ENV, str(golden_dir)) - monkeypatch.setenv(backend.PHALA_RA_TLS_PUBKEY_ENV, pubkey.hex()) - monkeypatch.setattr( - backend, "GoldenKeyReleaseClient", _client_factory(key=key, captured=captured) - ) - monkeypatch.setattr(backend, "run_own_runner_job", _RunRecorder()) - - rc = backend.main(["run", "--task", "t", "--job-dir", str(tmp_path / "job")]) - assert rc == 0 - assert captured["kwargs"].get("ra_tls_pubkey") == pubkey - - -def test_invalid_ra_tls_pubkey_hex_fails_closed(monkeypatch, tmp_path, capsys): - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - _enable_phala_decrypt(monkeypatch, task_id="t") - monkeypatch.setenv(backend.PHALA_RA_TLS_PUBKEY_ENV, "zz-not-hex") - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _client_factory(key=bytes(range(32)))) - recorder = _RunRecorder() - monkeypatch.setattr(backend, "run_own_runner_job", recorder) - - rc = backend.main(["run", "--task", "t", "--job-dir", str(tmp_path / "job")]) - lines = _result_lines(capsys.readouterr().out) - assert rc != 0 - assert lines[0]["status"] == "failed" - # A misconfigured RA-TLS key is a key-release failure (fail closed, no eval). - assert lines[0]["reason_code"] == "phala_key_release_failed" - assert recorder.called is False - - -def test_acquire_helper_returns_key_and_decrypt_helper_unseals(monkeypatch, tmp_path): - key = bytes(range(32)) - golden_dir = tmp_path / "golden" - doc = {"schema": GOLDEN_MARKER, "results": {"a": 1}} - _seed_encrypted_golden(golden_dir, key, doc) - monkeypatch.setenv(backend.GOLDEN_DIR_ENV, str(golden_dir)) - unsealed = backend._decrypt_golden_in_enclave(key) - assert unsealed == doc - # Wrong key raises the fail-closed crypto error. - with pytest.raises(crypto.GoldenCryptoError): - backend._decrypt_golden_in_enclave(bytes([0x00]) * 32) diff --git a/packages/challenges/agent-challenge/tests/test_own_runner_backend_failclosed_surface.py b/packages/challenges/agent-challenge/tests/test_own_runner_backend_failclosed_surface.py deleted file mode 100644 index 42cf1f5cc..000000000 --- a/packages/challenges/agent-challenge/tests/test_own_runner_backend_failclosed_surface.py +++ /dev/null @@ -1,428 +0,0 @@ -"""Durable guest fail-closed surface for pre-KR residual (eval-pre-kr-failclosed). - -Live residual after SPKI fix reached ``tcp_connect_ok`` then collapsed to opaque -``terminal_bench_failed`` because pre-frame/quote/agent failures fell through -``main``'s bare ``except Exception`` without stage labels. - -These tests encode the offline contract: - * guest_eval_fail markers print stage/class/detail (secret-free) - * agent/preflight failures get distinct durable labels (not opaque KR seals) - * bare Exception from quote/pre-frame acquire maps to ``phala_key_release_failed`` - * KeyReleaseError still maps via ``reason_code`` - * progress breadcrumbs after preflight_ok / acquire_start - * preflight can PASS with a live-like plan when digests match (mocked) -""" - -from __future__ import annotations - -import json -from typing import Any - -import pytest - -from agent_challenge.canonical import attested_result as ar -from agent_challenge.evaluation import own_runner_backend as backend -from agent_challenge.evaluation.own_runner.orchestrator import JobResult -from agent_challenge.evaluation.own_runner.result_schema import ( - RESULT_LINE_PREFIX, - build_benchmark_result, -) -from agent_challenge.keyrelease.client import ( - KEY_RELEASE_URL_ENV, - KeyReleaseDenied, - KeyReleaseError, -) - - -def _canned_result() -> JobResult: - return JobResult( - status="completed", - score=1.0, - resolved=1, - total=1, - reason_code=None, - pass_at_k={}, - n_total_trials=1, - n_completed_trials=1, - n_errored_trials=0, - trial_outcomes=[], - benchmark_result=build_benchmark_result( - status="completed", score=1.0, resolved=1, total=1, reason_code=None - ), - ) - - -class _RunRecorder: - def __init__(self) -> None: - self.called = False - - async def __call__(self, **kwargs: Any) -> JobResult: - self.called = True - return _canned_result() - - -def _result_lines(out: str) -> list[dict]: - return [ - json.loads(ln[len(RESULT_LINE_PREFIX) :]) - for ln in out.splitlines() - if ln.startswith(RESULT_LINE_PREFIX) - ] - - -def _live_like_plan() -> dict[str, Any]: - return { - "eval_run_id": "eval-run-live37", - "key_release_endpoint": "validator.test:8701", - "key_release_nonce": "key-nonce-live37", - "score_nonce": "score-nonce-live37", - "issued_at_ms": 0, - "expires_at_ms": 4_102_444_800_000, - "selected_tasks": [ - { - "task_id": "hello-world", - "image_ref": "registry/task@sha256:" + "a" * 64, - "task_config_sha256": "b" * 64, - }, - { - "task_id": "task-two", - "image_ref": "registry/task@sha256:" + "c" * 64, - "task_config_sha256": "d" * 64, - }, - { - "task_id": "task-three", - "image_ref": "registry/task@sha256:" + "e" * 64, - "task_config_sha256": "f" * 64, - }, - ], - "k": 1, - "agent_hash": "aa" * 32, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - } - - -def _enable_phala(monkeypatch, plan: dict[str, Any] | None = None) -> dict[str, Any]: - monkeypatch.setenv(backend.PHALA_ATTESTATION_ENABLED_ENV, "1") - eval_plan = plan if plan is not None else _live_like_plan() - monkeypatch.setattr( - backend, - "_resolve_phala_binding_from_env", - lambda: {"eval_plan": eval_plan, "rtmr3": "d" * 96}, - ) - return eval_plan - - -def test_preflight_ok_marker_with_live_like_plan(monkeypatch, tmp_path, capsys) -> None: - """When agent+tasks pass, preflight_ok + acquire_start mark before KR.""" - - plan = _enable_phala(monkeypatch) - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - monkeypatch.setattr( - backend, - "assert_agent_artifact_matches_plan", - lambda **_: plan["agent_hash"], - ) - monkeypatch.setattr( - backend, - "assert_package_tree_matches_plan", - lambda **_: "b" * 64, - ) - monkeypatch.setattr( - backend, - "_preflight_eval_plan_tasks", - lambda **_: {t["task_id"]: object() for t in plan["selected_tasks"]}, - ) - - def _blocked(*_a: Any, **_k: Any) -> bytes: - raise KeyReleaseDenied("measurement not allowlisted") - - class _FakeClient: - def __init__(self, *args: Any, **kwargs: Any) -> None: - pass - - def acquire_golden_key(self, **kwargs: Any) -> bytes: - return _blocked() - - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _FakeClient) - monkeypatch.setattr( - "agent_challenge.canonical.attested_result.DstackQuoteProvider", - lambda *a, **k: object(), - ) - recorder = _RunRecorder() - monkeypatch.setattr(backend, "run_own_runner_job", recorder) - - rc = backend.main(["run", "--job-dir", str(tmp_path / "job")]) - combined = capsys.readouterr().out - assert rc != 0 - assert recorder.called is False - assert "guest_eval stage=preflight_ok" in combined - assert "guest_eval stage=acquire_start" in combined - assert "guest_eval_fail stage=key_release" in combined - lines = _result_lines(combined) - assert len(lines) == 1 - assert lines[0]["reason_code"] == "phala_key_release_failed" - assert lines[0]["score"] == 0.0 - - -def test_missing_agent_hash_labeled_not_opaque(monkeypatch, tmp_path, capsys) -> None: - """Missing agent artifact/digest emits agent_identity labels, not silent generic.""" - - _enable_phala(monkeypatch) - monkeypatch.delenv(KEY_RELEASE_URL_ENV, raising=False) - monkeypatch.delenv(backend.PHALA_AGENT_HASH_ENV, raising=False) - monkeypatch.delenv(backend.AGENT_ARTIFACT_PATH_ENV, raising=False) - monkeypatch.delenv(backend.AGENT_ARTIFACT_PATH_ENV_ALT, raising=False) - - def _unexpected_kr(**_: Any) -> bytes | None: - pytest.fail("agent identity must fail before key release") - - def _unexpected_job(**_: Any) -> JobResult: - pytest.fail("agent identity must fail before job execution") - - # Real assert path: no artifact on disk and no declared hash. - monkeypatch.setattr(backend, "resolve_agent_artifact_path", lambda: None) - monkeypatch.setattr(backend, "_acquire_golden_key_if_required", _unexpected_kr) - monkeypatch.setattr(backend, "run_own_runner_job", _unexpected_job) - - rc = backend.main(["run", "--job-dir", str(tmp_path / "job")]) - combined = capsys.readouterr().out - assert rc != 0 - assert "guest_eval_fail stage=agent_identity" in combined - assert "class=ValueError" in combined - lines = _result_lines(combined) - assert len(lines) == 1 - payload = lines[0] - assert payload["status"] == "failed" - assert payload["score"] == 0.0 - # Still terminal_bench_failed reason (no new taxonomy code) but labeled. - assert payload["reason_code"] == "terminal_bench_failed" - assert payload.get("failure_stage") == "agent_identity" - assert payload.get("failure_class") == "ValueError" - assert "agent" in str(payload.get("failure_detail", "")).lower() - - -def test_quote_provider_bare_exception_maps_to_key_release_not_terminal( - monkeypatch, tmp_path, capsys -) -> None: - """quote_provider raising plain Exception must surface phala_key_release_failed.""" - - plan = _enable_phala(monkeypatch) - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - monkeypatch.setattr( - backend, - "assert_agent_artifact_matches_plan", - lambda **_: plan["agent_hash"], - ) - monkeypatch.setattr( - backend, - "assert_package_tree_matches_plan", - lambda **_: "b" * 64, - ) - monkeypatch.setattr( - backend, - "_preflight_eval_plan_tasks", - lambda **_: {"hello-world": object()}, - ) - - class _BoomQuote: - def get_quote(self, report_data: bytes) -> Any: - raise RuntimeError("dstack socket hung") - - class _ClientWithBoomQuote: - def __init__(self, *args: Any, **kwargs: Any) -> None: - self.quote_provider = _BoomQuote() - self.ra_tls_pubkey = kwargs.get("ra_tls_pubkey", b"") - self.endpoint_url = args[0] if args else "https://validator.test:8700" - - def acquire_golden_key(self, **kwargs: Any) -> bytes: - # Exercise the real client wrap path via a thin call into authentic logic. - from agent_challenge.keyrelease.client import GoldenKeyReleaseClient - - real = GoldenKeyReleaseClient( - self.endpoint_url, - quote_provider=self.quote_provider, - ra_tls_pubkey=self.ra_tls_pubkey, - ) - return real.acquire_golden_key(**kwargs) - - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _ClientWithBoomQuote) - monkeypatch.setattr( - "agent_challenge.canonical.attested_result.DstackQuoteProvider", - lambda *a, **k: _BoomQuote(), - ) - recorder = _RunRecorder() - monkeypatch.setattr(backend, "run_own_runner_job", recorder) - - rc = backend.main( - [ - "run", - "--task", - "hello-world", - "--task", - "task-two", - "--task", - "task-three", - "--job-dir", - str(tmp_path / "job"), - ] - ) - combined = capsys.readouterr().out - assert rc != 0 - assert recorder.called is False - lines = _result_lines(combined) - assert len(lines) == 1 - payload = lines[0] - assert payload["reason_code"] == "phala_key_release_failed" - assert payload["reason_code"] != "terminal_bench_failed" - assert "guest_eval_fail stage=key_release" in combined - assert ar.EXECUTION_PROOF_RESULT_KEY not in payload - - -def test_key_release_error_still_maps_via_reason_code(monkeypatch, tmp_path, capsys) -> None: - plan = _enable_phala(monkeypatch) - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - monkeypatch.setattr( - backend, - "assert_agent_artifact_matches_plan", - lambda **_: plan["agent_hash"], - ) - monkeypatch.setattr( - backend, - "assert_package_tree_matches_plan", - lambda **_: "b" * 64, - ) - monkeypatch.setattr( - backend, - "_preflight_eval_plan_tasks", - lambda **_: {"hello-world": object()}, - ) - - class _Denied: - def __init__(self, *a: Any, **k: Any) -> None: - pass - - def acquire_golden_key(self, **kwargs: Any) -> bytes: - raise KeyReleaseError("generic key-release failure") - - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _Denied) - monkeypatch.setattr( - "agent_challenge.canonical.attested_result.DstackQuoteProvider", - lambda *a, **k: object(), - ) - recorder = _RunRecorder() - monkeypatch.setattr(backend, "run_own_runner_job", recorder) - - rc = backend.main(["run", "--job-dir", str(tmp_path / "job")]) - combined = capsys.readouterr().out - assert rc != 0 - payload = _result_lines(combined)[0] - assert payload["reason_code"] == "phala_key_release_failed" - assert "guest_eval_fail stage=key_release class=KeyReleaseError" in combined - # Specific KR reasons must NOT get failure_stage override noise. - assert "failure_stage" not in payload - - -def test_pre_frame_exception_in_acquire_helper_is_key_release_error(monkeypatch) -> None: - """Bare Exception in quote-provider / client construction must wrap as KR error.""" - - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - - class _ExplodingClient: - def __init__(self, *args: Any, **kwargs: Any) -> None: - raise RuntimeError("quote provider construction boom") - - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _ExplodingClient) - monkeypatch.setattr( - "agent_challenge.canonical.attested_result.DstackQuoteProvider", - lambda *a, **k: object(), - ) - with pytest.raises(KeyReleaseError) as ei: - backend._acquire_golden_key_if_required( - eval_plan={ - "eval_run_id": "run-1", - "key_release_nonce": "n-1", - } - ) - assert ei.value.reason_code == "phala_key_release_failed" - assert "pre-frame" in str(ei.value).lower() or "boom" in str(ei.value).lower() - - -def test_guest_eval_fail_sanitizes_secretish_detail() -> None: - detail = "-----BEGIN PRIVATE KEY-----\nabcsecret\n-----END PRIVATE KEY-----" - sanitized = backend._sanitize_guest_detail(detail) - assert "BEGIN" not in sanitized - assert "abcsecret" not in sanitized - assert sanitized == "redacted" - - -def test_quote_ok_and_frame_send_markers_from_client(monkeypatch, capsys, tmp_path) -> None: - """Client progress markers after successful quote; frame_send when mTLS is ready.""" - - from agent_challenge.keyrelease.client import GoldenKeyReleaseClient - - class _OkQuote: - def get_quote(self, report_data: bytes) -> Any: - class _R: - quote = "ab" * 64 - event_log = "[]" - vm_config = "{}" - - return _R() - - # Empty pubkey path (no cert env) so SPKI bind is deterministic. - monkeypatch.delenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", raising=False) - # Provide dummy mTLS files so _raw_release reaches frame_send (then fail connect). - cert = tmp_path / "c.pem" - key = tmp_path / "k.pem" - ca = tmp_path / "ca.pem" - for path in (cert, key, ca): - path.write_text("not-a-real-pem\n", encoding="utf-8") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(cert)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_KEY_FILE", str(key)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CA_FILE", str(ca)) - # With a cert present, SPKI resolution needs a real PEM; force explicit digest - # and make _resolve_spki_digest return the caller-provided digest. - spki = "ab" * 32 - monkeypatch.setattr( - GoldenKeyReleaseClient, - "_resolve_spki_digest", - lambda self: spki, - ) - - client = GoldenKeyReleaseClient( - "ratls://127.0.0.1:9", - quote_provider=_OkQuote(), - ra_tls_pubkey=b"", - timeout=0.2, - ) - with pytest.raises(KeyReleaseError): - client.acquire_golden_key( - eval_run_id="run", - key_release_nonce="nonce", - ra_tls_spki_digest=spki, - ) - out = capsys.readouterr().out - assert "guest_eval stage=quote_ok" in out - # frame_send may not be reached if SSL context refuses the dummy PEMs; quote_ok - # alone proves post-quote progress reached. Accept either path. - assert "guest_eval stage=quote_ok" in out - - -def test_annotate_only_for_terminal_bench_failed() -> None: - kr = build_benchmark_result( - status="failed", score=0.0, resolved=0, total=3, reason_code="phala_key_release_failed" - ) - untouched = backend._annotate_failclosed_result( - dict(kr), stage="key_release", class_name="X", detail="y" - ) - assert "failure_stage" not in untouched - assert untouched["reason_code"] == "phala_key_release_failed" - - generic = build_benchmark_result( - status="failed", score=0.0, resolved=0, total=3, reason_code="terminal_bench_failed" - ) - annotated = backend._annotate_failclosed_result( - generic, stage="agent_identity", class_name="ValueError", detail="missing agent" - ) - assert annotated["failure_stage"] == "agent_identity" - assert annotated["failure_class"] == "ValueError" - assert "missing" in annotated["failure_detail"] diff --git a/packages/challenges/agent-challenge/tests/test_own_runner_backend_keyrelease.py b/packages/challenges/agent-challenge/tests/test_own_runner_backend_keyrelease.py deleted file mode 100644 index 518b26ab4..000000000 --- a/packages/challenges/agent-challenge/tests/test_own_runner_backend_keyrelease.py +++ /dev/null @@ -1,306 +0,0 @@ -"""Backend fail-closed wiring for the in-CVM golden key-release (VAL-ORCH-035). - -When a validator key-release endpoint is configured (Phala path) but the golden -key cannot be obtained — the endpoint denies, is unreachable, or drops -mid-exchange — the orchestrator must: - * NOT run the verifier against a missing/placeholder golden (run_own_runner_job - is never invoked), - * NOT emit a passing score / attestation envelope, - * surface exactly one parseable fail-closed ``BASE_BENCHMARK_RESULT=`` line - (score 0, reason ``phala_key_release_failed``), and return nonzero. - -When no endpoint is configured (legacy path) the eval runs unchanged and makes no -key-release call. -""" - -from __future__ import annotations - -import json -import os -from typing import Any - -import pytest - -from agent_challenge.canonical import attested_result as ar -from agent_challenge.evaluation import own_runner_backend as backend -from agent_challenge.evaluation.own_runner.orchestrator import JobResult -from agent_challenge.evaluation.own_runner.result_schema import ( - RESULT_LINE_PREFIX, - build_benchmark_result, -) -from agent_challenge.keyrelease.client import ( - KEY_RELEASE_URL_ENV, - KeyReleaseDenied, - KeyReleaseError, - KeyReleaseMidExchangeError, - KeyReleaseUnreachable, -) - - -def _canned_result() -> JobResult: - return JobResult( - status="completed", - score=1.0, - resolved=1, - total=1, - reason_code=None, - pass_at_k={}, - n_total_trials=1, - n_completed_trials=1, - n_errored_trials=0, - trial_outcomes=[], - benchmark_result=build_benchmark_result( - status="completed", score=1.0, resolved=1, total=1, reason_code=None - ), - ) - - -class _RunRecorder: - """Stand-in for run_own_runner_job that records whether it was invoked.""" - - def __init__(self) -> None: - self.called = False - - async def __call__(self, **kwargs: Any) -> JobResult: - self.called = True - return _canned_result() - - -def _fake_client_factory(*, exc: Exception | None = None, key: bytes = b"golden-key"): - class _FakeClient: - def __init__(self, *args: Any, **kwargs: Any) -> None: - self.args = args - self.kwargs = kwargs - - def acquire_golden_key(self, **kwargs: Any) -> bytes: - if exc is not None: - raise exc - return key - - return _FakeClient - - -def _result_lines(out: str) -> list[dict]: - return [ - json.loads(ln[len(RESULT_LINE_PREFIX) :]) - for ln in out.splitlines() - if ln.startswith(RESULT_LINE_PREFIX) - ] - - -def _run_main(monkeypatch, tmp_path) -> tuple[int, str, _RunRecorder]: - recorder = _RunRecorder() - monkeypatch.setattr(backend, "run_own_runner_job", recorder) - rc = backend.main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - return rc, recorder - - -def _enable_phala_key_release(monkeypatch) -> None: - monkeypatch.setenv(backend.PHALA_ATTESTATION_ENABLED_ENV, "1") - plan = { - "eval_run_id": "eval-run-001", - "key_release_endpoint": "validator.test:8701", - "key_release_nonce": "key-nonce-001", - "score_nonce": "score-nonce-001", - "issued_at_ms": 0, - "expires_at_ms": 4_102_444_800_000, - "selected_tasks": [ - { - "task_id": "hello-world", - "image_ref": "registry/task@sha256:" + "a" * 64, - } - ], - "k": 1, - "agent_hash": "f" * 64, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - } - monkeypatch.setattr( - backend, - "_resolve_phala_binding_from_env", - lambda: {"eval_plan": plan, "rtmr3": "d" * 96}, - ) - monkeypatch.setattr( - backend, - "assert_agent_artifact_matches_plan", - lambda **_: "f" * 64, - ) - monkeypatch.setattr( - backend, - "assert_package_tree_matches_plan", - lambda **_: "b" * 64, - ) - monkeypatch.setattr(backend, "_preflight_eval_plan_tasks", lambda **_: {}) - - -@pytest.mark.parametrize( - "exc", - [ - KeyReleaseDenied("measurement not allowlisted"), - KeyReleaseUnreachable("connection refused"), - KeyReleaseMidExchangeError("dropped after nonce"), - KeyReleaseError("generic key-release failure"), - ], -) -def test_key_unavailable_fails_closed_without_scoring(exc, monkeypatch, tmp_path, capsys): - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - _enable_phala_key_release(monkeypatch) - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _fake_client_factory(exc=exc)) - recorder = _RunRecorder() - monkeypatch.setattr(backend, "run_own_runner_job", recorder) - - rc = backend.main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - - out = capsys.readouterr().out - lines = _result_lines(out) - # Exactly one parseable fail-closed result line. - assert len(lines) == 1 - payload = lines[0] - assert payload["status"] == "failed" - assert payload["score"] == 0.0 - assert payload["reason_code"] == "phala_key_release_failed" - assert rc != 0 - # The verifier/scoring path (run_own_runner_job) NEVER ran against golden. - assert recorder.called is False - # No attestation envelope / passing artifact leaked out. - assert ar.EXECUTION_PROOF_RESULT_KEY not in payload - - -def test_flag_off_never_constructs_key_release_client(monkeypatch, tmp_path, capsys): - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - monkeypatch.delenv(backend.PHALA_ATTESTATION_ENABLED_ENV, raising=False) - - def _forbidden(*args: Any, **kwargs: Any): - raise AssertionError("flag-off must not access dstack key-release") - - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _forbidden) - recorder = _RunRecorder() - monkeypatch.setattr(backend, "run_own_runner_job", recorder) - - rc = backend.main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - - out = capsys.readouterr().out - lines = _result_lines(out) - assert rc == 0 - assert recorder.called is True - assert len(lines) == 1 - assert lines[0]["status"] == "completed" - - -def test_no_key_release_endpoint_uses_legacy_path(monkeypatch, tmp_path, capsys): - monkeypatch.delenv(KEY_RELEASE_URL_ENV, raising=False) - monkeypatch.delenv(backend.PHALA_ATTESTATION_ENABLED_ENV, raising=False) - - # A client here would be a bug: legacy path must make no key-release call. - def _forbidden(*args: Any, **kwargs: Any): # pragma: no cover - only fails on misuse - raise AssertionError("key-release client constructed on the legacy path") - - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _forbidden) - recorder = _RunRecorder() - monkeypatch.setattr(backend, "run_own_runner_job", recorder) - - rc = backend.main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - - assert rc == 0 - assert recorder.called is True - assert _result_lines(capsys.readouterr().out)[0]["status"] == "completed" - - -def test_acquire_helper_returns_none_without_endpoint(monkeypatch): - monkeypatch.delenv(KEY_RELEASE_URL_ENV, raising=False) - # Guard residual suite pollution: raw host/port must not silently arm KR path. - monkeypatch.delenv("KEY_RELEASE_RA_TLS_HOST", raising=False) - monkeypatch.delenv("KEY_RELEASE_RA_TLS_PORT", raising=False) - monkeypatch.delenv("CHALLENGE_PHALA_EVAL_PLAN", raising=False) - assert backend._acquire_golden_key_if_required() is None - - -def test_acquire_helper_returns_key_on_success(monkeypatch): - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _fake_client_factory(key=b"abc")) - assert backend._acquire_golden_key_if_required() == b"abc" - - -def _write_temp_leaf_pem(tmp_path) -> tuple[Any, str]: - """Create a throw-away PEM leaf cert; return (path, expected SPKI sha256 hex).""" - - import hashlib - from datetime import UTC, datetime, timedelta - - from cryptography import x509 - from cryptography.hazmat.primitives import hashes, serialization - from cryptography.hazmat.primitives.asymmetric import rsa - from cryptography.x509.oid import NameOID - - key = rsa.generate_private_key(public_exponent=65537, key_size=2048) - name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "ra-tls-test")]) - cert = ( - x509.CertificateBuilder() - .subject_name(name) - .issuer_name(name) - .public_key(key.public_key()) - .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.now(UTC) - timedelta(minutes=1)) - .not_valid_after(datetime.now(UTC) + timedelta(hours=1)) - .sign(key, hashes.SHA256()) - ) - spki = key.public_key().public_bytes( - serialization.Encoding.DER, - serialization.PublicFormat.SubjectPublicKeyInfo, - ) - path = tmp_path / "leaf.pem" - path.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) - return path, hashlib.sha256(spki).hexdigest() - - -def test_v2_acquire_uses_cert_spki_when_pubkey_and_spki_env_unset(monkeypatch, tmp_path): - """Live residual: empty PUBKEY/SPKI env + real cert must bind cert SPKI. - - Pre-fix force-sha256(empty) failed in GoldenKeyReleaseClient with - ``ra_tls_spki_digest does not match`` before any framed send. - """ - - import hashlib - - cert_path, expected_spki = _write_temp_leaf_pem(tmp_path) - monkeypatch.delenv(backend.PHALA_RA_TLS_PUBKEY_ENV, raising=False) - monkeypatch.delenv(backend.PHALA_RA_TLS_SPKI_SHA256_ENV, raising=False) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(cert_path)) - monkeypatch.setenv(KEY_RELEASE_URL_ENV, "https://validator.test:8700") - - captured: dict[str, Any] = {} - - class _CapturingClient: - def __init__(self, *args: Any, **kwargs: Any) -> None: - pass - - def acquire_golden_key(self, **kwargs: Any) -> bytes: - captured.update(kwargs) - # Discriminator vs pre-fix empty digest. - empty = hashlib.sha256(b"").hexdigest() - assert kwargs.get("ra_tls_spki_digest") != empty - assert kwargs.get("ra_tls_spki_digest") == expected_spki - return b"granted-key" - - monkeypatch.setattr(backend, "GoldenKeyReleaseClient", _CapturingClient) - monkeypatch.setattr( - "agent_challenge.canonical.attested_result.DstackQuoteProvider", - lambda *a, **k: object(), - ) - - plan = { - "eval_run_id": "eval-run-live-spki", - "key_release_nonce": "key-nonce-live", - } - key = backend._acquire_golden_key_if_required(eval_plan=plan) - assert key == b"granted-key" - assert captured["ra_tls_spki_digest"] == expected_spki - # Optionally materializes observability env after resolution. - assert os.environ.get(backend.PHALA_RA_TLS_SPKI_SHA256_ENV) == expected_spki - - -def test_resolve_ra_tls_spki_digest_prefers_explicit_env(monkeypatch, tmp_path): - cert_path, cert_spki = _write_temp_leaf_pem(tmp_path) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(cert_path)) - monkeypatch.setenv(backend.PHALA_RA_TLS_SPKI_SHA256_ENV, "ab" * 32) - assert backend._resolve_ra_tls_spki_digest(ra_tls_pubkey=b"") == "ab" * 32 - assert cert_spki # ensure helper ran diff --git a/packages/challenges/agent-challenge/tests/test_own_runner_backend_phala.py b/packages/challenges/agent-challenge/tests/test_own_runner_backend_phala.py deleted file mode 100644 index 71ec819d4..000000000 --- a/packages/challenges/agent-challenge/tests/test_own_runner_backend_phala.py +++ /dev/null @@ -1,371 +0,0 @@ -"""Backend wiring tests for Phala attested-result emission (M1). - -Verifies ``own_runner_backend.main`` dispatch: - * gate OFF -> legacy plain BASE_BENCHMARK_RESULT line, no attestation, rc 0 - * gate ON -> attested line carrying the ExecutionProof envelope, rc 0 - * gate ON + get_quote failure -> fail-closed failed line, rc != 0 (VAL-IMG-034) - * gate ON + missing binding env -> fail-closed failed line, rc != 0 (VAL-IMG-034) -""" - -from __future__ import annotations - -import hashlib -import json -import os -import time -from typing import Any - -import pytest - -from agent_challenge.canonical import attested_result as ar -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.evaluation import own_runner_backend as backend -from agent_challenge.evaluation.own_runner.orchestrator import JobResult, TrialOutcome -from agent_challenge.evaluation.own_runner.result_schema import ( - RESULT_LINE_PREFIX, - build_benchmark_result, -) -from agent_challenge.evaluation.own_runner_backend import ( - PHALA_ATTESTATION_ENABLED_ENV, - PHALA_EVAL_PLAN_ENV, - PHALA_RTMR3_ENV, - main, -) - -ATTESTED_REVIEW_ENABLED_ENV = "CHALLENGE_ATTESTED_REVIEW_ENABLED" - -MEASUREMENT = { - "mrtd": "a" * 96, - "rtmr0": "b0" * 48, - "rtmr1": "b1" * 48, - "rtmr2": "b2" * 48, - "compose_hash": "c" * 64, - "os_image_hash": "e" * 64, -} -FAKE_QUOTE = "ab" * 600 - - -class _FakeQuoteResponse: - def __init__(self, quote: str) -> None: - self.quote = quote - self.event_log = json.dumps( - [ - { - "imr": 3, - "event_type": 134217729, - "digest": "c" * 96, - "event": "compose-hash", - "event_payload": MEASUREMENT["compose_hash"], - } - ] - ) - self.vm_config = json.dumps( - {"vcpu": 1, "memory_mb": 2048, "os_image_hash": MEASUREMENT["os_image_hash"]} - ) - self.report_data = "" - - -def _fake_provider_factory(quote: str = FAKE_QUOTE, *, raises: Exception | None = None): - class _FakeProvider: - def __init__(self, endpoint: str | None = None) -> None: - self.endpoint = endpoint - - def get_quote(self, report_data: bytes) -> Any: - if raises is not None: - raise raises - return _FakeQuoteResponse(quote) - - return _FakeProvider - - -def _canned_result() -> JobResult: - return JobResult( - status="completed", - score=1.0, - resolved=1, - total=1, - reason_code=None, - pass_at_k={}, - n_total_trials=1, - n_completed_trials=1, - n_errored_trials=0, - trial_outcomes=[ - TrialOutcome( - task_name="hello-world", - trial_name="hello-world__attempt-0", - status="completed", - rewards={"reward": 1.0}, - ) - ], - benchmark_result=build_benchmark_result( - status="completed", score=1.0, resolved=1, total=1, reason_code=None - ), - ) - - -def _patch_run(monkeypatch) -> None: - async def _fake_run(**kwargs: Any) -> JobResult: - return _canned_result() - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.run_own_runner_job", _fake_run - ) - - -def _set_eval_plan_env(monkeypatch) -> None: - monkeypatch.setenv(PHALA_ATTESTATION_ENABLED_ENV, "1") - monkeypatch.setenv(ATTESTED_REVIEW_ENABLED_ENV, "1") - monkeypatch.setenv(PHALA_RTMR3_ENV, "d" * 96) - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - plan = { - "schema_version": 1, - "eval_run_id": "eval-run-001", - "submission_id": "submission-001", - "submission_version": 1, - "authorizing_review_digest": "1" * 64, - "agent_hash": "f" * 64, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "hello-world", - "image_ref": "registry.example/task@sha256:" + "d" * 64, - "task_config_sha256": "2" * 64, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "d" * 64, - "compose_hash": MEASUREMENT["compose_hash"], - "app_identity": "agent-challenge-eval", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "3" * 64, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("3" * 64)).hexdigest(), - "measurement": { - "mrtd": MEASUREMENT["mrtd"], - "rtmr0": MEASUREMENT["rtmr0"], - "rtmr1": MEASUREMENT["rtmr1"], - "rtmr2": MEASUREMENT["rtmr2"], - "os_image_hash": MEASUREMENT["os_image_hash"], - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "keyrelease.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-run-001/result", - "key_release_nonce": "key-nonce-001", - "score_nonce": "score-nonce-001", - "run_token_sha256": "5" * 64, - "issued_at_ms": (time.time_ns() // 1_000_000) - 1_000, - "expires_at_ms": (time.time_ns() // 1_000_000) + 60_000, - } - monkeypatch.setenv(PHALA_EVAL_PLAN_ENV, json.dumps(plan)) - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend._acquire_golden_key_if_required", - lambda **_: None, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.assert_agent_artifact_matches_plan", - lambda **_: "f" * 64, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.assert_package_tree_matches_plan", - lambda **_: "b" * 64, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend._preflight_eval_plan_tasks", - lambda **_: {}, - ) - - -def _result_line(out: str) -> dict: - lines = [ln for ln in out.splitlines() if ln.startswith(RESULT_LINE_PREFIX)] - assert len(lines) == 1 - return json.loads(lines[0][len(RESULT_LINE_PREFIX) :]) - - -def test_gate_off_emits_legacy_line_without_attestation(monkeypatch, tmp_path, capsys) -> None: - monkeypatch.delenv(PHALA_ATTESTATION_ENABLED_ENV, raising=False) - _patch_run(monkeypatch) - rc = main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - assert rc == 0 - payload = _result_line(capsys.readouterr().out) - assert ar.EXECUTION_PROOF_RESULT_KEY not in payload - assert payload["status"] == "completed" - - -def test_gate_on_emits_strict_eval_result_request(monkeypatch, tmp_path, capsys) -> None: - _patch_run(monkeypatch) - _set_eval_plan_env(monkeypatch) - monkeypatch.setattr(ar, "DstackQuoteProvider", _fake_provider_factory()) - rc = main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - assert rc == 0 - payload = _result_line(capsys.readouterr().out) - assert ew.validate_eval_result_request(payload) == payload - envelope = payload["execution_proof"] - assert envelope["tier"] == ar.PHALA_TDX_TIER - assert envelope["attestation"]["tdx_quote"] == FAKE_QUOTE - assert payload["agent_hash"] == "f" * 64 - - -def test_gate_on_quote_failure_fails_closed(monkeypatch, tmp_path, capsys) -> None: - _patch_run(monkeypatch) - _set_eval_plan_env(monkeypatch) - monkeypatch.setattr( - ar, "DstackQuoteProvider", _fake_provider_factory(raises=RuntimeError("no socket")) - ) - rc = main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - assert rc != 0 - out = capsys.readouterr().out - payload = _result_line(out) - assert "execution_proof" not in payload - assert payload["status"] == "failed" - assert payload["score"] == 0.0 - assert payload["reason_code"] == ar.PHALA_ATTESTATION_FAILED_REASON - assert FAKE_QUOTE not in out - assert "tdx_quote" not in out - - -def test_gate_on_missing_binding_env_fails_closed(monkeypatch, tmp_path, capsys) -> None: - _patch_run(monkeypatch) - monkeypatch.setenv(PHALA_ATTESTATION_ENABLED_ENV, "1") - monkeypatch.setenv(ATTESTED_REVIEW_ENABLED_ENV, "1") - # No immutable plan is provided. - monkeypatch.delenv(PHALA_EVAL_PLAN_ENV, raising=False) - rc = main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - assert rc != 0 - payload = _result_line(capsys.readouterr().out) - assert ar.EXECUTION_PROOF_RESULT_KEY not in payload - assert payload["status"] == "failed" - assert payload["reason_code"] == ar.PHALA_ATTESTATION_FAILED_REASON - - -@pytest.mark.parametrize( - "extra_args", - [ - ["--task", "unexpected-task"], - ["--n-attempts", "2"], - ], -) -def test_gate_on_rejects_cli_values_that_cross_the_immutable_plan( - monkeypatch, tmp_path, capsys, extra_args -) -> None: - _set_eval_plan_env(monkeypatch) - calls = {"key_release": 0, "runner": 0} - - def _no_key_release(**_: Any) -> None: - calls["key_release"] += 1 - pytest.fail("mismatched CLI values must fail before key release") - - async def _no_runner(**_: Any) -> JobResult: - calls["runner"] += 1 - pytest.fail("mismatched CLI values must fail before evaluation") - - monkeypatch.setattr(backend, "_acquire_golden_key_if_required", _no_key_release) - monkeypatch.setattr(backend, "run_own_runner_job", _no_runner) - rc = main(["run", "--task", "hello-world", *extra_args, "--job-dir", str(tmp_path / "job")]) - assert rc != 0 - assert calls == {"key_release": 0, "runner": 0} - assert _result_line(capsys.readouterr().out)["reason_code"] == "terminal_bench_failed" - - -def test_gate_on_uses_plan_for_key_release_images_and_attempts( - monkeypatch, tmp_path, capsys -) -> None: - _set_eval_plan_env(monkeypatch) - seen: dict[str, Any] = {} - - def _key_release(*, eval_plan: dict[str, Any] | None = None) -> None: - seen["key_release_plan"] = eval_plan - return None - - async def _run(**kwargs: Any) -> JobResult: - seen["run_kwargs"] = kwargs - return _canned_result() - - monkeypatch.setattr(backend, "_acquire_golden_key_if_required", _key_release) - monkeypatch.setattr(backend, "run_own_runner_job", _run) - monkeypatch.setattr(ar, "DstackQuoteProvider", _fake_provider_factory()) - rc = main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - assert rc == 0 - plan = seen["key_release_plan"] - assert plan is not None - assert seen["run_kwargs"]["n_attempts"] == plan["k"] == 1 - assert seen["run_kwargs"]["live_registry_refs"] == { - "hello-world": "registry.example/task@sha256:" + "d" * 64 - } - assert seen["run_kwargs"]["eval_plan"] == plan - assert ew.validate_eval_result_request(_result_line(capsys.readouterr().out)) - - -def test_expired_plan_blocks_key_release_and_evaluation(monkeypatch, tmp_path, capsys) -> None: - _set_eval_plan_env(monkeypatch) - plan = json.loads(os.environ[PHALA_EVAL_PLAN_ENV]) - plan["issued_at_ms"] = 1 - plan["expires_at_ms"] = 2 - monkeypatch.setenv(PHALA_EVAL_PLAN_ENV, json.dumps(plan)) - - def _unexpected(**_: Any) -> None: - pytest.fail("expired plans must fail before work or key release") - - monkeypatch.setattr(backend, "_acquire_golden_key_if_required", _unexpected) - monkeypatch.setattr(backend, "run_own_runner_job", _unexpected) - assert main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) != 0 - assert _result_line(capsys.readouterr().out)["reason_code"] == "terminal_bench_failed" - - -def test_task_config_mismatch_blocks_key_release_and_container_work( - monkeypatch, tmp_path, capsys -) -> None: - _set_eval_plan_env(monkeypatch) - - def _mismatch(**_: Any) -> dict[str, Any]: - raise ValueError("task content digest does not match Eval plan") - - def _unexpected(**_: Any) -> None: - pytest.fail("task mismatch must fail before key release or execution") - - monkeypatch.setattr(backend, "_preflight_eval_plan_tasks", _mismatch) - monkeypatch.setattr(backend, "_acquire_golden_key_if_required", _unexpected) - monkeypatch.setattr(backend, "run_own_runner_job", _unexpected) - assert main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) != 0 - assert _result_line(capsys.readouterr().out)["reason_code"] == "terminal_bench_failed" - - -def test_agent_source_hash_mismatch_blocks_key_release_and_execution( - monkeypatch, tmp_path, capsys -) -> None: - _set_eval_plan_env(monkeypatch) - - def _unexpected(**_: Any) -> None: - pytest.fail("agent mismatch must fail before key release or execution") - - def _mismatch(**_: Any) -> str: - raise ValueError("agent artifact does not match immutable Eval plan agent_hash") - - monkeypatch.setattr(backend, "assert_agent_artifact_matches_plan", _mismatch) - monkeypatch.setattr(backend, "_acquire_golden_key_if_required", _unexpected) - monkeypatch.setattr(backend, "run_own_runner_job", _unexpected) - assert main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) != 0 - assert _result_line(capsys.readouterr().out)["reason_code"] == "terminal_bench_failed" - - -def test_plan_agent_hash_matches_zip_artifact_domain(tmp_path) -> None: - zip_path = tmp_path / "agent.zip" - zip_path.write_bytes(b"submitted-bytes") - digest = hashlib.sha256(b"submitted-bytes").hexdigest() - assert backend.agent_artifact_sha256(zip_path) == digest - assert ( - backend.assert_agent_artifact_matches_plan( - artifact_path=zip_path, - plan_agent_hash=digest, - ) - == digest - ) diff --git a/packages/challenges/agent-challenge/tests/test_own_runner_dood_launch.py b/packages/challenges/agent-challenge/tests/test_own_runner_dood_launch.py index 148704b4e..631dfc116 100644 --- a/packages/challenges/agent-challenge/tests/test_own_runner_dood_launch.py +++ b/packages/challenges/agent-challenge/tests/test_own_runner_dood_launch.py @@ -244,13 +244,18 @@ def test_socket_mount_specs_no_false_positive_on_prefix_lookalikes() -> None: def test_socket_path_constants_are_single_sourced() -> None: - # DOCKER_SOCKET_PATH / DSTACK_SOCKET_PATH must be single-sourced: the compose - # generator imports the exact dood constants (same object identity), so the - # two definitions can never silently diverge. - from agent_challenge.canonical import compose as compose_mod + # T40 removed agent_challenge.canonical.compose (Phala CVM compose). + # Socket paths remain single-sourced from own_runner.dood and re-exported + # via own_runner package — pin stable values + package re-export identity. + from agent_challenge.evaluation import own_runner as own_runner_pkg + from agent_challenge.evaluation.own_runner import dood as dood_mod - assert compose_mod.DOCKER_SOCKET_PATH is DOCKER_SOCKET_PATH - assert compose_mod.DSTACK_SOCKET_PATH is DSTACK_SOCKET_PATH + assert DOCKER_SOCKET_PATH == "/var/run/docker.sock" + assert DSTACK_SOCKET_PATH == "/var/run/dstack.sock" + assert dood_mod.DOCKER_SOCKET_PATH is DOCKER_SOCKET_PATH + assert dood_mod.DSTACK_SOCKET_PATH is DSTACK_SOCKET_PATH + assert own_runner_pkg.DOCKER_SOCKET_PATH is DOCKER_SOCKET_PATH + assert own_runner_pkg.DSTACK_SOCKET_PATH is DSTACK_SOCKET_PATH def test_socket_mount_specs_empty_for_a_clean_task_launch() -> None: diff --git a/packages/challenges/agent-challenge/tests/test_package_tree_sha.py b/packages/challenges/agent-challenge/tests/test_package_tree_sha.py index 35f99251a..8f663130c 100644 --- a/packages/challenges/agent-challenge/tests/test_package_tree_sha.py +++ b/packages/challenges/agent-challenge/tests/test_package_tree_sha.py @@ -198,7 +198,7 @@ def test_build_plan_binds_submission_package_tree_sha() -> None: src = Path(auth.__file__).read_text(encoding="utf-8") assert "package_tree_sha" in src - assert 'plan = {' in src or "plan =" in src + assert "plan = {" in src or "plan =" in src def test_review_materials_bind_package_tree_sha() -> None: diff --git a/packages/challenges/agent-challenge/tests/test_phala_client_auth_region.py b/packages/challenges/agent-challenge/tests/test_phala_client_auth_region.py deleted file mode 100644 index 8def96351..000000000 --- a/packages/challenges/agent-challenge/tests/test_phala_client_auth_region.py +++ /dev/null @@ -1,285 +0,0 @@ -"""PhalaCloudClient auth headers + DEFAULT_REGION selection (ERR-02-002 residual). - -Live tee-proof-v4 residuals: - -* ``Authorization: Bearer `` → HTTP 401 Invalid/expired token -* bare region ``us-west`` → ERR-02-002 No teepod found (inventory is US-WEST-1 only) -* ``X-API-Key`` (+ ``X-Phala-Version``) matches Phala CLI and authenticates -* ``us-west-1`` / omitted region provisions successfully - -These unit tests pin the product client contract without live spend or secret logging. -""" - -from __future__ import annotations - -import json -from urllib.request import Request - -import pytest - -from agent_challenge.selfdeploy import eval as eval_mod -from agent_challenge.selfdeploy import plan as plan_mod -from agent_challenge.selfdeploy import review as review_mod -from agent_challenge.selfdeploy.phala import ( - DEFAULT_PHALA_API_VERSION, - PhalaApiError, - PhalaCloudClient, - normalize_phala_region, - select_phala_region, -) -from agent_challenge.selfdeploy.plan import ( - DEFAULT_REGION, - PHALA_API_KEY_ENV, - build_deploy_plan, -) - -DIGEST = "registry.example/agent-challenge@sha256:" + ("a" * 64) -KEY_RELEASE = "https://validator.example/v1/key-release" - - -class _CapturingOpener: - """Capture the outbound Request and return a fixed JSON body.""" - - def __init__(self, payload: dict | None = None) -> None: - self.requests: list[Request] = [] - self.payload = payload if payload is not None else {"ok": True} - - def __call__(self, request: Request, timeout: float = 0.0): # noqa: ARG002 - self.requests.append(request) - - class _Resp: - def __init__(self, body: bytes) -> None: - self._body = body - - def read(self, n: int = -1) -> bytes: # noqa: ARG002 - return self._body - - return _Resp(json.dumps(self.payload).encode()) - - -# --------------------------------------------------------------------------- # -# Auth headers: X-API-Key, never Bearer -# --------------------------------------------------------------------------- # - - -def test_phala_client_sends_x_api_key_not_bearer(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(PHALA_API_KEY_ENV, "phak_test_key_never_print_me") - opener = _CapturingOpener({"compose_hash": "abc", "app_id": "x"}) - client = PhalaCloudClient(api_key="phak_test_key_never_print_me", opener=opener) - - client.post("/cvms/provision", {"app_id": "x", "name": "x"}) - - assert len(opener.requests) == 1 - headers = {k.lower(): v for k, v in opener.requests[0].header_items()} - assert headers.get("x-api-key") == "phak_test_key_never_print_me" - assert "authorization" not in headers - assert not any( - (v or "").lower().startswith("bearer ") for _k, v in opener.requests[0].header_items() - ) - - -def test_phala_client_sends_x_phala_version_header(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(PHALA_API_KEY_ENV, "phak_test_key") - opener = _CapturingOpener({"ok": True}) - client = PhalaCloudClient(api_key="phak_test_key", opener=opener) - - client.post("/cvms", {"app_id": "x", "compose_hash": "h", "encrypted_env": "c", "env_keys": []}) - - headers = {k.lower(): v for k, v in opener.requests[0].header_items()} - assert headers.get("x-phala-version") == DEFAULT_PHALA_API_VERSION - assert DEFAULT_PHALA_API_VERSION # non-empty pin matching CLI Lo default - - -def test_phala_client_never_logs_or_echoes_api_key_in_errors( - monkeypatch: pytest.MonkeyPatch, -) -> None: - sentinel = "phak_super_secret_should_never_appear_in_errors" - monkeypatch.setenv(PHALA_API_KEY_ENV, sentinel) - - class _BoomOpener: - def __call__(self, request: Request, timeout: float = 0.0): # noqa: ARG002 - from urllib.error import HTTPError - - raise HTTPError( - url=request.full_url, - code=401, - msg="Unauthorized", - hdrs=None, # type: ignore[arg-type] - fp=None, - ) - - client = PhalaCloudClient(api_key=sentinel, opener=_BoomOpener()) - with pytest.raises(PhalaApiError) as excinfo: - client.post("/cvms/provision", {"app_id": "x"}) - message = str(excinfo.value) - assert sentinel not in message - assert "401" in message - assert "Bearer" not in message - - -def test_phala_client_rejects_http_base_url(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(PHALA_API_KEY_ENV, "phak_test") - with pytest.raises(PhalaApiError, match="https"): - PhalaCloudClient(api_key="phak_test", base_url="http://insecure.example/api/v1") - - -# --------------------------------------------------------------------------- # -# Region selection: us-west must not hard-fail; prefer us-west-1 / auto -# --------------------------------------------------------------------------- # - - -def test_default_region_is_not_bare_us_west_alias() -> None: - """Bare ``us-west`` hard-fails live with ERR-02-002 (no teepod).""" - - assert DEFAULT_REGION != "us-west" - assert DEFAULT_REGION in {"us-west-1", ""} - # Plan / Review / Eval keep the same product default (no drift). - assert review_mod.DEFAULT_REGION == DEFAULT_REGION - assert eval_mod.DEFAULT_REGION == DEFAULT_REGION - assert plan_mod.DEFAULT_REGION == DEFAULT_REGION - - -def test_normalize_phala_region_maps_us_west_to_us_west_1() -> None: - assert normalize_phala_region("us-west") == "us-west-1" - assert normalize_phala_region("US-WEST") == "us-west-1" - assert normalize_phala_region("us-west-1") == "us-west-1" - assert normalize_phala_region("US-WEST-1") == "us-west-1" - # Empty → auto (omit sentinel for callers that want no region key) - assert normalize_phala_region("") == "" - assert normalize_phala_region(None) == "" - - -def test_select_phala_region_prefers_available_capacity() -> None: - # When inventory lists only us-west-1, never pick bare us-west. - assert ( - select_phala_region("us-west", available_regions=["us-west-1", "eu-central-1"]) - == "us-west-1" - ) - # Explicit available region wins when present. - assert select_phala_region(None, available_regions=["eu-central-1"]) == "eu-central-1" - # Preferred fallback when inventory empty: us-west-1 (or empty auto). - assert select_phala_region(None, available_regions=[]) in {"us-west-1", ""} - # Explicit non-alias region passes through. - assert select_phala_region("eu-central-1", available_regions=["us-west-1"]) == "eu-central-1" - - -def test_build_deploy_plan_default_region_avoids_us_west_hard_fail() -> None: - plan = build_deploy_plan(image=DIGEST, key_release_url=KEY_RELEASE) - assert plan.region != "us-west" - assert plan.region == DEFAULT_REGION or plan.region == "" - - -def test_build_deploy_plan_normalizes_us_west_alias() -> None: - plan = build_deploy_plan(image=DIGEST, key_release_url=KEY_RELEASE, region="us-west") - assert plan.region != "us-west" - assert plan.region == "us-west-1" - - -def test_review_deployment_plan_default_region() -> None: - assert review_mod.DEFAULT_REGION != "us-west" - - -def test_phala_client_normalizes_region_in_provision_body( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv(PHALA_API_KEY_ENV, "phak_test_key") - opener = _CapturingOpener({"ok": True}) - client = PhalaCloudClient(api_key="phak_test_key", opener=opener) - - client.post( - "/cvms/provision", - { - "app_id": "x", - "name": "x", - "instance_type": "tdx.small", - "region": "us-west", - "env_keys": [], - }, - ) - - body = json.loads(opener.requests[0].data.decode()) - assert body["region"] == "us-west-1" - # Empty region is omitted (auto from teepods). - opener2 = _CapturingOpener({"ok": True}) - client2 = PhalaCloudClient(api_key="phak_test_key", opener=opener2) - client2.post( - "/cvms/provision", - {"app_id": "x", "name": "x", "region": "", "env_keys": []}, - ) - body2 = json.loads(opener2.requests[0].data.decode()) - assert "region" not in body2 - - -def test_review_provision_sends_image_and_accepts_matching_os_hash() -> None: - """Live teepod OS pin must be sent and checked (no auto-dev de9c drift).""" - from agent_challenge.selfdeploy.review import ( - DEFAULT_OS_IMAGE, - HttpReviewPhalaDeployment, - ReviewDeploymentError, - ReviewDeploymentPlan, - ) - - measurement = { - "mrtd": "01" * 48, - "rtmr0": "02" * 48, - "rtmr1": "03" * 48, - "rtmr2": "04" * 48, - "os_image_hash": "bd" + "0" * 62, - "key_provider": "phala", - "vm_shape": "tdx.small", - } - plan = ReviewDeploymentPlan( - assignment={"assignment_core": {"assignment_id": "a1"}}, - compose={"name": "agent-challenge-review-v1"}, - compose_text="{}", - compose_hash="ab" * 32, - app_identity="f024ea2315052843d0afd775b2b82b2d2455c798", - image_ref="registry.example/r@sha256:" + "a" * 64, - kms_public_key_hex="c" * 64, - kms_public_key_sha256="d" * 64, - measurement=measurement, - measurement_allowlist_sha256="e" * 64, - review_session_token="tok", - compose_name="agent-challenge-review-v1", - phala_app_nonce=0, - os_image=DEFAULT_OS_IMAGE, - ) - - HttpReviewPhalaDeployment._verify_provision_response( - plan, - { - "app_id": plan.app_identity, - "compose_hash": plan.compose_hash, - "app_env_encrypt_pubkey": plan.kms_public_key_hex, - "os_image_hash": measurement["os_image_hash"], - }, - ) - with pytest.raises(ReviewDeploymentError, match="os_image_hash"): - HttpReviewPhalaDeployment._verify_provision_response( - plan, - { - "app_id": plan.app_identity, - "compose_hash": plan.compose_hash, - "app_env_encrypt_pubkey": plan.kms_public_key_hex, - "os_image_hash": "de" + "9" * 62, # live-auto dev mismatch - }, - ) - with pytest.raises(ReviewDeploymentError, match="app identity"): - HttpReviewPhalaDeployment._verify_provision_response( - plan, - { - "app_id": "random-minted-hex", - "compose_hash": plan.compose_hash, - "app_env_encrypt_pubkey": plan.kms_public_key_hex, - "os_image_hash": measurement["os_image_hash"], - }, - ) - assert DEFAULT_OS_IMAGE == "dstack-0.5.9" - - -def test_default_os_image_is_live_available_non_dev() -> None: - from agent_challenge.selfdeploy.shapes import DEFAULT_OS_IMAGE - - assert DEFAULT_OS_IMAGE == "dstack-0.5.9" - assert "nvidia" not in DEFAULT_OS_IMAGE - assert "dev" not in DEFAULT_OS_IMAGE diff --git a/packages/challenges/agent-challenge/tests/test_phala_create_ack_and_cli_token.py b/packages/challenges/agent-challenge/tests/test_phala_create_ack_and_cli_token.py deleted file mode 100644 index 7a8124be9..000000000 --- a/packages/challenges/agent-challenge/tests/test_phala_create_ack_and_cli_token.py +++ /dev/null @@ -1,818 +0,0 @@ -"""TDD for tee-live-proof-v5 progressive residuals: - -1. PhalaCloudClient sends CLI-equivalent User-Agent so Cloudflare 1010 does - not block product urllib (keep X-API-Key / X-Phala-Version). -2. HttpReviewPhalaDeployment create ack: accept id/cvm_id as str or int, plus - alternate fields and safe GET /cvms list fallback by app_id before fail- - closed product_create_response_missing_cvm_id_field. -3. CLI review deploy consumes a review_retry one-time token path without - fail-closing solely because prepare already delivered (null re-prepare). -4. CLI eval deploy recovers EVAL_RUN_TOKEN via cancel+retry when prepare - returns token-less secret_delivery (production residual: submission 3). - -Never invent TEE measurements. Secrets never appear in errors or logs. -""" - -from __future__ import annotations - -import json -from types import SimpleNamespace -from typing import Any -from unittest.mock import MagicMock -from urllib.request import Request - -import pytest - -from agent_challenge.canonical import eval_wire -from agent_challenge.review.canonical import canonical_sha256 -from agent_challenge.review.compose import ( - generate_review_app_compose, - review_app_compose_hash, -) -from agent_challenge.review.schemas import ReviewInputConfig, build_review_assignment -from agent_challenge.selfdeploy import cli -from agent_challenge.selfdeploy import eval as eval_deploy -from agent_challenge.selfdeploy import review as review_mod -from agent_challenge.selfdeploy.client import RouteClientError -from agent_challenge.selfdeploy.phala import ( - DEFAULT_PHALA_USER_AGENT, - PhalaCloudClient, - extract_cvm_id_from_create_response, - resolve_cvm_id_from_list, -) -from agent_challenge.selfdeploy.plan import PHALA_API_KEY_ENV -from agent_challenge.selfdeploy.review import ( - ReviewDeploymentError, - ReviewPhalaDeployment, - build_review_deployment_plan, - encrypt_review_secrets, -) - -REVIEW_IMAGE = "registry.example/review@sha256:" + "a" * 64 -PUBLIC_KEY = "c" * 64 -MEASUREMENT = { - "mrtd": "01" * 48, - "rtmr0": "02" * 48, - "rtmr1": "03" * 48, - "rtmr2": "04" * 48, - "os_image_hash": "05" * 32, - "key_provider": "phala", - "vm_shape": "tdx.small", -} -TOKEN = "review-token-sentinel-create-ack" - - -class _CapturingOpener: - def __init__(self, payload: dict | list | None = None) -> None: - self.requests: list[Request] = [] - self.payload = payload if payload is not None else {"ok": True} - - def __call__(self, request: Request, timeout: float = 0.0): # noqa: ARG002 - self.requests.append(request) - - class _Resp: - def __init__(self, body: bytes) -> None: - self._body = body - - def read(self, n: int = -1) -> bytes: # noqa: ARG002 - return self._body - - return _Resp(json.dumps(self.payload).encode()) - - -def _assignment_and_plan() -> tuple[dict[str, Any], Any, Any]: - compose = generate_review_app_compose( - review_image=REVIEW_IMAGE, - app_identity="agent-challenge-review-v1", - ) - compose_hash = review_app_compose_hash(compose) - allowlist_entry = { - "mrtd": MEASUREMENT["mrtd"], - "rtmr0": MEASUREMENT["rtmr0"], - "rtmr1": MEASUREMENT["rtmr1"], - "rtmr2": MEASUREMENT["rtmr2"], - "compose_hash": compose_hash, - "os_image_hash": MEASUREMENT["os_image_hash"], - } - config = ReviewInputConfig( - image_ref=REVIEW_IMAGE, - compose_hash=compose_hash, - app_identity="agent-challenge-review-v1", - kms_public_key_hex=PUBLIC_KEY, - measurement=MEASUREMENT, - measurement_allowlist=(allowlist_entry,), - measurement_allowlist_sha256=canonical_sha256({"entries": [allowlist_entry]}), - ) - assignment, _, _ = build_review_assignment( - session_id="session-1", - assignment_id="assignment-1", - attempt=1, - submission_id="1", - artifact={ - "agent_hash": "10" * 32, - "zip_sha256": "20" * 32, - "zip_size_bytes": 1, - "manifest_sha256": "30" * 32, - "manifest_entries_sha256": "40" * 32, - "fetch_path": "/review/v1/assignments/assignment-1/artifact", - }, - rules_snapshot_sha256_value="50" * 32, - rules_revision_id="rules-v1", - review_nonce="nonce-review", - issued_at_ms=1, - expires_at_ms=2, - session_token_sha256=__import__("hashlib").sha256(TOKEN.encode()).hexdigest(), - config=config, - ) - plan = build_review_deployment_plan({"assignment": assignment, "review_session_token": TOKEN}) - encrypted = encrypt_review_secrets( - plan, - { - "OPENROUTER_API_KEY": "or-test-key-never-print", - "REVIEW_API_BASE_URL": "https://chain.joinbase.ai/challenges/agent-challenge", - "REVIEW_SESSION_TOKEN": TOKEN, - }, - ) - return assignment, plan, encrypted - - -# --------------------------------------------------------------------------- # -# 1) User-Agent header -# --------------------------------------------------------------------------- # - - -def test_phala_client_sends_cli_equivalent_user_agent(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(PHALA_API_KEY_ENV, "phak_test_key") - opener = _CapturingOpener({"ok": True}) - client = PhalaCloudClient(api_key="phak_test_key", opener=opener) - - client.post("/cvms/provision", {"app_id": "x", "name": "x"}) - - headers = {k.lower(): v for k, v in opener.requests[0].header_items()} - assert headers.get("user-agent") == DEFAULT_PHALA_USER_AGENT - assert DEFAULT_PHALA_USER_AGENT.startswith("phala-cli/") - # Keep auth contract: X-API-Key, never Bearer; no Python-urllib bare agent. - assert headers.get("x-api-key") == "phak_test_key" - assert "authorization" not in headers - assert "Python-urllib" not in (headers.get("user-agent") or "") - - -def test_phala_client_get_sends_same_auth_and_user_agent( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv(PHALA_API_KEY_ENV, "phak_test_key") - opener = _CapturingOpener({"items": []}) - client = PhalaCloudClient(api_key="phak_test_key", opener=opener) - - client.get("/cvms") - - assert opener.requests[0].get_method() == "GET" - headers = {k.lower(): v for k, v in opener.requests[0].header_items()} - assert headers.get("user-agent") == DEFAULT_PHALA_USER_AGENT - assert headers.get("x-api-key") == "phak_test_key" - assert headers.get("x-phala-version") - - -# --------------------------------------------------------------------------- # -# 2) Create response id mapping + list fallback -# --------------------------------------------------------------------------- # - - -@pytest.mark.parametrize( - "created,expected", - [ - ({"id": 4242, "app_id": "app-x", "status": "starting"}, "4242"), - ({"id": "cvm-str-1", "request_id": "req-1"}, "cvm-str-1"), - ({"cvm_id": "cvm-alt-9", "app_id": "app-x"}, "cvm-alt-9"), - ({"vm_uuid": "vm-uuid-7", "app_id": "app-x"}, "vm-uuid-7"), - ({"instance_id": "inst-3", "app_id": "app-x"}, "inst-3"), - ], -) -def test_extract_cvm_id_accepts_alternate_create_shapes( - created: dict[str, Any], expected: str -) -> None: - assert extract_cvm_id_from_create_response(created) == expected - - -def test_extract_cvm_id_numeric_id_wins_over_app_id_string() -> None: - # Live schema: id is int; app_id is the Phala app pin and is NOT the CVM id. - created = {"id": 99, "app_id": "f024ea2315052843d0afd775b2b82b2d2455c798"} - assert extract_cvm_id_from_create_response(created) == "99" - - -def test_extract_cvm_id_rejects_app_id_as_cvm_identity() -> None: - with pytest.raises(ValueError, match="does not identify"): - extract_cvm_id_from_create_response( - {"app_id": "f024ea2315052843d0afd775b2b82b2d2455c798", "status": "starting"} - ) - - -def test_resolve_cvm_id_from_list_by_app_id() -> None: - listing = { - "items": [ - {"id": 1, "app_id": "other", "status": "running"}, - {"id": 77, "app_id": "f024ea2315052843d0afd775b2b82b2d2455c798", "status": "starting"}, - ] - } - assert ( - resolve_cvm_id_from_list(listing, app_id="f024ea2315052843d0afd775b2b82b2d2455c798") == "77" - ) - - -def test_review_deploy_accepts_numeric_create_id() -> None: - _assignment, plan, encrypted = _assignment_and_plan() - deployment = ReviewPhalaDeployment( - provision_response={ - "app_id": plan.app_identity, - "compose_hash": plan.compose_hash, - "app_env_encrypt_pubkey": PUBLIC_KEY, - "os_image_hash": MEASUREMENT["os_image_hash"], - }, - # Live Phala create schema uses numeric id (CLI zod schema). - create_response={ - "id": 9175, - "name": plan.compose_name, - "status": "starting", - "app_id": plan.app_identity, - "created_at": "2026-07-15T00:00:00Z", - }, - ) - acknowledgement = deployment.deploy(plan, encrypted) - assert acknowledgement["cvm_id"] == "9175" - assert acknowledgement["phala_create_receipt"]["cvm_id"] == "9175" - - -def test_review_deploy_list_fallback_by_app_id_when_create_lacks_id() -> None: - _assignment, plan, encrypted = _assignment_and_plan() - - class _Api: - def __init__(self) -> None: - self.provision_requests: list[dict[str, Any]] = [] - self.create_requests: list[dict[str, Any]] = [] - self.get_paths: list[str] = [] - - def post(self, path: str, payload: dict[str, Any]) -> dict[str, Any]: - if path == "/cvms/provision": - self.provision_requests.append(dict(payload)) - return { - "app_id": plan.app_identity, - "compose_hash": plan.compose_hash, - "app_env_encrypt_pubkey": PUBLIC_KEY, - "os_image_hash": MEASUREMENT["os_image_hash"], - } - if path == "/cvms": - self.create_requests.append(dict(payload)) - # Residual shape: create 200 but no usable id field. - return { - "app_id": plan.app_identity, - "status": "processing", - "name": plan.compose_name, - } - raise AssertionError(path) - - def get(self, path: str) -> dict[str, Any]: - self.get_paths.append(path) - assert path == "/cvms" - return { - "items": [ - { - "id": 5511, - "app_id": plan.app_identity, - "status": "starting", - "name": plan.compose_name, - } - ] - } - - api = _Api() - acknowledgement = review_mod.HttpReviewPhalaDeployment(api).deploy(plan, encrypted) - assert api.get_paths == ["/cvms"] - assert acknowledgement["cvm_id"] == "5511" - - -def test_review_deploy_still_fail_closed_when_list_has_no_matching_cvm() -> None: - _assignment, plan, encrypted = _assignment_and_plan() - - class _Api: - def post(self, path: str, payload: dict[str, Any]) -> dict[str, Any]: - if path == "/cvms/provision": - return { - "app_id": plan.app_identity, - "compose_hash": plan.compose_hash, - "app_env_encrypt_pubkey": PUBLIC_KEY, - "os_image_hash": MEASUREMENT["os_image_hash"], - } - return {"app_id": plan.app_identity, "status": "processing"} - - def get(self, path: str) -> dict[str, Any]: # noqa: ARG002 - return {"items": [{"id": 1, "app_id": "someone-else", "status": "running"}]} - - with pytest.raises(ReviewDeploymentError, match="does not identify the review CVM"): - review_mod.HttpReviewPhalaDeployment(_Api()).deploy(plan, encrypted) - - -def test_extract_helpers_never_echo_secrets() -> None: - payload = { - "id": 1, - "encrypted_env": "CIPHERTEXT_SECRET_MUST_NOT_LEAK", - "app_id": "app", - } - assert extract_cvm_id_from_create_response(payload) == "1" - message = "" - try: - extract_cvm_id_from_create_response({"encrypted_env": "CIPHERTEXT_SECRET_MUST_NOT_LEAK"}) - except ValueError as exc: - message = str(exc) - assert "CIPHERTEXT_SECRET_MUST_NOT_LEAK" not in message - - -# --------------------------------------------------------------------------- # -# 3) CLI review deploy consumes retry-delivered token (no null re-prepare trap) -# --------------------------------------------------------------------------- # - - -def _fresh_assignment(*, assignment_id: str, attempt: int, token: str) -> dict[str, Any]: - compose = generate_review_app_compose( - review_image=REVIEW_IMAGE, - app_identity="agent-challenge-review-v1", - ) - compose_hash = review_app_compose_hash(compose) - allowlist_entry = { - "mrtd": MEASUREMENT["mrtd"], - "rtmr0": MEASUREMENT["rtmr0"], - "rtmr1": MEASUREMENT["rtmr1"], - "rtmr2": MEASUREMENT["rtmr2"], - "compose_hash": compose_hash, - "os_image_hash": MEASUREMENT["os_image_hash"], - } - config = ReviewInputConfig( - image_ref=REVIEW_IMAGE, - compose_hash=compose_hash, - app_identity="agent-challenge-review-v1", - kms_public_key_hex=PUBLIC_KEY, - measurement=MEASUREMENT, - measurement_allowlist=(allowlist_entry,), - measurement_allowlist_sha256=canonical_sha256({"entries": [allowlist_entry]}), - ) - import hashlib - - assignment, _, _ = build_review_assignment( - session_id="session-1", - assignment_id=assignment_id, - attempt=attempt, - submission_id="1", - artifact={ - "agent_hash": "10" * 32, - "zip_sha256": "20" * 32, - "zip_size_bytes": 1, - "manifest_sha256": "30" * 32, - "manifest_entries_sha256": "40" * 32, - "fetch_path": f"/review/v1/assignments/{assignment_id}/artifact", - }, - rules_snapshot_sha256_value="50" * 32, - rules_revision_id="rules-v1", - review_nonce=f"nonce-{assignment_id}", - issued_at_ms=1, - expires_at_ms=2, - session_token_sha256=hashlib.sha256(token.encode()).hexdigest(), - config=config, - ) - return assignment - - -def test_cli_review_deploy_uses_retry_when_prepare_token_already_delivered( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Reused submission: prepare returns null token; product cancel+retry recovers. - - Discriminator residual: CLI always called review_prepare only, received - null token on re-prepare, and fail-closed before Phala create. - """ - assignment, plan, _encrypted = _assignment_and_plan() - retry_token = "review-retry-token-fresh" - retry_assignment = _fresh_assignment( - assignment_id="assignment-2", - attempt=2, - token=retry_token, - ) - - fake_client = MagicMock() - fake_client.review_prepare.return_value = { - "session_id": "session-1", - "assignment_id": "assignment-1", - "attempt": 1, - "assignment": assignment, - "review_session_token": None, # already delivered (one-time) - } - fake_client.review_cancel.return_value = { - "assignment_id": "assignment-1", - "phase": "review_cancelled", - } - fake_client.review_retry.return_value = { - "session_id": "session-1", - "assignment_id": "assignment-2", - "attempt": 2, - "assignment": retry_assignment, - "review_session_token": retry_token, - } - fake_client.review_deployed.return_value = {"ok": True} - - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - monkeypatch.setenv("OPENROUTER_API_KEY", "or-test-key") - - class _FixedDeploy: - def __init__(self, _api: object) -> None: - pass - - def deploy(self, plan_obj: Any, encrypted_obj: Any) -> dict[str, Any]: - assert plan_obj.review_session_token == retry_token - assert encrypted_obj.assignment_id == "assignment-2" - return { - "schema_version": 1, - "assignment_id": "assignment-2", - "cvm_id": "cvm-from-deploy", - "phala_create_receipt": { - "request_id": "req", - "app_id": plan.app_identity, - "cvm_id": "cvm-from-deploy", - "receipt_sha256": "a" * 64, - "created_at_ms": 1, - }, - "compose_identity": { - "image_ref": plan.image_ref, - "compose_hash": plan.compose_hash, - "app_kms_public_key_sha256": plan.kms_public_key_sha256, - }, - } - - monkeypatch.setattr(review_mod, "HttpReviewPhalaDeployment", _FixedDeploy) - monkeypatch.setattr(cli, "PhalaCloudClient", lambda **_k: object()) - printed: list[Any] = [] - monkeypatch.setattr(cli, "_print", printed.append) - - args = SimpleNamespace( - review_command="deploy", - submission_id=1, - dry_run=False, - auto_sign=True, - signature=None, - nonce=None, - prepare_response=None, - review_instance_type=plan.instance_type, - eval_instance_type="tdx.small", - review_runtime_hours=1.0, - eval_runtime_hours=1.0, - money_cap_usd=20.0, - openrouter_key_env="OPENROUTER_API_KEY", - phala_api=None, - base_url="https://chain.joinbase.ai/challenges/agent-challenge", - hotkey="hk", - timestamp=None, - ) - code = cli._ordered_review_command(args) - assert code == 0, printed - fake_client.review_prepare.assert_called_once_with(1) - fake_client.review_cancel.assert_called_once_with(1, "assignment-1") - fake_client.review_retry.assert_called_once() - assert fake_client.review_retry.call_args.args[0] == 1 - assert fake_client.review_retry.call_args.args[1] == "assignment-1" - fake_client.review_deployed.assert_called_once() - # Capability bytes never printed. - assert retry_token not in json.dumps(printed) - - -def test_cli_review_deploy_uses_prepare_token_when_fresh( - monkeypatch: pytest.MonkeyPatch, -) -> None: - assignment, plan, _encrypted = _assignment_and_plan() - fake_client = MagicMock() - fake_client.review_prepare.return_value = { - "session_id": "session-1", - "assignment_id": "assignment-1", - "attempt": 1, - "assignment": assignment, - "review_session_token": TOKEN, - } - fake_client.review_deployed.return_value = {"ok": True} - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - monkeypatch.setenv("OPENROUTER_API_KEY", "or-test-key") - - class _FixedDeploy: - def __init__(self, _api: object) -> None: - pass - - def deploy(self, plan_obj: Any, encrypted_obj: Any) -> dict[str, Any]: - assert plan_obj.review_session_token == TOKEN - return { - "schema_version": 1, - "assignment_id": "assignment-1", - "cvm_id": "cvm-1", - "phala_create_receipt": { - "request_id": "req", - "app_id": plan.app_identity, - "cvm_id": "cvm-1", - "receipt_sha256": "b" * 64, - "created_at_ms": 1, - }, - "compose_identity": { - "image_ref": plan.image_ref, - "compose_hash": plan.compose_hash, - "app_kms_public_key_sha256": plan.kms_public_key_sha256, - }, - } - - monkeypatch.setattr(review_mod, "HttpReviewPhalaDeployment", _FixedDeploy) - monkeypatch.setattr(cli, "PhalaCloudClient", lambda **_k: object()) - monkeypatch.setattr(cli, "_print", lambda _p: None) - - args = SimpleNamespace( - review_command="deploy", - submission_id=9, - dry_run=False, - auto_sign=True, - signature=None, - nonce=None, - prepare_response=None, - review_instance_type=plan.instance_type, - eval_instance_type="tdx.small", - review_runtime_hours=1.0, - eval_runtime_hours=1.0, - money_cap_usd=20.0, - openrouter_key_env="OPENROUTER_API_KEY", - phala_api=None, - base_url="https://chain.joinbase.ai/challenges/agent-challenge", - hotkey="hk", - timestamp=None, - ) - assert cli._ordered_review_command(args) == 0 - fake_client.review_cancel.assert_not_called() - fake_client.review_retry.assert_not_called() - - -def test_cli_review_deploy_skips_cancel_when_prepare_redelivers( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Dry-run prepare then live deploy must reuse redelivered token (no cancel burn). - - Residual review-v9 token path: previous CLI always cancel+retried whenever - prepare returned null and burned undepoyed active attempts 1-2. With product - redelivery, prepare after dry-run yields the same token → zero cancel/retry. - """ - - assignment, plan, _encrypted = _assignment_and_plan() - fake_client = MagicMock() - # Simulate: first prepare (dry) already delivered; second prepare redelivers. - fake_client.review_prepare.return_value = { - "session_id": "session-1", - "assignment_id": "assignment-1", - "attempt": 1, - "assignment": assignment, - "review_session_token": TOKEN, - } - fake_client.review_deployed.return_value = {"ok": True} - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - monkeypatch.setenv("OPENROUTER_API_KEY", "or-test-key") - - class _FixedDeploy: - def __init__(self, _api: object) -> None: - pass - - def deploy(self, plan_obj: Any, encrypted_obj: Any) -> dict[str, Any]: - assert plan_obj.review_session_token == TOKEN - assert encrypted_obj.assignment_id == "assignment-1" - return { - "schema_version": 1, - "assignment_id": "assignment-1", - "cvm_id": "cvm-reuse", - "phala_create_receipt": { - "request_id": "req", - "app_id": plan.app_identity, - "cvm_id": "cvm-reuse", - "receipt_sha256": "c" * 64, - "created_at_ms": 1, - }, - "compose_identity": { - "image_ref": plan.image_ref, - "compose_hash": plan.compose_hash, - "app_kms_public_key_sha256": plan.kms_public_key_sha256, - }, - } - - monkeypatch.setattr(review_mod, "HttpReviewPhalaDeployment", _FixedDeploy) - monkeypatch.setattr(cli, "PhalaCloudClient", lambda **_k: object()) - printed: list[Any] = [] - monkeypatch.setattr(cli, "_print", printed.append) - - args = SimpleNamespace( - review_command="deploy", - submission_id=3, - dry_run=False, - auto_sign=True, - signature=None, - nonce=None, - prepare_response=None, - review_instance_type=plan.instance_type, - eval_instance_type="tdx.small", - review_runtime_hours=1.0, - eval_runtime_hours=1.0, - money_cap_usd=12.0, - openrouter_key_env="OPENROUTER_API_KEY", - phala_api=None, - base_url="https://chain.joinbase.ai/challenges/agent-challenge", - hotkey="hk", - timestamp=None, - ) - assert cli._ordered_review_command(args) == 0 - fake_client.review_prepare.assert_called_once_with(3) - fake_client.review_cancel.assert_not_called() - fake_client.review_retry.assert_not_called() - fake_client.review_deployed.assert_called_once() - - -def test_obtain_review_prepare_only_cancel_retries_when_token_null_and_terminal( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Cancel+retry remains for sticky-null (post-deploy / terminal) only.""" - - assignment, _plan, _enc = _assignment_and_plan() - retry_token = "review-retry-after-sticky-null" - retry_assignment = _fresh_assignment( - assignment_id="assignment-3", - attempt=3, - token=retry_token, - ) - fake_client = MagicMock() - fake_client.review_prepare.return_value = { - "session_id": "session-1", - "assignment_id": "assignment-1", - "attempt": 1, - "assignment": assignment, - "review_session_token": None, # sticky after deploy or terminal - "phase": "review_cvm_running", - } - fake_client.review_cancel.return_value = { - "assignment_id": "assignment-1", - "phase": "review_cancelled", - } - fake_client.review_retry.return_value = { - "session_id": "session-1", - "assignment_id": "assignment-3", - "attempt": 3, - "assignment": retry_assignment, - "review_session_token": retry_token, - } - out = cli._obtain_review_prepare_with_token(fake_client, 7) - assert out["review_session_token"] == retry_token - fake_client.review_cancel.assert_called_once_with(7, "assignment-1") - fake_client.review_retry.assert_called_once() - - -def _eval_prepare_wrapper(*, token: str | None, eval_run_id: str = "eval-1") -> dict[str, Any]: - """Minimal signed-shape Eval prepare wrapper for CLI recovery unit tests.""" - - import hashlib - - from agent_challenge.canonical.compose import ( - generate_app_compose, - render_app_compose, - ) - - eval_image = "registry.example/eval@sha256:" + "b" * 64 - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - compose = generate_app_compose( - orchestrator_image=eval_image, - name="eval-v1", - key_release_url="validator.example:8701", - allowed_envs=eval_deploy.EVAL_ALLOWED_ENVS, - ) - compose_hash = hashlib.sha256(render_app_compose(compose).encode()).hexdigest() - run_token = token if isinstance(token, str) and token else "placeholder-token" - plan = { - "schema_version": 1, - "eval_run_id": eval_run_id, - "submission_id": "1", - "submission_version": 1, - "authorizing_review_digest": "d" * 64, - "agent_hash": "e" * 64, - "package_tree_sha": "b" * 64, - "selected_tasks": [ - { - "task_id": "task-1", - "image_ref": "registry.example/task@sha256:" + "f" * 64, - "task_config_sha256": "1" * 64, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": eval_wire.scoring_policy_digest(policy), - "eval_app": { - "image_ref": eval_image, - "compose_hash": compose_hash, - "app_identity": "eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": PUBLIC_KEY, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex(PUBLIC_KEY)).hexdigest(), - "measurement": MEASUREMENT, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": f"/evaluation/v1/runs/{eval_run_id}/result", - "key_release_nonce": "key-release-nonce", - "score_nonce": "score-nonce", - "run_token_sha256": hashlib.sha256(run_token.encode()).hexdigest(), - "issued_at_ms": 1, - "expires_at_ms": 2, - } - validated = eval_wire.validate_eval_plan(plan) - delivery: dict[str, str] | None - if isinstance(token, str) and token: - delivery = {"env_key": "EVAL_RUN_TOKEN", "token": token} - else: - delivery = None - return { - "schema_version": 1, - "plan": validated, - "plan_sha256": hashlib.sha256(eval_wire.canonical_json_v1(validated)).hexdigest(), - "secret_delivery": delivery, - } - - -def test_eval_token_present_requires_exact_env_key_token_shape() -> None: - """Presence predicate matches eval.py secret_delivery contract; no relaxation.""" - - assert cli._eval_token_present(None) is False - assert cli._eval_token_present({}) is False - assert cli._eval_token_present({"secret_delivery": None}) is False - assert cli._eval_token_present({"secret_delivery": {"env_key": "EVAL_RUN_TOKEN"}}) is False - assert ( - cli._eval_token_present( - {"secret_delivery": {"env_key": "EVAL_RUN_TOKEN", "token": "", "extra": 1}} - ) - is False - ) - assert ( - cli._eval_token_present( - {"secret_delivery": {"env_key": "EVAL_RUN_TOKEN", "token": "run-tok"}} - ) - is True - ) - # Validation shape in build_eval_deployment_plan remains fail-closed. - bare = _eval_prepare_wrapper(token=None) - with pytest.raises(eval_deploy.EvalDeploymentError, match="EVAL_RUN_TOKEN capability"): - eval_deploy.build_eval_deployment_plan(bare) - - -def test_obtain_eval_prepare_returns_first_prepare_when_token_present() -> None: - """Token already delivered on prepare → never cancel or retry.""" - - fresh = _eval_prepare_wrapper(token="eval-fresh-token", eval_run_id="eval-run-a") - fake_client = MagicMock() - fake_client.eval_prepare.return_value = fresh - out = cli._obtain_eval_prepare_with_token(fake_client, 3) - assert out is fresh - assert out["secret_delivery"]["token"] == "eval-fresh-token" - fake_client.eval_prepare.assert_called_once_with(3) - fake_client.eval_cancel.assert_not_called() - fake_client.eval_retry.assert_not_called() - - -def test_obtain_eval_prepare_cancel_retries_when_token_absent() -> None: - """Spent one-shot token: prepare token-less → cancel+retry recovers capability. - - Production residual (submission 3): standalone ``eval prepare`` or - ``eval retry`` spends EVAL_RUN_TOKEN; subsequent ``eval deploy`` saw - secret_delivery=None and hard-failed with no recovery path. - """ - - spent = _eval_prepare_wrapper(token=None, eval_run_id="eval-run-1") - recovered_token = "eval-retry-token-fresh" - recovered = _eval_prepare_wrapper(token=recovered_token, eval_run_id="eval-run-2") - fake_client = MagicMock() - fake_client.eval_prepare.return_value = spent - fake_client.eval_cancel.return_value = {"phase": "eval_cancelled"} - fake_client.eval_retry.return_value = recovered - out = cli._obtain_eval_prepare_with_token(fake_client, 3) - assert out["secret_delivery"]["token"] == recovered_token - fake_client.eval_prepare.assert_called_once_with(3) - fake_client.eval_cancel.assert_called_once_with(3, "eval-run-1") - fake_client.eval_retry.assert_called_once_with(3, "eval-run-1") - - -def test_obtain_eval_prepare_raises_when_token_still_absent_after_retry() -> None: - """Sticky token-less after cancel+retry → typed RouteClientError.""" - - spent = _eval_prepare_wrapper(token=None, eval_run_id="eval-run-stuck") - still_spent = _eval_prepare_wrapper(token=None, eval_run_id="eval-run-stuck") - fake_client = MagicMock() - fake_client.eval_prepare.return_value = spent - fake_client.eval_cancel.return_value = {"phase": "eval_cancelled"} - fake_client.eval_retry.return_value = still_spent - with pytest.raises(RouteClientError, match="eval run token unavailable"): - cli._obtain_eval_prepare_with_token(fake_client, 3) - fake_client.eval_cancel.assert_called_once_with(3, "eval-run-stuck") - fake_client.eval_retry.assert_called_once_with(3, "eval-run-stuck") diff --git a/packages/challenges/agent-challenge/tests/test_post_grant_stage_markers_and_score_quote_normalize.py b/packages/challenges/agent-challenge/tests/test_post_grant_stage_markers_and_score_quote_normalize.py deleted file mode 100644 index c4a530c53..000000000 --- a/packages/challenges/agent-challenge/tests/test_post_grant_stage_markers_and_score_quote_normalize.py +++ /dev/null @@ -1,492 +0,0 @@ -"""Post-grant stage markers + score-path GetQuote KR normalize (offline). - -Live residual (image@sha256:6e163501 + host KR 32ed505b): key_release_state=granted, -guest frame_send, then BASE_BENCHMARK_RESULT reason_code=phala_attestation_failed with -NO trial/post-grant stage markers. Ranked cause: score-path obtain_quote/emit does not -reuse KR GetQuote normalizers (quote_hex lower/0x strip, event_log coerce + empty IMR3 -runtime_event_digest fill), so RTMR3 self-check / validate_eval_phala_attestation fails; -also no decrypt_ok/job_*/emit_* markers. - -Discriminators (would fail a wrong implementation): - * score obtain_quote must inherit KR normalize so empty-IMR3 dstack GetQuote passes - RTMR3 self-check used by schema-v2 emit; - * happy-path fakes must surface guest_eval stages decrypt_ok → job_start → - job_done → emit_start → score_quote_ok in order; - * emit failures surface guest_eval_fail stage=emit class/detail without secrets. -No Phala create; targeted offline only. -""" - -from __future__ import annotations - -import hashlib -import json -import time -from typing import Any - -import pytest - -from agent_challenge.canonical import attested_result as ar -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.evaluation.own_runner.orchestrator import JobResult, TrialOutcome -from agent_challenge.evaluation.own_runner.result_schema import ( - RESULT_LINE_PREFIX, - build_benchmark_result, -) -from agent_challenge.evaluation.own_runner_backend import ( - PHALA_ATTESTATION_ENABLED_ENV, - PHALA_EVAL_PLAN_ENV, - PHALA_RTMR3_ENV, - main, -) -from agent_challenge.keyrelease.quote import ( - COMPOSE_HASH_EVENT, - KEY_PROVIDER_EVENT, - build_rtmr3_event_log, - build_tdx_quote, - parse_tdx_quote_v4, - replay_rtmr3, - runtime_event_digest, -) - -ATTESTED_REVIEW_ENABLED_ENV = "CHALLENGE_ATTESTED_REVIEW_ENABLED" - -MEASUREMENT = { - "mrtd": "a" * 96, - "rtmr0": "b0" * 48, - "rtmr1": "b1" * 48, - "rtmr2": "b2" * 48, - "compose_hash": "c" * 64, - "os_image_hash": "e" * 64, -} - - -def _identity_events_empty_digests() -> tuple[list[dict[str, Any]], str]: - """dstack-shaped IMR3 log with empty digests (live GetQuote residual).""" - - compose_payload = bytes.fromhex(MEASUREMENT["compose_hash"]) - provider_payload = b'{"name":"phala"}' - bootstrap = bytes.fromhex("11" * 8) - filled, rtmr3 = build_rtmr3_event_log( - [ - ("instance-id", bootstrap), - (COMPOSE_HASH_EVENT, compose_payload), - ("boot-mr-done", bootstrap), - (KEY_PROVIDER_EVENT, provider_payload), - ] - ) - empty: list[dict[str, Any]] = [] - for entry in filled: - raw = dict(entry) - raw["digest"] = "" # live residual: IMR3 digests blank until fill - raw["event_payload"] = "0x" + str(raw["event_payload"]).upper() - raw["extra_dstack_field"] = "drop-me" - empty.append(raw) - return empty, rtmr3 - - -class _DstackShapedScoreQuoteResponse: - """Mimics live dstack GetQuote: 0x mixed-case quote + empty IMR3 digests.""" - - def __init__(self, quote_hex: str, event_log: list[dict[str, Any]]) -> None: - if quote_hex.startswith(("0x", "0X")): - self.quote = quote_hex - else: - self.quote = "0x" + quote_hex.upper() - self.event_log = json.dumps(event_log) - self.vm_config = json.dumps( - {"vcpu": 1, "memory_mb": 2048, "os_image_hash": MEASUREMENT["os_image_hash"]} - ) - self.report_data = "" - - -class _DstackShapedScoreProvider: - def __init__(self, response: _DstackShapedScoreQuoteResponse) -> None: - self._response = response - self.calls: list[bytes] = [] - - def get_quote(self, report_data: bytes) -> _DstackShapedScoreQuoteResponse: - self.calls.append(report_data) - return self._response - - -def test_score_path_obtain_quote_normalizes_empty_imr3_and_passes_rtmr3_self_check() -> None: - """Score-path GetQuote reuses KR normalize so empty IMR3 digests self-check.""" - - empty_log, expected_rtmr3 = _identity_events_empty_digests() - # Discriminator vs shallow coerce: empty digests fail closed register schema - # (non-empty 96-char hex required by validate_eval_phala_attestation). - for entry in empty_log: - assert entry["digest"] == "" - with pytest.raises(ew.EvalWireError): - ew.validate_eval_phala_attestation( - { - "tdx_quote": "ab" * 600, - "event_log": [ - { - "imr": e["imr"], - "event_type": e["event_type"], - "digest": e["digest"], - "event": e["event"], - "event_payload": str(e["event_payload"]).removeprefix("0x").lower() - if isinstance(e["event_payload"], str) - else e["event_payload"], - } - for e in empty_log - ], - "report_data": "00" * 64, - "measurement": { - **MEASUREMENT, - "rtmr3": expected_rtmr3, - }, - "vm_config": { - "vcpu": 1, - "memory_mb": 2048, - "os_image_hash": MEASUREMENT["os_image_hash"], - }, - } - ) - - quote_hex = build_tdx_quote( - mrtd=MEASUREMENT["mrtd"], - rtmr0=MEASUREMENT["rtmr0"], - rtmr1=MEASUREMENT["rtmr1"], - rtmr2=MEASUREMENT["rtmr2"], - rtmr3=expected_rtmr3, - report_data=b"z" * 32, - ) - provider = _DstackShapedScoreProvider(_DstackShapedScoreQuoteResponse(quote_hex, empty_log)) - result = ar.obtain_quote(provider, b"z" * 32) - - # quote_hex lower / 0x stripped - assert result.quote == quote_hex.lower().removeprefix("0x") - assert not result.quote.startswith(("0x", "0X")) - assert result.quote == result.quote.lower() - - # Closed 5-key projection + filled IMR3 digests match runtime_event_digest. - closed = {"imr", "event_type", "digest", "event", "event_payload"} - assert all(set(entry) == closed for entry in result.event_log) - for entry in result.event_log: - expected = runtime_event_digest( - str(entry["event"]), - bytes.fromhex(str(entry["event_payload"])), - ).hex() - assert entry["digest"] == expected - assert entry["event_payload"] == str(entry["event_payload"]).lower() - assert not str(entry["event_payload"]).startswith("0x") - - # RTMR3 self-check used by schema-v2 emit must pass after normalize. - parsed_rtmr3 = parse_tdx_quote_v4(result.quote).rtmr3 - replayed = replay_rtmr3(result.event_log).rtmr3 - assert parsed_rtmr3 == replayed == expected_rtmr3 - - -def test_score_path_schema_v2_emit_with_empty_imr3_event_log(monkeypatch) -> None: - """schema-v2 emit obtains a quote, self-checks RTMR3, validates wire after normalize.""" - - empty_log, expected_rtmr3 = _identity_events_empty_digests() - report_data_holder: dict[str, bytes] = {} - - class _Provider: - def get_quote(self, report_data: bytes) -> _DstackShapedScoreQuoteResponse: - report_data_holder["rd"] = report_data - # Pin quote report_data to the call so parse path is consistent. - quote_hex = build_tdx_quote( - mrtd=MEASUREMENT["mrtd"], - rtmr0=MEASUREMENT["rtmr0"], - rtmr1=MEASUREMENT["rtmr1"], - rtmr2=MEASUREMENT["rtmr2"], - rtmr3=expected_rtmr3, - report_data=report_data, - ) - return _DstackShapedScoreQuoteResponse(quote_hex, empty_log) - - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - task_ids = ["task-a", "task-b"] - record = ew.build_canonical_score_record( - eval_run_id="eval-score-normalize", - policy=policy, - trial_scores_by_task={"task-a": [1.0], "task-b": [0.0]}, - ) - line = ar.emit_attested_benchmark_result( - benchmark_result={ - "status": "completed", - "score": 0.5, - "resolved": 1, - "total": 2, - "reason_code": None, - }, - canonical_measurement=dict(MEASUREMENT), - rtmr3="0" * 96, # supplier is overridden by quote parse after self-check - agent_hash="f" * 64, - task_ids=task_ids, - scores={}, - quote_provider=_Provider(), - manifest_sha256="1" * 64, - eval_run_id="eval-score-normalize", - submission_id="submission-score-normalize", - score_nonce="score-nonce-normalize", - score_record=record, - image_digest="registry.example/eval@sha256:" + "d" * 64, - vm_config={ - "vcpu": 1, - "memory_mb": 2048, - "os_image_hash": MEASUREMENT["os_image_hash"], - }, - ) - payload = json.loads(line.split("=", 1)[1]) - assert ew.validate_eval_result_request(payload) == payload - attestation = payload["execution_proof"]["attestation"] - assert attestation["measurement"]["rtmr3"] == expected_rtmr3 - # Digests were filled and project closed keys. - for entry in attestation["event_log"]: - assert set(entry) == {"imr", "event_type", "digest", "event", "event_payload"} - assert entry["digest"] and len(entry["digest"]) == 96 - - -def _canned_result(*, trials: int = 1) -> JobResult: - outcomes = [ - TrialOutcome( - task_name="hello-world", - trial_name=f"hello-world__attempt-{i}", - status="completed", - rewards={"reward": 1.0}, - ) - for i in range(trials) - ] - return JobResult( - status="completed", - score=1.0, - resolved=1, - total=1, - reason_code=None, - pass_at_k={}, - n_total_trials=trials, - n_completed_trials=trials, - n_errored_trials=0, - trial_outcomes=outcomes, - benchmark_result=build_benchmark_result( - status="completed", score=1.0, resolved=1, total=1, reason_code=None - ), - ) - - -def _fake_provider_factory(quote: str = "ab" * 600, *, raises: Exception | None = None): - class _FakeProvider: - def __init__(self, endpoint: str | None = None) -> None: - self.endpoint = endpoint - - def get_quote(self, report_data: bytes) -> Any: - if raises is not None: - raise raises - - class _Resp: - def __init__(self) -> None: - self.quote = quote - self.event_log = json.dumps( - [ - { - "imr": 3, - "event_type": 134217729, - "digest": "c" * 96, - "event": "compose-hash", - "event_payload": MEASUREMENT["compose_hash"], - } - ] - ) - self.vm_config = json.dumps( - { - "vcpu": 1, - "memory_mb": 2048, - "os_image_hash": MEASUREMENT["os_image_hash"], - } - ) - self.report_data = "" - - return _Resp() - - return _FakeProvider - - -def _set_happy_phala_env(monkeypatch, *, trials: int = 1) -> None: - monkeypatch.setenv(PHALA_ATTESTATION_ENABLED_ENV, "1") - monkeypatch.setenv(ATTESTED_REVIEW_ENABLED_ENV, "1") - monkeypatch.setenv(PHALA_RTMR3_ENV, "d" * 96) - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - plan = { - "schema_version": 1, - "eval_run_id": "eval-run-markers", - "submission_id": "submission-markers", - "submission_version": 1, - "authorizing_review_digest": "1" * 64, - "agent_hash": "f" * 64, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "hello-world", - "image_ref": "registry.example/task@sha256:" + "d" * 64, - "task_config_sha256": "2" * 64, - } - ], - "k": trials, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "d" * 64, - "compose_hash": MEASUREMENT["compose_hash"], - "app_identity": "agent-challenge-eval", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "3" * 64, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("3" * 64)).hexdigest(), - "measurement": { - "mrtd": MEASUREMENT["mrtd"], - "rtmr0": MEASUREMENT["rtmr0"], - "rtmr1": MEASUREMENT["rtmr1"], - "rtmr2": MEASUREMENT["rtmr2"], - "os_image_hash": MEASUREMENT["os_image_hash"], - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "keyrelease.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-run-markers/result", - "key_release_nonce": "key-nonce-markers", - "score_nonce": "score-nonce-markers", - "run_token_sha256": "5" * 64, - "issued_at_ms": (time.time_ns() // 1_000_000) - 1_000, - "expires_at_ms": (time.time_ns() // 1_000_000) + 60_000, - } - monkeypatch.setenv(PHALA_EVAL_PLAN_ENV, json.dumps(plan)) - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.assert_agent_artifact_matches_plan", - lambda **_: "f" * 64, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend._preflight_eval_plan_tasks", - lambda **_: {}, - ) - - -def test_post_grant_stage_markers_order_on_happy_path(monkeypatch, tmp_path, capsys) -> None: - """decrypt_ok → job_start → job_done → emit_start → score_quote_ok on fakes.""" - - trials = 2 - _set_happy_phala_env(monkeypatch, trials=trials) - sentinel_key = b"super-secret-golden-key-SENTINEL-markers-xyz" - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend._acquire_golden_key_if_required", - lambda **_: sentinel_key, - ) - - def _decrypt_ok(key: bytes) -> dict[str, Any]: - if key != sentinel_key: - raise RuntimeError("bad key") - return {"tasks": {}} - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend._decrypt_golden_in_enclave", - _decrypt_ok, - ) - - async def _fake_run(**kwargs: Any) -> JobResult: - return _canned_result(trials=trials) - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.run_own_runner_job", - _fake_run, - ) - monkeypatch.setattr(ar, "DstackQuoteProvider", _fake_provider_factory()) - - rc = main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - out = capsys.readouterr().out - assert rc == 0 - - stages = [] - for line in out.splitlines(): - if line.startswith("guest_eval stage="): - # "guest_eval stage=NAME ..." - token = line.split()[1] - stages.append(token.split("=", 1)[1]) - - required = ["decrypt_ok", "job_start", "job_done", "emit_start", "score_quote_ok"] - positions = [stages.index(name) for name in required] - assert positions == sorted(positions), f"stage order wrong: {stages}" - - job_done_line = next(ln for ln in out.splitlines() if "guest_eval stage=job_done" in ln) - assert f"trials={trials}" in job_done_line - # Secret-free: sentinel key must never appear. - assert "SENTINEL" not in out - assert sentinel_key.decode() not in out - - result_lines = [ln for ln in out.splitlines() if ln.startswith(RESULT_LINE_PREFIX)] - assert len(result_lines) == 1 - payload = json.loads(result_lines[0][len(RESULT_LINE_PREFIX) :]) - assert "execution_proof" in payload - - -def test_emit_failure_surfaces_guest_eval_fail_stage_emit(monkeypatch, tmp_path, capsys) -> None: - """Emit failures print guest_eval_fail stage=emit class/detail without secrets.""" - - _set_happy_phala_env(monkeypatch, trials=1) - sentinel = "leak-me-SENTINEL-api-key-should-not-appear" - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend._acquire_golden_key_if_required", - lambda **_: b"granted-key", - ) - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend._decrypt_golden_in_enclave", - lambda _key: {"tasks": {}}, - ) - - async def _fake_run(**kwargs: Any) -> JobResult: - return _canned_result(trials=1) - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.run_own_runner_job", - _fake_run, - ) - monkeypatch.setattr( - ar, - "DstackQuoteProvider", - _fake_provider_factory(raises=RuntimeError(f"dstack boom {sentinel}")), - ) - - rc = main(["run", "--task", "hello-world", "--job-dir", str(tmp_path / "job")]) - out = capsys.readouterr().out - assert rc != 0 - - assert "guest_eval stage=decrypt_ok" in out - assert "guest_eval stage=job_start" in out - assert "guest_eval stage=job_done" in out - assert "guest_eval stage=emit_start" in out - fail_line = next(ln for ln in out.splitlines() if ln.startswith("guest_eval_fail ")) - assert "stage=emit" in fail_line - assert "class=" in fail_line - assert "detail=" in fail_line - # Secret-free: raw api-key-ish detail is redacted or class-only; sentinel not dumped. - assert sentinel not in out - assert "SENTINEL" not in out - - result_lines = [ - ln[len(RESULT_LINE_PREFIX) :] - for ln in out.splitlines() - if ln.startswith(RESULT_LINE_PREFIX) - ] - assert len(result_lines) == 1 - payload = json.loads(result_lines[0]) - assert payload["status"] == "failed" - assert payload["reason_code"] == ar.PHALA_ATTESTATION_FAILED_REASON - assert "execution_proof" not in payload diff --git a/packages/challenges/agent-challenge/tests/test_product_blackbox_attestation_chain.py b/packages/challenges/agent-challenge/tests/test_product_blackbox_attestation_chain.py deleted file mode 100644 index 90295df33..000000000 --- a/packages/challenges/agent-challenge/tests/test_product_blackbox_attestation_chain.py +++ /dev/null @@ -1,455 +0,0 @@ -"""VAL-ACAT-020 + VAL-ACAT-X001..X012: black-box e2e attestation chain. - -Product Mode B: scripted re-verifiable package covering -miner ZIP+script → rules → OR digests → ≤24h → eval RA-TLS → score → raw weights, -with independent re-verify fail-closed on mutilation and zero Base gateway -critical path. Weights path is challenge-push / master aggregate only. -""" - -from __future__ import annotations - -import copy - -import pytest - -from agent_challenge.evaluation.blackbox_attestation_chain import ( - FRESHNESS_WINDOW_MS, - HOP_ASSERTIONS, - HOP_EVAL_AGENT_OR, - HOP_EVAL_CONJUNCTION, - HOP_FRESHNESS_24H, - HOP_GATEWAY_FREE, - HOP_HONESTY, - HOP_KEY_RELEASE, - HOP_MINER_SUBMIT, - HOP_OR_OBSERVED, - HOP_OR_PLANNED, - HOP_ORDER, - HOP_RAW_WEIGHTS, - HOP_RULES_LOAD, - HOP_SCORE_DOMAIN, - MS_23H, - REFUSE_BURN_TRACKED, - REFUSE_CHAIN_MUTILATED, - REFUSE_GATEWAY_RESIDUAL, - REFUSE_PACKAGE_TAMPER, - REFUSE_SET_WEIGHTS, - T0_DEFAULT, - BlackboxChainError, - build_complete_package, - chronological_evidence_index, - mutilate_package, - package_digest, - require_attestation_chain, - reverify_package, - run_attestation_chain, - scan_gateway_residuals, -) -from agent_challenge.evaluation.eval_agent_llm import MODE_MEASURED_OPENROUTER - - -def _hop(admission, hop_id: str): - for hop in admission.hops: - if hop.hop_id == hop_id: - return hop - raise AssertionError(f"missing hop {hop_id}") - - -# --------------------------------------------------------------------------- -# VAL-ACAT-020 control path -# --------------------------------------------------------------------------- - - -def test_full_e2e_chain_admits_and_maps_all_hops() -> None: - decision = run_attestation_chain() - assert decision.admitted is True - assert decision.production_emit is True - assert decision.reverify_ok is True - assert decision.gateway_free is True - assert decision.reason_code == "e2e_chain_verified" - assert decision.side_effects["set_weights"] == 0 - assert decision.side_effects["burn_weights_24h"] == 0 - assert list(h.hop_id for h in decision.hops) == list(HOP_ORDER) - assert all(h.status == "pass" for h in decision.hops) - for hop_id, assertion in HOP_ASSERTIONS.items(): - hop = _hop(decision, hop_id) - assert hop.assertion_id == assertion - assert hop.gateway_dependency is False - assert hop.set_weights_invoked is False - assert decision.raw_weights - assert "payload_digest" in decision.package["raw_weights_payload"] - index = chronological_evidence_index(decision) - assert index["assertion_table"]["VAL-ACAT-020"]["status"] == "PASS" - for assertion in HOP_ASSERTIONS.values(): - assert index["assertion_table"][assertion]["status"] == "PASS" - assert index["no_set_weights"] is True - assert index["gateway_free"] is True - assert index["weights_path"].startswith("challenge-raw-weights") - - -def test_reverify_is_idempotent_on_same_package() -> None: - package = build_complete_package() - a = reverify_package(package) - b = reverify_package(package) - assert a.admitted is True - assert b.admitted is True - assert a.package_digest == b.package_digest == package["package_digest"] - - -def test_require_raises_on_refuse() -> None: - with pytest.raises(BlackboxChainError) as exc: - require_attestation_chain(dual_flags_on=False) - # dual flags off fails score / agent path - assert exc.value.code - - -# --------------------------------------------------------------------------- -# VAL-ACAT-X001 miner submit gateway free -# --------------------------------------------------------------------------- - - -def test_x001_gateway_env_on_master_refuses() -> None: - decision = run_attestation_chain( - master_env={"BASE_GATEWAY_TOKEN": "should-never-exist"}, - ) - assert decision.admitted is False - assert _hop(decision, HOP_MINER_SUBMIT).status == "refuse" - assert _hop(decision, HOP_MINER_SUBMIT).gateway_dependency is True - assert decision.reason_code == REFUSE_GATEWAY_RESIDUAL - - -def test_x001_llm_v1_route_refuses_gateway_free_hop() -> None: - decision = run_attestation_chain( - master_routes=("/health", "/llm/v1/chat/completions"), - ) - assert decision.admitted is False - # residual is on gateway inventory; miner submit or gateway-free hop refuses - assert any( - h.status == "refuse" - for h in decision.hops - if h.hop_id - in { - HOP_MINER_SUBMIT, - HOP_GATEWAY_FREE, - } - ) - - -# --------------------------------------------------------------------------- -# VAL-ACAT-X002 rules -# --------------------------------------------------------------------------- - - -def test_x002_rules_digest_bound_into_review() -> None: - decision = run_attestation_chain() - hop = _hop(decision, HOP_RULES_LOAD) - assert hop.status == "pass" - assert hop.digests["rules_version"] - core = decision.package["review_envelope"]["review_core"] - assert core["rules_observation"]["rules_version"] == hop.digests["rules_version"] - - -def test_x002_empty_rules_refuses_entry() -> None: - decision = run_attestation_chain(rules_files={}) - assert decision.admitted is False - assert decision.package.get("identity_error") is not None - - -# --------------------------------------------------------------------------- -# VAL-ACAT-X003 / X004 OpenRouter digests -# --------------------------------------------------------------------------- - - -def test_x003_x004_planned_observed_digests_present() -> None: - decision = run_attestation_chain() - planned = _hop(decision, HOP_OR_PLANNED) - observed = _hop(decision, HOP_OR_OBSERVED) - assert planned.status == "pass" - assert observed.status == "pass" - assert len(planned.digests["planned_request_sha256"]) == 64 - assert len(observed.digests["transport_observation_sha256"]) == 64 - assert decision.package["gateway_inventory"]["openrouter_destination"].startswith( - "https://openrouter.ai" - ) - - -def test_x003_mutilated_planned_digest_fail_closed() -> None: - package = build_complete_package() - core = package["review_envelope"]["review_core"] - core = copy.deepcopy(core) - core["openrouter_observation"]["planned_request_sha256"] = "00" * 32 - package = mutilate_package( - package, - path=("review_envelope", "review_core"), - value=core, - ) - # also fix report_data so we exercise OR check not package digest alone - # (package_digest mismatch is also refuse; both acceptable fail-closed) - decision = reverify_package(package) - assert decision.admitted is False - assert decision.reason_code in { - REFUSE_PACKAGE_TAMPER, - "review_or_planned_digest_missing", - "score_refused_review_chain", - "review_attestation_reverify_failed", - "eval_cvm_refused_no_fresh_review", - "score_refused_tampered", - } - - -# --------------------------------------------------------------------------- -# VAL-ACAT-X005 24h window -# --------------------------------------------------------------------------- - - -def test_x005_stale_over_24h_refuses() -> None: - decision = run_attestation_chain( - issued_at_ms=T0_DEFAULT, - received_at_ms=T0_DEFAULT + FRESHNESS_WINDOW_MS + 1, - ) - assert decision.admitted is False - hop = _hop(decision, HOP_FRESHNESS_24H) - assert hop.status == "refuse" - assert "24h" in hop.reason_code or hop.reason_code == "attestation_stale_over_24h" - - -def test_x005_exactly_24h_boundary_admits() -> None: - decision = run_attestation_chain( - issued_at_ms=T0_DEFAULT, - received_at_ms=T0_DEFAULT + FRESHNESS_WINDOW_MS, - ) - assert decision.admitted is True - assert _hop(decision, HOP_FRESHNESS_24H).status == "pass" - - -def test_x005_23h_admits() -> None: - decision = run_attestation_chain( - issued_at_ms=T0_DEFAULT, - received_at_ms=T0_DEFAULT + MS_23H, - ) - assert decision.admitted is True - - -# --------------------------------------------------------------------------- -# VAL-ACAT-X006 conjunction ablation -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - "kwargs", - [ - {"skip_review": True}, - {"verdict": "reject"}, - { - "issued_at_ms": T0_DEFAULT, - "received_at_ms": T0_DEFAULT + FRESHNESS_WINDOW_MS + 1, - }, - ], -) -def test_x006_missing_conjunct_blocks_eval(kwargs: dict) -> None: - decision = run_attestation_chain(**kwargs) - assert decision.admitted is False - # eval conjunction or earlier hop refused - eval_hop = _hop(decision, HOP_EVAL_CONJUNCTION) - fresh_hop = _hop(decision, HOP_FRESHNESS_24H) - or_hop = _hop(decision, HOP_OR_PLANNED) - assert eval_hop.status == "refuse" or fresh_hop.status == "refuse" or or_hop.status == "refuse" - - -# --------------------------------------------------------------------------- -# VAL-ACAT-X007 RA-TLS key release (no Base gateway mint) -# --------------------------------------------------------------------------- - - -def test_x007_key_release_present_and_no_gateway_mint() -> None: - decision = run_attestation_chain() - hop = _hop(decision, HOP_KEY_RELEASE) - assert hop.status == "pass" - grant = decision.package["key_release_grant"] - assert grant["domain"] == "base-agent-challenge-keyrelease-v1" - assert grant["base_gateway_token_minted"] is False - assert grant["ra_tls_spki_digest"] - - -def test_x007_missing_key_release_refuses() -> None: - decision = run_attestation_chain(skip_key_release=True) - assert decision.admitted is False - assert _hop(decision, HOP_KEY_RELEASE).status == "refuse" - - -# --------------------------------------------------------------------------- -# VAL-ACAT-X008 eval-agent OR measured or tools-only -# --------------------------------------------------------------------------- - - -def test_x008_tools_only_default_admits() -> None: - decision = run_attestation_chain() - hop = _hop(decision, HOP_EVAL_AGENT_OR) - assert hop.status == "pass" - assert hop.digests["mode"] == "tools_only" - - -def test_x008_measured_agent_or_admits() -> None: - decision = run_attestation_chain( - agent_llm_mode=MODE_MEASURED_OPENROUTER, - include_measured_agent_or=True, - ) - assert decision.admitted is True - hop = _hop(decision, HOP_EVAL_AGENT_OR) - assert hop.status == "pass" - assert hop.digests["mode"] == MODE_MEASURED_OPENROUTER - assert hop.digests.get("planned_request_sha256") - - -# --------------------------------------------------------------------------- -# VAL-ACAT-X009 score domain -# --------------------------------------------------------------------------- - - -def test_x009_score_admits_after_full_chain() -> None: - decision = run_attestation_chain() - hop = _hop(decision, HOP_SCORE_DOMAIN) - assert hop.status == "pass" - assert "base-agent-challenge-review-v1" in hop.digests["domains"] - assert "base-agent-challenge-keyrelease-v1" in hop.digests["domains"] - assert "base-agent-challenge-v1" in hop.digests["domains"] - - -def test_x009_skip_score_materials_refuses() -> None: - decision = run_attestation_chain(skip_score=True) - assert decision.admitted is False - assert _hop(decision, HOP_SCORE_DOMAIN).status == "refuse" - - -# --------------------------------------------------------------------------- -# VAL-ACAT-X010 raw weights / no set_weights -# --------------------------------------------------------------------------- - - -def test_x010_raw_weights_path_only() -> None: - decision = run_attestation_chain() - hop = _hop(decision, HOP_RAW_WEIGHTS) - assert hop.status == "pass" - assert "raw-weights" in hop.digests["path"] - assert decision.side_effects["set_weights"] == 0 - assert decision.package["raw_weights_payload"]["master_set_weights"] is False - - -def test_x010_set_weights_refuses() -> None: - decision = run_attestation_chain(set_weights_count=1) - assert decision.admitted is False - hop = _hop(decision, HOP_RAW_WEIGHTS) - assert hop.status == "refuse" - assert hop.reason_code == REFUSE_SET_WEIGHTS - assert hop.set_weights_invoked is True - - -def test_x010_burn_invoked_refuses() -> None: - decision = run_attestation_chain(burn_invoked=True) - assert decision.admitted is False - assert _hop(decision, HOP_RAW_WEIGHTS).reason_code == REFUSE_BURN_TRACKED - - -# --------------------------------------------------------------------------- -# VAL-ACAT-X011 zero Base gateway -# --------------------------------------------------------------------------- - - -def test_x011_gateway_free_pass() -> None: - decision = run_attestation_chain() - assert decision.gateway_free is True - assert _hop(decision, HOP_GATEWAY_FREE).status == "pass" - assert scan_gateway_residuals(env={}) == [] - - -def test_x011_gateway_token_scrapes() -> None: - hits = scan_gateway_residuals(env={"BASE_LLM_GATEWAY_URL": "http://x/llm/v1"}) - assert hits - hits2 = scan_gateway_residuals(urls=["https://master.example/llm/v1/chat"]) - assert hits2 - - -# --------------------------------------------------------------------------- -# VAL-ACAT-X012 honesty -# --------------------------------------------------------------------------- - - -def test_x012_real_provider_blocked_on_pass() -> None: - decision = run_attestation_chain() - assert decision.tee_labels["REAL-PROVIDER"] == "BLOCKED" - assert _hop(decision, HOP_HONESTY).status == "pass" - - -def test_x012_invented_real_provider_pass_refuses() -> None: - decision = run_attestation_chain(real_provider_pass_claimed=True) - assert decision.admitted is False - hop = _hop(decision, HOP_HONESTY) - assert hop.status == "refuse" - assert "real_provider" in hop.reason_code - - -# --------------------------------------------------------------------------- -# Independent re-verify fail-closed on mutilation samples -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - "path,value", - [ - (("review_envelope", "report_data_hex"), ("ff" * 32) + ("00" * 32)), - ( - ("key_release_grant", "report_data_hex"), - ("00" * 32) + ("ff" * 32), - ), - (("score_report_data_hex",), "aa" * 64), - (("issued_at_ms",), T0_DEFAULT - 10_000_000), - (("raw_weights_payload", "payload_digest"), "00" * 32), - ], -) -def test_mutilation_samples_fail_closed(path: tuple[str, ...], value: object) -> None: - package = build_complete_package() - control = reverify_package(package) - assert control.admitted is True - mutilated = mutilate_package(package, path=path, value=value) - decision = reverify_package(mutilated) - assert decision.admitted is False - assert decision.production_emit is False - assert decision.raw_weights == {} or decision.reason_code != "e2e_chain_verified" - assert decision.reason_code in { - REFUSE_PACKAGE_TAMPER, - REFUSE_CHAIN_MUTILATED, - "score_refused_review_chain", - "score_refused_tampered", - "score_refused_key_release_mismatch", - "score_refused_score_domain", - "score_refused_incomplete_chain", - "attestation_stale_over_24h", - "review_attestation_reverify_failed", - "eval_cvm_refused_no_fresh_review", - "review_report_data_mismatch", - "score_refused_domain_confusion", - } - - -def test_package_digest_detects_any_material_change() -> None: - package = build_complete_package() - d1 = package_digest(package) - package2 = copy.deepcopy(package) - package2["issued_at_ms"] = package["issued_at_ms"] + 1 - d2 = package_digest(package2) - assert d1 != d2 - - -def test_dual_flags_off_cannot_emit() -> None: - decision = run_attestation_chain(dual_flags_on=False) - assert decision.admitted is False - assert decision.production_emit is False - - -def test_index_lists_all_val_acat_x_assertions() -> None: - decision = run_attestation_chain() - index = chronological_evidence_index(decision) - for i in range(1, 13): - key = f"VAL-ACAT-X{i:03d}" - assert key in index["assertion_table"] - assert index["assertion_table"]["VAL-ACAT-020"]["status"] == "PASS" diff --git a/packages/challenges/agent-challenge/tests/test_product_durable_kr_grant_score.py b/packages/challenges/agent-challenge/tests/test_product_durable_kr_grant_score.py deleted file mode 100644 index 9ad9844ef..000000000 --- a/packages/challenges/agent-challenge/tests/test_product_durable_kr_grant_score.py +++ /dev/null @@ -1,564 +0,0 @@ -"""Durable RA-TLS KR grant materials → score admission (VAL-ACAT-036/037/040). - -Scrutiny residual on acat-eval-gate: production KR must stamp reconstructible -grant JSON on EvalRun; score admission must load that durable column (not only -the process-local registry). Multi-worker restart simulation: clear the -in-process map and still admit from durable JSON; missing grant refuse even -when key_granted_at alone is set. -""" - -from __future__ import annotations - -import hashlib -import json -from datetime import UTC, datetime, timedelta -from typing import Any - -import pytest -from sqlalchemy import select - -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.core.models import ( - AgentSubmission, - EvalRun, - ReviewAssignment, - ReviewSession, -) -from agent_challenge.evaluation.authorization import ( - build_key_release_grant_materials, - create_eval_run, - load_eval_run_plan, - mark_eval_key_granted, - persist_key_release_grant_materials, - receipt_eval_key_release, -) -from agent_challenge.evaluation.direct_result import _key_release_grant_from_result -from agent_challenge.evaluation.score_chain_gate import ( - KEY_RELEASE_DOMAIN, - REFUSE_MISSING_KEY_RELEASE, - admit_production_score_for_eval_result, - admit_production_score_from_chain, - build_score_binding_from_plan_and_digest, - clear_key_release_grant_for_score, - load_durable_key_release_grant, - lookup_key_release_grant_for_score, - recompute_key_release_report_data_hex, - register_key_release_grant_for_score, - verify_key_release_grant, -) -from agent_challenge.review.or_outcome_bind import ( - REVIEW_REPORT_DOMAIN, - build_decision, - build_observed_openrouter_transport, - build_openrouter_observation, - build_planned_openrouter_request, - build_policy_observation, - build_review_core_minimal, - planned_request_sha256, - review_digest, - review_report_data_hex, - sha256_hex, -) -from agent_challenge.sdk.config import ChallengeSettings - -_T0 = 1_700_000_000_000 -SPKI = "aa" * 32 -COMPOSE_HASH = "ab" * 32 -MRTD = "11" * 48 -RTMR0 = "22" * 48 -RTMR1 = "33" * 48 -RTMR2 = "44" * 48 -MEASUREMENT = { - "mrtd": MRTD, - "rtmr0": RTMR0, - "rtmr1": RTMR1, - "rtmr2": RTMR2, - "os_image_hash": "66" * 32, - "key_provider": "validator-kms", - "vm_shape": "tdx-small", -} -_ROUTING = sha256_hex(b'{"order":["durable-kr"]}') -_BODY = b'{"model":"x-ai/grok-4.5","messages":[]}' -_BODY_SHA = sha256_hex(_BODY) -_RESP = b'{"id":"gen-durable-kr","model":"x-ai/grok-4.5","choices":[]}' -_RESP_SHA = sha256_hex(_RESP) -_META = sha256_hex(b"meta-durable-kr") - -_SUBMISSION_SEQ = 0 - - -def _settings() -> ChallengeSettings: - return ChallengeSettings( - attested_review_enabled=True, - phala_attestation_enabled=True, - eval_app_image_ref="registry.example/eval@sha256:" + "a" * 64, - eval_app_compose_hash=COMPOSE_HASH, - eval_app_identity="agent-challenge-eval-v1", - eval_app_kms_public_key_hex="07" * 32, - eval_app_measurement=MEASUREMENT, - eval_app_measurement_allowlist=( - { - "mrtd": MEASUREMENT["mrtd"], - "rtmr0": MEASUREMENT["rtmr0"], - "rtmr1": MEASUREMENT["rtmr1"], - "rtmr2": MEASUREMENT["rtmr2"], - "compose_hash": COMPOSE_HASH, - "os_image_hash": MEASUREMENT["os_image_hash"], - }, - ), - eval_key_release_endpoint="validator.example:8701", - eval_k=1, - evaluation_task_count=1, - ) - - -def _patch_tasks(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - "agent_challenge.evaluation.authorization.load_benchmark_tasks", - lambda: [ - type( - "Task", - (), - { - "task_id": "task-a", - "docker_image": "registry.example/task@sha256:" + "b" * 64, - "prompt": "", - "benchmark": "terminal_bench", - "metadata": {"content_digest_sha256": "aa" * 32}, - }, - )() - ], - ) - - -def _fresh_review_envelope( - *, - session_id: str = "rs-durable-kr", - assignment_id: str = "ra-durable-kr", - submission_id: str = "sub-durable-kr", - review_nonce: str = "nonce-durable-kr", -) -> tuple[str, str, str, dict[str, Any]]: - """Full receipted envelope for create_eval_run re-verify (not cache-only).""" - - planned = build_planned_openrouter_request( - body_sha256=_BODY_SHA, - body_length=len(_BODY), - routing_sha256=_ROUTING, - ) - p_digest = planned_request_sha256(planned) - observed = build_observed_openrouter_transport( - planned_request_sha256_=p_digest, - response_body_sha256=_RESP_SHA, - response_body_length=len(_RESP), - metadata_sha256=_META, - ) - or_obs = build_openrouter_observation( - planned=planned, - observed=observed, - request_body_sha256=_BODY_SHA, - request_body_length=len(_BODY), - response_id="gen-durable-kr", - ) - policy = build_policy_observation( - prompt_version="review-policy-prompt-v1", - prompt_bytes=b"prompt-durable-kr", - tool_schema_version="review-policy-tool-v1", - tool_schema_bytes=b"tools-durable-kr", - verifier_version="review-policy-verifier-v1", - verifier_bytes=b"verifier-durable-kr", - routing_sha256=_ROUTING, - ) - core = build_review_core_minimal( - session_id=session_id, - assignment_id=assignment_id, - submission_id=submission_id, - review_nonce=review_nonce, - assignment_digest="13" * 32, - rules_observation={ - "rules_version": "rules-v1", - "rules_bundle_sha256": "11" * 32, - "rules_files": [".rules/acceptance.md"], - "rules_file_digests": {".rules/acceptance.md": "22" * 32}, - "rules_policy_text_sha256": "33" * 32, - }, - policy_observation=policy, - openrouter_observation=or_obs, - decision=build_decision(verdict="allow"), - times={ - "issued_at_ms": _T0, - "started_at_ms": _T0, - "model_call_marked_at_ms": _T0 + 1, - "request_started_at_ms": _T0 + 2, - "request_finished_at_ms": _T0 + 3, - "verifier_finished_at_ms": _T0 + 4, - "report_finished_at_ms": _T0 + 5, - "expires_at_ms": _T0 + 3_600_000, - "submission_received_at_ms": _T0 + 60_000, - }, - ) - digest = review_digest(core) - rd = review_report_data_hex(core) - env: dict[str, Any] = { - "schema_version": 1, - "domain": REVIEW_REPORT_DOMAIN, - "review_digest": digest, - "report_data_hex": rd, - "review_core": core, - } - # AGATE: create_eval_run / score dual-flag path requires residual + tree sha. - from agent_challenge.evaluation.llm_rules_residual import ( - MEASURED_RESIDUAL_KIND, - bind_package_residual_into_review_materials, - build_package_residual_materials, - ) - - materials = build_package_residual_materials( - residual_verdict="allow", - rules_bundle_sha256="11" * 32, - rules_version="rules-v1", - rules_file_digests={".rules/acceptance.md": "22" * 32}, - package_tree_sha="bb" * 32, - residual_kind=MEASURED_RESIDUAL_KIND, - rules_policy_text_sha256="33" * 32, - harness_kind="measured_review_cvm_script_zip", - ) - env = bind_package_residual_into_review_materials( - envelope=env, - materials=materials, - )["envelope"] - return json.dumps(env, sort_keys=True, separators=(",", ":")), rd, digest, env - - -async def _authorized_submission(database_session) -> tuple[int, dict[str, Any]]: - global _SUBMISSION_SEQ - _SUBMISSION_SEQ += 1 - salt = f"kr-score-{_SUBMISSION_SEQ}".encode() - sid = f"rs-durable-kr-{_SUBMISSION_SEQ}" - aid = f"ra-durable-kr-{_SUBMISSION_SEQ}" - rid = f"nonce-durable-kr-{_SUBMISSION_SEQ}" - sub_label = f"sub-durable-kr-{_SUBMISSION_SEQ}" - envelope_json, report_data_hex, digest, env = _fresh_review_envelope( - session_id=sid, - assignment_id=aid, - submission_id=sub_label, - review_nonce=rid, - ) - async with database_session() as session: - submission = AgentSubmission( - miner_hotkey=f"kr-score-miner-{_SUBMISSION_SEQ}", - name=f"kr-score-agent-{_SUBMISSION_SEQ}", - agent_hash=hashlib.sha256(b"agent-" + salt).hexdigest(), - artifact_uri=f"/tmp/agent-kr-{_SUBMISSION_SEQ}.zip", - artifact_path=f"/tmp/agent-kr-{_SUBMISSION_SEQ}.zip", - zip_sha256=hashlib.sha256(b"zip-" + salt).hexdigest(), - package_tree_sha="bb" * 32, - zip_size_bytes=3, - raw_status="review_allowed", - status="queued", - effective_status="queued", - version_number=1, - ) - session.add(submission) - await session.flush() - review_session = ReviewSession( - session_id=sid, - submission_id=submission.id, - artifact_sha256=submission.zip_sha256, - artifact_size_bytes=3, - manifest_sha256="11" * 32, - manifest_entries_sha256="12" * 32, - authorizing_assignment_id=aid, - current_assignment_id=aid, - ) - session.add(review_session) - await session.flush() - assignment = ReviewAssignment( - session_id=review_session.id, - assignment_id=aid, - attempt=1, - assignment_bytes="{}", - assignment_digest="13" * 32, - artifact_sha256=submission.zip_sha256, - rules_snapshot_sha256="14" * 32, - rules_revision_id="rules-1", - review_nonce=rid, - session_token_sha256="15" * 32, - capability_state="revoked", - phase="review_allowed", - issued_at=datetime.now(UTC), - expires_at=datetime.now(UTC) + timedelta(hours=1), - review_report_envelope_json=envelope_json, - review_report_data_hex=report_data_hex, - review_digest=digest, - review_verification_outcome_json=json.dumps( - { - "status": "verified_allow", - "terminal": True, - "retryable": False, - "nonce_consumed": True, - "package_residual": env.get("package_residual"), - }, - separators=(",", ":"), - ), - ) - session.add(assignment) - await session.commit() - return submission.id, env - - -async def _create_run(database_session, monkeypatch: pytest.MonkeyPatch): - submission_id, env = await _authorized_submission(database_session) - _patch_tasks(monkeypatch) - async with database_session() as session: - submission = await session.get(AgentSubmission, submission_id) - assert submission is not None - created = await create_eval_run(session, submission, settings=_settings()) - await session.commit() - return created.run.eval_run_id, created.plan, submission_id, env - - -def test_build_key_release_grant_materials_closed_shape() -> None: - grant = build_key_release_grant_materials( - eval_run_id="eval-1", - key_release_nonce="kr-nonce-1", - ra_tls_spki_digest=SPKI, - agent_hash="55" * 32, - ) - assert grant["domain"] == KEY_RELEASE_DOMAIN - assert grant["schema_version"] == 2 - assert grant["eval_run_id"] == "eval-1" - assert grant["key_release_nonce"] == "kr-nonce-1" - assert grant["ra_tls_spki_digest"] == SPKI - assert grant["agent_hash"] == "55" * 32 - expected = recompute_key_release_report_data_hex( - eval_run_id="eval-1", - key_release_nonce="kr-nonce-1", - ra_tls_spki_digest=SPKI, - ) - assert grant["report_data_hex"] == expected - err, rd = verify_key_release_grant( - grant=grant, - eval_plan={ - "eval_run_id": "eval-1", - "key_release_nonce": "kr-nonce-1", - "score_nonce": "score-nonce-1", - "agent_hash": "55" * 32, - "package_tree_sha": "bb" * 32, - }, - key_granted_flag=True, - ) - assert err is None - assert rd == expected - - -@pytest.mark.asyncio -async def test_mark_eval_key_granted_persists_durable_grant_json( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Live KR success stamping: durable grant_json + process registry.""" - - clear_key_release_grant_for_score() - eval_run_id, plan, _, _env = await _create_run(database_session, monkeypatch) - digest = hashlib.sha256(b"kr-grant-frame").hexdigest() - async with database_session() as session: - await receipt_eval_key_release(session, eval_run_id=eval_run_id, body_sha256=digest) - granted = await mark_eval_key_granted( - session, - eval_run_id=eval_run_id, - ra_tls_spki_digest=SPKI, - ) - assert granted.key_granted_at is not None - assert granted.key_release_state == "granted" - assert isinstance(granted.key_release_grant_json, str) - assert granted.key_release_grant_json - durable = json.loads(granted.key_release_grant_json) - assert durable["domain"] == KEY_RELEASE_DOMAIN - assert durable["eval_run_id"] == eval_run_id - assert durable["key_release_nonce"] == plan["key_release_nonce"] - assert durable["ra_tls_spki_digest"] == SPKI - assert durable["agent_hash"] == plan["agent_hash"] - assert durable["report_data_hex"] == recompute_key_release_report_data_hex( - eval_run_id=eval_run_id, - key_release_nonce=str(plan["key_release_nonce"]), - ra_tls_spki_digest=SPKI, - ) - cached = lookup_key_release_grant_for_score(eval_run_id) - assert cached is not None - assert cached["ra_tls_spki_digest"] == SPKI - await session.commit() - - # Restart simulation: wipe process-local map; reload from durable column. - clear_key_release_grant_for_score(eval_run_id) - assert lookup_key_release_grant_for_score(eval_run_id) is None - async with database_session() as session: - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == eval_run_id)) - assert run is not None - reloaded = load_durable_key_release_grant(run) - assert reloaded is not None - assert reloaded["ra_tls_spki_digest"] == SPKI - assert lookup_key_release_grant_for_score(eval_run_id) is not None - - -@pytest.mark.asyncio -async def test_score_admission_from_durable_grant_after_process_restart( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """KR→score: after clearing process registry, durable JSON still admits.""" - - clear_key_release_grant_for_score() - eval_run_id, plan, _, env = await _create_run(database_session, monkeypatch) - digest = hashlib.sha256(b"kr-score-admit").hexdigest() - async with database_session() as session: - await receipt_eval_key_release(session, eval_run_id=eval_run_id, body_sha256=digest) - await mark_eval_key_granted( - session, - eval_run_id=eval_run_id, - ra_tls_spki_digest=SPKI, - ) - await session.commit() - - # Simulate multi-worker / process bounce: only DB remains. - clear_key_release_grant_for_score() - assert lookup_key_release_grant_for_score(eval_run_id) is None - - async with database_session() as session: - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == eval_run_id)) - assert run is not None - plan_reloaded = load_eval_run_plan(run) - grant = _key_release_grant_from_result( - plan=plan_reloaded, - validated={}, - run=run, - raw_request=None, - ) - assert grant is not None, "must load durable grant, not process dict alone" - assert grant["domain"] == KEY_RELEASE_DOMAIN - plan_for_score = dict(plan_reloaded) - plan_for_score["authorizing_review_digest"] = env["review_digest"] - # Synthetic scores_digest (hex) — gate re-checks binding equality only. - scores_digest = "cd" * 32 - binding = build_score_binding_from_plan_and_digest( - eval_plan=plan_for_score, - scores_digest=scores_digest, - ) - decision = admit_production_score_for_eval_result( - settings_dual_flags_on=True, - eval_plan=plan_for_score, - review_envelope=env, - key_release_grant=grant, - key_granted_flag=run.key_granted_at is not None, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=scores_digest, - score_nonce_outstanding=True, - ) - assert decision.admitted is True, decision.reason_code - assert decision.partial_score is False - assert decision.production_emit is True - assert KEY_RELEASE_DOMAIN in decision.domains_checked - - -@pytest.mark.asyncio -async def test_missing_durable_grant_refuses_even_with_key_granted_at( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """VAL-ACAT-037: key_granted_at alone is never enough under dual flags.""" - - clear_key_release_grant_for_score() - eval_run_id, plan, _, env = await _create_run(database_session, monkeypatch) - digest = hashlib.sha256(b"kr-legacy-no-grant").hexdigest() - async with database_session() as session: - await receipt_eval_key_release(session, eval_run_id=eval_run_id, body_sha256=digest) - # Legacy path: grant without SPKI materials. - granted = await mark_eval_key_granted(session, eval_run_id=eval_run_id) - assert granted.key_granted_at is not None - assert granted.key_release_grant_json is None - await session.commit() - - clear_key_release_grant_for_score() - async with database_session() as session: - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == eval_run_id)) - assert run is not None - plan_reloaded = load_eval_run_plan(run) - grant = _key_release_grant_from_result( - plan=plan_reloaded, - validated={}, - run=run, - raw_request=None, - ) - assert grant is None - plan_for_score = dict(plan_reloaded) - plan_for_score["authorizing_review_digest"] = env["review_digest"] - scores_digest = "cd" * 32 - binding = build_score_binding_from_plan_and_digest( - eval_plan=plan_for_score, - scores_digest=scores_digest, - ) - decision = admit_production_score_from_chain( - dual_flags_on=True, - review_envelope=env, - key_release_grant=grant, - key_granted_flag=True, - eval_plan=plan_for_score, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=scores_digest, - score_nonce_state="outstanding", - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_MISSING_KEY_RELEASE - assert decision.score is None - assert decision.partial_score is False - _ = plan - - -@pytest.mark.asyncio -async def test_process_local_dict_alone_insufficient_after_restart( - database_session, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """If only the process dict held materials and was cleared, load fails closed.""" - - clear_key_release_grant_for_score() - eval_run_id, plan, _, _env = await _create_run(database_session, monkeypatch) - # Process-only stamp (simulates pre-fix listener that never wrote DB). - register_key_release_grant_for_score( - eval_run_id, - build_key_release_grant_materials( - eval_run_id=eval_run_id, - key_release_nonce=str(plan["key_release_nonce"]), - ra_tls_spki_digest=SPKI, - agent_hash=str(plan["agent_hash"]), - ), - ) - assert lookup_key_release_grant_for_score(eval_run_id) is not None - clear_key_release_grant_for_score(eval_run_id) - assert lookup_key_release_grant_for_score(eval_run_id) is None - - async with database_session() as session: - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == eval_run_id)) - assert run is not None - assert run.key_release_grant_json is None - grant = _key_release_grant_from_result( - plan=load_eval_run_plan(run), - validated={}, - run=run, - raw_request=None, - ) - assert grant is None - materials = build_key_release_grant_materials( - eval_run_id=eval_run_id, - key_release_nonce=str(plan["key_release_nonce"]), - ra_tls_spki_digest=SPKI, - agent_hash=str(plan["agent_hash"]), - ) - persist_key_release_grant_materials(run, materials) - assert run.key_release_grant_json is not None - clear_key_release_grant_for_score(eval_run_id) - reloaded = load_durable_key_release_grant(run) - assert reloaded is not None - assert reloaded["ra_tls_spki_digest"] == SPKI - await session.commit() diff --git a/packages/challenges/agent-challenge/tests/test_product_eval_agent_openrouter_measured.py b/packages/challenges/agent-challenge/tests/test_product_eval_agent_openrouter_measured.py deleted file mode 100644 index 26a3b0a0f..000000000 --- a/packages/challenges/agent-challenge/tests/test_product_eval_agent_openrouter_measured.py +++ /dev/null @@ -1,888 +0,0 @@ -"""VAL-ACAT-016 / 050–054: eval-agent OpenRouter only inside measured eval CVM. - -- Guest-only measured OpenRouter with planned+observed digests bound into score materials -- Base /llm/v1 and BASE_GATEWAY_TOKEN refuse (GatewayConfigError never production success) -- Tools-only mode fails closed on model egress -- Dual-flag matrix: only ON/ON may emit production scores -- create_review_session retains harness identity -- live helpers require_real_or_digests / admit_production_from_bound_outcome -""" - -from __future__ import annotations - -from dataclasses import fields -from hashlib import sha256 -from typing import Any - -import pytest - -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.evaluation.direct_result import extract_agent_llm_materials_for_score -from agent_challenge.evaluation.eval_agent_llm import ( - BASE_GATEWAY_ENV_NAMES, - BASE_MASTER_KIND, - MEASURED_EVAL_CVM_KIND, - MINER_LAPTOP_KIND, - MODE_MEASURED_OPENROUTER, - MODE_TOOLS_ONLY, - REFUSE_BASE_GATEWAY, - REFUSE_BASE_GATEWAY_URL, - REFUSE_DIGEST_MISMATCH, - REFUSE_DIGEST_UNBOUND, - REFUSE_FLAGS_OFF, - REFUSE_MEASUREMENT, - REFUSE_TOOLS_ONLY_EGRESS, - REFUSE_UNMEASURED_OR, - SIDECAR_PROXY_KIND, - UNMEASURED_HOST_KIND, - EvalAgentLlmError, - admit_eval_agent_llm_for_score, - assert_no_base_gateway_agent_env, - assert_no_base_gateway_url, - bind_eval_agent_or_digests_into_score_materials, - build_eval_agent_observed_transport, - build_eval_agent_planned_request, - flag_matrix_production_emit, - refuse_base_gateway_assignment_payload, - require_eval_agent_llm_for_score, - require_eval_agent_or_digests, -) -from agent_challenge.evaluation.gateway import ( - GATEWAY_TOKEN_ENV, - LLM_GATEWAY_PATH, - GatewayConfigError, - GatewayExecutionConfig, - agent_gateway_config_from_settings, -) -from agent_challenge.evaluation.score_chain_gate import ( - KEY_RELEASE_DOMAIN, - admit_production_score_for_eval_result, - admit_production_score_from_chain, - build_score_binding_from_plan_and_digest, - recompute_key_release_report_data_hex, -) -from agent_challenge.evaluation.score_chain_gate import ( - REFUSE_FLAGS_OFF as SCORE_REFUSE_FLAGS_OFF, -) -from agent_challenge.review.attested_times import FRESHNESS_WINDOW_MS -from agent_challenge.review.canonical import canonical_json_v1 -from agent_challenge.review.or_outcome_bind import ( - REVIEW_REPORT_DOMAIN, - build_decision, - build_openrouter_observation, - build_planned_openrouter_request, - build_policy_observation, - build_review_core_minimal, - planned_request_sha256, - require_real_or_digests, - review_digest, - review_report_data_hex, - sha256_hex, - transport_observation_sha256, -) -from agent_challenge.review.or_outcome_bind import ( - build_observed_openrouter_transport as build_review_observed, -) -from agent_challenge.review.sessions import CreatedReviewSession, create_review_session -from agent_challenge.sdk.config import ChallengeSettings -from agent_challenge.selfdeploy.eval import EVAL_REQUIRED_SECRET_ENVS - - -def _h(data: bytes) -> str: - return sha256(data).hexdigest() - - -ROUTING = _h(b'{"order":["eval-agent"]}') -BODY = b'{"model":"x-ai/grok-4.5","messages":[{"role":"user","content":"x"}]}' -BODY_SHA = _h(BODY) -RESP = b'{"id":"gen-eval-agent","model":"x-ai/grok-4.5","choices":[]}' -RESP_SHA = _h(RESP) -META = _h(b"eval-agent-or-meta") - -MEASUREMENT = { - "compose_hash": "aa" * 32, - "os_image_hash": "bb" * 32, - "mrtd": "cc" * 48, - "key_provider": "phala-kms", - "vm_shape": "2c-4g", -} -ALLOWLIST = [dict(MEASUREMENT)] - - -def _materials() -> dict: - planned = build_eval_agent_planned_request( - body_sha256=BODY_SHA, - body_length=len(BODY), - routing_sha256=ROUTING, - model="x-ai/grok-4.5", - ) - observed = build_eval_agent_observed_transport( - planned=planned, - response_body_sha256=RESP_SHA, - response_body_length=len(RESP), - metadata_sha256=META, - ) - return bind_eval_agent_or_digests_into_score_materials(planned=planned, observed=observed) - - -# --------------------------------------------------------------------------- -# VAL-ACAT-050 — OpenRouter only inside measured eval CVM -# --------------------------------------------------------------------------- - - -def test_measured_eval_cvm_openrouter_admits_with_digests() -> None: - mats = _materials() - decision = admit_eval_agent_llm_for_score( - mode=MODE_MEASURED_OPENROUTER, - dual_flags_on=True, - runtime_kind=MEASURED_EVAL_CVM_KIND, - measurement=MEASUREMENT, - allowlist=ALLOWLIST, - claims_model_call=True, - agent_or_materials=mats, - ) - assert decision.admitted is True - assert decision.production_emit_eligible is True - assert decision.digests_bound is True - assert decision.planned_request_sha256 == mats["planned_request_sha256"] - assert decision.transport_observation_sha256 == mats["transport_observation_sha256"] - assert decision.base_gateway_used is False - - -def test_gateway_module_unused_for_production_success() -> None: - """VAL-ACAT-050: Base gateway config is never the production success path.""" - - assert agent_gateway_config_from_settings(ChallengeSettings()) is None - # Residual from_assignment_payload may raise GatewayConfigError — not success. - with pytest.raises(GatewayConfigError): - GatewayExecutionConfig.from_assignment_payload({}) - # Production path refuses gateway payload keys instead of succeeding. - with pytest.raises(EvalAgentLlmError) as exc: - refuse_base_gateway_assignment_payload( - {"gateway_token": "t", "gateway_url": "https://master.example"} - ) - assert exc.value.code == REFUSE_BASE_GATEWAY - - -def test_unmeasured_runtimes_refuse_openrouter_score_credit() -> None: - mats = _materials() - for kind in ( - UNMEASURED_HOST_KIND, - BASE_MASTER_KIND, - MINER_LAPTOP_KIND, - SIDECAR_PROXY_KIND, - "host_python", - "", - ): - decision = admit_eval_agent_llm_for_score( - mode=MODE_MEASURED_OPENROUTER, - dual_flags_on=True, - runtime_kind=kind or None, - measurement=MEASUREMENT, - allowlist=ALLOWLIST, - claims_model_call=True, - agent_or_materials=mats, - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_UNMEASURED_OR - assert decision.production_emit_eligible is False - - -def test_eval_required_secrets_exclude_base_gateway_pair() -> None: - """VAL-ACAT-050 evidence: EVAL_REQUIRED_SECRET_ENVS has no Base gateway pair.""" - - assert "BASE_GATEWAY_TOKEN" not in EVAL_REQUIRED_SECRET_ENVS - assert "BASE_LLM_GATEWAY_URL" not in EVAL_REQUIRED_SECRET_ENVS - for name in BASE_GATEWAY_ENV_NAMES: - assert name not in EVAL_REQUIRED_SECRET_ENVS - - -# --------------------------------------------------------------------------- -# VAL-ACAT-051 / 052 — planned + observed digests bind into eval/score materials -# --------------------------------------------------------------------------- - - -def test_planned_and_observed_digests_bound_into_score_materials() -> None: - mats = _materials() - assert len(mats["planned_request_sha256"]) == 64 - assert len(mats["transport_observation_sha256"]) == 64 - digests = require_eval_agent_or_digests(mats) - assert digests["planned_request_sha256"] == mats["planned_request_sha256"] - assert digests["transport_observation_sha256"] == mats["transport_observation_sha256"] - # Live helper used on review path is also exercised (feature wire note). - require_real_or_digests( - planned=mats["planned"], - observed=mats["observed"], - openrouter_observation=mats["openrouter_observation"], - ) - - -def test_forged_planned_digest_refuses() -> None: - mats = _materials() - forged = dict(mats) - planned = dict(mats["planned"]) - planned["body_sha256"] = "ff" * 32 # mutates planned digest vs observed link - forged["planned"] = planned - with pytest.raises(EvalAgentLlmError) as exc: - require_eval_agent_or_digests(forged) - assert exc.value.code in {REFUSE_DIGEST_MISMATCH, REFUSE_DIGEST_UNBOUND} - - -def test_missing_observed_materials_refuse_score() -> None: - decision = admit_eval_agent_llm_for_score( - mode=MODE_MEASURED_OPENROUTER, - dual_flags_on=True, - runtime_kind=MEASURED_EVAL_CVM_KIND, - measurement=MEASUREMENT, - allowlist=ALLOWLIST, - claims_model_call=True, - agent_or_materials=None, - ) - assert decision.admitted is False - assert decision.production_emit_eligible is False - assert decision.reason_code in { - REFUSE_DIGEST_UNBOUND, - "eval_agent_llm_claim_missing_digests", - } - - -def test_mismatched_observed_planned_link_refuses() -> None: - planned = build_eval_agent_planned_request( - body_sha256=BODY_SHA, - body_length=len(BODY), - routing_sha256=ROUTING, - model="x-ai/grok-4.5", - ) - observed = build_eval_agent_observed_transport( - planned=planned, - response_body_sha256=RESP_SHA, - response_body_length=len(RESP), - metadata_sha256=META, - ) - bad_observed = dict(observed) - bad_observed["planned_request_sha256"] = "00" * 32 - with pytest.raises(EvalAgentLlmError) as exc: - bind_eval_agent_or_digests_into_score_materials(planned=planned, observed=bad_observed) - assert exc.value.code == REFUSE_DIGEST_MISMATCH - - -def test_score_empty_on_digest_reject_via_require() -> None: - with pytest.raises(EvalAgentLlmError): - require_eval_agent_llm_for_score( - mode=MODE_MEASURED_OPENROUTER, - dual_flags_on=True, - runtime_kind=MEASURED_EVAL_CVM_KIND, - measurement=MEASUREMENT, - allowlist=ALLOWLIST, - claims_model_call=True, - agent_or_materials=None, - ) - - -# --------------------------------------------------------------------------- -# VAL-ACAT-053 — forbid Base gateway URLs and tokens -# --------------------------------------------------------------------------- - - -def test_base_llm_v1_url_refuses() -> None: - with pytest.raises(EvalAgentLlmError) as exc: - assert_no_base_gateway_url(f"https://master.example{LLM_GATEWAY_PATH}") - assert exc.value.code == REFUSE_BASE_GATEWAY_URL - - decision = admit_eval_agent_llm_for_score( - mode=MODE_MEASURED_OPENROUTER, - dual_flags_on=True, - runtime_kind=MEASURED_EVAL_CVM_KIND, - measurement=MEASUREMENT, - allowlist=ALLOWLIST, - claims_model_call=True, - agent_or_materials=_materials(), - gateway_url=f"https://master.example{LLM_GATEWAY_PATH}", - gateway_token_present=True, - used_base_llm_v1=True, - ) - assert decision.admitted is False - assert decision.reason_code in {REFUSE_BASE_GATEWAY, REFUSE_BASE_GATEWAY_URL} - assert decision.base_gateway_used is True - assert decision.production_emit_eligible is False - - -def test_base_gateway_token_env_refuses() -> None: - with pytest.raises(EvalAgentLlmError) as exc: - assert_no_base_gateway_agent_env({GATEWAY_TOKEN_ENV: "scoped-token"}) - assert exc.value.code == REFUSE_BASE_GATEWAY - - -def test_llm_gateway_path_constant_is_forbidden_surface() -> None: - assert LLM_GATEWAY_PATH == "/llm/v1" - with pytest.raises(EvalAgentLlmError): - assert_no_base_gateway_url(f"http://127.0.0.1:8000{LLM_GATEWAY_PATH}/chat/completions") - - -# --------------------------------------------------------------------------- -# VAL-ACAT-054 — flag-off residual cannot emit production scores -# --------------------------------------------------------------------------- - - -def test_flag_matrix_only_dual_on_emits() -> None: - matrix = flag_matrix_production_emit( - phala_attestation_enabled=True, - attested_review_enabled=True, - ) - assert matrix["dual_flags_on"] is True - assert matrix["production_emit"] is True - assert matrix["refuse_code"] is None - by = { - (r["phala_attestation_enabled"], r["attested_review_enabled"]): r for r in matrix["matrix"] - } - assert by[(True, True)]["production_emit"] is True - assert by[(True, False)]["production_emit"] is False - assert by[(False, True)]["production_emit"] is False - assert by[(False, False)]["production_emit"] is False - assert by[(False, False)]["refuse_code"] == REFUSE_FLAGS_OFF - - -def test_flag_off_refuses_agent_llm_and_score_chain() -> None: - mats = _materials() - decision = admit_eval_agent_llm_for_score( - mode=MODE_MEASURED_OPENROUTER, - dual_flags_on=False, - runtime_kind=MEASURED_EVAL_CVM_KIND, - measurement=MEASUREMENT, - allowlist=ALLOWLIST, - claims_model_call=True, - agent_or_materials=mats, - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_FLAGS_OFF - assert decision.production_emit_eligible is False - - chain = admit_production_score_from_chain( - dual_flags_on=False, - cached_review_allow=True, - key_granted_flag=True, - offline_ast_pass=True, - master_status_green=True, - cached_score_ok=True, - eval_plan={"eval_run_id": "x"}, - ) - assert chain.admitted is False - assert chain.production_emit is False - assert chain.score is None - assert chain.reason_code == SCORE_REFUSE_FLAGS_OFF - - -def test_flag_matrix_any_off_has_no_weight_material() -> None: - for phala, review in ((False, False), (True, False), (False, True)): - m = flag_matrix_production_emit( - phala_attestation_enabled=phala, - attested_review_enabled=review, - ) - assert m["production_emit"] is False - assert m["refuse_code"] == REFUSE_FLAGS_OFF - - -# --------------------------------------------------------------------------- -# VAL-ACAT-016 — tools-only vs measured mode -# --------------------------------------------------------------------------- - - -def test_tools_only_admits_without_model_call() -> None: - decision = admit_eval_agent_llm_for_score( - mode=MODE_TOOLS_ONLY, - dual_flags_on=True, - runtime_kind=MEASURED_EVAL_CVM_KIND, - claims_model_call=False, - ) - assert decision.admitted is True - assert decision.reason_code == "eval_agent_tools_only" - assert decision.digests_bound is False - - -def test_tools_only_fails_closed_on_model_egress() -> None: - decision = admit_eval_agent_llm_for_score( - mode=MODE_TOOLS_ONLY, - dual_flags_on=True, - claims_model_call=True, - agent_or_materials=_materials(), - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_TOOLS_ONLY_EGRESS - - -def test_measurement_allowlist_miss_refuses() -> None: - decision = admit_eval_agent_llm_for_score( - mode=MODE_MEASURED_OPENROUTER, - dual_flags_on=True, - runtime_kind=MEASURED_EVAL_CVM_KIND, - measurement={**MEASUREMENT, "compose_hash": "dd" * 32}, - allowlist=ALLOWLIST, - claims_model_call=True, - agent_or_materials=_materials(), - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_MEASUREMENT - - -# --------------------------------------------------------------------------- -# Live call-site wire checks (feature text) -# --------------------------------------------------------------------------- - - -def test_create_review_session_retains_harness_identity_contract() -> None: - assert callable(create_review_session) - names = {f.name for f in fields(CreatedReviewSession)} - assert "harness_identity" in names - - -def test_digest_builders_roundtrip_sha() -> None: - planned = build_eval_agent_planned_request( - body_sha256=BODY_SHA, - body_length=len(BODY), - routing_sha256=ROUTING, - model="x-ai/grok-4.5", - ) - observed = build_eval_agent_observed_transport( - planned=planned, - response_body_sha256=RESP_SHA, - response_body_length=len(RESP), - metadata_sha256=META, - ) - assert planned_request_sha256(planned) == sha256_hex(canonical_json_v1(planned)) - assert transport_observation_sha256(observed) == sha256_hex(canonical_json_v1(observed)) - assert observed["planned_request_sha256"] == planned_request_sha256(planned) - - -# --------------------------------------------------------------------------- -# Emission-wrapper wire (VAL-ACAT-016/050–052 fix): for_eval_result + extract -# --------------------------------------------------------------------------- - -_T0 = 1_700_000_000_000 -_MS_23H = FRESHNESS_WINDOW_MS - 60_000 -_REGS = { - "mrtd": "11" * 48, - "rtmr0": "22" * 48, - "rtmr1": "33" * 48, - "rtmr2": "44" * 48, -} -_AGENT_HASH = "55" * 32 -_SPKI = "aa" * 32 -_REVIEW_ROUTING = sha256_hex(b'{"order":["score-chain-agent-or"]}') -_REVIEW_BODY = b'{"model":"x-ai/grok-4.5","messages":[]}' -_REVIEW_BODY_SHA = sha256_hex(_REVIEW_BODY) -_REVIEW_RESP = b'{"id":"gen-score-agent","model":"x-ai/grok-4.5","choices":[]}' -_REVIEW_RESP_SHA = sha256_hex(_REVIEW_RESP) -_REVIEW_META = sha256_hex(b"meta-score-agent-or") - - -def _os_image_hash() -> str: - from agent_challenge.keyrelease.quote import os_image_hash_from_registers - - return os_image_hash_from_registers(_REGS["mrtd"], _REGS["rtmr1"], _REGS["rtmr2"]) - - -def _review_times(*, issued: int = _T0, received: int = _T0 + _MS_23H) -> dict[str, int]: - base = min(issued, received) - return { - "issued_at_ms": issued, - "started_at_ms": base, - "model_call_marked_at_ms": base + 1, - "request_started_at_ms": base + 2, - "request_finished_at_ms": base + 3, - "verifier_finished_at_ms": base + 4, - "report_finished_at_ms": base + 5, - "expires_at_ms": max(issued, received) + 3_600_000, - "submission_received_at_ms": received, - } - - -def _emission_review_envelope() -> dict[str, Any]: - planned = build_planned_openrouter_request( - body_sha256=_REVIEW_BODY_SHA, - body_length=len(_REVIEW_BODY), - routing_sha256=_REVIEW_ROUTING, - ) - p_digest = planned_request_sha256(planned) - observed = build_review_observed( - planned_request_sha256_=p_digest, - response_body_sha256=_REVIEW_RESP_SHA, - response_body_length=len(_REVIEW_RESP), - metadata_sha256=_REVIEW_META, - ) - or_obs = build_openrouter_observation( - planned=planned, - observed=observed, - request_body_sha256=_REVIEW_BODY_SHA, - request_body_length=len(_REVIEW_BODY), - response_id="gen-score-agent", - ) - policy = build_policy_observation( - prompt_version="review-policy-prompt-v1", - prompt_bytes=b"prompt-agent-or", - tool_schema_version="review-policy-tool-v1", - tool_schema_bytes=b"tools-agent-or", - verifier_version="review-policy-verifier-v1", - verifier_bytes=b"verifier-agent-or", - routing_sha256=_REVIEW_ROUTING, - ) - rules = { - "rules_version": "rules-v1", - "rules_bundle_sha256": "11" * 32, - "rules_files": [".rules/acceptance.md"], - "rules_file_digests": {".rules/acceptance.md": "22" * 32}, - "rules_policy_text_sha256": "33" * 32, - } - core = build_review_core_minimal( - session_id="rs-agent-or-emit", - assignment_id="ra-agent-or-emit", - submission_id="sub-agent-or-emit", - review_nonce="nonce-agent-or-emit", - assignment_digest="aa" * 32, - rules_observation=rules, - policy_observation=policy, - openrouter_observation=or_obs, - decision=build_decision(verdict="allow"), - times=_review_times(), - ) - env = { - "schema_version": 1, - "domain": REVIEW_REPORT_DOMAIN, - "review_digest": review_digest(core), - "report_data_hex": review_report_data_hex(core), - "review_core": core, - } - # AGATE: dual-flag score path requires measured package residual + tree sha. - from agent_challenge.evaluation.llm_rules_residual import ( - MEASURED_RESIDUAL_KIND, - bind_package_residual_into_review_materials, - build_package_residual_materials, - ) - - materials = build_package_residual_materials( - residual_verdict="allow", - rules_bundle_sha256="11" * 32, - rules_version="rules-v1", - rules_file_digests={".rules/acceptance.md": "22" * 32}, - package_tree_sha="bb" * 32, - residual_kind=MEASURED_RESIDUAL_KIND, - rules_policy_text_sha256="33" * 32, - harness_kind="measured_review_cvm_script_zip", - ) - return bind_package_residual_into_review_materials( - envelope=env, - materials=materials, - )["envelope"] - - -def _emission_plan(*, authorizing_review_digest: str) -> dict[str, Any]: - os_hash = _os_image_hash() - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - return ew.validate_eval_plan( - { - "schema_version": 1, - "eval_run_id": "eval-agent-or-emit-1", - "submission_id": "submission-agent-or-emit-1", - "submission_version": 1, - "authorizing_review_digest": authorizing_review_digest, - "agent_hash": _AGENT_HASH, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "task-a", - "image_ref": "registry.example/task@sha256:" + "77" * 32, - "task_config_sha256": "88" * 32, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "99" * 32, - "compose_hash": "ab" * 32, - "app_identity": "agent-challenge-eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "aa" * 32, - "kms_public_key_sha256": sha256(bytes.fromhex("aa" * 32)).hexdigest(), - "measurement": { - **_REGS, - "os_image_hash": os_hash, - "key_provider": "phala", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-agent-or-emit-1/result", - "key_release_nonce": "key-release-agent-or-emit-1", - "score_nonce": "score-nonce-agent-or-emit-1", - "run_token_sha256": "bb" * 32, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - ) - - -def _emission_grant(plan: dict[str, Any]) -> dict[str, Any]: - rd = recompute_key_release_report_data_hex( - eval_run_id=str(plan["eval_run_id"]), - key_release_nonce=str(plan["key_release_nonce"]), - ra_tls_spki_digest=_SPKI, - ) - return { - "domain": KEY_RELEASE_DOMAIN, - "schema_version": 2, - "eval_run_id": plan["eval_run_id"], - "key_release_nonce": plan["key_release_nonce"], - "ra_tls_spki_digest": _SPKI, - "report_data_hex": rd, - "agent_hash": plan.get("agent_hash"), - } - - -def _emission_chain_kit() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], str, dict]: - env = _emission_review_envelope() - plan = _emission_plan(authorizing_review_digest=env["review_digest"]) - from agent_challenge.evaluation.plan_scoring import build_score_record_from_eval_plan - - record = build_score_record_from_eval_plan(plan, {"task-a": [1.0]}) - scores_digest = ew.score_record_digest(record) - binding = build_score_binding_from_plan_and_digest( - eval_plan=plan, - scores_digest=scores_digest, - ) - return plan, env, _emission_grant(plan), scores_digest, binding - - -def test_for_eval_result_admits_measured_agent_openrouter() -> None: - """Emission wrapper succeeds when agent OR digests + measured runtime are wired.""" - - plan, env, grant, sd, binding = _emission_chain_kit() - mats = _materials() - decision = admit_production_score_for_eval_result( - settings_dual_flags_on=True, - eval_plan=plan, - review_envelope=env, - key_release_grant=grant, - key_granted_flag=True, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=sd, - score_nonce_outstanding=True, - agent_llm_mode=MODE_MEASURED_OPENROUTER, - agent_or_materials=mats, - agent_llm_runtime_kind=MEASURED_EVAL_CVM_KIND, - agent_llm_measurement=MEASUREMENT, - agent_llm_allowlist=ALLOWLIST, - claims_agent_model_call=True, - ) - assert decision.admitted is True, decision.reason_code - assert decision.production_emit is True - assert decision.partial_score is False - assert decision.score is None - - -def test_for_eval_result_refuses_claim_without_digests() -> None: - """Live dual-flag emission refuses model claim when planned/observed missing.""" - - plan, env, grant, sd, binding = _emission_chain_kit() - decision = admit_production_score_for_eval_result( - settings_dual_flags_on=True, - eval_plan=plan, - review_envelope=env, - key_release_grant=grant, - key_granted_flag=True, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=sd, - score_nonce_outstanding=True, - agent_llm_mode=MODE_MEASURED_OPENROUTER, - claims_agent_model_call=True, - agent_or_materials=None, - agent_llm_runtime_kind=MEASURED_EVAL_CVM_KIND, - agent_llm_measurement=MEASUREMENT, - agent_llm_allowlist=ALLOWLIST, - ) - assert decision.admitted is False - assert decision.production_emit is False - assert decision.partial_score is False - assert decision.score is None - assert decision.reason_code in { - REFUSE_DIGEST_UNBOUND, - "eval_agent_llm_claim_missing_digests", - } - - -def test_for_eval_result_refuses_unmeasured_runtime() -> None: - plan, env, grant, sd, binding = _emission_chain_kit() - decision = admit_production_score_for_eval_result( - settings_dual_flags_on=True, - eval_plan=plan, - review_envelope=env, - key_release_grant=grant, - key_granted_flag=True, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=sd, - score_nonce_outstanding=True, - agent_llm_mode=MODE_MEASURED_OPENROUTER, - claims_agent_model_call=True, - agent_or_materials=_materials(), - agent_llm_runtime_kind=UNMEASURED_HOST_KIND, - agent_llm_measurement=MEASUREMENT, - agent_llm_allowlist=ALLOWLIST, - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_UNMEASURED_OR - assert decision.production_emit is False - - -def test_for_eval_result_refuses_base_gateway_residue() -> None: - plan, env, grant, sd, binding = _emission_chain_kit() - decision = admit_production_score_for_eval_result( - settings_dual_flags_on=True, - eval_plan=plan, - review_envelope=env, - key_release_grant=grant, - key_granted_flag=True, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=sd, - score_nonce_outstanding=True, - agent_llm_mode=MODE_MEASURED_OPENROUTER, - claims_agent_model_call=True, - agent_or_materials=_materials(), - agent_llm_runtime_kind=MEASURED_EVAL_CVM_KIND, - agent_llm_measurement=MEASUREMENT, - agent_llm_allowlist=ALLOWLIST, - agent_gateway_url=f"https://master.example{LLM_GATEWAY_PATH}", - agent_gateway_token_present=True, - agent_used_base_llm_v1=True, - ) - assert decision.admitted is False - assert decision.reason_code in {REFUSE_BASE_GATEWAY, REFUSE_BASE_GATEWAY_URL} - assert decision.production_emit is False - assert decision.score is None - - -def test_for_eval_result_tools_only_default_without_claim() -> None: - """No claim / materials / residue → tools-only default still admits chain.""" - - plan, env, grant, sd, binding = _emission_chain_kit() - decision = admit_production_score_for_eval_result( - settings_dual_flags_on=True, - eval_plan=plan, - review_envelope=env, - key_release_grant=grant, - key_granted_flag=True, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=sd, - score_nonce_outstanding=True, - # Explicit no claim; tools-only path does not require digests. - claims_agent_model_call=False, - agent_or_materials=None, - ) - assert decision.admitted is True, decision.reason_code - assert decision.production_emit is True - - -def test_extract_agent_llm_materials_forwards_claim_and_digests() -> None: - """process_direct_eval_result extractor surfaces agent_or bags into kwargs.""" - - plan, env, grant, sd, binding = _emission_chain_kit() - _ = env, grant, sd, binding - mats = _materials() - raw = { - "agent_llm": { - "mode": MODE_MEASURED_OPENROUTER, - "claims_model_call": True, - "runtime_kind": MEASURED_EVAL_CVM_KIND, - "materials": mats, - "measurement": MEASUREMENT, - "allowlist": ALLOWLIST, - } - } - extracted = extract_agent_llm_materials_for_score( - plan=plan, - validated={"scores_digest": "00" * 32}, - raw_request=raw, - settings=None, - ) - assert extracted["claims_agent_model_call"] is True - assert extracted["agent_llm_mode"] == MODE_MEASURED_OPENROUTER - assert extracted["agent_llm_runtime_kind"] == MEASURED_EVAL_CVM_KIND - assert extracted["agent_or_materials"] is not None - assert ( - extracted["agent_or_materials"]["planned_request_sha256"] == mats["planned_request_sha256"] - ) - - # Emission path driven by extractor: must re-check digests via for_eval_result. - plan2, env2, grant2, sd2, binding2 = _emission_chain_kit() - decision = admit_production_score_for_eval_result( - settings_dual_flags_on=True, - eval_plan=plan2, - review_envelope=env2, - key_release_grant=grant2, - key_granted_flag=True, - score_binding=binding2, - score_report_data_hex=ew.score_report_data_hex(binding2), - scores_digest=sd2, - score_nonce_outstanding=True, - **extracted, - ) - assert decision.admitted is True, decision.reason_code - - -def test_extract_agent_llm_materials_surfaces_gateway_residue() -> None: - extracted = extract_agent_llm_materials_for_score( - plan={}, - validated={}, - raw_request={ - "agent_env": {"BASE_GATEWAY_TOKEN": "scoped"}, - "gateway_url": f"https://master.example{LLM_GATEWAY_PATH}/chat/completions", - "used_base_llm_v1": True, - }, - ) - assert extracted["agent_gateway_token_present"] is True - assert extracted["agent_used_base_llm_v1"] is True - assert extracted["agent_gateway_url"] is not None - assert "/llm/v1" in str(extracted["agent_gateway_url"]) - - -def test_from_chain_matches_for_eval_result_on_agent_refuse() -> None: - """from_chain and for_eval_result must agree (no orphan helper-only path).""" - - plan, env, grant, sd, binding = _emission_chain_kit() - common = dict( - eval_plan=plan, - review_envelope=env, - key_release_grant=grant, - key_granted_flag=True, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=sd, - claims_agent_model_call=True, - agent_or_materials=None, - agent_llm_mode=MODE_MEASURED_OPENROUTER, - agent_llm_runtime_kind=MEASURED_EVAL_CVM_KIND, - agent_llm_measurement=MEASUREMENT, - agent_llm_allowlist=ALLOWLIST, - ) - via_wrapper = admit_production_score_for_eval_result( - settings_dual_flags_on=True, - score_nonce_outstanding=True, - **common, - ) - via_chain = admit_production_score_from_chain( - dual_flags_on=True, - score_nonce_state="outstanding", - **common, - ) - assert via_wrapper.admitted is False - assert via_chain.admitted is False - assert via_wrapper.reason_code == via_chain.reason_code diff --git a/packages/challenges/agent-challenge/tests/test_product_gateway_free_policy.py b/packages/challenges/agent-challenge/tests/test_product_gateway_free_policy.py deleted file mode 100644 index 9d9126213..000000000 --- a/packages/challenges/agent-challenge/tests/test_product_gateway_free_policy.py +++ /dev/null @@ -1,200 +0,0 @@ -"""VAL-ACAT-013/014/015: gateway-free eval secrets, policy, static analyzer matrix. - -Product Mode B: Base LLM gateway is not a required secret and not a legal -agent LLM path. Measured OpenRouter under harness is not auto-cheated solely -for avoiding Base gateway. GatewayRulesReviewer must not consume Base gateway. -""" - -from __future__ import annotations - -from pathlib import Path - -from agent_challenge.analyzer.lifecycle import ( - build_configured_rules_reviewer, - gateway_llm_base_url, -) -from agent_challenge.analyzer.pipeline import run_rules_analyzer -from agent_challenge.canonical.compose import DEFAULT_ALLOWED_ENVS -from agent_challenge.evaluation.gateway import agent_gateway_config_from_settings -from agent_challenge.review.sessions import create_review_session -from agent_challenge.sdk.config import ChallengeSettings -from agent_challenge.selfdeploy.eval import ( - EVAL_REQUIRED_SECRET_ENVS, -) - -FORBIDDEN_GATEWAY_ENV_NAMES = frozenset( - { - "BASE_GATEWAY_TOKEN", - "BASE_LLM_GATEWAY_URL", - "GATEWAY_TOKEN", - "CENTRAL_GATEWAY_TOKEN", - "CHALLENGE_LLM_GATEWAY_TOKEN", - "CHALLENGE_LLM_GATEWAY_BASE_URL", - "CHALLENGE_LLM_GATEWAY_TOKEN_FILE", - "CHALLENGE_AGENT_GATEWAY_TOKEN", - } -) - - -class _SilentReviewer: - def review(self, request): # noqa: ANN001, ARG002 - from agent_challenge.analyzer.schemas import ReviewerResult - - return ReviewerResult(verdict="valid", reason_codes=["rules_passed"], notes="ok") - - -def test_eval_required_secrets_exclude_base_gateway() -> None: - """VAL-ACAT-013: EVAL_REQUIRED_SECRET_ENVS must not mandate Base gateway vars.""" - - assert "BASE_GATEWAY_TOKEN" not in EVAL_REQUIRED_SECRET_ENVS - assert "BASE_LLM_GATEWAY_URL" not in EVAL_REQUIRED_SECRET_ENVS - for name in FORBIDDEN_GATEWAY_ENV_NAMES: - assert name not in EVAL_REQUIRED_SECRET_ENVS - # Still require the attested run capability material. - assert "EVAL_RUN_TOKEN" in EVAL_REQUIRED_SECRET_ENVS - assert "CHALLENGE_PHALA_EVAL_PLAN" in EVAL_REQUIRED_SECRET_ENVS - - -def test_default_allowed_envs_exclude_base_gateway_vars() -> None: - """VAL-ACAT-013: compose encrypted_env allowlist omits Base gateway names.""" - - allowed = set(DEFAULT_ALLOWED_ENVS) - for name in ("BASE_GATEWAY_TOKEN", "BASE_LLM_GATEWAY_URL"): - assert name not in allowed - - -def test_settings_instantiate_without_gateway_vars(monkeypatch) -> None: - """VAL-ACAT-013: unit surfaces start healthy with gateway env unset.""" - - for name in ( - "CHALLENGE_LLM_GATEWAY_BASE_URL", - "CHALLENGE_LLM_GATEWAY_TOKEN", - "CHALLENGE_LLM_GATEWAY_TOKEN_FILE", - "CHALLENGE_AGENT_GATEWAY_TOKEN", - "CHALLENGE_AGENT_GATEWAY_TOKEN_FILE", - "BASE_GATEWAY_TOKEN", - "BASE_LLM_GATEWAY_URL", - ): - monkeypatch.delenv(name, raising=False) - settings = ChallengeSettings() - assert settings.llm_gateway_base_url is None - assert settings.llm_gateway_token is None - assert settings.agent_gateway_token is None - # Residual gateway config must never inject agent sandbox env. - assert agent_gateway_config_from_settings(settings) is None - - -def test_agent_gateway_config_never_injects_even_when_residual_url_set() -> None: - """VAL-ACAT-013: residual Settings URL is inert — no Base gateway routing.""" - - settings = ChallengeSettings( - llm_gateway_base_url="https://master.example", - agent_gateway_token="should-not-inject", - llm_gateway_token="analyzer-should-not-inject", - ) - assert agent_gateway_config_from_settings(settings) is None - - -def test_gateway_rules_reviewer_does_not_consume_base_gateway() -> None: - """VAL-ACAT-015: configured rules reviewer must not wire Base gateway.""" - - reviewer = build_configured_rules_reviewer() - # Production path returns None (no Base gateway consumer). - assert reviewer is None - - -def test_gateway_llm_base_url_helper_pure_path_only() -> None: - """Helper may join path for residual tests but is not a product agent injector.""" - - url = gateway_llm_base_url("https://master.example") - assert url.endswith("/llm/v1") - - -def test_static_analyzer_flags_base_gateway_client(tmp_path: Path) -> None: - """VAL-ACAT-015: residual Base gateway client patterns fail offline.""" - - workspace = tmp_path / "gateway-agent" - workspace.mkdir() - (workspace / "agent.py").write_text( - "import os\n" - "import httpx\n" - "\n" - "BASE = os.environ['BASE_LLM_GATEWAY_URL']\n" - "TOKEN = os.environ['BASE_GATEWAY_TOKEN']\n" - "httpx.post(BASE + '/chat/completions', headers={'X-Gateway-Token': TOKEN})\n" - "url = 'https://master.example/llm/v1/chat/completions'\n", - encoding="utf-8", - ) - report = run_rules_analyzer(workspace, reviewer=_SilentReviewer()) - assert report.overall_verdict == "invalid" - codes = {f.reason_code for f in report.hardcoding_findings} - assert "base_gateway_forbidden" in codes or "unauthorized_llm_provider" in codes - # Message semantics: Base gateway is the forbidden residue. - joined = " ".join(f.description for f in report.hardcoding_findings) - assert "BASE_LLM_GATEWAY" in joined or "Base gateway" in joined or "/llm/v1" in joined - - -def test_static_analyzer_measured_openrouter_not_auto_cheated(tmp_path: Path) -> None: - """VAL-ACAT-015: measured OpenRouter path is not spoiled for avoiding Base gateway.""" - - workspace = tmp_path / "measured-or-agent" - workspace.mkdir() - (workspace / "agent.py").write_text( - '"""Agent that relies on measured OpenRouter inside review/eval CVM."""\n' - "\n" - "def solve(task: str) -> str:\n" - " # tools-only score path; LLM judge is measured review under .rules\n" - " return task.upper()\n", - encoding="utf-8", - ) - report = run_rules_analyzer(workspace, reviewer=_SilentReviewer()) - codes = {f.reason_code for f in report.hardcoding_findings} - # Pure tools-only / mention of openrouter.ai in docs-context comments for - # the measured path must not be auto-flagged solely for avoiding gateway. - assert "unauthorized_llm_provider" not in codes or "openrouter" not in " ".join( - f.description.lower() for f in report.hardcoding_findings - ) - assert report.overall_verdict in {"valid", "suspicious"} # not forced invalid by gateway avoids - - -def test_static_analyzer_does_not_false_flag_openrouter_host_alone(tmp_path: Path) -> None: - """Mention of openrouter.ai host for measured path is not auto-cheat.""" - - workspace = tmp_path / "or-mention" - workspace.mkdir() - (workspace / "agent.py").write_text( - "ORIGIN = 'https://openrouter.ai'\ndef solve():\n return ORIGIN\n", - encoding="utf-8", - ) - report = run_rules_analyzer(workspace, reviewer=_SilentReviewer()) - codes = {f.reason_code for f in report.hardcoding_findings} - assert "unauthorized_llm_provider" not in codes - assert "base_gateway_forbidden" not in codes - - -def test_create_review_session_retains_harness_identity_export() -> None: - """Feature wire check: create_review_session export retains harness materials.""" - - # Import surface must keep create_review_session for API/product intake. - assert callable(create_review_session) - # Document that harness_identity is part of the CreatedReviewSession contract. - from dataclasses import fields - - from agent_challenge.review.sessions import CreatedReviewSession - - names = {f.name for f in fields(CreatedReviewSession)} - assert "harness_identity" in names - - -def test_policy_rules_forbid_base_gateway_text() -> None: - """VAL-ACAT-014: .rules forbid Base gateway; allow measured OR / tools-only.""" - - root = Path(__file__).resolve().parents[1] - rules = "\n".join(p.read_text(encoding="utf-8") for p in sorted((root / ".rules").glob("*.md"))) - # Residual Base gateway names may appear only as forbidden material. - assert "BASE_LLM_GATEWAY_URL" in rules - assert "BASE_GATEWAY_TOKEN" in rules - assert "must route all LLM traffic through the platform gateway" not in rules - assert "measured OpenRouter" in rules - assert "tools-only" in rules - assert "/llm/v1" in rules # named as forbidden diff --git a/packages/challenges/agent-challenge/tests/test_product_or_outcome_bind.py b/packages/challenges/agent-challenge/tests/test_product_or_outcome_bind.py deleted file mode 100644 index b7b9baaf9..000000000 --- a/packages/challenges/agent-challenge/tests/test_product_or_outcome_bind.py +++ /dev/null @@ -1,567 +0,0 @@ -"""Product OpenRouter digests + policy equality + outcome→report_data bind. - -Ports VAL-ACAT-003..006 into agent-challenge production. -No Base gateway. No Base OpenRouter keys. Fail-closed codes from wire freeze. -""" - -from __future__ import annotations - -import pytest - -from agent_challenge.review.or_outcome_bind import ( - BASE_MASTER_KIND, - MEASURED_CVM_KIND, - OPENROUTER_ORIGIN, - OPENROUTER_PATH, - OPENROUTER_TLS_HOSTNAME, - REFUSE_BASE_MASTER_OR, - REFUSE_BASE_OR_KEY, - REFUSE_FAKE_ALLOW, - REFUSE_MEASUREMENT_UNALLOWLISTED, - REFUSE_MISSING_OBSERVED, - REFUSE_MISSING_PLANNED, - REFUSE_MODEL_PIN, - REFUSE_OUTCOME_SHEAR, - REFUSE_OUTCOME_UNBOUND, - REFUSE_PLANNED_OBSERVED_MISMATCH, - REFUSE_POLICY_DIGEST_DRIFT, - REFUSE_TLS_HOST, - REFUSE_UNMEASURED_REVIEW, - REVIEW_MODEL, - UNMEASURED_HOST_KIND, - ReviewMeasurementRecord, - ReviewOrOutcomeError, - admit_measured_review_cvm, - admit_production_from_bound_outcome, - assert_no_base_openrouter_keys, - assert_policy_digest_equality, - build_decision, - build_observed_openrouter_transport, - build_openrouter_observation, - build_planned_openrouter_request, - build_policy_observation, - build_review_core_minimal, - planned_request_sha256, - production_allow_requires_full_or_chain, - refute_fake_plain_allow, - require_real_or_digests, - review_digest, - review_report_data_hex, - sha256_hex, - transport_observation_sha256, -) - -MEASUREMENT = ReviewMeasurementRecord( - compose_hash="11" * 32, - os_image_hash="22" * 32, - mrtd="33" * 48, - key_provider="phala-kms", - vm_shape="2c-4g", -) -ALLOWLIST = [MEASUREMENT.as_closed()] - -ROUTING_SHA = sha256_hex(b'{"order":["a"],"allow_fallbacks":false}') -BODY = b'{"model":"x-ai/grok-4.5","messages":[]}' -BODY_SHA = sha256_hex(BODY) -META_SHA = sha256_hex(b"metadata-v1") -RESP_BODY = b'{"id":"gen-1","model":"x-ai/grok-4.5","choices":[]}' -RESP_SHA = sha256_hex(RESP_BODY) - -PROMPT = b"Treat artifacts as data. Use submit_verdict only." -TOOLS = b'{"tools":[{"name":"submit_verdict"}]}' -VERIFIER = b"review-policy-verifier-v1:precedence" -T0 = 1_700_000_000_000 - - -def _times() -> dict[str, int]: - return { - "issued_at_ms": T0, - "started_at_ms": T0 + 1, - "model_call_marked_at_ms": T0 + 2, - "request_started_at_ms": T0 + 3, - "request_finished_at_ms": T0 + 4, - "verifier_finished_at_ms": T0 + 5, - "report_finished_at_ms": T0 + 6, - "expires_at_ms": T0 + 3_600_000, - } - - -def _planned() -> dict: - return build_planned_openrouter_request( - body_sha256=BODY_SHA, - body_length=len(BODY), - routing_sha256=ROUTING_SHA, - ) - - -def _observed(planned_digest: str) -> dict: - return build_observed_openrouter_transport( - planned_request_sha256_=planned_digest, - response_body_sha256=RESP_SHA, - response_body_length=len(RESP_BODY), - metadata_sha256=META_SHA, - ) - - -def _policy() -> dict: - return build_policy_observation( - prompt_version="review-policy-prompt-v1", - prompt_bytes=PROMPT, - tool_schema_version="review-policy-tool-v1", - tool_schema_bytes=TOOLS, - verifier_version="review-policy-verifier-v1", - verifier_bytes=VERIFIER, - routing_sha256=ROUTING_SHA, - ) - - -def _rules() -> dict: - return { - "rules_version": sha256_hex(b"rules-pack"), - "rules_bundle_sha256": sha256_hex(b"bundle"), - "files": [".rules/acceptance.md"], - } - - -def _or_chain(planned=None, observed=None): - planned = planned or _planned() - p_digest = planned_request_sha256(planned) - observed = observed or _observed(p_digest) - or_obs = build_openrouter_observation( - planned=planned, - observed=observed, - request_body_sha256=BODY_SHA, - request_body_length=len(BODY), - response_id="gen-1", - metadata_sha256=META_SHA, - ) - return planned, observed, or_obs - - -def _core(*, verdict: str = "allow", or_obs=None, policy=None, decision=None): - planned, observed, computed_or = _or_chain() - or_obs = or_obs if or_obs is not None else computed_or - policy = policy or _policy() - decision = decision or build_decision(verdict=verdict) - return build_review_core_minimal( - session_id="rs-or-1", - assignment_id="ra-or-1", - submission_id="42", - review_nonce="rn-or-1", - assignment_digest="cd" * 32, - rules_observation=_rules(), - policy_observation=policy, - openrouter_observation=or_obs, - decision=decision, - times=_times(), - ) - - -# --------------------------------------------------------------------------- -# VAL-ACAT-003 — measured review CVM only -# --------------------------------------------------------------------------- - - -def test_measured_review_cvm_allowlisted_admits() -> None: - result = admit_measured_review_cvm( - runtime_kind=MEASURED_CVM_KIND, - measurement=MEASUREMENT, - allowlist=ALLOWLIST, - base_env={"WHATEVER": "ok"}, - ) - assert result["measurement_allowlisted"] is True - assert result["openrouter_allowed_from"] == "review_cvm_guest" - - -def test_base_master_openrouter_refuses() -> None: - with pytest.raises(ReviewOrOutcomeError) as exc: - admit_measured_review_cvm( - runtime_kind=BASE_MASTER_KIND, - measurement=MEASUREMENT, - allowlist=ALLOWLIST, - ) - assert exc.value.code == REFUSE_BASE_MASTER_OR - - -def test_unmeasured_host_python_refuses() -> None: - with pytest.raises(ReviewOrOutcomeError) as exc: - admit_measured_review_cvm( - runtime_kind=UNMEASURED_HOST_KIND, - measurement=MEASUREMENT, - allowlist=ALLOWLIST, - ) - assert exc.value.code == REFUSE_UNMEASURED_REVIEW - - -def test_measurement_not_on_allowlist_refuses() -> None: - other = ReviewMeasurementRecord( - compose_hash="aa" * 32, - os_image_hash="bb" * 32, - mrtd="cc" * 48, - key_provider="other", - vm_shape="1c-1g", - ) - with pytest.raises(ReviewOrOutcomeError) as exc: - admit_measured_review_cvm( - runtime_kind=MEASURED_CVM_KIND, - measurement=other, - allowlist=ALLOWLIST, - ) - assert exc.value.code == REFUSE_MEASUREMENT_UNALLOWLISTED - - -def test_empty_allowlist_matches_nothing() -> None: - with pytest.raises(ReviewOrOutcomeError) as exc: - admit_measured_review_cvm( - runtime_kind=MEASURED_CVM_KIND, - measurement=MEASUREMENT, - allowlist=[], - ) - assert exc.value.code == REFUSE_MEASUREMENT_UNALLOWLISTED - - -def test_base_env_must_not_hold_openrouter_keys() -> None: - with pytest.raises(ReviewOrOutcomeError) as exc: - assert_no_base_openrouter_keys({"OPENROUTER_API_KEY": "sk-redacted"}) - assert exc.value.code == REFUSE_BASE_OR_KEY - with pytest.raises(ReviewOrOutcomeError) as exc: - assert_no_base_openrouter_keys({"BASE_GATEWAY_TOKEN": "tok"}) - assert exc.value.code == REFUSE_BASE_OR_KEY - - -# --------------------------------------------------------------------------- -# VAL-ACAT-004 — planned + observed real digests -# --------------------------------------------------------------------------- - - -def test_planned_pins_openrouter_origin_model_routing() -> None: - planned = _planned() - assert planned["origin"] == OPENROUTER_ORIGIN - assert planned["path"] == OPENROUTER_PATH - assert planned["model"] == REVIEW_MODEL - assert planned["method"] == "POST" - digest = planned_request_sha256(planned) - assert len(digest) == 64 - - -def test_wrong_model_pin_refuses() -> None: - with pytest.raises(ReviewOrOutcomeError) as exc: - build_planned_openrouter_request( - body_sha256=BODY_SHA, - body_length=len(BODY), - routing_sha256=ROUTING_SHA, - model="openai/gpt-4o", - ) - assert exc.value.code == REFUSE_MODEL_PIN - - -def test_observed_tls_must_be_openrouter_ai() -> None: - planned = _planned() - p_digest = planned_request_sha256(planned) - with pytest.raises(ReviewOrOutcomeError) as exc: - build_observed_openrouter_transport( - planned_request_sha256_=p_digest, - response_body_sha256=RESP_SHA, - response_body_length=len(RESP_BODY), - metadata_sha256=META_SHA, - tls_hostname="evil.example", - ) - assert exc.value.code == REFUSE_TLS_HOST - with pytest.raises(ReviewOrOutcomeError) as exc: - build_observed_openrouter_transport( - planned_request_sha256_=p_digest, - response_body_sha256=RESP_SHA, - response_body_length=len(RESP_BODY), - metadata_sha256=META_SHA, - redirected=True, - ) - assert exc.value.code == REFUSE_TLS_HOST - - -def test_plan_observe_match_accepts() -> None: - planned, observed, or_obs = _or_chain() - digests = require_real_or_digests( - planned=planned, - observed=observed, - openrouter_observation=or_obs, - ) - assert digests["planned_request_sha256"] == planned_request_sha256(planned) - assert digests["transport_observation_sha256"] == transport_observation_sha256(observed) - assert observed["tls_hostname"] == OPENROUTER_TLS_HOSTNAME - assert observed["tls_hostname_verified"] is True - - -def test_observed_mismatch_plan_refuses() -> None: - planned = _planned() - other_planned = build_planned_openrouter_request( - body_sha256=sha256_hex(b"different-body"), - body_length=14, - routing_sha256=ROUTING_SHA, - ) - wrong_obs = _observed(planned_request_sha256(other_planned)) - or_obs = { - "planned_request_sha256": planned_request_sha256(planned), - "transport_observation_sha256": transport_observation_sha256(wrong_obs), - "request_body_sha256": BODY_SHA, - "response_body_sha256": RESP_SHA, - "cache_hit": False, - } - with pytest.raises(ReviewOrOutcomeError) as exc: - require_real_or_digests( - planned=planned, - observed=wrong_obs, - openrouter_observation=or_obs, - ) - assert exc.value.code == REFUSE_PLANNED_OBSERVED_MISMATCH - - -def test_missing_planned_or_observed_refuses() -> None: - planned, observed, or_obs = _or_chain() - with pytest.raises(ReviewOrOutcomeError) as exc: - require_real_or_digests(planned=None, observed=observed, openrouter_observation=or_obs) - assert exc.value.code == REFUSE_MISSING_PLANNED - with pytest.raises(ReviewOrOutcomeError) as exc: - require_real_or_digests(planned=planned, observed=None, openrouter_observation=or_obs) - assert exc.value.code == REFUSE_MISSING_OBSERVED - - -def test_fake_arm_chair_allow_without_or_refuses() -> None: - decision = build_decision(verdict="allow") - fake_or = { - "planned_request_sha256": "00" * 32, - "transport_observation_sha256": "00" * 32, - "request_body_sha256": "00" * 32, - "request_body_length": 1, - "response_status": 200, - "response_content_encoding": "identity", - "response_body_sha256": "00" * 32, - "response_body_length": 1, - "response_id": "fake", - "returned_model": REVIEW_MODEL, - "metadata_sha256": "00" * 32, - "observed_provider": None, - "provider_provenance": "unavailable", - "cache_hit": False, - } - core = build_review_core_minimal( - session_id="rs-fake", - assignment_id="ra-fake", - submission_id="1", - review_nonce="rn-fake", - assignment_digest="ee" * 32, - rules_observation=_rules(), - policy_observation=_policy(), - openrouter_observation=fake_or, - decision=decision, - times=_times(), - ) - rd = review_report_data_hex(core) - with pytest.raises(ReviewOrOutcomeError) as exc: - admit_production_from_bound_outcome( - review_core=core, - reported_report_data_hex=rd, - plain_status_allow=True, - ) - assert exc.value.code == REFUSE_FAKE_ALLOW - - -# --------------------------------------------------------------------------- -# VAL-ACAT-005 — policy digest equality -# --------------------------------------------------------------------------- - - -def test_policy_digests_equal_across_assignment_report_quote() -> None: - policy = _policy() - equal = assert_policy_digest_equality( - assignment_policy=policy, - report_policy=dict(policy), - quote_bound_policy=dict(policy), - ) - assert equal["prompt_sha256"] == sha256_hex(PROMPT) - assert equal["tool_schema_sha256"] == sha256_hex(TOOLS) - assert equal["verifier_sha256"] == sha256_hex(VERIFIER) - assert equal["model"] == REVIEW_MODEL - - -def test_policy_digest_drift_refuses() -> None: - policy = _policy() - mutated = dict(policy) - mutated["prompt_sha256"] = "ff" * 32 - with pytest.raises(ReviewOrOutcomeError) as exc: - assert_policy_digest_equality( - assignment_policy=policy, - report_policy=mutated, - quote_bound_policy=policy, - ) - assert exc.value.code == REFUSE_POLICY_DIGEST_DRIFT - - -def test_free_text_allow_without_verifier_pass_refuses() -> None: - with pytest.raises(ReviewOrOutcomeError) as exc: - build_decision(verdict="allow", verifier_result="reject") - assert exc.value.code == REFUSE_FAKE_ALLOW - - -# --------------------------------------------------------------------------- -# VAL-ACAT-006 — outcome bound into report_data; plain status insufficient -# --------------------------------------------------------------------------- - - -def test_allow_outcome_bound_into_report_data() -> None: - core = _core(verdict="allow") - rd = review_report_data_hex(core) - assert len(rd) == 128 - assert rd.endswith("00" * 32) - reject_core = _core(verdict="reject") - assert review_digest(core) != review_digest(reject_core) - assert review_report_data_hex(reject_core) != rd - - admission = admit_production_from_bound_outcome( - review_core=core, - reported_report_data_hex=rd, - ) - assert admission.admitted is True - assert admission.status == "verified_allow" - assert admission.verdict == "allow" - assert admission.report_data_hex == rd - - -def test_reject_is_terminal_not_score_eligible() -> None: - core = _core(verdict="reject") - rd = review_report_data_hex(core) - admission = admit_production_from_bound_outcome( - review_core=core, - reported_report_data_hex=rd, - ) - assert admission.admitted is False - assert admission.status == "verified_reject" - assert admission.verdict == "reject" - - -def test_escalate_bound_not_auto_allow() -> None: - core = _core(verdict="escalate") - rd = review_report_data_hex(core) - admission = admit_production_from_bound_outcome( - review_core=core, - reported_report_data_hex=rd, - ) - assert admission.admitted is False - assert admission.status == "verified_escalate" - - -def test_plain_status_allow_with_reject_quote_shears() -> None: - core = _core(verdict="reject") - rd = review_report_data_hex(core) - with pytest.raises(ReviewOrOutcomeError) as exc: - admit_production_from_bound_outcome( - review_core=core, - reported_report_data_hex=rd, - plain_status_allow=True, - ) - assert exc.value.code == REFUSE_OUTCOME_SHEAR - - -def test_report_data_mismatch_fails_closed() -> None: - core = _core(verdict="allow") - with pytest.raises(ReviewOrOutcomeError) as exc: - admit_production_from_bound_outcome( - review_core=core, - reported_report_data_hex="00" * 64, - ) - assert exc.value.code == REFUSE_OUTCOME_UNBOUND - - -def test_plain_allow_without_core_or_report_data_fails() -> None: - with pytest.raises(ReviewOrOutcomeError) as exc: - refute_fake_plain_allow( - bound_verdict="reject", - plain_status="allow", - report_data_hex_value=None, - review_core=None, - ) - assert exc.value.code == REFUSE_OUTCOME_SHEAR - with pytest.raises(ReviewOrOutcomeError) as exc: - refute_fake_plain_allow( - bound_verdict="allow", - plain_status="allow", - report_data_hex_value=None, - review_core=None, - ) - assert exc.value.code == REFUSE_OUTCOME_UNBOUND - - -def test_outcome_mutation_invalidates_report_data() -> None: - core = _core(verdict="allow") - rd = review_report_data_hex(core) - flipped = dict(core) - flipped["decision"] = build_decision(verdict="reject") - assert review_report_data_hex(flipped) != rd - with pytest.raises(ReviewOrOutcomeError) as exc: - admit_production_from_bound_outcome( - review_core=flipped, - reported_report_data_hex=rd, - ) - assert exc.value.code == REFUSE_OUTCOME_UNBOUND - - -def test_production_allow_conjunction() -> None: - measured = admit_measured_review_cvm( - runtime_kind=MEASURED_CVM_KIND, - measurement=MEASUREMENT, - allowlist=ALLOWLIST, - ) - policy = _policy() - assert_policy_digest_equality( - assignment_policy=policy, - report_policy=policy, - quote_bound_policy=policy, - ) - planned, observed, or_obs = _or_chain() - digests = require_real_or_digests( - planned=planned, - observed=observed, - openrouter_observation=or_obs, - ) - core = _core(verdict="allow", or_obs=or_obs, policy=policy) - admission = admit_production_from_bound_outcome( - review_core=core, - reported_report_data_hex=review_report_data_hex(core), - ) - assert production_allow_requires_full_or_chain( - measured=measured, - policy_equal=policy, - or_digests=digests, - admission=admission, - ) - - -def test_production_allow_fails_if_any_conjunct_missing() -> None: - assert ( - production_allow_requires_full_or_chain( - measured=None, - policy_equal=_policy(), - or_digests={ - "planned_request_sha256": "ab" * 32, - "transport_observation_sha256": "cd" * 32, - }, - admission=None, - ) - is False - ) - core = _core(verdict="reject") - admission = admit_production_from_bound_outcome( - review_core=core, - reported_report_data_hex=review_report_data_hex(core), - ) - assert ( - production_allow_requires_full_or_chain( - measured={"measurement_allowlisted": True}, - policy_equal=_policy(), - or_digests={ - "planned_request_sha256": "ab" * 32, - "transport_observation_sha256": "cd" * 32, - }, - admission=admission, - ) - is False - ) diff --git a/packages/challenges/agent-challenge/tests/test_product_score_chain_failclosed.py b/packages/challenges/agent-challenge/tests/test_product_score_chain_failclosed.py deleted file mode 100644 index 5fd62b9d3..000000000 --- a/packages/challenges/agent-challenge/tests/test_product_score_chain_failclosed.py +++ /dev/null @@ -1,839 +0,0 @@ -"""VAL-ACAT-011/012/017/018/030–037/039/040: full score-chain fail-closed admission. - -Production score emission under dual flags ON requires conjunction: - review (bound times + ≤24h + OR digests + allow) - + key-release RA-TLS grant re-checked at admission - + score-domain report_data / nonce / domain separation -Any single ablation refuses with zero partial score. AST-only never emits. -""" - -from __future__ import annotations - -import hashlib -from typing import Any - -import pytest - -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.evaluation.llm_rules_residual import ( - MEASURED_RESIDUAL_KIND, - bind_package_residual_into_review_materials, - build_package_residual_materials, -) -from agent_challenge.evaluation.score_chain_gate import ( - KEY_RELEASE_DOMAIN, - REFUSE_AST_ONLY, - REFUSE_DOMAIN_CONFUSION, - REFUSE_FLAGS_OFF, - REFUSE_INCOMPLETE_CHAIN, - REFUSE_KEY_RELEASE_MISMATCH, - REFUSE_MISSING_KEY_RELEASE, - REFUSE_NONCE_REPLAY, - REFUSE_NONCE_STALE, - REFUSE_REVIEW, - REFUSE_REVIEW_STALE, - REFUSE_SCORE_DOMAIN, - REFUSE_STICKY, - REFUSE_TAMPERED, - REVIEW_DOMAIN, - SCORE_DOMAIN, - ScoreChainAdmissionError, - admit_production_score_for_eval_result, - admit_production_score_from_chain, - build_score_binding_from_plan_and_digest, - recompute_key_release_report_data_hex, - require_production_score_from_chain, - verify_key_release_grant, -) -from agent_challenge.review.attested_times import FRESHNESS_WINDOW_MS -from agent_challenge.review.or_outcome_bind import ( - REVIEW_REPORT_DOMAIN, - build_decision, - build_observed_openrouter_transport, - build_openrouter_observation, - build_planned_openrouter_request, - build_policy_observation, - build_review_core_minimal, - planned_request_sha256, - review_digest, - review_report_data_hex, - sha256_hex, -) - -PACKAGE_TREE_SHA = "bb" * 32 - -T0 = 1_700_000_000_000 -MS_23H = FRESHNESS_WINDOW_MS - 60_000 -MS_24H = FRESHNESS_WINDOW_MS -MS_24H_PLUS = FRESHNESS_WINDOW_MS + 1 - -REGS = { - "mrtd": "11" * 48, - "rtmr0": "22" * 48, - "rtmr1": "33" * 48, - "rtmr2": "44" * 48, -} -COMPOSE_HASH = "ab" * 32 -AGENT_HASH = "55" * 32 -OS_IMAGE_HASH = "66" * 32 # synthetic; plan validation may only check hex form -ROUTING = sha256_hex(b'{"order":["score-chain"]}') -BODY = b'{"model":"x-ai/grok-4.5","messages":[]}' -BODY_SHA = sha256_hex(BODY) -RESP = b'{"id":"gen-score","model":"x-ai/grok-4.5","choices":[]}' -RESP_SHA = sha256_hex(RESP) -META = sha256_hex(b"meta-score-chain") -SPKI = "aa" * 32 - - -def _times(*, issued: int = T0, received: int = T0 + MS_23H) -> dict[str, int]: - base = min(issued, received) - return { - "issued_at_ms": issued, - "started_at_ms": base, - "model_call_marked_at_ms": base + 1, - "request_started_at_ms": base + 2, - "request_finished_at_ms": base + 3, - "verifier_finished_at_ms": base + 4, - "report_finished_at_ms": base + 5, - "expires_at_ms": max(issued, received) + 3_600_000, - "submission_received_at_ms": received, - } - - -def _rules() -> dict: - return { - "rules_version": "rules-v1", - "rules_bundle_sha256": "11" * 32, - "rules_files": [".rules/acceptance.md"], - "rules_file_digests": {".rules/acceptance.md": "22" * 32}, - "rules_policy_text_sha256": "33" * 32, - } - - -def _review_core( - *, - verdict: str = "allow", - issued: int = T0, - received: int = T0 + MS_23H, -) -> dict[str, Any]: - planned = build_planned_openrouter_request( - body_sha256=BODY_SHA, - body_length=len(BODY), - routing_sha256=ROUTING, - ) - p_digest = planned_request_sha256(planned) - observed = build_observed_openrouter_transport( - planned_request_sha256_=p_digest, - response_body_sha256=RESP_SHA, - response_body_length=len(RESP), - metadata_sha256=META, - ) - or_obs = build_openrouter_observation( - planned=planned, - observed=observed, - request_body_sha256=BODY_SHA, - request_body_length=len(BODY), - response_id="gen-score-chain", - ) - policy = build_policy_observation( - prompt_version="review-policy-prompt-v1", - prompt_bytes=b"prompt-score-chain", - tool_schema_version="review-policy-tool-v1", - tool_schema_bytes=b"tools-score-chain", - verifier_version="review-policy-verifier-v1", - verifier_bytes=b"verifier-score-chain", - routing_sha256=ROUTING, - ) - return build_review_core_minimal( - session_id="rs-score-chain", - assignment_id="ra-score-chain", - submission_id="sub-score-chain", - review_nonce="nonce-score-chain-review", - assignment_digest="aa" * 32, - rules_observation=_rules(), - policy_observation=policy, - openrouter_observation=or_obs, - decision=build_decision(verdict=verdict), - times=_times(issued=issued, received=received), - ) - - -def _review_envelope( - *, - verdict: str = "allow", - issued: int = T0, - received: int = T0 + MS_23H, - mutilate: bool = False, - domain: str = REVIEW_REPORT_DOMAIN, - include_package_residual: bool = True, - package_tree_sha: str = PACKAGE_TREE_SHA, -) -> dict[str, Any]: - core = _review_core(verdict=verdict, issued=issued, received=received) - rd = review_report_data_hex(core) - if mutilate: - rd = ("ff" * 32) + ("00" * 32) - env: dict[str, Any] = { - "schema_version": 1, - "domain": domain, - "review_digest": review_digest(core), - "report_data_hex": rd, - "review_core": core, - } - if include_package_residual and verdict == "allow" and not mutilate: - materials = build_package_residual_materials( - residual_verdict="allow", - rules_bundle_sha256="11" * 32, - rules_version="rules-v1", - rules_file_digests={".rules/acceptance.md": "22" * 32}, - package_tree_sha=package_tree_sha, - residual_kind=MEASURED_RESIDUAL_KIND, - rules_policy_text_sha256="33" * 32, - harness_kind="measured_review_cvm_script_zip", - ) - env = bind_package_residual_into_review_materials( - envelope=env, - materials=materials, - )["envelope"] - return env - - -def _os_image_hash() -> str: - from agent_challenge.keyrelease.quote import os_image_hash_from_registers - - return os_image_hash_from_registers(REGS["mrtd"], REGS["rtmr1"], REGS["rtmr2"]) - - -def _plan(*, authorizing_review_digest: str | None = None) -> dict[str, Any]: - os_hash = _os_image_hash() - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - core = _review_core() - review_d = authorizing_review_digest or review_digest(core) - return ew.validate_eval_plan( - { - "schema_version": 1, - "eval_run_id": "eval-score-chain-1", - "submission_id": "submission-score-chain-1", - "submission_version": 1, - "authorizing_review_digest": review_d, - "agent_hash": AGENT_HASH, - "package_tree_sha": PACKAGE_TREE_SHA, - "selected_tasks": [ - { - "task_id": "task-a", - "image_ref": "registry.example/task@sha256:" + "77" * 32, - "task_config_sha256": "88" * 32, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "99" * 32, - "compose_hash": COMPOSE_HASH, - "app_identity": "agent-challenge-eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "aa" * 32, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), - "measurement": { - **REGS, - "os_image_hash": os_hash, - "key_provider": "phala", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-score-chain-1/result", - "key_release_nonce": "key-release-score-chain-1", - "score_nonce": "score-nonce-score-chain-1", - "run_token_sha256": "bb" * 32, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - ) - - -def _grant( - plan: dict[str, Any], - *, - eval_run_id: str | None = None, - domain: str = KEY_RELEASE_DOMAIN, - spki: str = SPKI, - mutilate_rd: bool = False, - agent_hash: str | None = None, -) -> dict[str, Any]: - er = eval_run_id if eval_run_id is not None else plan["eval_run_id"] - nonce = plan["key_release_nonce"] - rd = recompute_key_release_report_data_hex( - eval_run_id=er, - key_release_nonce=nonce, - ra_tls_spki_digest=spki, - ) - if mutilate_rd: - rd = ("00" * 32) + ("ff" * 32) - out: dict[str, Any] = { - "domain": domain, - "schema_version": 2, - "eval_run_id": er, - "key_release_nonce": nonce, - "ra_tls_spki_digest": spki, - "report_data_hex": rd, - } - if agent_hash is not None: - out["agent_hash"] = agent_hash - return out - - -def _scores_digest(plan: dict[str, Any]) -> str: - from agent_challenge.evaluation.plan_scoring import build_score_record_from_eval_plan - - record = build_score_record_from_eval_plan(plan, {"task-a": [1.0]}) - return ew.score_record_digest(record) - - -def _score_binding(plan: dict[str, Any], scores_digest: str) -> dict[str, Any]: - return build_score_binding_from_plan_and_digest( - eval_plan=plan, - scores_digest=scores_digest, - ) - - -def _full_ok(**overrides: Any) -> dict[str, Any]: - plan = _plan() - env = _review_envelope() - # Force authorizing_review_digest to match envelope. - plan = dict(plan) - plan["authorizing_review_digest"] = env["review_digest"] - sd = _scores_digest(plan) - binding = _score_binding(plan, sd) - grant = _grant(plan) - kwargs: dict[str, Any] = { - "dual_flags_on": True, - "review_envelope": env, - "key_release_grant": grant, - "key_granted_flag": True, - "eval_plan": plan, - "score_binding": binding, - "score_report_data_hex": ew.score_report_data_hex(binding), - "scores_digest": sd, - "score_nonce_state": "outstanding", - "offline_ast_pass": False, - } - kwargs.update(overrides) - return kwargs - - -# --------------------------------------------------------------------------- -# VAL-ACAT-030 / 040: only all-on control admits -# --------------------------------------------------------------------------- - - -def test_full_chain_control_admits_production() -> None: - decision = admit_production_score_from_chain(**_full_ok()) - assert decision.admitted is True - assert decision.production_emit is True - assert decision.partial_score is False - assert decision.score is None # gate is binary; score set elsewhere - assert decision.reverify_exercised is True - assert REVIEW_DOMAIN in decision.domains_checked - assert KEY_RELEASE_DOMAIN in decision.domains_checked - assert SCORE_DOMAIN in decision.domains_checked - assert decision.reason_code == "score_chain_verified" - - -def test_require_raises_on_refuse() -> None: - with pytest.raises(ScoreChainAdmissionError) as exc: - require_production_score_from_chain( - **_full_ok(key_release_grant=None, key_granted_flag=False) - ) - assert exc.value.code == REFUSE_MISSING_KEY_RELEASE - - -# --------------------------------------------------------------------------- -# VAL-ACAT-031: ablation of each required artifact → refuse, no partial -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - "mutation,expected_code", - [ - ({"review_envelope": None, "cached_review_allow": True}, REFUSE_INCOMPLETE_CHAIN), - ({"key_release_grant": None, "key_granted_flag": False}, REFUSE_MISSING_KEY_RELEASE), - ({"score_binding": None}, REFUSE_INCOMPLETE_CHAIN), - ({"score_report_data_hex": None, "score_binding": None}, REFUSE_INCOMPLETE_CHAIN), - ], -) -def test_missing_chain_material_refuses_zero_partial( - mutation: dict[str, Any], expected_code: str -) -> None: - decision = admit_production_score_from_chain(**_full_ok(**mutation)) - assert decision.admitted is False - assert decision.production_emit is False - assert decision.partial_score is False - assert decision.score is None - assert decision.reason_code == expected_code - - -def test_db_key_granted_flag_alone_without_grant_materials_refuses() -> None: - """VAL-ACAT-037: historic env-inject / DB key_granted alone is insufficient.""" - - decision = admit_production_score_from_chain( - **_full_ok(key_release_grant=None, key_granted_flag=True) - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_MISSING_KEY_RELEASE - assert decision.partial_score is False - - -# --------------------------------------------------------------------------- -# VAL-ACAT-032: tamper / domain confusion fail closed -# --------------------------------------------------------------------------- - - -def test_tampered_review_report_data_refuses() -> None: - decision = admit_production_score_from_chain( - **_full_ok(review_envelope=_review_envelope(mutilate=True)) - ) - assert decision.admitted is False - assert decision.partial_score is False - assert decision.score is None - assert decision.reverify_exercised is True - # re-verify fail class (review) - assert decision.reason_code in { - REFUSE_REVIEW, - REFUSE_TAMPERED, - "review_attestation_reverify_failed", - } - - -def test_tampered_key_release_report_data_refuses() -> None: - plan = _plan() - env = _review_envelope() - plan = dict(plan) - plan["authorizing_review_digest"] = env["review_digest"] - sd = _scores_digest(plan) - binding = _score_binding(plan, sd) - decision = admit_production_score_from_chain( - dual_flags_on=True, - review_envelope=env, - key_release_grant=_grant(plan, mutilate_rd=True), - key_granted_flag=True, - eval_plan=plan, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=sd, - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_TAMPERED - assert decision.score is None - - -def test_domain_confusion_review_as_score_refuses() -> None: - plan = _plan() - env = _review_envelope() - plan = dict(plan) - plan["authorizing_review_digest"] = env["review_digest"] - sd = _scores_digest(plan) - # Use review domain on score binding. - bad_binding = _score_binding(plan, sd) - bad_binding = dict(bad_binding) - bad_binding["domain"] = REVIEW_DOMAIN - decision = admit_production_score_from_chain( - dual_flags_on=True, - review_envelope=env, - key_release_grant=_grant(plan), - key_granted_flag=True, - eval_plan=plan, - score_binding=bad_binding, - score_report_data_hex=ew.score_report_data_hex( - _score_binding(plan, sd) - ), # mismatch + domain - scores_digest=sd, - ) - assert decision.admitted is False - assert decision.reason_code in {REFUSE_DOMAIN_CONFUSION, REFUSE_SCORE_DOMAIN, REFUSE_TAMPERED} - assert decision.partial_score is False - - -def test_domain_confusion_score_materials_as_key_release_refuses() -> None: - grant = { - "domain": SCORE_DOMAIN, - "eval_run_id": "eval-score-chain-1", - "key_release_nonce": "key-release-score-chain-1", - "ra_tls_spki_digest": SPKI, - "report_data_hex": "00" * 64, - } - decision = admit_production_score_from_chain(**_full_ok(key_release_grant=grant)) - assert decision.admitted is False - assert decision.reason_code == REFUSE_DOMAIN_CONFUSION - - -def test_cross_session_key_release_grant_swap_refuses() -> None: - """VAL-ACAT-037: grant for another eval_run_id does not authorize.""" - - plan = _plan() - env = _review_envelope() - plan = dict(plan) - plan["authorizing_review_digest"] = env["review_digest"] - sd = _scores_digest(plan) - binding = _score_binding(plan, sd) - foreign = _grant(plan, eval_run_id="eval-FOREIGN-session") - decision = admit_production_score_from_chain( - dual_flags_on=True, - review_envelope=env, - key_release_grant=foreign, - key_granted_flag=True, - eval_plan=plan, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=sd, - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_KEY_RELEASE_MISMATCH - - -def test_agent_hash_mismatch_on_grant_refuses() -> None: - plan = _plan() - env = _review_envelope() - plan = dict(plan) - plan["authorizing_review_digest"] = env["review_digest"] - grant = _grant(plan, agent_hash="ff" * 32) - sd = _scores_digest(plan) - binding = _score_binding(plan, sd) - decision = admit_production_score_from_chain( - dual_flags_on=True, - review_envelope=env, - key_release_grant=grant, - key_granted_flag=True, - eval_plan=plan, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=sd, - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_KEY_RELEASE_MISMATCH - - -# --------------------------------------------------------------------------- -# VAL-ACAT-033 / 017: nonce replay / stale -# --------------------------------------------------------------------------- - - -def test_replayed_score_nonce_refuses() -> None: - decision = admit_production_score_from_chain(**_full_ok(score_nonce_state="consumed")) - assert decision.admitted is False - assert decision.reason_code == REFUSE_NONCE_REPLAY - assert decision.score is None - - -def test_expired_score_nonce_refuses() -> None: - decision = admit_production_score_from_chain(**_full_ok(score_nonce_state="expired")) - assert decision.admitted is False - assert decision.reason_code == REFUSE_NONCE_STALE - - -def test_stale_review_over_24h_refuses_score() -> None: - env = _review_envelope(received=T0 + MS_24H_PLUS) - decision = admit_production_score_from_chain(**_full_ok(review_envelope=env)) - assert decision.admitted is False - assert decision.reason_code == REFUSE_REVIEW_STALE - assert decision.partial_score is False - - -def test_exactly_24h_review_still_allows_score_chain() -> None: - env = _review_envelope(received=T0 + MS_24H) - plan = _plan() - plan = dict(plan) - plan["authorizing_review_digest"] = env["review_digest"] - sd = _scores_digest(plan) - binding = _score_binding(plan, sd) - decision = admit_production_score_from_chain( - dual_flags_on=True, - review_envelope=env, - key_release_grant=_grant(plan), - key_granted_flag=True, - eval_plan=plan, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=sd, - score_nonce_state="outstanding", - ) - assert decision.admitted is True - - -# --------------------------------------------------------------------------- -# VAL-ACAT-034 / 035: offline AST / tutte cannot score alone -# --------------------------------------------------------------------------- - - -def test_ast_only_green_without_attestation_refuses() -> None: - decision = admit_production_score_from_chain( - dual_flags_on=True, - eval_plan=_plan(), - review_envelope=None, - key_release_grant=None, - score_binding=None, - offline_ast_pass=True, - cached_review_allow=True, - master_status_green=True, - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_AST_ONLY - assert decision.production_emit is False - assert decision.score is None - assert decision.partial_score is False - - -def test_ast_pass_does_not_upgrade_missing_key_release() -> None: - env = _review_envelope() - plan = _plan() - plan = dict(plan) - plan["authorizing_review_digest"] = env["review_digest"] - sd = _scores_digest(plan) - binding = _score_binding(plan, sd) - decision = admit_production_score_from_chain( - dual_flags_on=True, - review_envelope=env, - key_release_grant=None, - key_granted_flag=False, - eval_plan=plan, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=sd, - offline_ast_pass=True, - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_AST_ONLY - assert decision.score is None - - -# --------------------------------------------------------------------------- -# VAL-ACAT-018: flag-off cannot emit production scores -# --------------------------------------------------------------------------- - - -def test_flags_off_refuses_production_even_with_perfect_chain() -> None: - decision = admit_production_score_from_chain(**_full_ok(dual_flags_on=False)) - assert decision.admitted is False - assert decision.reason_code == REFUSE_FLAGS_OFF - assert decision.production_emit is False - assert decision.score is None - - -# --------------------------------------------------------------------------- -# VAL-ACAT-012: master green + bad materials refuses -# --------------------------------------------------------------------------- - - -def test_master_green_with_bad_quote_materials_refuses() -> None: - decision = admit_production_score_from_chain( - **_full_ok( - review_envelope=_review_envelope(mutilate=True), - master_status_green=True, - cached_score_ok=True, - ) - ) - assert decision.admitted is False - assert decision.production_emit is False - - -# --------------------------------------------------------------------------- -# VAL-ACAT-039: sticky refuse, no silent downgrade -# --------------------------------------------------------------------------- - - -def test_prior_reverify_failed_sticky_refuses() -> None: - decision = admit_production_score_from_chain( - **_full_ok(prior_reverify_failed=True, offline_ast_pass=True) - ) - assert decision.admitted is False - assert decision.reason_code == REFUSE_STICKY - assert decision.sticky is True - assert decision.score is None - - -# --------------------------------------------------------------------------- -# VAL-ACAT-017: domain-separated nonces - score_nonce ≠ key_release_nonce -# --------------------------------------------------------------------------- - - -def test_key_release_and_score_nonce_must_differ_in_plan_constructor() -> None: - """eval_wire plan validation already enforces nonce separation.""" - - with pytest.raises(ew.EvalWireError): - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - os_hash = _os_image_hash() - ew.validate_eval_plan( - { - "schema_version": 1, - "eval_run_id": "eval-collide", - "submission_id": "sub-collide", - "submission_version": 1, - "authorizing_review_digest": "66" * 32, - "agent_hash": AGENT_HASH, - "package_tree_sha": "bb" * 32, - "selected_tasks": [ - { - "task_id": "task-a", - "image_ref": "registry.example/task@sha256:" + "77" * 32, - "task_config_sha256": "88" * 32, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "99" * 32, - "compose_hash": COMPOSE_HASH, - "app_identity": "agent-challenge-eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "aa" * 32, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), - "measurement": { - **REGS, - "os_image_hash": os_hash, - "key_provider": "phala", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-collide/result", - "key_release_nonce": "SAME-NONCE", - "score_nonce": "SAME-NONCE", - "run_token_sha256": "bb" * 32, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - ) - - -def test_verify_key_release_grant_rejects_colliding_score_nonce() -> None: - plan = { - "eval_run_id": "eval-x", - "key_release_nonce": "nonce-shared", - "score_nonce": "nonce-shared", - "agent_hash": AGENT_HASH, - "package_tree_sha": PACKAGE_TREE_SHA, - } - grant = { - "domain": KEY_RELEASE_DOMAIN, - "eval_run_id": "eval-x", - "key_release_nonce": "nonce-shared", - "ra_tls_spki_digest": SPKI, - "report_data_hex": recompute_key_release_report_data_hex( - eval_run_id="eval-x", - key_release_nonce="nonce-shared", - ra_tls_spki_digest=SPKI, - ), - } - err, _ = verify_key_release_grant(grant=grant, eval_plan=plan, key_granted_flag=True) - assert err == REFUSE_DOMAIN_CONFUSION - - -# --------------------------------------------------------------------------- -# VAL-ACAT-040: single-factor ablation matrix (conjunction all-or-nothing) -# --------------------------------------------------------------------------- - - -def test_conjunction_ablation_matrix_only_all_on_admits() -> None: - control = admit_production_score_from_chain(**_full_ok()) - assert control.admitted is True - - ablations: list[tuple[str, dict[str, Any]]] = [ - ("flags_off", {"dual_flags_on": False}), - ("no_review", {"review_envelope": None}), - ("stale_review", {"review_envelope": _review_envelope(received=T0 + MS_24H_PLUS)}), - ("reject_review", {"review_envelope": _review_envelope(verdict="reject")}), - ("no_key_release", {"key_release_grant": None, "key_granted_flag": False}), - ("foreign_grant", {}), # filled below - ("no_score_binding", {"score_binding": None, "score_report_data_hex": None}), - ("nonce_replay", {"score_nonce_state": "consumed"}), - ("nonce_expired", {"score_nonce_state": "expired"}), - ("sticky", {"prior_reverify_failed": True}), - ("ast_only", {"review_envelope": None, "offline_ast_pass": True}), - ] - # Foreign grant ablation needs plan. - base = _full_ok() - plan = base["eval_plan"] - ablations[5] = ( - "foreign_grant", - {"key_release_grant": _grant(plan, eval_run_id="EVAL-OTHER")}, - ) - - for name, mut in ablations: - decision = admit_production_score_from_chain(**_full_ok(**mut)) - assert decision.admitted is False, f"ablation {name} unexpectedly admitted" - assert decision.score is None, f"ablation {name} leaked score" - assert decision.partial_score is False, f"ablation {name} partial" - assert decision.production_emit is False, f"ablation {name} production_emit" - - -# --------------------------------------------------------------------------- -# Convenience wrapper (direct-result path surface) -# --------------------------------------------------------------------------- - - -def test_admit_production_score_for_eval_result_wrapper() -> None: - plan = _plan() - env = _review_envelope() - plan = dict(plan) - plan["authorizing_review_digest"] = env["review_digest"] - sd = _scores_digest(plan) - binding = _score_binding(plan, sd) - decision = admit_production_score_for_eval_result( - settings_dual_flags_on=True, - eval_plan=plan, - review_envelope=env, - key_release_grant=_grant(plan), - key_granted_flag=True, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=sd, - score_nonce_outstanding=True, - ) - assert decision.admitted is True - decision_fail = admit_production_score_for_eval_result( - settings_dual_flags_on=True, - eval_plan=plan, - review_envelope=env, - key_release_grant=_grant(plan), - key_granted_flag=True, - score_binding=binding, - score_report_data_hex=ew.score_report_data_hex(binding), - scores_digest=sd, - score_nonce_outstanding=False, - ) - assert decision_fail.admitted is False - assert decision_fail.reason_code == REFUSE_NONCE_REPLAY - - -def test_refuse_codes_are_stable_nonsecret_strings() -> None: - for code in ( - REFUSE_INCOMPLETE_CHAIN, - REFUSE_AST_ONLY, - REFUSE_FLAGS_OFF, - REFUSE_MISSING_KEY_RELEASE, - REFUSE_KEY_RELEASE_MISMATCH, - REFUSE_NONCE_REPLAY, - REFUSE_NONCE_STALE, - REFUSE_DOMAIN_CONFUSION, - REFUSE_TAMPERED, - REFUSE_STICKY, - REFUSE_REVIEW_STALE, - ): - assert isinstance(code, str) - assert code - assert " " not in code - assert "secret" not in code.lower() - assert "key=" not in code.lower() diff --git a/packages/challenges/agent-challenge/tests/test_public_client_fullchain_export.py b/packages/challenges/agent-challenge/tests/test_public_client_fullchain_export.py deleted file mode 100644 index c3050f2ff..000000000 --- a/packages/challenges/agent-challenge/tests/test_public_client_fullchain_export.py +++ /dev/null @@ -1,423 +0,0 @@ -"""Public-only client fullchain export surface (product residual Mode B). - -After GetTlsKey material_ready, live Path B could not harvest public client.crt -(SSH disabled, SCP fail, secret-free logs). These tests pin: - -1. Export emits public fullchain (leaf + intermediates) for ----BEGIN CERT----. -2. Private key is never present in any export surface (log + path). -3. Fail closed when the chain is missing/empty. -4. Entrypoint bootstrap path invokes export after material_ready. -""" - -from __future__ import annotations - -import hashlib -import os -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import pytest -from cryptography import x509 -from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID - -from agent_challenge.canonical import entrypoint -from agent_challenge.canonical import public_client_fullchain as pubfc - -# --------------------------------------------------------------------------- # -# Crypto helpers (match bootstrap suite style; no invent roots for production) -# --------------------------------------------------------------------------- # - - -def _rsa_key(size: int = 2048): - return rsa.generate_private_key(public_exponent=65537, key_size=size) - - -def _make_ca(*, cn: str = "export-test-ca") -> tuple[Any, x509.Certificate]: - key = _rsa_key() - name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, cn)]) - cert = ( - x509.CertificateBuilder() - .subject_name(name) - .issuer_name(name) - .public_key(key.public_key()) - .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.now(UTC) - timedelta(minutes=1)) - .not_valid_after(datetime.now(UTC) + timedelta(days=1)) - .add_extension(x509.BasicConstraints(ca=True, path_length=1), critical=True) - .sign(key, hashes.SHA256()) - ) - return key, cert - - -def _sign_cert( - *, - subject_cn: str, - issuer_key, - issuer_cert: x509.Certificate, - client_auth: bool = True, -) -> tuple[Any, x509.Certificate]: - key = _rsa_key() - name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, subject_cn)]) - builder = ( - x509.CertificateBuilder() - .subject_name(name) - .issuer_name(issuer_cert.subject) - .public_key(key.public_key()) - .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.now(UTC) - timedelta(minutes=1)) - .not_valid_after(datetime.now(UTC) + timedelta(hours=1)) - ) - if client_auth: - builder = builder.add_extension( - x509.ExtendedKeyUsage([ExtendedKeyUsageOID.CLIENT_AUTH]), - critical=False, - ) - cert = builder.sign(issuer_key, hashes.SHA256()) - return key, cert - - -def _pem_cert(cert: x509.Certificate) -> str: - return cert.public_bytes(serialization.Encoding.PEM).decode() - - -def _pem_key(key) -> str: - return key.private_bytes( - serialization.Encoding.PEM, - serialization.PrivateFormat.TraditionalOpenSSL, - serialization.NoEncryption(), - ).decode() - - -def _chain_files(tmp_path: Path) -> tuple[Path, Path, str, str]: - """Write leaf+intermediate chain file + private key; return paths + PEMs.""" - - ca_key, ca_cert = _make_ca(cn="dstack-inter") - leaf_key, leaf = _sign_cert( - subject_cn="guest-client", - issuer_key=ca_key, - issuer_cert=ca_cert, - client_auth=True, - ) - leaf_pem = _pem_cert(leaf) - inter_pem = _pem_cert(ca_cert) - chain = leaf_pem + inter_pem - key_pem = _pem_key(leaf_key) - cert_path = tmp_path / "client.crt" - key_path = tmp_path / "client.key" - cert_path.write_text(chain, encoding="utf-8") - key_path.write_text(key_pem, encoding="utf-8") - os.chmod(key_path, 0o600) - return cert_path, key_path, chain, key_pem - - -# --------------------------------------------------------------------------- # -# Unit: extract + assert public-only + fail closed -# --------------------------------------------------------------------------- # - - -def test_extract_public_fullchain_leaf_and_intermediate(): - ca_key, ca_cert = _make_ca() - _lk, leaf = _sign_cert(subject_cn="leaf", issuer_key=ca_key, issuer_cert=ca_cert) - chain = _pem_cert(leaf) + _pem_cert(ca_cert) - - pem = pubfc.extract_public_fullchain_pem(chain) - assert pem.count("BEGIN CERTIFICATE") == 2 - assert pem.count("END CERTIFICATE") == 2 - assert "PRIVATE KEY" not in pem.upper() - # Order preserved: leaf first (PEM header is -----BEGIN CERTIFICATE-----). - assert pem.startswith("-----BEGIN CERTIFICATE-----") - - -def test_extract_fails_closed_when_chain_empty(): - with pytest.raises(pubfc.PublicFullchainExportError, match="public_chain_missing"): - pubfc.extract_public_fullchain_pem("") - with pytest.raises(pubfc.PublicFullchainExportError, match="public_chain_missing"): - pubfc.extract_public_fullchain_pem(" ") - with pytest.raises(pubfc.PublicFullchainExportError, match="public_chain_missing"): - pubfc.extract_public_fullchain_pem("not a pem at all") - - -def test_extract_fails_closed_when_private_key_in_input(): - ca_key, ca_cert = _make_ca() - leaf_key, leaf = _sign_cert(subject_cn="leaf", issuer_key=ca_key, issuer_cert=ca_cert) - blob = _pem_cert(leaf) + _pem_key(leaf_key) - with pytest.raises(pubfc.PublicFullchainExportError, match="private_key_in_export"): - pubfc.extract_public_fullchain_pem(blob) - - -def test_assert_public_only_rejects_private_key_variants(): - for header in ( - "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----\n", - "-----BEGIN RSA PRIVATE KEY-----\nabc\n-----END RSA PRIVATE KEY-----\n", - "-----BEGIN EC PRIVATE KEY-----\nabc\n-----END EC PRIVATE KEY-----\n", - "-----BEGIN ENCRYPTED PRIVATE KEY-----\nabc\n-----END ENCRYPTED PRIVATE KEY-----\n", - ): - with pytest.raises(pubfc.PublicFullchainExportError, match="private_key_in_export"): - pubfc.assert_public_only(header) - - -def test_export_surface_contains_private_key_predicate(): - assert pubfc.export_surface_contains_private_key("-----BEGIN PRIVATE KEY-----\n") is True - assert pubfc.export_surface_contains_private_key("-----BEGIN CERTIFICATE-----\n") is False - assert pubfc.export_surface_contains_private_key("") is False - - -# --------------------------------------------------------------------------- # -# Unit: full export path + log surface -# --------------------------------------------------------------------------- # - - -def test_export_writes_public_file_and_emits_log(tmp_path, capsys): - cert_path, key_path, chain, key_pem = _chain_files(tmp_path) - export_path = tmp_path / "public" / "client-fullchain.pem" - - result = pubfc.export_public_client_fullchain( - cert_path=cert_path, - export_path=export_path, - emit_log=True, - write_file=True, - ) - - assert result.cert_count == 2 - assert result.pem_len > 0 - assert result.path_written is True - assert export_path.is_file() - written = export_path.read_text(encoding="utf-8") - assert written == result.pem - assert "BEGIN CERTIFICATE" in written - assert "PRIVATE KEY" not in written.upper() - # Private key file must never leak into the public path. - assert key_pem[:40] not in written - assert key_path.read_text(encoding="utf-8") != written - - expected_sha = hashlib.sha256(result.pem.encode("utf-8")).hexdigest() - assert result.sha256_hex == expected_sha - assert result.leaf_spki_sha256 is not None - assert len(result.leaf_spki_sha256) == 64 - - out = capsys.readouterr().out - assert f"{pubfc.PUBLIC_FULLCHAIN_MARKER} stage=export_ready" in out - assert "cert_count=2" in out - assert f"sha256={expected_sha}" in out - assert f"leaf_spki_sha256={result.leaf_spki_sha256}" in out - assert "path_written=yes" in out - assert "BEGIN CERTIFICATE" in out - assert "END CERTIFICATE" in out - assert f"{pubfc.PUBLIC_FULLCHAIN_MARKER} stage=export_end" in out - assert "PRIVATE KEY" not in out.upper() - assert key_pem[:40] not in out - # Export surface object inspection - assert pubfc.export_surface_contains_private_key(out) is False - assert pubfc.export_surface_contains_private_key(written) is False - - -def test_export_fails_closed_when_cert_file_missing(tmp_path): - missing = tmp_path / "nope.crt" - with pytest.raises(pubfc.PublicFullchainExportError, match="public_chain_missing"): - pubfc.export_public_client_fullchain( - cert_path=missing, - emit_log=False, - write_file=False, - ) - - -def test_export_fails_closed_when_cert_file_empty(tmp_path): - empty = tmp_path / "empty.crt" - empty.write_text("", encoding="utf-8") - with pytest.raises(pubfc.PublicFullchainExportError, match="public_chain_missing"): - pubfc.export_public_client_fullchain( - cert_path=empty, - emit_log=False, - write_file=False, - ) - - -def test_export_log_only_when_path_not_writable(tmp_path, capsys, monkeypatch): - """Path write may fail; log surface still exposes public PEM for phala logs.""" - - cert_path, _key_path, _chain, key_pem = _chain_files(tmp_path) - - def _fail_write(pem: str, path: Path) -> bool: # noqa: ARG001 - return False - - monkeypatch.setattr(pubfc, "write_public_fullchain_file", _fail_write) - result = pubfc.export_public_client_fullchain( - cert_path=cert_path, - export_path=tmp_path / "blocked" / "client-fullchain.pem", - emit_log=True, - write_file=True, - ) - assert result.path_written is False - out = capsys.readouterr().out - assert "path_written=no" in out - assert "BEGIN CERTIFICATE" in out - assert "PRIVATE KEY" not in out.upper() - assert key_pem[:40] not in out - - -def test_write_public_fullchain_file_refuses_private_key(tmp_path): - bad = "-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY-----\n" - target = tmp_path / "should-not-write.pem" - with pytest.raises(pubfc.PublicFullchainExportError, match="private_key_in_export"): - pubfc.write_public_fullchain_file(bad, target) - assert not target.exists() - - -def test_resolve_public_export_path_env_override(monkeypatch, tmp_path): - custom = tmp_path / "custom-export.pem" - monkeypatch.setenv(pubfc.PUBLIC_EXPORT_PATH_ENV, str(custom)) - assert pubfc.resolve_public_export_path() == custom - monkeypatch.delenv(pubfc.PUBLIC_EXPORT_PATH_ENV, raising=False) - assert pubfc.resolve_public_export_path() == pubfc.DEFAULT_PUBLIC_EXPORT_PATH - - -# --------------------------------------------------------------------------- # -# Entrypoint integration: export runs after material_ready; private key absent -# --------------------------------------------------------------------------- # - - -def _patch_tcp_ok(monkeypatch) -> None: - class _Conn: - def __enter__(self): - return self - - def __exit__(self, *args: object) -> None: - return None - - def close(self) -> None: - return None - - def _fake_create_connection(address, timeout=None, **_kwargs): # noqa: ANN001, ARG001 - return _Conn() - - monkeypatch.setattr(entrypoint.socket, "create_connection", _fake_create_connection) - - -def test_entrypoint_exports_public_fullchain_after_material_ready(monkeypatch, tmp_path, capsys): - cert_path = tmp_path / "client.crt" - key_path = tmp_path / "client.key" - ca_path = tmp_path / "ca.crt" - export_path = tmp_path / "varlog" / "client-fullchain.pem" - - ca_key, ca_cert = _make_ca(cn="server-trust-ca") - leaf_key, leaf = _sign_cert( - subject_cn="guest-client", - issuer_key=ca_key, - issuer_cert=ca_cert, - client_auth=True, - ) - # Leaf first, then intermediate (issuer) for fullchain length == 2. - leaf_pem = _pem_cert(leaf) - inter_pem = _pem_cert(ca_cert) - key_pem = _pem_key(leaf_key) - server_ca_pem = _pem_cert(ca_cert) - - class _Resp: - key = key_pem - certificate_chain = [leaf_pem, inter_pem] - - class _FakeDstack: - def __init__(self, timeout: int = 90) -> None: - assert timeout >= 60 - - def get_tls_key(self, **kwargs: Any): - assert kwargs.get("usage_ra_tls") is True - return _Resp() - - import dstack_sdk - - monkeypatch.setattr(dstack_sdk, "DstackClient", _FakeDstack, raising=False) - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "84.32.70.61") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "8701") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(cert_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_KEY_FILE", str(key_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CA_FILE", str(ca_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", server_ca_pem) - monkeypatch.setenv(pubfc.PUBLIC_EXPORT_PATH_ENV, str(export_path)) - _patch_tcp_ok(monkeypatch) - - def fake_backend(args: list[str]) -> int: # noqa: ARG001 - return 0 - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.main", - fake_backend, - raising=True, - ) - - rc = entrypoint.main(["run", "--job-dir", "/tmp/job"]) - out = capsys.readouterr().out - - assert rc == 0 - assert "ra_tls_bootstrap stage=material_ready" in out - assert f"{pubfc.PUBLIC_FULLCHAIN_MARKER} stage=export_ready" in out - assert "BEGIN CERTIFICATE" in out - assert "END CERTIFICATE" in out - assert f"{pubfc.PUBLIC_FULLCHAIN_MARKER} stage=export_end" in out - assert "ra_tls_bootstrap stage=tcp_connect_ok" in out - - # Private key never on export surfaces (stdout + public path). - assert "PRIVATE KEY" not in out.upper() - assert key_pem[:48] not in out - assert export_path.is_file() - surface = export_path.read_text(encoding="utf-8") - assert "BEGIN CERTIFICATE" in surface - assert "PRIVATE KEY" not in surface.upper() - assert key_pem[:48] not in surface - # Secret paths still hold the key for mTLS dial only. - assert key_path.is_file() - assert "PRIVATE KEY" in key_path.read_text(encoding="utf-8").upper() - # Public export path is distinct from secret client.crt (same certs, no key). - assert cert_path.read_text(encoding="utf-8").strip() - assert "PRIVATE KEY" not in cert_path.read_text(encoding="utf-8").upper() - - -def test_entrypoint_fails_closed_when_public_chain_export_impossible(monkeypatch, tmp_path, capsys): - """If cert file becomes empty after write checks, export fails closed.""" - - cert_path = tmp_path / "client.crt" - key_path = tmp_path / "client.key" - ca_path = tmp_path / "ca.crt" - ca_key, ca_cert = _make_ca() - leaf_key, leaf = _sign_cert( - subject_cn="prebaked", - issuer_key=ca_key, - issuer_cert=ca_cert, - client_auth=True, - ) - # Pre-write empty cert so material check for is_file() passes... actually - # empty fails later at extract. Use pe-filled then overwrite via monkeypatch. - cert_path.write_text(_pem_cert(leaf), encoding="utf-8") - key_path.write_text(_pem_key(leaf_key), encoding="utf-8") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_HOST", "127.0.0.1") - monkeypatch.setenv("KEY_RELEASE_RA_TLS_PORT", "8701") - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CERT_FILE", str(cert_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_KEY_FILE", str(key_path)) - monkeypatch.setenv("CHALLENGE_PHALA_RA_TLS_CA_FILE", str(ca_path)) - monkeypatch.setenv( - "CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", - _pem_cert(ca_cert), - ) - - def _export_raises(**_kwargs): - raise pubfc.PublicFullchainExportError("public_chain_missing: forced") - - monkeypatch.setattr(entrypoint, "export_public_client_fullchain", _export_raises) - - def _must_not_run(*_a, **_k): # pragma: no cover - raise AssertionError("backend must not run when public export fails closed") - - monkeypatch.setattr( - "agent_challenge.evaluation.own_runner_backend.main", - _must_not_run, - raising=True, - ) - - rc = entrypoint.main(["run", "--job-dir", "/tmp/job"]) - out = capsys.readouterr().out - assert rc == 1 - assert "ra_tls_bootstrap stage=fail" in out - assert "reason=public_chain_missing" in out diff --git a/packages/challenges/agent-challenge/tests/test_public_tee_math.py b/packages/challenges/agent-challenge/tests/test_public_tee_math.py deleted file mode 100644 index b4c0993a6..000000000 --- a/packages/challenges/agent-challenge/tests/test_public_tee_math.py +++ /dev/null @@ -1,944 +0,0 @@ -"""Public TEE math route + serializer unit locks (VAL-ACATM-001..010). - -Covers: -- available:false locked closed form -- available:true field allowlist + required safe classes -- redaction deny-list (nonce/tokens/evidence/model IO/KEY material) -- dual-flag status review.* independent of list queued status -- OpenAPI path present -""" - -from __future__ import annotations - -import hashlib -import json -from datetime import UTC, datetime -from typing import Any - -import pytest - -from agent_challenge.api import routes as api_routes -from agent_challenge.app import app -from agent_challenge.core.models import AgentSubmission, ReviewAssignment, ReviewSession -from agent_challenge.keyrelease.quote import build_rtmr3_event_log, build_tdx_quote -from agent_challenge.review.public_tee import ( - PUBLIC_TEE_DENY_KEYS, - PUBLIC_TEE_TOP_LEVEL_ALLOWLIST, - assert_public_tee_safe, - build_public_tee_math, - build_public_tee_math_from_assignment, - public_tee_assignment_qualifies, - public_tee_unavailable, -) -from agent_challenge.review.report import ( - REVIEW_REPORT_DOMAIN, - ReviewVerificationOutcome, - build_review_envelope, - review_report_data_hex, -) -from agent_challenge.review.report import _public_projection as miner_public_projection -from agent_challenge.review.schemas import ReviewInputConfig, build_review_assignment - - -def _routing() -> dict[str, object]: - return { - "order": ["alpha", "beta"], - "only": ["alpha", "beta"], - "ignore": [], - "quantizations": [], - "sort": None, - "allow_fallbacks": False, - "require_parameters": True, - "data_collection": "deny", - } - - -def _assignment() -> tuple[dict[str, Any], ReviewInputConfig]: - measurement = { - "mrtd": "11" * 48, - "rtmr0": "22" * 48, - "rtmr1": "33" * 48, - "rtmr2": "44" * 48, - "os_image_hash": hashlib.sha256( - bytes.fromhex(("11" * 48) + ("33" * 48) + ("44" * 48)) - ).hexdigest(), - "key_provider": "phala", - "vm_shape": "tdx.small", - } - config = ReviewInputConfig( - routing=_routing(), - image_ref="docker.io/example/reviewer@sha256:" + ("a" * 64), - compose_hash="ab" * 32, - kms_public_key_hex="cd" * 32, - measurement=measurement, - ) - assignment, _bytes, _digest = build_review_assignment( - session_id="rs-public-tee", - assignment_id="ra-public-tee", - attempt=1, - submission_id="42", - artifact={ - "agent_hash": "10" * 32, - "zip_sha256": "20" * 32, - "zip_size_bytes": 9, - "manifest_sha256": "30" * 32, - "manifest_entries_sha256": "40" * 32, - "fetch_path": "/review/v1/assignments/ra-public-tee/artifact", - }, - rules_snapshot_sha256_value="50" * 32, - rules_revision_id="rules-v1", - review_nonce="rn-public-tee-secret-nonce", - issued_at_ms=1_000, - expires_at_ms=9_000, - session_token_sha256="60" * 32, - config=config, - ) - return assignment, config - - -def _review_core(assignment: dict[str, Any]) -> dict[str, Any]: - core = assignment["assignment_core"] - policy = core["policy"] - return { - "schema_version": 1, - "session_id": core["session_id"], - "assignment_id": core["assignment_id"], - "assignment_digest": assignment["assignment_digest"], - "submission_id": core["submission_id"], - "artifact_observation": { - "agent_hash": core["artifact"]["agent_hash"], - "zip_sha256": core["artifact"]["zip_sha256"], - "zip_size_bytes": core["artifact"]["zip_size_bytes"], - "manifest_sha256": core["artifact"]["manifest_sha256"], - "manifest_entries_sha256": core["artifact"]["manifest_entries_sha256"], - }, - "rules_observation": { - "snapshot_sha256": core["rules"]["snapshot_sha256"], - "revision_id": core["rules"]["revision_id"], - }, - "policy_observation": { - "model": policy["model"], - "routing_sha256": policy["routing_sha256"], - "prompt_version": policy["prompt_version"], - "prompt_sha256": policy["prompt_sha256"], - "tool_schema_version": policy["tool_schema_version"], - "tool_schema_sha256": policy["tool_schema_sha256"], - "verifier_version": policy["verifier_version"], - "verifier_sha256": policy["verifier_sha256"], - }, - "openrouter_observation": { - "planned_request_sha256": "70" * 32, - "transport_observation_sha256": "71" * 32, - "request_body_sha256": "72" * 32, - "request_body_length": 7, - "response_status": 200, - "response_content_encoding": "identity", - "response_body_sha256": "73" * 32, - "response_body_length": 11, - "response_id": "or-response", - "returned_model": "x-ai/grok-4.5", - "metadata_sha256": "74" * 32, - "observed_provider": "openrouter", - "provider_provenance": "openrouter_metadata", - "cache_hit": False, - }, - "decision": { - "static_findings_sha256": "75" * 32, - "parsed_output_sha256": "76" * 32, - "verifier_input_sha256": "77" * 32, - "verifier_output_sha256": "78" * 32, - "verifier_result": "pass", - "verdict": "allow", - "reason_codes": ["alpha_reason", "zeta_reason"], - "evidence_digests": ["79" * 32, "80" * 32], - }, - "times": { - "issued_at_ms": 1_000, - "started_at_ms": 1_000, - "model_call_marked_at_ms": 1_001, - "request_started_at_ms": 1_002, - "request_finished_at_ms": 1_003, - "verifier_finished_at_ms": 1_004, - "report_finished_at_ms": 1_005, - "expires_at_ms": 9_000, - "submission_received_at_ms": 1_000, - }, - "review_nonce": core["review_nonce"], - } - - -def _envelope() -> tuple[dict[str, Any], dict[str, Any], ReviewVerificationOutcome]: - assignment, config = _assignment() - core = _review_core(assignment) - event_log, rtmr3 = build_rtmr3_event_log( - [ - ("compose-hash", bytes.fromhex(config.compose_hash)), - ("key-provider", b"phala"), - ] - ) - measurement = { - **config.resolved_measurement(), - "rtmr3": rtmr3, - "compose_hash": config.compose_hash, - } - quote = build_tdx_quote( - mrtd=measurement["mrtd"], - rtmr0=measurement["rtmr0"], - rtmr1=measurement["rtmr1"], - rtmr2=measurement["rtmr2"], - rtmr3=measurement["rtmr3"], - report_data=review_report_data_hex(core), - ) - envelope = build_review_envelope( - review_core=core, - tdx_quote_hex=quote, - event_log=event_log, - measurement=measurement, - vm_config={ - "vcpu": 1, - "memory_mb": 2048, - "os_image_hash": measurement["os_image_hash"], - }, - ) - outcome = ReviewVerificationOutcome( - status="verified_allow", - terminal=True, - retryable=False, - reason_code="policy_allowed", - nonce_consumed=True, - measurement_allowlisted=True, - report_data_matched=True, - verified_at_ms=1_700, - ) - return envelope, assignment, outcome - - -def test_public_tee_unavailable_locked_closed_form() -> None: - assert public_tee_unavailable() == {"available": False} - assert build_public_tee_math(submission_id=1, envelope=None) == {"available": False} - assert build_public_tee_math(submission_id=1, envelope={}) == {"available": False} - assert build_public_tee_math_from_assignment( - submission_id=1, - envelope_json=None, - ) == {"available": False} - assert_public_tee_safe({"available": False}) - - -def test_public_tee_envelope_without_projection_is_unavailable() -> None: - """Envelope alone (review_verifying pre-verify) must not yield available:true.""" - - envelope, _assignment, _outcome = _envelope() - assert build_public_tee_math(submission_id=42, envelope=envelope) == {"available": False} - assert build_public_tee_math( - submission_id=42, - envelope=envelope, - verification_outcome=None, - public_projection=None, - ) == {"available": False} - assert build_public_tee_math_from_assignment( - submission_id=42, - envelope_json=json.dumps(envelope), - outcome_json=None, - public_projection_json=None, - ) == {"available": False} - assert ( - public_tee_assignment_qualifies( - envelope_json=json.dumps(envelope), - outcome_json=None, - public_projection_json=None, - ) - is False - ) - - -def test_public_tee_envelope_verifier_unavailable_is_unavailable() -> None: - """Parked verifier_unavailable must not yield available:true without projection.""" - - envelope, _assignment, _outcome = _envelope() - parked = { - "status": "verifier_unavailable", - "terminal": False, - "retryable": True, - "reason_code": "dcap_qvl_missing", - "nonce_consumed": False, - "measurement_allowlisted": False, - "report_data_matched": False, - "verified_at_ms": None, - } - assert build_public_tee_math( - submission_id=42, - envelope=envelope, - verification_outcome=parked, - public_projection=None, - ) == {"available": False} - assert build_public_tee_math_from_assignment( - submission_id=42, - envelope_json=json.dumps(envelope), - outcome_json=json.dumps(parked), - public_projection_json=None, - ) == {"available": False} - assert ( - public_tee_assignment_qualifies( - envelope_json=json.dumps(envelope), - outcome_json=json.dumps(parked), - public_projection_json=None, - ) - is False - ) - - -def test_public_tee_envelope_verified_with_projection_is_available() -> None: - """Envelope + verified_* + projection → available:true (positive gate).""" - - envelope, _assignment, outcome = _envelope() - projection = miner_public_projection(envelope=envelope, outcome=outcome) - payload = build_public_tee_math( - submission_id=42, - envelope=envelope, - verification_outcome=outcome.as_dict(), - public_projection=projection, - ) - assert payload["available"] is True - assert payload["verification_outcome"]["status"] == "verified_allow" - assert ( - public_tee_assignment_qualifies( - envelope_json=json.dumps(envelope), - outcome_json=json.dumps(outcome.as_dict()), - public_projection_json=json.dumps(projection), - ) - is True - ) - # Verified outcome alone (no projection) is still eligible — align with - # verified_* dual-flag; builder projects safe math without miner digests. - verified_only = build_public_tee_math( - submission_id=42, - envelope=envelope, - verification_outcome=outcome.as_dict(), - public_projection=None, - ) - assert verified_only["available"] is True - assert verified_only["verification_outcome"]["status"] == "verified_allow" - assert ( - public_tee_assignment_qualifies( - envelope_json=json.dumps(envelope), - outcome_json=json.dumps(outcome.as_dict()), - public_projection_json=None, - ) - is True - ) - - -def test_public_tee_math_available_true_field_allowlist() -> None: - envelope, _assignment, outcome = _envelope() - projection = miner_public_projection(envelope=envelope, outcome=outcome) - payload = build_public_tee_math( - submission_id=42, - envelope=envelope, - verification_outcome=outcome.as_dict(), - public_projection=projection, - ) - assert payload["available"] is True - assert set(payload.keys()) <= PUBLIC_TEE_TOP_LEVEL_ALLOWLIST - # Required safe classes (VAL-ACATM-004 / 010) - assert payload["domain"] == REVIEW_REPORT_DOMAIN - assert payload["review_digest"] == envelope["review_digest"] - assert payload["report_data_hex"] == envelope["report_data_hex"] - assert isinstance(payload["report_data_preimage"], dict) - assert "review_nonce" not in payload["report_data_preimage"] - assert "review_nonce_sha256" in payload["report_data_preimage"] - measurement = payload["measurement"] - assert isinstance(measurement, dict) - for key in ( - "mrtd", - "rtmr0", - "rtmr1", - "rtmr2", - "rtmr3", - "compose_hash", - "os_image_hash", - "key_provider", - "vm_shape", - ): - assert key in measurement - assert isinstance(measurement[key], str) and measurement[key] - assert isinstance(payload["tdx_quote_hex"], str) and payload["tdx_quote_hex"] - assert isinstance(payload.get("event_log"), list) - outcome_pub = payload["verification_outcome"] - assert outcome_pub["status"] == "verified_allow" - assert outcome_pub["measurement_allowlisted"] is True - assert outcome_pub["report_data_matched"] is True - assert outcome_pub["verified_at_ms"] == 1_700 - assert outcome_pub["reason_code"] == "policy_allowed" - assert payload["quote_fingerprint_sha256"] == projection["quote_fingerprint_sha256"] - assert_public_tee_safe(payload) - - -def test_public_tee_math_redacts_nonce_tokens_evidence_and_keys() -> None: - envelope, _assignment, outcome = _envelope() - # Smuggle secrets into a hostile envelope copy — builder must not echo them. - # Keep review_core schema-valid so preimage projection still runs. - hostile = json.loads(json.dumps(envelope)) - hostile["session_token"] = "session-token-plaintext-secret" - hostile["attestation"]["capability"] = "cap-bearer-secret" - hostile["evidence_objects"] = {"request_body": "not-public-body"} - hostile["openrouter_api_key"] = "sk-leaked-openrouter" - hostile["encryption_key"] = "KEY_FILE_MATERIAL" - payload = build_public_tee_math( - submission_id=42, - envelope=hostile, - verification_outcome={ - **outcome.as_dict(), - "nonce_consumed": True, - "terminal": True, - "retryable": False, - "private_key": "should-not-leak", - }, - ) - serialized = json.dumps(payload, sort_keys=True) - assert "rn-public-tee-secret-nonce" not in serialized - assert "session-token-plaintext-secret" not in serialized - assert "cap-bearer-secret" not in serialized - assert "sk-leaked-openrouter" not in serialized - assert "KEY_FILE_MATERIAL" not in serialized - assert "private_key" not in serialized - assert "nonce_consumed" not in serialized - assert "request_body" not in serialized - for key in PUBLIC_TEE_DENY_KEYS: - assert f'"{key}":' not in serialized - # Hash of nonce is OK (inspectability without plaintext). - expected_hash = hashlib.sha256(b"rn-public-tee-secret-nonce").hexdigest() - assert payload["report_data_preimage"]["review_nonce_sha256"] == expected_hash - assert '"review_nonce":' not in serialized - assert_public_tee_safe(payload) - - -def test_public_tee_math_from_assignment_json_columns() -> None: - envelope, _assignment, outcome = _envelope() - projection = miner_public_projection(envelope=envelope, outcome=outcome) - payload = build_public_tee_math_from_assignment( - submission_id=7, - envelope_json=json.dumps(envelope), - outcome_json=json.dumps(outcome.as_dict()), - public_projection_json=json.dumps(projection), - ) - assert payload["available"] is True - assert payload["submission_id"] in {7, 42, "42"} - assert payload["verdict"] == "allow" - - -async def test_get_review_tee_available_false_when_no_report( - client, - database_session, - monkeypatch, -) -> None: - monkeypatch.setattr(api_routes.settings, "attested_review_enabled", True) - monkeypatch.setattr(api_routes.settings, "phala_attestation_enabled", True) - async with database_session() as session: - submission = AgentSubmission( - miner_hotkey="hk-tee-absent", - name="tee-absent", - agent_hash="tee-absent-hash", - artifact_uri="/tmp/tee-absent.zip", - status="received", - raw_status="received", - effective_status="received", - ) - session.add(submission) - await session.commit() - submission_id = submission.id - - response = await client.get(f"/submissions/{submission_id}/review/tee") - assert response.status_code == 200 - assert response.json() == {"available": False} - - v1 = await client.get(f"/v1/submissions/{submission_id}/review/tee") - assert v1.status_code == 200 - assert v1.json() == {"available": False} - - -async def test_get_review_tee_envelope_only_not_available( - client, - database_session, - monkeypatch, -) -> None: - """Route must not treat durable envelope alone as available:true.""" - - monkeypatch.setattr(api_routes.settings, "attested_review_enabled", True) - monkeypatch.setattr(api_routes.settings, "phala_attestation_enabled", True) - envelope, assignment_payload, _outcome = _envelope() - now = datetime(2026, 7, 20, 12, 0, tzinfo=UTC) - - async with database_session() as session: - submission = AgentSubmission( - miner_hotkey="hk-tee-env-only", - name="tee-env-only", - agent_hash="tee-env-only-hash", - artifact_uri="/tmp/tee-env-only.zip", - status="queued", - raw_status="queued", - effective_status="queued", - ) - session.add(submission) - await session.flush() - review_session = ReviewSession( - session_id="rs-tee-env-only", - submission_id=submission.id, - artifact_sha256="ab" * 32, - artifact_size_bytes=12, - manifest_sha256="cd" * 32, - manifest_entries_sha256="ef" * 32, - current_assignment_id="ra-tee-env-only", - authorizing_assignment_id=None, - ) - session.add(review_session) - await session.flush() - # Durable envelope as written during review_verifying — no projection / - # no verified outcome yet. - row = ReviewAssignment( - session_id=review_session.id, - assignment_id="ra-tee-env-only", - attempt=1, - phase="review_verifying", - assignment_bytes=json.dumps(assignment_payload), - assignment_digest=assignment_payload["assignment_digest"], - artifact_sha256="ab" * 32, - rules_snapshot_sha256="11" * 32, - rules_revision_id="rules-v1", - review_nonce="nonce-ra-tee-env-only", - session_token_sha256="bb" * 32, - capability_state="active", - issued_at=now, - expires_at=now, - review_report_envelope_json=json.dumps(envelope), - review_digest=envelope["review_digest"], - review_report_data_hex=envelope["report_data_hex"], - review_verification_outcome_json=None, - review_public_projection_json=None, - reason_code=None, - finished_at=None, - ) - session.add(row) - await session.commit() - submission_id = submission.id - - response = await client.get(f"/submissions/{submission_id}/review/tee") - assert response.status_code == 200 - assert response.json() == {"available": False} - - status = await client.get(f"/submissions/{submission_id}/status") - assert status.status_code == 200 - review = status.json()["review"] - assert review is not None - assert review["report_available"] is False - assert review["verified"] is False - - -async def test_get_review_tee_verifier_unavailable_not_available( - client, - database_session, - monkeypatch, -) -> None: - """Route must not surface available:true for parked verifier_unavailable.""" - - monkeypatch.setattr(api_routes.settings, "attested_review_enabled", True) - monkeypatch.setattr(api_routes.settings, "phala_attestation_enabled", True) - envelope, assignment_payload, _outcome = _envelope() - now = datetime(2026, 7, 20, 12, 0, tzinfo=UTC) - parked = { - "status": "verifier_unavailable", - "terminal": False, - "retryable": True, - "reason_code": "dcap_qvl_missing", - "nonce_consumed": False, - "measurement_allowlisted": False, - "report_data_matched": False, - "verified_at_ms": None, - } - - async with database_session() as session: - submission = AgentSubmission( - miner_hotkey="hk-tee-vu", - name="tee-vu", - agent_hash="tee-vu-hash", - artifact_uri="/tmp/tee-vu.zip", - status="queued", - raw_status="queued", - effective_status="queued", - ) - session.add(submission) - await session.flush() - review_session = ReviewSession( - session_id="rs-tee-vu", - submission_id=submission.id, - artifact_sha256="ab" * 32, - artifact_size_bytes=12, - manifest_sha256="cd" * 32, - manifest_entries_sha256="ef" * 32, - current_assignment_id="ra-tee-vu", - authorizing_assignment_id="ra-tee-vu", - ) - session.add(review_session) - await session.flush() - row = ReviewAssignment( - session_id=review_session.id, - assignment_id="ra-tee-vu", - attempt=1, - phase="review_verifying", - assignment_bytes=json.dumps(assignment_payload), - assignment_digest=assignment_payload["assignment_digest"], - artifact_sha256="ab" * 32, - rules_snapshot_sha256="11" * 32, - rules_revision_id="rules-v1", - review_nonce="nonce-ra-tee-vu", - session_token_sha256="bb" * 32, - capability_state="active", - issued_at=now, - expires_at=now, - review_report_envelope_json=json.dumps(envelope), - review_digest=envelope["review_digest"], - review_report_data_hex=envelope["report_data_hex"], - review_verification_outcome_json=json.dumps(parked), - review_public_projection_json=None, - reason_code="dcap_qvl_missing", - finished_at=None, - ) - session.add(row) - await session.commit() - submission_id = submission.id - - response = await client.get(f"/submissions/{submission_id}/review/tee") - assert response.status_code == 200 - assert response.json() == {"available": False} - - status = await client.get(f"/submissions/{submission_id}/status") - assert status.status_code == 200 - review = status.json()["review"] - assert review is not None - assert review["report_available"] is False - assert review["verified"] is False - - -async def test_get_review_tee_available_true_safe_fields( - client, - database_session, - monkeypatch, -) -> None: - monkeypatch.setattr(api_routes.settings, "attested_review_enabled", True) - monkeypatch.setattr(api_routes.settings, "phala_attestation_enabled", True) - envelope, assignment_payload, outcome = _envelope() - projection = miner_public_projection(envelope=envelope, outcome=outcome) - now = datetime(2026, 7, 20, 12, 0, tzinfo=UTC) - - async with database_session() as session: - submission = AgentSubmission( - miner_hotkey="hk-tee-present", - name="tee-present", - agent_hash="tee-present-hash", - artifact_uri="/tmp/tee-present.zip", - status="queued", - raw_status="queued", - effective_status="queued", - ) - session.add(submission) - await session.flush() - review_session = ReviewSession( - session_id="rs-tee-route", - submission_id=submission.id, - artifact_sha256="ab" * 32, - artifact_size_bytes=12, - manifest_sha256="cd" * 32, - manifest_entries_sha256="ef" * 32, - current_assignment_id="ra-tee-route", - authorizing_assignment_id="ra-tee-route", - ) - session.add(review_session) - await session.flush() - row = ReviewAssignment( - session_id=review_session.id, - assignment_id="ra-tee-route", - attempt=1, - phase="review_allowed", - assignment_bytes=json.dumps(assignment_payload), - assignment_digest=assignment_payload["assignment_digest"], - artifact_sha256="ab" * 32, - rules_snapshot_sha256="11" * 32, - rules_revision_id="rules-v1", - review_nonce="nonce-ra-tee-route", - session_token_sha256="bb" * 32, - capability_state="revoked", - issued_at=now, - expires_at=now, - review_report_envelope_json=json.dumps(envelope), - review_digest=envelope["review_digest"], - review_report_data_hex=envelope["report_data_hex"], - review_verification_outcome_json=json.dumps(outcome.as_dict()), - review_public_projection_json=json.dumps(projection), - reason_code="policy_allowed", - finished_at=now, - ) - session.add(row) - await session.commit() - submission_id = submission.id - - response = await client.get(f"/submissions/{submission_id}/review/tee") - assert response.status_code == 200 - body = response.json() - assert body["available"] is True - assert body["domain"] == REVIEW_REPORT_DOMAIN - assert body["review_digest"] == envelope["review_digest"] - assert body["report_data_hex"] == envelope["report_data_hex"] - serialized = json.dumps(body) - assert '"review_nonce":' not in serialized - assert "rn-public-tee-secret-nonce" not in serialized - assert body["measurement"]["key_provider"] == "phala" - assert body["tdx_quote_hex"] - assert body["verification_outcome"]["status"] == "verified_allow" - assert body["quote_fingerprint_sha256"] - assert set(body.keys()) <= PUBLIC_TEE_TOP_LEVEL_ALLOWLIST - assert_public_tee_safe(body) - - -@pytest.mark.parametrize( - ("outcome_status", "expected_verdict"), - [ - ("verified_allow", "allow"), - ("verified_reject", "reject"), - ("verified_escalate", "escalate"), - ], -) -async def test_dual_flag_status_review_projection_independent_of_queued( - client, - database_session, - monkeypatch, - outcome_status: str, - expected_verdict: str, -) -> None: - """VAL-ACATM-001/002/003/008: dual-flag review.* stays even when list is queued.""" - - monkeypatch.setattr(api_routes.settings, "attested_review_enabled", True) - monkeypatch.setattr(api_routes.settings, "phala_attestation_enabled", True) - now = datetime(2026, 7, 20, 12, 0, tzinfo=UTC) - phase = { - "verified_allow": "review_allowed", - "verified_reject": "review_rejected", - "verified_escalate": "review_escalated", - }[outcome_status] - - async with database_session() as session: - submission = AgentSubmission( - miner_hotkey=f"hk-dual-{expected_verdict}", - name=f"dual-{expected_verdict}", - agent_hash=f"dual-hash-{expected_verdict}", - artifact_uri=f"/tmp/dual-{expected_verdict}.zip", - # Intentionally non-terminal queue-looking lifecycle status. - status="queued", - raw_status="queued", - effective_status="queued", - ) - session.add(submission) - await session.flush() - # Durable status event so public status stays queued-family without - # requiring a full happy-path transition graph. - from agent_challenge.models import SubmissionStatusEvent - - session.add( - SubmissionStatusEvent( - submission_id=submission.id, - sequence=1, - from_status=None, - to_status="queued", - actor="test", - reason="seed_queued", - ) - ) - review_session = ReviewSession( - session_id=f"rs-dual-{expected_verdict}", - submission_id=submission.id, - artifact_sha256="ab" * 32, - artifact_size_bytes=12, - manifest_sha256="cd" * 32, - manifest_entries_sha256="ef" * 32, - current_assignment_id=f"ra-dual-{expected_verdict}", - authorizing_assignment_id=( - f"ra-dual-{expected_verdict}" if outcome_status == "verified_allow" else None - ), - ) - session.add(review_session) - await session.flush() - assignment = ReviewAssignment( - session_id=review_session.id, - assignment_id=f"ra-dual-{expected_verdict}", - attempt=1, - phase=phase, - assignment_bytes="{}", - assignment_digest="aa" * 32, - artifact_sha256="ab" * 32, - rules_snapshot_sha256="11" * 32, - rules_revision_id="rules-v1", - review_nonce=f"nonce-dual-{expected_verdict}", - session_token_sha256="bb" * 32, - capability_state="revoked", - issued_at=now, - expires_at=now, - review_public_projection_json=json.dumps( - {"schema_version": 1, "verdict": expected_verdict} - ), - review_verification_outcome_json=json.dumps( - { - "status": outcome_status, - "terminal": True, - "retryable": False, - "reason_code": f"{expected_verdict}_ok", - "nonce_consumed": True, - "measurement_allowlisted": True, - "report_data_matched": True, - "verified_at_ms": 99, - } - ), - reason_code=f"{expected_verdict}_ok", - finished_at=now, - ) - session.add(assignment) - await session.commit() - submission_id = submission.id - session_public_id = review_session.session_id - assignment_public_id = assignment.assignment_id - - status = await client.get(f"/submissions/{submission_id}/status") - assert status.status_code == 200 - body = status.json() - # Queue/list status may remain queued-family; dual-flag must still expose review. - assert body["status"] in {"queued", "pending", "assigned"} or isinstance(body["status"], str) - review = body["review"] - assert review is not None - assert review["verified"] is True - assert review["verdict"] == expected_verdict - assert review["terminal"] is True - assert review["retryable"] is False - assert review["report_available"] is True - assert review["phase"] == phase - assert review["reason_code"] == f"{expected_verdict}_ok" - assert review["session_id"] == session_public_id - assert review["assignment_id"] == assignment_public_id - # Secret classes never on status surface. - serialized = json.dumps(body) - for forbidden in ( - "session_token", - "rn-public-tee-secret-nonce", - "OPENROUTER_API_KEY", - "encryption_key", - "request_body", - "response_body", - ): - assert forbidden not in serialized - - -async def test_dual_flag_status_report_available_false_without_projection( - client, - database_session, - monkeypatch, -) -> None: - """Optional lock: status.report_available is false without projection.""" - - monkeypatch.setattr(api_routes.settings, "attested_review_enabled", True) - monkeypatch.setattr(api_routes.settings, "phala_attestation_enabled", True) - now = datetime(2026, 7, 20, 12, 0, tzinfo=UTC) - - async with database_session() as session: - submission = AgentSubmission( - miner_hotkey="hk-dual-no-proj", - name="dual-no-proj", - agent_hash="dual-hash-no-proj", - artifact_uri="/tmp/dual-no-proj.zip", - status="queued", - raw_status="queued", - effective_status="queued", - ) - session.add(submission) - await session.flush() - from agent_challenge.models import SubmissionStatusEvent - - session.add( - SubmissionStatusEvent( - submission_id=submission.id, - sequence=1, - from_status=None, - to_status="queued", - actor="test", - reason="seed_queued", - ) - ) - review_session = ReviewSession( - session_id="rs-dual-no-proj", - submission_id=submission.id, - artifact_sha256="ab" * 32, - artifact_size_bytes=12, - manifest_sha256="cd" * 32, - manifest_entries_sha256="ef" * 32, - current_assignment_id="ra-dual-no-proj", - authorizing_assignment_id=None, - ) - session.add(review_session) - await session.flush() - assignment = ReviewAssignment( - session_id=review_session.id, - assignment_id="ra-dual-no-proj", - attempt=1, - phase="review_verifying", - assignment_bytes="{}", - assignment_digest="aa" * 32, - artifact_sha256="ab" * 32, - rules_snapshot_sha256="11" * 32, - rules_revision_id="rules-v1", - review_nonce="nonce-dual-no-proj", - session_token_sha256="bb" * 32, - capability_state="active", - issued_at=now, - expires_at=now, - review_public_projection_json=None, - review_verification_outcome_json=json.dumps( - { - "status": "verifier_unavailable", - "terminal": False, - "retryable": True, - "reason_code": "dcap_qvl_missing", - "nonce_consumed": False, - "measurement_allowlisted": False, - "report_data_matched": False, - "verified_at_ms": None, - } - ), - reason_code="dcap_qvl_missing", - finished_at=None, - ) - session.add(assignment) - await session.commit() - submission_id = submission.id - - status = await client.get(f"/submissions/{submission_id}/status") - assert status.status_code == 200 - review = status.json()["review"] - assert review is not None - assert review["report_available"] is False - assert review["verified"] is False - assert review["verdict"] is None - - -def test_openapi_documents_public_tee_path() -> None: - schema = app.openapi() - paths = schema["paths"] - assert "/submissions/{submission_id}/review/tee" in paths - assert "/v1/submissions/{submission_id}/review/tee" in paths - get_op = paths["/submissions/{submission_id}/review/tee"]["get"] - assert get_op["responses"]["200"] - # response model is registered - components = schema.get("components", {}).get("schemas", {}) - assert any("PublicTeeMath" in name for name in components) - - -def test_frontend_contract_public_paths_include_tee() -> None: - from _routing import public_route_paths - - public_paths = public_route_paths(app) - assert "/submissions/{submission_id}/review/tee" in public_paths - assert "/v1/submissions/{submission_id}/review/tee" in public_paths diff --git a/packages/challenges/agent-challenge/tests/test_quote_golden_vector.py b/packages/challenges/agent-challenge/tests/test_quote_golden_vector.py deleted file mode 100644 index f3ddc0fe7..000000000 --- a/packages/challenges/agent-challenge/tests/test_quote_golden_vector.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Shared cross-repo golden TDX quote/measurement/RTMR3 anti-drift vector (AC). - -agent-challenge ``src/agent_challenge/keyrelease/quote.py`` and base -``src/base/worker/phala_quote.py`` re-implement the same TDX quote parsing (register -byte-offsets, ``os_image_hash = sha256(MRTD||RTMR1||RTMR2)``, dstack RTMR3 -event-log replay) because base cannot import this lean in-CVM module. That -duplication is drift-prone: a one-sided offset/hash tweak could let a real dstack -quote verify in one repo and silently fail in the other. - -This test pins a FIXED quote + event log (``tests/phala_quote_golden_vector.json``) -to the exact registers / os_image_hash / RTMR3 / compose-hash a correct parser -must reproduce, and asserts this repo's parser reproduces them. The SAME fixture -bytes and the SAME :data:`GOLDEN_VECTOR_SHA256` are asserted in base -(``tests/unit/test_phala_quote_golden_vector.py``); because the pinned expected -values come from the frozen fixture (not recomputed with the same offsets under -test), a one-sided offset/hash change diverges from these values and fails here or -there. Do NOT edit one repo's copy without the other (AGENTS.md -'TDX-quote-parse / measurement / RTMR3 anti-drift'). -""" - -from __future__ import annotations - -import hashlib -import json -from pathlib import Path - -from agent_challenge.keyrelease.quote import ( - _MRTD_OFFSET, - REGISTER_LEN, - build_rtmr3_event_log, - os_image_hash_from_registers, - parse_td_report, - replay_rtmr3, -) - -# Same literal in BOTH repos: the SHA-256 of the byte-identical golden fixture. -# If either repo's fixture is edited, that side's pin fails; a reviewer can grep -# this constant across both repos to confirm they still match. -GOLDEN_VECTOR_SHA256 = "053979e8445c147798ec6f9165f9849a38ff5797e4dccd2ee38aa329b7f673bf" - -_VECTOR_PATH = Path(__file__).parent / "phala_quote_golden_vector.json" - - -def _vector() -> dict: - return json.loads(_VECTOR_PATH.read_text(encoding="utf-8")) - - -def test_golden_fixture_is_byte_identical_across_repos() -> None: - digest = hashlib.sha256(_VECTOR_PATH.read_bytes()).hexdigest() - assert digest == GOLDEN_VECTOR_SHA256 - - -def test_golden_quote_parses_to_expected_registers() -> None: - vector = _vector() - expected = vector["expected"] - report = parse_td_report(vector["quote_hex"]) - assert report.mrtd == expected["mrtd"] - assert report.rtmr0 == expected["rtmr0"] - assert report.rtmr1 == expected["rtmr1"] - assert report.rtmr2 == expected["rtmr2"] - assert report.rtmr3 == expected["rtmr3"] - assert report.report_data.hex() == expected["report_data_hex"] - - -def test_golden_os_image_hash_matches() -> None: - vector = _vector() - expected = vector["expected"] - assert ( - os_image_hash_from_registers(expected["mrtd"], expected["rtmr1"], expected["rtmr2"]) - == expected["os_image_hash"] - ) - - -def test_golden_event_log_replays_to_expected_rtmr3() -> None: - vector = _vector() - expected = vector["expected"] - replay = replay_rtmr3(vector["event_log"]) - assert replay.rtmr3 == expected["rtmr3"] - assert replay.compose_hash == expected["compose_hash"] - assert replay.key_provider == expected["key_provider"] - # The event-log replay reproduces the RTMR3 the fixed quote carries. - assert replay.rtmr3 == parse_td_report(vector["quote_hex"]).rtmr3 - - -def test_golden_vector_offset_sensitivity_discriminator() -> None: - # Non-vacuity: the pinned MRTD is not read from a constant. Reading one byte - # off the register offset (a simulated one-sided off-by-one) yields a value - # that differs from the golden -- exactly the divergence the pin above catches. - vector = _vector() - raw = bytes.fromhex(vector["quote_hex"]) - correct = raw[_MRTD_OFFSET : _MRTD_OFFSET + REGISTER_LEN].hex() - tweaked = raw[_MRTD_OFFSET + 1 : _MRTD_OFFSET + 1 + REGISTER_LEN].hex() - assert correct == vector["expected"]["mrtd"] - assert tweaked != vector["expected"]["mrtd"] - - -def test_golden_vector_replay_sensitivity_discriminator() -> None: - # Non-vacuity: a different compose payload replays to a different RTMR3, so the - # pinned RTMR3 genuinely binds the event-log digest/extend formula. - vector = _vector() - _log, other_rtmr3 = build_rtmr3_event_log([("compose-hash", bytes.fromhex("00" * 32))]) - assert other_rtmr3 != vector["expected"]["rtmr3"] diff --git a/packages/challenges/agent-challenge/tests/test_quote_measurement_mismatch_diag.py b/packages/challenges/agent-challenge/tests/test_quote_measurement_mismatch_diag.py deleted file mode 100644 index 412a621cb..000000000 --- a/packages/challenges/agent-challenge/tests/test_quote_measurement_mismatch_diag.py +++ /dev/null @@ -1,174 +0,0 @@ -"""quote_measurement_mismatch field-class diag (message words → closed token). - -Live residual after or_outcome_bind tip (7e5497c3): dry-run allowlist was IN-LIST -but measured guest returned durable reason_code=quote_measurement_mismatch with -public_logs that did not say which register/field mismatched. - -Product Mode B: map known ValueError message words from review_runtime -``_measurement_from_quote`` onto a closed diag token -{compose,key_provider,os,mrtd,rtmr0,rtmr1,rtmr2,rtmr3,event_log,other} -attached to OpenRouterTransportError + bounded_review_failure_surface. -Never re-emit digests/secrets. No invent allow/KR/MRTD. -""" - -from __future__ import annotations - -import importlib.util -from pathlib import Path - -import pytest - -from agent_challenge.review.openrouter import ( - OpenRouterTransportError, - infrastructure_failure_reason, - short_quote_measurement_diag_class, -) - -# Exact / known product ValueError strings from review_runtime._measurement_from_quote -# plus residual pattern messages that openrouter already classifies as mismatch. -_MESSAGE_TO_DIAG: tuple[tuple[str, str], ...] = ( - ("quoted compose hash mismatches assignment", "compose"), - ("quoted key provider mismatches assignment", "key_provider"), - ("quoted os image hash mismatches assignment", "os"), - ("quoted mrtd mismatches assignment", "mrtd"), - ("quoted rtmr0 mismatches assignment", "rtmr0"), - ("quoted rtmr1 mismatches assignment", "rtmr1"), - ("quoted rtmr2 mismatches assignment", "rtmr2"), - ("quoted rtmr3 mismatches assignment", "rtmr3"), - # Soft residual wording still attributeable by closed keywords: - ("measurement register rtmr0 mismatches assignment", "rtmr0"), - ("quote measurement mismatch without field token", "other"), - ("", "other"), -) - - -def _load_review_runtime(): - path = Path(__file__).resolve().parents[1] / "docker" / "review" / "review_runtime.py" - spec = importlib.util.spec_from_file_location("review_runtime_quote_diag_under_test", path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -@pytest.mark.parametrize(("message", "expected_diag"), _MESSAGE_TO_DIAG) -def test_message_words_map_to_closed_quote_diag(message: str, expected_diag: str) -> None: - """TDD: every known mismatch message collapses to one allowlisted token.""" - - token = short_quote_measurement_diag_class(ValueError(message)) - assert token == expected_diag - # Closed set only; never free-form or secret fragments. - assert token in { - "compose", - "key_provider", - "os", - "mrtd", - "rtmr0", - "rtmr1", - "rtmr2", - "rtmr3", - "event_log", - "other", - } - assert "sha256" not in token - assert " " not in token - - -def test_diag_never_reemits_raw_digests_or_secrets() -> None: - """Token is closed; digest-looking residue in message never becomes diag text.""" - - toxic = ( - "quoted compose hash mismatches assignment " - "got=deadbeefcafebabe00112233445566778899aabbccddeeff0011223344556677 " - "want=ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff " - "api_key=sk-or-v1-SENSITIVE" - ) - token = short_quote_measurement_diag_class(ValueError(toxic)) - assert token == "compose" - assert "deadbeef" not in token - assert "sk-or" not in token - assert "SENSITIVE" not in token - - -def test_openrouter_transport_error_accepts_quote_diag_token() -> None: - """OpenRouterTransportError must retain quote field-class diag tokens.""" - - err = OpenRouterTransportError( - "quote_measurement_mismatch", - "quoted compose hash mismatches assignment", - diag="compose", - ) - assert err.reason_code == "quote_measurement_mismatch" - assert err.diag == "compose" - # Unknown free-form collapses to other (never accepted as surface free text). - bad = OpenRouterTransportError( - "quote_measurement_mismatch", - "quoted compose hash mismatches assignment", - diag="compose_hash_0xdeadbeef", - ) - assert bad.diag == "other" - - -def test_infrastructure_reason_still_quote_measurement_mismatch() -> None: - """Field diag is additive; reason_code stays quote_measurement_mismatch.""" - - for message, _diag in _MESSAGE_TO_DIAG[:7]: - assert infrastructure_failure_reason(ValueError(message)) == ("quote_measurement_mismatch") - wrapped = OpenRouterTransportError( - "quote_measurement_mismatch", - message, - diag=short_quote_measurement_diag_class(ValueError(message)), - ) - assert infrastructure_failure_reason(wrapped) == "quote_measurement_mismatch" - - -@pytest.mark.parametrize( - ("message", "expected_diag"), - [ - ("quoted compose hash mismatches assignment", "compose"), - ("quoted key provider mismatches assignment", "key_provider"), - ("quoted os image hash mismatches assignment", "os"), - ("quoted mrtd mismatches assignment", "mrtd"), - ("quoted rtmr0 mismatches assignment", "rtmr0"), - ("quoted rtmr1 mismatches assignment", "rtmr1"), - ("quoted rtmr2 mismatches assignment", "rtmr2"), - ("quoted rtmr3 mismatches assignment", "rtmr3"), - ], -) -def test_bounded_surface_exposes_quote_diag_from_value_error( - message: str, expected_diag: str -) -> None: - """public_logs residual must include the field-class diag for mismatch.""" - - runtime = _load_review_runtime() - surface = runtime.bounded_review_failure_surface(ValueError(message)) - assert surface["error"] == "review_failed" - assert surface["reason"] == "ValueError" - assert surface["reason_code"] == "quote_measurement_mismatch" - assert surface.get("diag") == expected_diag - # Never leak the full mismatch wording or registry digests into the surface. - assert "mismatches assignment" not in str(surface) - assert "quoted" not in str(surface) - - -def test_bounded_surface_exposes_diag_from_transport_error() -> None: - runtime = _load_review_runtime() - surface = runtime.bounded_review_failure_surface( - OpenRouterTransportError( - "quote_measurement_mismatch", - "quoted os image hash mismatches assignment", - diag="os", - ) - ) - assert surface["reason_code"] == "quote_measurement_mismatch" - assert surface.get("diag") == "os" - assert "os image" not in str(surface) - - -def test_bounded_surface_unknown_measurement_mismatch_uses_other() -> None: - runtime = _load_review_runtime() - surface = runtime.bounded_review_failure_surface( - ValueError("measurement mismatch without a named field") - ) - assert surface["reason_code"] == "quote_measurement_mismatch" - assert surface.get("diag") == "other" diff --git a/packages/challenges/agent-challenge/tests/test_quote_mismatch_prefix_emit.py b/packages/challenges/agent-challenge/tests/test_quote_mismatch_prefix_emit.py deleted file mode 100644 index d4221d285..000000000 --- a/packages/challenges/agent-challenge/tests/test_quote_mismatch_prefix_emit.py +++ /dev/null @@ -1,367 +0,0 @@ -"""quote_measurement_mismatch: allowlisted short hex prefixes on guest public_logs. - -SPEED residual after tip 920b3ed6 / sub22: guest surface had diag=os but no -actual/expected digests, so offline dstack-mr pin packs could not be compared -to live Phala quote registers for honest repin. - -Product Mode B: when reason_code=quote_measurement_mismatch and diag is in -{os,mrtd,rtmr0,rtmr1,rtmr2,compose}, append only short hex prefixes -(actual_prefix / expected_prefix, length 12–16) for the mismatched field. -Never full digests/secrets in the default residual surface. TDD; no invent allow. -""" - -from __future__ import annotations - -import importlib -import importlib.util -import re -import sys -from pathlib import Path - -import pytest - -from agent_challenge.review.openrouter import OpenRouterTransportError - -_PREFIX_RE = re.compile(r"^[0-9a-f]{12,16}$") -_PREFIX_FIELDS = frozenset({"os", "mrtd", "rtmr0", "rtmr1", "rtmr2", "compose"}) - - -def _load_review_runtime(): - path = Path(__file__).resolve().parents[1] / "docker" / "review" / "review_runtime.py" - spec = importlib.util.spec_from_file_location("review_runtime_prefix_under_test", path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def _canonical_keyrelease_quote_module(): - """Return the quote module instance late-imports will bind. - - ``test_keyrelease_package_init`` can leave package-attribute + sys.modules - dual identities for ``keyrelease.quote``. String monkeypatches then miss - the instance ``_measurement_from_quote`` late-imports. Prefer the - ``sys.modules`` entry (what ``from ... import`` uses) and rebind package - ``.quote`` — never delete modules mid-suite (that breaks later DCAP class - identity / maps). - """ - - importlib.import_module("agent_challenge.keyrelease.quote") - quote = sys.modules["agent_challenge.keyrelease.quote"] - pkg = sys.modules.get("agent_challenge.keyrelease") - if pkg is not None and getattr(pkg, "quote", None) is not quote: - pkg.quote = quote # type: ignore[attr-defined] - return quote - - -def test_short_hex_prefix_helper_trims_and_lowercases() -> None: - runtime = _load_review_runtime() - full = "BD369A8C2F9EDB2B52DAD48AC8E0B32DDE5F1337C423A506B48D07403A7D8033" - prefix = runtime.short_quote_field_hex_prefix(full) - assert _PREFIX_RE.fullmatch(prefix) - assert 12 <= len(prefix) <= 16 - assert prefix == full.lower()[: len(prefix)] - # Full digest must never equal the prefix (unless artificially short). - assert prefix != full.lower() - # 0x prefix and whitespace stripped. - assert runtime.short_quote_field_hex_prefix("0x" + full) == prefix - assert runtime.short_quote_field_hex_prefix(" " + full.lower() + " ") == prefix - - -def test_short_hex_prefix_rejects_non_hex() -> None: - runtime = _load_review_runtime() - assert runtime.short_quote_field_hex_prefix("not-a-digest") is None - assert runtime.short_quote_field_hex_prefix("") is None - assert runtime.short_quote_field_hex_prefix("zzzz") is None - assert runtime.short_quote_field_hex_prefix(None) is None # type: ignore[arg-type] - - -@pytest.mark.parametrize("diag", sorted(_PREFIX_FIELDS)) -def test_bounded_surface_emits_prefixes_for_prefix_diag_fields(diag: str) -> None: - """public_logs residual must include short prefixes for the mismatched field.""" - - runtime = _load_review_runtime() - actual = ("a1" * 32) if diag != "mrtd" else ("b3" * 48) - expected = ("c4" * 32) if diag != "mrtd" else ("d5" * 48) - # SHA-384 registers are 96 hex chars; hashes are 64. Either ok. - if diag in {"mrtd", "rtmr0", "rtmr1", "rtmr2"}: - actual = "ab" * 48 - expected = "cd" * 48 - else: - actual = "11" * 32 - expected = "22" * 32 - - exc = runtime.QuoteMeasurementMismatchError( - f"quoted {diag} mismatches assignment", - diag=diag, - actual=actual, - expected=expected, - ) - surface = runtime.bounded_review_failure_surface(exc) - assert surface["error"] == "review_failed" - assert surface["reason_code"] == "quote_measurement_mismatch" - assert surface.get("diag") == diag - assert "actual_prefix" in surface - assert "expected_prefix" in surface - assert _PREFIX_RE.fullmatch(surface["actual_prefix"]) - assert _PREFIX_RE.fullmatch(surface["expected_prefix"]) - assert surface["actual_prefix"] == actual.lower()[: len(surface["actual_prefix"])] - assert surface["expected_prefix"] == expected.lower()[: len(surface["expected_prefix"])] - # Never full digests on default surface. - assert actual.lower() not in str(surface) - assert expected.lower() not in str(surface) - # Only prefixes (not raw message wording). - assert "mismatches assignment" not in str(surface) - - -def test_os_mismatch_surface_uses_computed_os_image_hash_prefixes() -> None: - """Prefer actual computed os_image_hash (sha256(mrtd∥rtmr1∥rtmr2)) prefixes.""" - - runtime = _load_review_runtime() - actual_os = "bd369a8c2f9edb2b52dad48ac8e0b32dde5f1337c423a506b48d07403a7d8033" - expected_os = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - exc = runtime.QuoteMeasurementMismatchError( - "quoted os image hash mismatches assignment", - diag="os", - actual=actual_os, - expected=expected_os, - ) - surface = runtime.bounded_review_failure_surface(exc) - assert surface["diag"] == "os" - assert surface["actual_prefix"] == actual_os[:16] - assert surface["expected_prefix"] == expected_os[:16] - assert len(surface["actual_prefix"]) == 16 - assert "bd369a8c2f9edb2b52dad48ac8e0b32d" not in str(surface) # not ≥20 chars full - - -def test_bounded_surface_never_emits_full_digests_when_prefixes_present() -> None: - runtime = _load_review_runtime() - actual = "0123456789abcdef" * 4 # 64 hex - expected = "fedcba9876543210" * 4 - exc = runtime.QuoteMeasurementMismatchError( - "quoted compose hash mismatches assignment", - diag="compose", - actual=actual, - expected=expected, - ) - surface = runtime.bounded_review_failure_surface(exc) - blob = str(surface) - assert actual not in blob - assert expected not in blob - assert surface["actual_prefix"] != actual - assert surface["expected_prefix"] != expected - assert 12 <= len(surface["actual_prefix"]) <= 16 - - -def test_bounded_surface_skips_prefixes_for_non_prefix_diags() -> None: - """key_provider / event_log / other must not mint free-form digest prefixes.""" - - runtime = _load_review_runtime() - for diag, message in ( - ("key_provider", "quoted key provider mismatches assignment"), - ("event_log", "quote event log soft mismatch"), - ("other", "quote measurement mismatch without field token"), - ): - # Even if someone passes actual/expected, non-prefix diags omit them. - try: - exc = runtime.QuoteMeasurementMismatchError( - message, - diag=diag, - actual="aa" * 32, - expected="bb" * 32, - ) - except ValueError: - # Implementation may refuse non-prefix diags on the typed error — - # plain ValueError path must also stay prefix-free. - surface = runtime.bounded_review_failure_surface(ValueError(message)) - assert "actual_prefix" not in surface - assert "expected_prefix" not in surface - continue - surface = runtime.bounded_review_failure_surface(exc) - assert "actual_prefix" not in surface - assert "expected_prefix" not in surface - - -def test_plain_value_error_os_still_has_diag_without_prefixes() -> None: - """Legacy plain ValueError (no attrs) keeps field diag but no invented prefixes.""" - - runtime = _load_review_runtime() - surface = runtime.bounded_review_failure_surface( - ValueError("quoted os image hash mismatches assignment") - ) - assert surface["reason_code"] == "quote_measurement_mismatch" - assert surface.get("diag") == "os" - assert "actual_prefix" not in surface - assert "expected_prefix" not in surface - - -def test_openrouter_transport_error_with_prefix_attrs_surfaces_them() -> None: - """Transport-wrapped mismatch may also carry prefix attrs for residual emit.""" - - runtime = _load_review_runtime() - err = OpenRouterTransportError( - "quote_measurement_mismatch", - "quoted mrtd mismatches assignment", - diag="mrtd", - ) - err.actual_prefix = "deadbeefcafebabe"[:16] # type: ignore[attr-defined] - err.expected_prefix = "0011223344556677"[:16] # type: ignore[attr-defined] - surface = runtime.bounded_review_failure_surface(err) - assert surface["diag"] == "mrtd" - assert surface["actual_prefix"] == "deadbeefcafebabe"[:16] - assert surface["expected_prefix"] == "0011223344556677"[:16] - - -def test_measurement_from_quote_os_mismatch_raises_with_prefixes( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """_measurement_from_quote must attach short prefixes on os mismatch.""" - - quote_mod = _canonical_keyrelease_quote_module() - runtime = _load_review_runtime() - # Minimal structural stubs so only the os compare path fires. - assignment = { - "assignment_core": { - "review_app": { - "compose_hash": "cc" * 32, - "measurement": { - "key_provider": "phala", - "os_image_hash": "ee" * 32, - "mrtd": "11" * 48, - "rtmr0": "22" * 48, - "rtmr1": "33" * 48, - "rtmr2": "44" * 48, - "vm_shape": "tdx.small", - }, - } - } - } - - class _Report: - mrtd = "11" * 48 - rtmr0 = "22" * 48 - rtmr1 = "33" * 48 - rtmr2 = "44" * 48 - rtmr3 = "55" * 48 - - class _Replay: - rtmr3 = "55" * 48 - compose_hash = "cc" * 32 - key_provider = "7068616c61" # "phala" hex — decode path handled below - - def _parse(_hex: str) -> _Report: - return _Report() - - def _validate(event_log: list) -> list: - return list(event_log) - - def _replay(_validated: list) -> _Replay: - return _Replay() - - # os_image_hash computed from registers will not equal ee*32 → os mismatch. - fixed_os = "aa" * 32 - - monkeypatch.setattr(quote_mod, "parse_tdx_quote_v4", _parse) - monkeypatch.setattr(quote_mod, "validate_rtmr3_event_log", _validate) - monkeypatch.setattr(quote_mod, "replay_rtmr3", _replay) - monkeypatch.setattr( - quote_mod, - "os_image_hash_from_registers", - lambda *a, **k: fixed_os, - ) - monkeypatch.setattr( - "agent_challenge.review.report._decode_key_provider", - lambda _v: "phala", - ) - - with pytest.raises(runtime.QuoteMeasurementMismatchError) as ei: - runtime._measurement_from_quote( - assignment=assignment, - tdx_quote_hex="00", - event_log=[{"event": "compose-hash"}], - ) - err = ei.value - assert err.diag == "os" - assert err.actual_prefix == fixed_os[:16] - assert err.expected_prefix == ("ee" * 32)[:16] - assert len(err.actual_prefix) <= 16 - assert fixed_os not in str(err) or fixed_os[:16] in str(err) - # Full digests must not appear as standalone free text in message (closed wording). - assert "mismatches assignment" in str(err).lower() - - surface = runtime.bounded_review_failure_surface(err) - assert surface["diag"] == "os" - assert surface["actual_prefix"] == fixed_os[:16] - assert surface["expected_prefix"] == ("ee" * 32)[:16] - assert fixed_os not in str(surface) - assert ("ee" * 32) not in str(surface) - - -def test_measurement_from_quote_rtmr1_mismatch_prefixes( - monkeypatch: pytest.MonkeyPatch, -) -> None: - quote_mod = _canonical_keyrelease_quote_module() - runtime = _load_review_runtime() - assignment = { - "assignment_core": { - "review_app": { - "compose_hash": "cc" * 32, - "measurement": { - "key_provider": "phala", - "os_image_hash": "aa" * 32, - "mrtd": "11" * 48, - "rtmr0": "22" * 48, - "rtmr1": "dd" * 48, # expected; report will differ - "rtmr2": "44" * 48, - "vm_shape": "tdx.small", - }, - } - } - } - - class _Report: - mrtd = "11" * 48 - rtmr0 = "22" * 48 - rtmr1 = "33" * 48 # actual - rtmr2 = "44" * 48 - rtmr3 = "55" * 48 - - class _Replay: - rtmr3 = "55" * 48 - compose_hash = "cc" * 32 - key_provider = "7068616c61" - - monkeypatch.setattr(quote_mod, "parse_tdx_quote_v4", lambda _h: _Report()) - monkeypatch.setattr( - quote_mod, - "validate_rtmr3_event_log", - lambda e: list(e), - ) - monkeypatch.setattr(quote_mod, "replay_rtmr3", lambda _v: _Replay()) - # Match os so we reach register loop. - monkeypatch.setattr( - quote_mod, - "os_image_hash_from_registers", - lambda *a, **k: "aa" * 32, - ) - monkeypatch.setattr( - "agent_challenge.review.report._decode_key_provider", - lambda _v: "phala", - ) - - with pytest.raises(runtime.QuoteMeasurementMismatchError) as ei: - runtime._measurement_from_quote( - assignment=assignment, - tdx_quote_hex="00", - event_log=[{"event": "compose-hash"}], - ) - err = ei.value - assert err.diag == "rtmr1" - assert err.actual_prefix == ("33" * 48)[:16] - assert err.expected_prefix == ("dd" * 48)[:16] - surface = runtime.bounded_review_failure_surface(err) - assert surface["diag"] == "rtmr1" - assert surface["actual_prefix"] == err.actual_prefix - assert surface["expected_prefix"] == err.expected_prefix - assert ("33" * 48) not in str(surface) - assert ("dd" * 48) not in str(surface) diff --git a/packages/challenges/agent-challenge/tests/test_replay_audit_attested_population.py b/packages/challenges/agent-challenge/tests/test_replay_audit_attested_population.py deleted file mode 100644 index f8cc8a272..000000000 --- a/packages/challenges/agent-challenge/tests/test_replay_audit_attested_population.py +++ /dev/null @@ -1,462 +0,0 @@ -"""Production replay-audit population and labelled seam regressions.""" - -from __future__ import annotations - -import copy -import hashlib -from datetime import UTC, datetime, timedelta -from types import SimpleNamespace - -import pytest -from sqlalchemy import select - -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.core.models import EvalRun, EvaluationJob, TaskAttestation, TaskResult -from agent_challenge.evaluation.plan_scoring import ( - build_score_record_from_eval_plan, - canonical_eval_plan_json, - persist_canonical_score_record, - persist_eval_plan, -) -from agent_challenge.evaluation.replay_audit import ( - REPLAY_AUDIT_LABEL, - AggregationSpec, - AuditCandidate, - InvalidReplayTrialsError, - ReplayAuditDispute, - ReplayAuditWireError, - accepted_verified_replay_population, - compare_replay_trials, - persist_replay_dispute, - replay_request_for_candidate, - replay_result_from_mapping, -) -from agent_challenge.models import AgentSubmission - - -def _plan(eval_run_id: str = "eval-replay-population-1") -> dict: - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - return ew.validate_eval_plan( - { - "schema_version": 1, - "eval_run_id": eval_run_id, - "submission_id": "1", - "submission_version": 1, - "authorizing_review_digest": "2" * 64, - "agent_hash": "3" * 64, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "task-a", - "image_ref": "registry.example/task@sha256:" + "4" * 64, - "task_config_sha256": "5" * 64, - } - ], - "k": 2, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "6" * 64, - "compose_hash": "7" * 64, - "app_identity": "agent-challenge-eval", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "8" * 64, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("8" * 64)).hexdigest(), - "measurement": { - "mrtd": "a" * 96, - "rtmr0": "b" * 96, - "rtmr1": "c" * 96, - "rtmr2": "d" * 96, - "os_image_hash": "e" * 64, - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "validator.example:8700", - "result_endpoint": f"/evaluation/v1/runs/{eval_run_id}/result", - "key_release_nonce": "key-replay", - "score_nonce": "score-replay", - "run_token_sha256": "f" * 64, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - ) - - -async def _seed_population_row( - session, - *, - plan: dict, - phase: str = "eval_accepted", - verified: bool = True, - result_available: bool = True, - reward_eligible: bool = True, - job_plan: dict | None = None, - task_attested: bool = True, -) -> tuple[AgentSubmission, EvaluationJob, EvalRun]: - now = datetime.now(UTC) - submission = AgentSubmission( - miner_hotkey=f"hotkey-{plan['eval_run_id']}", - name=f"agent-{plan['eval_run_id']}", - agent_hash=plan["agent_hash"], - artifact_uri=f"/tmp/{plan['eval_run_id']}.zip", - raw_status="tb_completed", - status="tb_completed", - effective_status="valid", - version_number=1, - submitted_at=now, - created_at=now, - ) - session.add(submission) - await session.flush() - plan = copy.deepcopy(plan) - plan["submission_id"] = str(submission.id) - plan = ew.validate_eval_plan(plan) - job = EvaluationJob( - job_id=f"job-{plan['eval_run_id']}", - submission_id=submission.id, - status="completed", - selected_tasks_json="[]", - score=0.75, - passed_tasks=0, - total_tasks=1, - ) - persist_eval_plan(job, job_plan or plan) - record = build_score_record_from_eval_plan(plan, {"task-a": [0.5, 1.0]}) - persist_canonical_score_record(job, record) - session.add(job) - await session.flush() - session.add( - TaskResult( - job_id=job.id, - task_id="task-a", - docker_image=plan["selected_tasks"][0]["image_ref"], - status="completed", - score=ew.decode_score_f64be(record["tasks"][0]["aggregate_score_f64be"]), - returncode=0, - stdout="", - stderr="", - duration_seconds=0.0, - ) - ) - session.add( - TaskAttestation( - job_id=job.id, - task_id="task-a", - verified=task_attested, - reason=None if task_attested else "missing", - retryable=False, - ) - ) - run = EvalRun( - eval_run_id=plan["eval_run_id"], - submission_id=submission.id, - submission_version=1, - authorizing_review_digest=plan["authorizing_review_digest"], - plan_json=canonical_eval_plan_json(plan), - plan_sha256=hashlib.sha256(canonical_eval_plan_json(plan).encode()).hexdigest(), - token_sha256="9" * 64, - phase=phase, - verified=verified, - result_available=result_available, - reward_eligible=reward_eligible, - retryable=False, - key_granted_at=now, - issued_at=now, - expires_at=now + timedelta(hours=1), - result_job_id=job.id, - ) - session.add(run) - await session.flush() - return submission, job, run - - -async def _review_allow(*_args, **_kwargs): - return SimpleNamespace(review_digest="2" * 64) - - -async def test_population_requires_durable_accepted_full_attested_evidence( - database_session, monkeypatch -) -> None: - monkeypatch.setattr( - "agent_challenge.evaluation.replay_audit.verified_review_assignment_for_submission", - _review_allow, - ) - plan = _plan() - async with database_session() as session: - submission, job, run = await _seed_population_row(session, plan=plan) - await session.commit() - - async with database_session() as session: - population = await accepted_verified_replay_population(session, enabled=True) - assert [candidate.eval_run_id for candidate in population] == [run.eval_run_id] - assert population[0].attested_score == pytest.approx(0.75) - assert population[0].n_attempts == 2 - assert population[0].eval_plan["scoring_policy"] == plan["scoring_policy"] - - # Caller-controlled candidate flags cannot substitute for durable eligibility. - assert AuditCandidate("spoofed", attested=True, verified=True).population_eligible is None - assert submission.id == job.submission_id - - -@pytest.mark.parametrize( - ("phase", "verified", "result_available", "reward_eligible", "task_attested"), - [ - ("eval_rejected", False, False, False, True), - ("eval_error", False, False, False, True), - ("eval_accepted", False, True, False, True), - ("eval_accepted", True, False, False, True), - ("eval_accepted", True, True, True, False), - ], -) -async def test_failed_unverified_liveness_and_incomplete_runs_are_inert( - database_session, - monkeypatch, - phase, - verified, - result_available, - reward_eligible, - task_attested, -) -> None: - monkeypatch.setattr( - "agent_challenge.evaluation.replay_audit.verified_review_assignment_for_submission", - _review_allow, - ) - plan = _plan(f"eval-ineligible-{phase}-{verified}-{result_available}-{task_attested}") - async with database_session() as session: - await _seed_population_row( - session, - plan=plan, - phase=phase, - verified=verified, - result_available=result_available, - reward_eligible=reward_eligible, - task_attested=task_attested, - ) - await session.commit() - - async with database_session() as session: - assert await accepted_verified_replay_population(session, enabled=True) == [] - assert await accepted_verified_replay_population(session, enabled=False) == [] - - -def test_labelled_request_and_result_require_exact_plan_identity() -> None: - plan = _plan() - candidate = AuditCandidate( - "1", - attested_score=0.75, - n_attempts=2, - eval_plan=plan, - eval_run_id=plan["eval_run_id"], - plan_sha256=hashlib.sha256(canonical_eval_plan_json(plan).encode()).hexdigest(), - population_eligible=True, - ) - request = replay_request_for_candidate(candidate) - body = request.to_dict() - assert body["audit_label"] == REPLAY_AUDIT_LABEL - assert body["kind"] == "replay_audit_request" - assert body["k"] == plan["k"] - assert body["scoring_policy"] == plan["scoring_policy"] - - result = { - "schema_version": 1, - "audit_label": REPLAY_AUDIT_LABEL, - "kind": "replay_audit_result", - "audit_id": request.audit_id, - "submission_id": "1", - "eval_run_id": plan["eval_run_id"], - "replay_attempt": 1, - "plan_sha256": request.plan_sha256, - "trial_scores_by_task": {"task-a": [0.5, 1.0]}, - } - assert replay_result_from_mapping(result).trial_scores_by_task["task-a"] == [0.5, 1.0] - malformed = copy.deepcopy(result) - malformed["audit_label"] = "unlabelled" - with pytest.raises(ReplayAuditWireError): - replay_result_from_mapping(malformed) - - tampered = copy.deepcopy(plan) - tampered["k"] = 1 - with pytest.raises(ReplayAuditWireError): - replay_request_for_candidate( - AuditCandidate( - "1", - attested_score=0.75, - eval_plan=tampered, - eval_run_id=plan["eval_run_id"], - plan_sha256=candidate.plan_sha256, - population_eligible=True, - ) - ) - - -@pytest.mark.parametrize( - "trial_scores_by_task", - [ - {"task-a": [0.5, 1.0]}, - {"task-b": [0.5, 1.0], "task-a": [0.5, 1.0]}, - {"task-a": [0.5, 1.0], "task-b": [0.5]}, - {"task-a": [0.5, 1.0], "task-b": [0.5, 1.0], "task-extra": [1.0, 1.0]}, - ], -) -def test_replay_result_validation_requires_complete_ordered_selected_set( - trial_scores_by_task, -) -> None: - plan = _plan("eval-replay-result-shape-1") - plan["selected_tasks"].append( - { - "task_id": "task-b", - "image_ref": "registry.example/task-b@sha256:" + "9" * 64, - "task_config_sha256": "a" * 64, - } - ) - plan = ew.validate_eval_plan(plan) - candidate = AuditCandidate( - "1", - attested_score=0.75, - eval_plan=plan, - eval_run_id=plan["eval_run_id"], - plan_sha256=hashlib.sha256(canonical_eval_plan_json(plan).encode()).hexdigest(), - population_eligible=True, - ) - request = replay_request_for_candidate(candidate) - result = replay_result_from_mapping( - { - "schema_version": 1, - "audit_label": REPLAY_AUDIT_LABEL, - "kind": "replay_audit_result", - "audit_id": request.audit_id, - "submission_id": request.submission_id, - "eval_run_id": request.eval_run_id, - "replay_attempt": request.replay_attempt, - "plan_sha256": request.plan_sha256, - "trial_scores_by_task": trial_scores_by_task, - } - ) - with pytest.raises(ReplayAuditWireError): - result.validate_against(request) - - -def test_plan_comparison_rejects_digest_mutation_and_uses_immutable_k() -> None: - plan = _plan() - candidate = AuditCandidate( - "1", - attested_score=0.75, - eval_plan=plan, - eval_run_id=plan["eval_run_id"], - plan_sha256=hashlib.sha256(canonical_eval_plan_json(plan).encode()).hexdigest(), - population_eligible=True, - ) - matching = compare_replay_trials( - candidate, - {"task-a": [0.5, 1.0]}, - spec=AggregationSpec(), - tolerance=0.2, - ) - assert matching.delta == 0.0 - with pytest.raises(InvalidReplayTrialsError): - compare_replay_trials( - candidate, - {"task-a": [0.5]}, - spec=AggregationSpec(), - tolerance=0.2, - ) - - -async def test_replay_request_endpoint_exposes_only_sampled_plan(client, monkeypatch) -> None: - plan = _plan("eval-replay-endpoint-1") - candidate = AuditCandidate( - "1", - attested_score=0.75, - eval_plan=plan, - eval_run_id=plan["eval_run_id"], - plan_sha256=hashlib.sha256(canonical_eval_plan_json(plan).encode()).hexdigest(), - population_eligible=True, - ) - monkeypatch.setattr( - "agent_challenge.api.routes.settings.phala_attestation_enabled", - True, - ) - monkeypatch.setattr( - "agent_challenge.api.routes.settings.attested_review_enabled", - True, - ) - monkeypatch.setattr( - "agent_challenge.api.routes.accepted_verified_replay_population", - lambda *_args, **_kwargs: _async_value([candidate]), - ) - monkeypatch.setattr( - "agent_challenge.api.routes.replay_audit_sampler_from_settings", - lambda _settings: SimpleNamespace(sample=lambda _candidates: ["1"]), - ) - - response = await client.get( - "/internal/v1/replay-audits/requests", - headers={ - "Authorization": "Bearer test-token", - "X-Base-Challenge-Slug": "agent-challenge", - }, - ) - - assert response.status_code == 200 - body = response.json() - assert len(body["requests"]) == 1 - assert body["requests"][0]["eval_plan"] == plan - assert body["requests"][0]["k"] == 2 - - -async def _async_value(value): - return value - - -async def test_mismatch_dispute_is_idempotent_and_does_not_change_job( - database_session, monkeypatch -) -> None: - monkeypatch.setattr( - "agent_challenge.evaluation.replay_audit.verified_review_assignment_for_submission", - _review_allow, - ) - plan = _plan("eval-dispute-1") - async with database_session() as session: - _submission, job, _run = await _seed_population_row(session, plan=plan) - await session.commit() - - candidate = AuditCandidate( - "1", - attested_score=0.75, - eval_plan=plan, - eval_run_id=plan["eval_run_id"], - plan_sha256=hashlib.sha256(canonical_eval_plan_json(plan).encode()).hexdigest(), - population_eligible=True, - ) - comparison = compare_replay_trials( - candidate, - {"task-a": [0.0, 0.0]}, - spec=AggregationSpec(), - tolerance=0.2, - ) - assert comparison.flagged - async with database_session() as session: - first = await persist_replay_dispute( - session, - candidate=candidate, - comparison=comparison, - now=datetime(2026, 1, 1, tzinfo=UTC), - ) - second = await persist_replay_dispute(session, candidate=candidate, comparison=comparison) - await session.commit() - disputes = (await session.scalars(select(ReplayAuditDispute))).all() - persisted_job = await session.scalar( - select(EvaluationJob).where(EvaluationJob.id == job.id) - ) - assert first is not None - assert second is not None - assert first.id == second.id - assert len(disputes) == 1 - assert persisted_job.score == pytest.approx(0.75) diff --git a/packages/challenges/agent-challenge/tests/test_replay_audit_compare.py b/packages/challenges/agent-challenge/tests/test_replay_audit_compare.py index 140bbf87c..178f8133a 100644 --- a/packages/challenges/agent-challenge/tests/test_replay_audit_compare.py +++ b/packages/challenges/agent-challenge/tests/test_replay_audit_compare.py @@ -381,9 +381,11 @@ def test_run_replay_audit_is_inert_when_sampler_disabled() -> None: def test_replay_audit_from_settings_wires_sampler_spec_and_tolerance() -> None: + # T40/T41: dual flags permanently OFF — sampler stays inert; aggregation + # wiring still comes from host-trust settings. settings = ChallengeSettings( - attested_review_enabled=True, - phala_attestation_enabled=True, + attested_review_enabled=False, + phala_attestation_enabled=False, per_task_aggregation="best-of-k", keep_good_tasks_policy="drop-lowest-n", keep_good_tasks_drop_lowest=2, @@ -395,18 +397,40 @@ def test_replay_audit_from_settings_wires_sampler_spec_and_tolerance() -> None: assert audit.spec.per_task_aggregation == "best-of-k" assert audit.spec.keep_policy == "drop-lowest-n" assert audit.spec.drop_lowest_n == 2 - assert audit.sampler.enabled is True + assert audit.sampler.enabled is False def test_replay_audit_run_flags_beyond_tolerance_end_to_end() -> None: + # T40/T41: dual flags cannot be enabled. Construct host-trust settings and + # drive the audit with an explicitly enabled sampler (unit path) so the + # beyond-tolerance flagging contract stays covered without Phala. + from pydantic import ValidationError + + with pytest.raises(ValidationError, match="Phala TEE dual flags"): + ChallengeSettings( + attested_review_enabled=True, + phala_attestation_enabled=True, + ) + settings = ChallengeSettings( - attested_review_enabled=True, - phala_attestation_enabled=True, + attested_review_enabled=False, + phala_attestation_enabled=False, replay_audit_attested_rate=1.0, replay_audit_unverified_rate=0.0, replay_audit_tolerance=0.2, ) - audit = ReplayAudit.from_settings(settings) + # from_settings keeps sampler off under host-trust; build an enabled + # sampler explicitly for the tolerance contract (frozen dataclass). + audit = ReplayAudit( + sampler=ReplayAuditSampler( + attested_rate=1.0, + unverified_rate=0.0, + seed=settings.replay_audit_seed, + enabled=True, + ), + spec=AggregationSpec.from_settings(settings), + tolerance=settings.replay_audit_tolerance, + ) broker = RecordingBroker([0.1, 0.1]) candidates = [AuditCandidate("sub-1", attested_score=0.9, n_attempts=1)] diff --git a/packages/challenges/agent-challenge/tests/test_replay_audit_sampler.py b/packages/challenges/agent-challenge/tests/test_replay_audit_sampler.py index d4abb6a33..b8c524176 100644 --- a/packages/challenges/agent-challenge/tests/test_replay_audit_sampler.py +++ b/packages/challenges/agent-challenge/tests/test_replay_audit_sampler.py @@ -220,14 +220,24 @@ def test_from_settings_is_inert_when_flag_off() -> None: def test_from_settings_enables_sampler_when_flag_on() -> None: + # T40/T41: dual flags permanently rejected. from_settings never enables + # the Phala-gated sampler; host-trust keeps it inert. + from pydantic import ValidationError + + with pytest.raises(ValidationError, match="Phala TEE dual flags"): + ChallengeSettings( + attested_review_enabled=True, + phala_attestation_enabled=True, + ) + settings = ChallengeSettings( - attested_review_enabled=True, - phala_attestation_enabled=True, + attested_review_enabled=False, + phala_attestation_enabled=False, ) sampler = replay_audit_sampler_from_settings(settings) - assert sampler.enabled is True - assert len(sampler.sample(_attested_population(5000))) > 0 + assert sampler.enabled is False + assert sampler.sample(_attested_population(5000)) == [] # --------------------------------------------------------------------------- # diff --git a/packages/challenges/agent-challenge/tests/test_report_http_status_envelope.py b/packages/challenges/agent-challenge/tests/test_report_http_status_envelope.py index 9d8a18231..8def78057 100644 --- a/packages/challenges/agent-challenge/tests/test_report_http_status_envelope.py +++ b/packages/challenges/agent-challenge/tests/test_report_http_status_envelope.py @@ -350,36 +350,23 @@ async def _wrap_once() -> None: def test_require_review_evidence_encryption_fail_closed_when_attested() -> None: - from agent_challenge.sdk.config import ChallengeSettings + """T40/T41: dual flags permanently rejected; host-trust skips encryption gate.""" + from pydantic import ValidationError - settings = ChallengeSettings( - database_url="sqlite+aiosqlite:///:memory:", - shared_token="test-shared-token-not-evidence", - attested_review_enabled=True, - phala_attestation_enabled=True, - review_evidence_encryption_key=None, - review_evidence_encryption_key_file=None, - eval_result_signer_mnemonic="test mnemonic phrase for offline only", - ) - with pytest.raises(ValueError, match="review evidence encryption key"): - settings.require_review_evidence_encryption_for_production() - try: - settings.require_review_evidence_encryption_for_production() - except ValueError as exc: - msg = str(exc).lower() - assert "sk-or" not in msg - assert "test-shared" not in msg - - ok = ChallengeSettings( - database_url="sqlite+aiosqlite:///:memory:", - shared_token="test-shared-token-not-evidence", - attested_review_enabled=True, - phala_attestation_enabled=True, - review_evidence_encryption_key="dedicated-evidence-key-material", - eval_result_signer_mnemonic="test mnemonic phrase for offline only", - ) - ok.require_review_evidence_encryption_for_production() # no raise + from agent_challenge.sdk.config import ChallengeSettings + with pytest.raises(ValidationError, match="Phala TEE dual flags"): + ChallengeSettings( + database_url="sqlite+aiosqlite:///:memory:", + shared_token="test-shared-token-not-evidence", + attested_review_enabled=True, + phala_attestation_enabled=True, + review_evidence_encryption_key=None, + review_evidence_encryption_key_file=None, + eval_result_signer_mnemonic="test mnemonic phrase for offline only", + ) + + # Host-trust product path: dual flags off → encryption gate is a no-op. legacy = ChallengeSettings( database_url="sqlite+aiosqlite:///:memory:", shared_token="test-shared-token-not-evidence", diff --git a/packages/challenges/agent-challenge/tests/test_residual_orch_probes.py b/packages/challenges/agent-challenge/tests/test_residual_orch_probes.py deleted file mode 100644 index 6aeeed8b7..000000000 --- a/packages/challenges/agent-challenge/tests/test_residual_orch_probes.py +++ /dev/null @@ -1,499 +0,0 @@ -"""Offline residual ORCH probe marker formats (VAL-ORCH-009/010/014/022). - -Public_logs residual path without guest SSH: secret-free marker lines prove -concurrency bound, concurrent running samples, DooD inspect fields, network-none -seal egress fail, and gateway-only / fail-closed posture. - -Discriminators (would fail a wrong implementation): -* marker prefix is always residual_orch kind=... -* secrets / PEM / tcp docker hosts never appear in emitted lines -* concurrency_bound logs the bound used by the job -* ps_sample / inflight encode running vs bound and gt_one flags -* task_inspect encodes NetworkMode, Privileged=false, no tcp 2375/2376 -* network_none_seal proves network=none + egress blocked/fail_closed -* gateway_posture fails closed when no gateway is configured - -No Phala create; pure offline fakes with injectable docker runner. -""" - -from __future__ import annotations - -import json -from typing import Any - -import pytest - -from agent_challenge.evaluation.own_runner import residual_orch_probes as probes -from agent_challenge.evaluation.own_runner.dood import DOOD_DOCKER_HOST - - -def _cp(returncode: int = 0, stdout: str = "", stderr: str = "") -> Any: - return type( - "CP", - (), - {"returncode": returncode, "stdout": stdout, "stderr": stderr}, - )() - - -class _RecordingRun: - def __init__(self, decide: Any) -> None: - self.decide = decide - self.calls: list[list[str]] = [] - - def __call__(self, argv: list[str], **kwargs: Any) -> Any: - self.calls.append(list(argv)) - return self.decide(argv, **kwargs) - - -def test_residual_probes_disabled_by_default(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv(probes.RESIDUAL_ORCH_PROBES_ENV, raising=False) - assert probes.residual_orch_probes_enabled() is False - assert probes.maybe_make_probe_controller(bound=2) is None - - -def test_residual_probes_enabled_truthy(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(probes.RESIDUAL_ORCH_PROBES_ENV, "1") - assert probes.residual_orch_probes_enabled() is True - ctrl = probes.maybe_make_probe_controller(bound=2) - assert ctrl is not None - assert ctrl.enabled is True - - -def test_format_residual_marker_has_prefix_and_kind() -> None: - line = probes.format_residual_marker("concurrency_bound", bound=2, nproc=2) - assert line.startswith(f"{probes.RESIDUAL_ORCH_MARKER} kind=concurrency_bound") - assert "bound=2" in line - assert "nproc=2" in line - - -def test_format_residual_marker_redacts_secrets_and_pems() -> None: - line = probes.format_residual_marker( - "task_inspect", - note="BEGIN CERTIFICATE abc", - token="FAKESECRET_a2b3c4d5e6f7g8h9i0j1", - detail="password=hunter2", - ) - assert "BEGIN CERTIFICATE" not in line - assert "sk-live" not in line - assert "hunter2" not in line - assert "redacted" in line - - -def test_format_residual_marker_redacts_tcp_docker_host() -> None: - line = probes.format_residual_marker( - "task_inspect", - docker_host="tcp://127.0.0.1:2375", - ) - assert "tcp://127.0.0.1:2375" not in line - assert "tcp_docker_redacted" in line - - -def test_log_concurrency_bound_emits_bound(capsys: pytest.CaptureFixture[str]) -> None: - line = probes.log_concurrency_bound(4, nproc=8, source="auto") - out = capsys.readouterr().out - assert "residual_orch kind=concurrency_bound" in line - assert "bound=4" in line - assert "bound=4" in out - assert "nproc=8" in out - - -def test_parse_and_count_task_containers() -> None: - names = probes.parse_docker_ps_names( - "own-runner-task-aaa\nown-runner-task-bbb\ndstack-orchestrator-1\nresidual-orch-seal-xyz\n" - ) - assert len(names) == 4 - assert probes.count_task_containers(names) == 2 - - -def test_sample_task_running_count_filters_prefixes() -> None: - def decide(argv: list[str], **_kwargs: Any) -> Any: - assert "ps" in argv - assert any("label=base.own_runner=1" in a for a in argv) - return _cp( - 0, - "own-runner-task-1\nown-runner-task-2\nresidual-orch-seal-x\n", - ) - - runner = _RecordingRun(decide) - running, names = probes.sample_task_running_count(runner=runner) - assert running == 2 - assert names == ["own-runner-task-1", "own-runner-task-2"] - - -def test_log_ps_sample_flags(capsys: pytest.CaptureFixture[str]) -> None: - line = probes.log_ps_sample( - bound=2, - running=2, - names=["own-runner-task-aa", "own-runner-task-bb"], - sample_index=3, - ) - out = capsys.readouterr().out - assert "kind=ps_sample" in line - assert "running=2" in line - assert "bound=2" in line - assert "within_bound=true" in line - assert "gt_one=true" in line - assert "i=3" in out - - -def test_extract_inspect_fields_dood_safe_sibling() -> None: - payload = { - "HostConfig": { - "NetworkMode": "none", - "Privileged": False, - "Binds": ["/opt/agent-challenge/task-cache:/opt/agent-challenge/task-cache:ro"], - }, - "NetworkSettings": {"Networks": {}}, - } - fields = probes.extract_inspect_fields(payload, docker_host=DOOD_DOCKER_HOST) - assert fields["network_mode"] == "none" - assert fields["privileged"] is False - assert fields["binds_count"] == 1 - assert fields["has_docker_sock_bind"] is False - assert fields["has_dstack_sock_bind"] is False - assert fields["docker_host_is_unix"] is True - assert fields["docker_host_has_tcp_2375_2376"] is False - - -def test_extract_inspect_fields_detects_tcp_and_privileged() -> None: - payload = { - "HostConfig": { - "NetworkMode": "bridge", - "Privileged": True, - "Binds": ["/var/run/docker.sock:/var/run/docker.sock"], - } - } - fields = probes.extract_inspect_fields( - payload, - docker_host="tcp://10.0.0.2:2376", - ) - assert fields["privileged"] is True - assert fields["has_docker_sock_bind"] is True - assert fields["docker_host_is_unix"] is False - assert fields["docker_host_has_tcp_2375_2376"] is True - - -def test_log_task_inspect_emits_contract_fields(capsys: pytest.CaptureFixture[str]) -> None: - fields = { - "container": "own-runner-task-deadbeef", - "network_mode": "none", - "privileged": False, - "binds_count": 1, - "has_docker_sock_bind": False, - "has_dstack_sock_bind": False, - "docker_host_is_unix": True, - "docker_host_has_tcp_2375_2376": False, - } - line = probes.log_task_inspect(fields, task_id="adaptive-rejection-sampler") - out = capsys.readouterr().out - assert "kind=task_inspect" in line - assert "NetworkMode=none" in line - assert "Privileged=false" in line - assert "docker_host_has_tcp_2375_2376=false" in line - assert "task_id=adaptive-rejection-sampler" in out - assert "BEGIN " not in out - - -def test_run_network_none_seal_probe_blocked(capsys: pytest.CaptureFixture[str]) -> None: - container = {"name": None} - - def decide(argv: list[str], **_kwargs: Any) -> Any: - if len(argv) >= 2 and argv[1] == "run": - # --name - idx = argv.index("--name") - container["name"] = argv[idx + 1] - assert "--network" in argv - assert argv[argv.index("--network") + 1] == "none" - return _cp(0, "cid\n") - if len(argv) >= 2 and argv[1] == "inspect": - body = [ - { - "HostConfig": { - "NetworkMode": "none", - "Privileged": False, - "Binds": [], - } - } - ] - return _cp(0, json.dumps(body)) - if len(argv) >= 2 and argv[1] == "exec": - return _cp(0, "EGRESS_BLOCKED\n") - if len(argv) >= 2 and argv[1] == "rm": - return _cp(0, "") - return _cp(1, "", "unexpected") - - runner = _RecordingRun(decide) - result = probes.run_network_none_seal_probe(runner=runner) - assert result["network_mode"] == "none" - assert result["egress"] == "blocked" - assert result["ok"] is True - assert result["container"].startswith(probes.RESIDUAL_SEAL_NAME_PREFIX) - line = probes.log_network_none_seal(result) - out = capsys.readouterr().out - assert "kind=network_none_seal" in line - assert "NetworkMode=none" in line - assert "egress=blocked" in out - assert "ok=true" in out - - -def test_gateway_posture_fail_closed_without_gateway( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - monkeypatch.delenv("BASE_LLM_GATEWAY_URL", raising=False) - line = probes.log_gateway_posture(env={}) - out = capsys.readouterr().out - assert "kind=gateway_posture" in line - assert "gateway_configured=false" in line - assert "fail_closed_no_nongateway_egress" in out - - -def test_gateway_posture_with_configured_host( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - env = {"BASE_LLM_GATEWAY_URL": "https://gateway.example.test/v1"} - line = probes.log_gateway_posture(env=env, probe_result="reachable") - out = capsys.readouterr().out - assert "gateway_configured=true" in line - assert "gateway_host=gateway.example.test" in out - assert "allowlist_host=reachable" in out - assert "sk-" not in out - - -def test_controller_emits_full_residual_set( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - monkeypatch.setenv(probes.RESIDUAL_ORCH_PROBES_ENV, "1") - - def decide(argv: list[str], **_kwargs: Any) -> Any: - if len(argv) >= 2 and argv[1] == "ps": - return _cp(0, "own-runner-task-a\nown-runner-task-b\n") - if len(argv) >= 2 and argv[1] == "run": - return _cp(0, "cid\n") - if len(argv) >= 2 and argv[1] == "inspect": - body = [ - { - "HostConfig": { - "NetworkMode": "none", - "Privileged": False, - "Binds": [], - } - } - ] - return _cp(0, json.dumps(body)) - if len(argv) >= 2 and argv[1] == "exec": - return _cp(0, "EGRESS_BLOCKED\n") - if len(argv) >= 2 and argv[1] == "rm": - return _cp(0, "") - return _cp(0, "") - - runner = _RecordingRun(decide) - ctrl = probes.ResidualOrchProbeController( - bound=2, - env={probes.RESIDUAL_ORCH_PROBES_ENV: "1"}, - sample_interval_sec=0.2, - runner=runner, - nproc=2, - ) - assert ctrl.enabled - ctrl.on_job_start() - # Give the sampler at least one tick. - import time - - time.sleep(0.45) - ctrl.on_container_launched("own-runner-task-deadbeef", task_id="demo-task") - ctrl.on_container_exited() - ctrl.on_job_done() - out = capsys.readouterr().out - assert "kind=concurrency_bound" in out - assert "bound=2" in out - assert "kind=ps_sample" in out or "kind=ps_sample_summary" in out - assert "kind=task_inspect" in out - assert "NetworkMode=none" in out - assert "Privileged=false" in out - assert "kind=network_none_seal" in out - assert "kind=gateway_posture" in out - assert "kind=inflight" in out - # Secret surface must stay clean. - assert "BEGIN CERTIFICATE" not in out - assert "OPENROUTER" not in out - assert "tcp://127.0.0.1:2375" not in out - - -def test_compose_allows_residual_probe_env() -> None: - from agent_challenge.canonical.compose import DEFAULT_ALLOWED_ENVS - - assert probes.RESIDUAL_ORCH_PROBES_ENV in DEFAULT_ALLOWED_ENVS - - -def test_flag_off_byte_identical_no_controller(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv(probes.RESIDUAL_ORCH_PROBES_ENV, raising=False) - assert probes.maybe_make_probe_controller(bound=4, nproc=4) is None - - -def test_sample_task_running_count_engine_api(monkeypatch: pytest.MonkeyPatch) -> None: - def fake_request(method: str, path: str, **_kwargs: Any) -> tuple[int, Any]: - assert method == "GET" - assert path.startswith("/containers/json") - return 200, [ - {"Names": ["/own-runner-task-aaa"]}, - {"Names": ["/own-runner-task-bbb"]}, - {"Names": ["/residual-orch-seal-x"]}, - ] - - monkeypatch.setattr(probes, "docker_engine_request", fake_request) - running, names = probes.sample_task_running_count() - assert running == 2 - assert names == ["own-runner-task-aaa", "own-runner-task-bbb"] - - -def test_inspect_task_container_engine_api(monkeypatch: pytest.MonkeyPatch) -> None: - def fake_request(method: str, path: str, **_kwargs: Any) -> tuple[int, Any]: - assert method == "GET" - assert path.startswith("/containers/") - return 200, { - "HostConfig": { - "NetworkMode": "none", - "Privileged": False, - "Binds": ["/opt/task-cache:/opt/task-cache:ro"], - } - } - - monkeypatch.setattr(probes, "docker_engine_request", fake_request) - fields = probes.inspect_task_container("own-runner-task-deadbeef") - assert fields["network_mode"] == "none" - assert fields["privileged"] is False - assert fields["docker_host_is_unix"] is True - assert fields["docker_host_has_tcp_2375_2376"] is False - - -def test_run_concurrent_loader_probe_engine_api( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - created: list[str] = [] - - def fake_request( - method: str, - path: str, - body: Any = None, - **_kwargs: Any, - ) -> tuple[int, Any]: - if method == "GET" and path.startswith("/images/json"): - return 200, [ - { - "Id": "sha256:orch", - "RepoTags": [ - "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:deadbeef" - ], - } - ] - if method == "POST" and path.startswith("/containers/create"): - assert isinstance(body, dict) - assert body.get("HostConfig", {}).get("NetworkMode") == "none" - assert body.get("HostConfig", {}).get("Privileged") is False - assert body.get("Labels", {}).get("base.own_runner") == "1" - created.append(path) - return 201, {"Id": "cid"} - if method == "POST" and path.endswith("/start"): - return 204, None - if method == "GET" and path.endswith("/json"): - return 200, { - "HostConfig": { - "NetworkMode": "none", - "Privileged": False, - "Binds": [], - } - } - if method == "GET" and path.startswith("/containers/json"): - # Sampler after loaders: report started residual loaders. - names = [{"Names": [f"/own-runner-task-residual-{i}"]} for i in range(len(created))] - return 200, names or [{"Names": ["/own-runner-task-residual-a"]}] - if method == "DELETE": - return 204, None - return 500, {"message": f"unexpected {method} {path}"} - - monkeypatch.setattr(probes, "docker_engine_request", fake_request) - result = probes.run_concurrent_loader_probe(bound=3, count=2, runner=None) - assert result["ok"] is True - assert result["started"] == 2 - assert len(result["names"]) == 2 - assert all(n.startswith(probes.RESIDUAL_LOADER_NAME_PREFIX) for n in result["names"]) - line = probes.log_concurrent_loaders(result, bound=3) - out = capsys.readouterr().out - assert "kind=concurrent_loaders" in line - assert "started=2" in out - assert "bound=3" in out - assert "ok=true" in out - - -def test_controller_spawns_loaders_and_emits_gt_one( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - monkeypatch.setenv(probes.RESIDUAL_ORCH_PROBES_ENV, "1") - created = 0 - - def fake_request( - method: str, - path: str, - body: Any = None, - **_kwargs: Any, - ) -> tuple[int, Any]: - nonlocal created - if method == "GET" and path.startswith("/images/json"): - return 200, [{"Id": "sha256:local", "RepoTags": ["local-orch:latest"]}] - if method == "POST" and path.startswith("/containers/create"): - created += 1 - return 201, {"Id": f"c{created}"} - if method == "POST" and "/start" in path: - return 204, None - if method == "POST" and "/exec" in path: - return 201, {"Id": "exec1"} - if method == "GET" and path.startswith("/containers/json"): - # At least two residual loaders for concurrent >1 evidence. - return 200, [ - {"Names": ["/own-runner-task-residual-aaa"]}, - {"Names": ["/own-runner-task-residual-bbb"]}, - ] - if method == "GET" and path.endswith("/json"): - return 200, { - "HostConfig": { - "NetworkMode": "none", - "Privileged": False, - "Binds": [], - } - } - if method == "DELETE": - return 204, None - return 200, None - - monkeypatch.setattr(probes, "docker_engine_request", fake_request) - ctrl = probes.ResidualOrchProbeController( - bound=3, - env={probes.RESIDUAL_ORCH_PROBES_ENV: "1"}, - sample_interval_sec=0.25, - runner=None, - nproc=4, - ) - ctrl.on_job_start() - import time - - time.sleep(0.35) - ctrl.on_job_done() - out = capsys.readouterr().out - assert "kind=concurrency_bound" in out - assert "nproc=4" in out - assert "kind=concurrent_loaders" in out - assert "kind=task_inspect" in out - assert "NetworkMode=none" in out - assert "Privileged=false" in out - assert "docker_host_has_tcp_2375_2376=false" in out - assert "kind=network_none_seal" in out - assert "kind=gateway_posture" in out - assert "kind=ps_sample" in out - # Concurrent >1 is required for VAL-ORCH-009 multi-vCPU residual. - assert "running=2" in out or "gt_one=true" in out - assert created >= 2 diff --git a/packages/challenges/agent-challenge/tests/test_review_api_base_hardcode.py b/packages/challenges/agent-challenge/tests/test_review_api_base_hardcode.py deleted file mode 100644 index 492feed1d..000000000 --- a/packages/challenges/agent-challenge/tests/test_review_api_base_hardcode.py +++ /dev/null @@ -1,331 +0,0 @@ -"""Anti-cheat: REVIEW_API_BASE_URL hard-pin to joinbase agent-challenge. - -Covers VAL-ACURL-001..007,012..014: constant pin, prod refuse matrix, deploy -encrypt refuse, default joinbase, honest accept, allowed_envs force, dev flag. -""" - -from __future__ import annotations - -import importlib.util -import io -import json -from pathlib import Path -from unittest import mock - -import pytest -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey - -from agent_challenge.review import compose as review_compose -from agent_challenge.review.canonical import canonical_sha256 -from agent_challenge.review.schemas import ReviewInputConfig, build_review_assignment -from agent_challenge.review.urls import ( - ALLOW_DEV_REVIEW_URLS_ENV, - DEFAULT_REVIEW_API_BASE_URL, - PINNED_REVIEW_API_BASE_URL, - ReviewApiBaseUrlError, - assert_pinned_review_api_base_url, - is_pinned_review_api_base_url, - resolve_review_api_base_url, -) -from agent_challenge.selfdeploy.review import ( - REVIEW_ALLOWED_ENVS, - ReviewDeploymentError, - build_review_deployment_plan, - encrypt_review_secrets, -) - -JOINBASE = "https://chain.joinbase.ai/challenges/agent-challenge" -REVIEW_IMAGE = "docker.io/example/agent-challenge-review@sha256:" + ("a" * 64) -MEASUREMENT = { - "mrtd": "01" * 48, - "rtmr0": "02" * 48, - "rtmr1": "03" * 48, - "rtmr2": "04" * 48, - "os_image_hash": "05" * 32, - "key_provider": "phala", - "vm_shape": "tdx.small", -} - -REFUSE_MATRIX: tuple[str, ...] = ( - "http://chain.joinbase.ai/challenges/agent-challenge", - "https://evil.example/challenges/agent-challenge", - "https://chain.platform.network/challenges/agent-challenge", - "https://joinbase.ai.evil.example/challenges/agent-challenge", - "https://chain.joinbase.ai.evil/challenges/agent-challenge", - "https://127.0.0.1/challenges/agent-challenge", - "https://localhost/challenges/agent-challenge", - "https://86.38.238.235/challenges/agent-challenge", - "https://chain.joinbase.ai/challenges/prism", - "https://chain.joinbase.ai/", - "//chain.joinbase.ai/challenges/agent-challenge", - "chain.joinbase.ai/challenges/agent-challenge", -) - - -def _load_review_runtime(): - path = Path(__file__).resolve().parents[1] / "docker" / "review" / "review_runtime.py" - spec = importlib.util.spec_from_file_location("review_runtime_hardcode_under_test", path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def _review_compose() -> dict[str, object]: - return review_compose.generate_review_app_compose( - review_image=REVIEW_IMAGE, - app_identity="agent-challenge-review-v1", - ) - - -def _allowlisted(compose_hash: str | None = None) -> dict[str, str]: - return { - "mrtd": MEASUREMENT["mrtd"], - "rtmr0": MEASUREMENT["rtmr0"], - "rtmr1": MEASUREMENT["rtmr1"], - "rtmr2": MEASUREMENT["rtmr2"], - "compose_hash": compose_hash or review_compose.review_app_compose_hash(_review_compose()), - "os_image_hash": MEASUREMENT["os_image_hash"], - } - - -def _assignment(*, public_key_hex: str) -> tuple[dict[str, object], str]: - compose_hash = review_compose.review_app_compose_hash(_review_compose()) - entries = (_allowlisted(compose_hash),) - config = ReviewInputConfig( - image_ref=REVIEW_IMAGE, - compose_hash=compose_hash, - app_identity="agent-challenge-review-v1", - kms_public_key_hex=public_key_hex, - measurement=MEASUREMENT, - measurement_allowlist=entries, - measurement_allowlist_sha256=canonical_sha256({"entries": list(entries)}), - ) - token = "review-session-token-sentinel" - assignment, _bytes, _digest = build_review_assignment( - session_id="rs-review-url", - assignment_id="ra-review-url", - attempt=1, - submission_id="42", - artifact={ - "agent_hash": "10" * 32, - "zip_sha256": "20" * 32, - "zip_size_bytes": 1, - "manifest_sha256": "30" * 32, - "manifest_entries_sha256": "40" * 32, - "fetch_path": "/review/v1/assignments/ra-review-url/artifact", - }, - rules_snapshot_sha256_value="50" * 32, - rules_revision_id="rules-v1", - review_nonce="rn-review-url", - issued_at_ms=1, - expires_at_ms=2, - session_token_sha256=__import__("hashlib").sha256(token.encode()).hexdigest(), - config=config, - ) - return assignment, token - - -def test_pinned_constant_is_joinbase_agent_challenge() -> None: - """VAL-ACURL-001/004: product constant is exactly joinbase challenge path.""" - - assert PINNED_REVIEW_API_BASE_URL == JOINBASE - assert DEFAULT_REVIEW_API_BASE_URL == JOINBASE - runtime = _load_review_runtime() - assert runtime.DEFAULT_REVIEW_API_BASE_URL == JOINBASE - assert runtime.PINNED_REVIEW_API_BASE_URL == JOINBASE - assert "platform.network" not in PINNED_REVIEW_API_BASE_URL - - -def test_default_resolve_without_env_is_joinbase(monkeypatch: pytest.MonkeyPatch) -> None: - """VAL-ACURL-004: unset env → joinbase pin.""" - - monkeypatch.delenv("REVIEW_API_BASE_URL", raising=False) - monkeypatch.delenv(ALLOW_DEV_REVIEW_URLS_ENV, raising=False) - assert resolve_review_api_base_url() == JOINBASE - assert resolve_review_api_base_url(explicit=None, environ={}) == JOINBASE - - -def test_honest_joinbase_accepted_with_trailing_slash() -> None: - """VAL-ACURL-005: exact pin and trailing-slash form accepted.""" - - assert assert_pinned_review_api_base_url(JOINBASE) == JOINBASE - assert assert_pinned_review_api_base_url(JOINBASE + "/") == JOINBASE - assert is_pinned_review_api_base_url(JOINBASE + "/") - assert resolve_review_api_base_url(environ={"REVIEW_API_BASE_URL": JOINBASE + "/"}) == JOINBASE - - -@pytest.mark.parametrize("bad", REFUSE_MATRIX) -def test_prod_assert_refuses_non_joinbase_matrix(bad: str) -> None: - """VAL-ACURL-002/006/013: refuse host/scheme/path cheat classes in prod.""" - - with pytest.raises(ReviewApiBaseUrlError, match="exactly"): - assert_pinned_review_api_base_url(bad, allow_dev=False) - with pytest.raises(ReviewApiBaseUrlError): - resolve_review_api_base_url( - explicit=bad, - environ={ALLOW_DEV_REVIEW_URLS_ENV: "0"}, - ) - - -@pytest.mark.parametrize("bad", REFUSE_MATRIX) -def test_encrypt_review_secrets_refuses_non_joinbase(bad: str) -> None: - """VAL-ACURL-003/013: selfdeploy encrypt fails closed on non-joinbase.""" - - private_key = X25519PrivateKey.generate() - public_key_hex = ( - private_key.public_key() - .public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) - .hex() - ) - assignment, token = _assignment(public_key_hex=public_key_hex) - plan = build_review_deployment_plan({"assignment": assignment, "review_session_token": token}) - with pytest.raises(ReviewDeploymentError, match="exactly|joinbase|REVIEW_API_BASE_URL"): - encrypt_review_secrets( - plan, - { - "OPENROUTER_API_KEY": "or-key", - "REVIEW_API_BASE_URL": bad, - "REVIEW_SESSION_TOKEN": token, - }, - ) - - -def test_encrypt_review_secrets_accepts_honest_joinbase() -> None: - """VAL-ACURL-005/014: honest joinbase encrypt succeeds and stores pin.""" - - private_key = X25519PrivateKey.generate() - public_key_hex = ( - private_key.public_key() - .public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) - .hex() - ) - assignment, token = _assignment(public_key_hex=public_key_hex) - plan = build_review_deployment_plan({"assignment": assignment, "review_session_token": token}) - encrypted = encrypt_review_secrets( - plan, - { - "OPENROUTER_API_KEY": "or-key", - "REVIEW_API_BASE_URL": JOINBASE + "/", - "REVIEW_SESSION_TOKEN": token, - }, - ) - assert encrypted.env_keys == REVIEW_ALLOWED_ENVS - assert "REVIEW_API_BASE_URL" in REVIEW_ALLOWED_ENVS - # Ciphertext opaque; force path already validated joinbase. - - -def test_review_allowed_envs_still_lists_review_api_but_authority_forced() -> None: - """VAL-ACURL-007: REVIEW_API_BASE_URL stays in allowlist for compose_hash, - but cannot change authority (force/refuse in encrypt).""" - - assert "REVIEW_API_BASE_URL" in review_compose.REVIEW_ALLOWED_ENVS - assert review_compose.REVIEW_ALLOWED_ENVS == ( - "OPENROUTER_API_KEY", - "REVIEW_API_BASE_URL", - "REVIEW_SESSION_TOKEN", - ) - - -def test_runtime_main_refuses_non_joinbase_env(monkeypatch: pytest.MonkeyPatch) -> None: - """VAL-ACURL-002: measured runtime main exits refuse on evil env.""" - - runtime = _load_review_runtime() - env = { - "REVIEW_SESSION_TOKEN": "tok", - "OPENROUTER_API_KEY": "or", - "REVIEW_API_BASE_URL": "https://evil.example/callback", - } - stderr = io.StringIO() - with ( - mock.patch.dict("os.environ", env, clear=False), - mock.patch.object(runtime.sys, "stderr", new=stderr), - mock.patch.object(runtime, "run_assignment") as run_assignment, - ): - rc = runtime.main(["--run-assignment"]) - assert rc == 2 - run_assignment.assert_not_called() - payload = json.loads(stderr.getvalue()) - assert payload["error"] == "review_api_base_url_refused" - assert payload["pinned"] == JOINBASE - - -def test_runtime_main_defaults_to_joinbase_when_env_unset( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """VAL-ACURL-004/014: unset REVIEW_API_BASE_URL → joinbase run_assignment.""" - - runtime = _load_review_runtime() - monkeypatch.delenv("REVIEW_API_BASE_URL", raising=False) - monkeypatch.delenv(ALLOW_DEV_REVIEW_URLS_ENV, raising=False) - env = { - "REVIEW_SESSION_TOKEN": "tok", - "OPENROUTER_API_KEY": "or", - } - captured: dict[str, str] = {} - - def _fake_run(*, api_base_url: str, **kwargs: object) -> dict[str, object]: - del kwargs - captured["api_base_url"] = api_base_url - return {"report_status": 200} - - with ( - mock.patch.dict("os.environ", env, clear=False), - mock.patch.object(runtime, "run_assignment", side_effect=_fake_run), - mock.patch.object(runtime.sys, "stdout", new=io.StringIO()), - ): - # Ensure env map does not reintroduce REVIEW_API_BASE_URL. - monkeypatch.delenv("REVIEW_API_BASE_URL", raising=False) - rc = runtime.main(["--run-assignment"]) - assert rc == 0 - assert captured["api_base_url"] == JOINBASE - - -def test_runtime_main_accepts_honest_joinbase_env() -> None: - """VAL-ACURL-005: honest joinbase env accepted into run_assignment.""" - - runtime = _load_review_runtime() - env = { - "REVIEW_SESSION_TOKEN": "tok", - "OPENROUTER_API_KEY": "or", - "REVIEW_API_BASE_URL": JOINBASE, - } - captured: dict[str, str] = {} - - def _fake_run(*, api_base_url: str, **kwargs: object) -> dict[str, object]: - del kwargs - captured["api_base_url"] = api_base_url - return {"report_status": 202} - - with ( - mock.patch.dict("os.environ", env, clear=False), - mock.patch.object(runtime, "run_assignment", side_effect=_fake_run), - mock.patch.object(runtime.sys, "stdout", new=io.StringIO()), - ): - rc = runtime.main(["--run-assignment"]) - assert rc == 0 - assert captured["api_base_url"] == JOINBASE - - -def test_dev_flag_allows_https_override(monkeypatch: pytest.MonkeyPatch) -> None: - """VAL-ACURL-012: only explicit CHALLENGE_ALLOW_DEV_URLS=1 unlocks override.""" - - monkeypatch.setenv(ALLOW_DEV_REVIEW_URLS_ENV, "1") - assert ( - resolve_review_api_base_url(explicit="https://review.dev.local/ac") - == "https://review.dev.local/ac" - ) - monkeypatch.setenv(ALLOW_DEV_REVIEW_URLS_ENV, "0") - with pytest.raises(ReviewApiBaseUrlError): - resolve_review_api_base_url(explicit="https://review.dev.local/ac") - - -def test_dev_flag_still_rejects_http() -> None: - """Dev override still requires https (no plaintext callback).""" - - with pytest.raises(ReviewApiBaseUrlError, match="https"): - assert_pinned_review_api_base_url( - "http://review.dev.local/ac", - allow_dev=True, - ) diff --git a/packages/challenges/agent-challenge/tests/test_review_deployment.py b/packages/challenges/agent-challenge/tests/test_review_deployment.py deleted file mode 100644 index ed73e3454..000000000 --- a/packages/challenges/agent-challenge/tests/test_review_deployment.py +++ /dev/null @@ -1,575 +0,0 @@ -"""Offline contract tests for the isolated encrypted attested-review deployment. - -These tests intentionally exercise only deterministic local seams. They do -not claim that a CVM was deployed or that encrypted-env confinement was -observed on hardware. -""" - -from __future__ import annotations - -import copy -import hashlib -import importlib.util -import json -from dataclasses import replace -from datetime import UTC, datetime - -import pytest -import yaml -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey, X25519PublicKey -from cryptography.hazmat.primitives.ciphers.aead import AESGCM - -from agent_challenge.core.models import AgentSubmission -from agent_challenge.review import compose as review_compose -from agent_challenge.review.build import validate_review_build_definition -from agent_challenge.review.canonical import canonical_sha256 -from agent_challenge.review.deployment import ( - ReviewDeploymentError, - build_review_deployed_acknowledgement, - review_input_config_from_settings, - validate_review_deployed_acknowledgement, -) -from agent_challenge.review.schemas import ReviewInputConfig, build_review_assignment -from agent_challenge.review.sessions import ( - ReviewConflict, - create_review_session, - mark_review_deployed, -) -from agent_challenge.sdk.config import ChallengeSettings -from agent_challenge.selfdeploy.review import ( - REVIEW_ALLOWED_ENVS, - ReviewPhalaDeployment, - build_review_deployment_plan, - encrypt_review_secrets, -) - -REVIEW_IMAGE = "docker.io/example/agent-challenge-review@sha256:" + ("a" * 64) -EVAL_IMAGE = "docker.io/example/agent-challenge-canonical@sha256:" + ("b" * 64) -MEASUREMENT = { - "mrtd": "01" * 48, - "rtmr0": "02" * 48, - "rtmr1": "03" * 48, - "rtmr2": "04" * 48, - "os_image_hash": "05" * 32, - "key_provider": "phala", - "vm_shape": "tdx.small", -} -EVAL_MEASUREMENT = { - "mrtd": "11" * 48, - "rtmr0": "12" * 48, - "rtmr1": "13" * 48, - "rtmr2": "14" * 48, - "compose_hash": "15" * 32, - "os_image_hash": "16" * 32, -} - - -def _review_compose() -> dict[str, object]: - return review_compose.generate_review_app_compose( - review_image=REVIEW_IMAGE, - app_identity="agent-challenge-review-v1", - ) - - -def _allowlisted(compose_hash: str | None = None) -> dict[str, str]: - return { - "mrtd": MEASUREMENT["mrtd"], - "rtmr0": MEASUREMENT["rtmr0"], - "rtmr1": MEASUREMENT["rtmr1"], - "rtmr2": MEASUREMENT["rtmr2"], - "compose_hash": compose_hash or review_compose.review_app_compose_hash(_review_compose()), - "os_image_hash": MEASUREMENT["os_image_hash"], - } - - -def _review_config(public_key_hex: str) -> ReviewInputConfig: - compose_hash = review_compose.review_app_compose_hash(_review_compose()) - entries = (_allowlisted(compose_hash),) - return ReviewInputConfig( - image_ref=REVIEW_IMAGE, - compose_hash=compose_hash, - app_identity="agent-challenge-review-v1", - kms_public_key_hex=public_key_hex, - measurement=MEASUREMENT, - measurement_allowlist=entries, - measurement_allowlist_sha256=canonical_sha256({"entries": list(entries)}), - ) - - -def _assignment(*, public_key_hex: str) -> tuple[dict[str, object], str]: - config = _review_config(public_key_hex) - token = "review-session-token-sentinel" - assignment, _bytes, _digest = build_review_assignment( - session_id="rs-review", - assignment_id="ra-review", - attempt=1, - submission_id="17", - artifact={ - "agent_hash": "10" * 32, - "zip_sha256": "20" * 32, - "zip_size_bytes": 1, - "manifest_sha256": "30" * 32, - "manifest_entries_sha256": "40" * 32, - "fetch_path": "/review/v1/assignments/ra-review/artifact", - }, - rules_snapshot_sha256_value="50" * 32, - rules_revision_id="rules-v1", - review_nonce="rn-review", - issued_at_ms=1, - expires_at_ms=2, - session_token_sha256=hashlib.sha256(token.encode()).hexdigest(), - config=config, - ) - return assignment, token - - -def _decrypt_ciphertext(ciphertext: str, private_key: X25519PrivateKey) -> dict[str, object]: - raw = bytes.fromhex(ciphertext) - ephemeral = raw[:32] - nonce = raw[32:44] - encrypted = raw[44:] - shared = private_key.exchange(X25519PublicKey.from_public_bytes(ephemeral)) - return json.loads(AESGCM(shared).decrypt(nonce, encrypted, None)) - - -def _test_settings(public_key_hex: str) -> ChallengeSettings: - compose_hash = review_compose.review_app_compose_hash(_review_compose()) - allowlisted = _allowlisted(compose_hash) - return ChallengeSettings( - attested_review_enabled=True, - phala_attestation_enabled=True, - review_app_image_ref=REVIEW_IMAGE, - review_app_compose_hash=compose_hash, - review_app_identity="agent-challenge-review-v1", - review_app_kms_public_key_hex=public_key_hex, - review_app_measurement=MEASUREMENT, - review_app_measurement_allowlist=(allowlisted,), - eval_app_image_ref=EVAL_IMAGE, - eval_app_compose_hash=EVAL_MEASUREMENT["compose_hash"], - eval_app_identity="agent-challenge-eval-v1", - eval_app_kms_public_key_hex="e" * 64, - eval_app_measurement_allowlist=(EVAL_MEASUREMENT,), - ) - - -def test_review_compose_is_deterministic_digest_pinned_and_capability_confined() -> None: - first = _review_compose() - second = _review_compose() - - assert review_compose.render_review_app_compose( - first - ) == review_compose.render_review_app_compose(second) - assert review_compose.review_app_compose_hash(first) == review_compose.review_app_compose_hash( - second - ) - assert first["name"] == "agent-challenge-review-v1" - assert first["allowed_envs"] == [ - "OPENROUTER_API_KEY", - "REVIEW_API_BASE_URL", - "REVIEW_SESSION_TOKEN", - ] - assert first["gateway_enabled"] is False - assert first["public_logs"] is True - assert first["public_sysinfo"] is False - - service = yaml.safe_load(str(first["docker_compose_file"]))["services"]["reviewer"] - assert set(service) == review_compose.REVIEWER_SERVICE_KEYS - assert service["image"] == REVIEW_IMAGE - assert service["environment"] == [ - "OPENROUTER_API_KEY", - "REVIEW_API_BASE_URL", - "REVIEW_SESSION_TOKEN", - ] - assert service["volumes"] == ["/var/run/dstack.sock:/var/run/dstack.sock:ro"] - inventory = json.dumps(first, sort_keys=True) - for forbidden in ( - "docker.sock", - "golden", - "task-cache", - "BASE_LLM_GATEWAY_URL", - "BASE_GATEWAY_TOKEN", - "KEY_RELEASE", - "EVAL_RUN", - "weight", - ): - assert forbidden not in inventory - - -@pytest.mark.parametrize( - "extra_key", - [ - "privileged", - "cap_add", - "devices", - "network_mode", - "ports", - "secrets", - "configs", - "pid", - "ipc", - "security_opt", - ], -) -def test_review_compose_rejects_unexpected_service_capability_keys(extra_key: str) -> None: - compose = _review_compose() - services = yaml.safe_load(compose["docker_compose_file"]) - services["services"]["reviewer"][extra_key] = True if extra_key != "cap_add" else ["SYS_ADMIN"] - if extra_key == "ports": - services["services"]["reviewer"][extra_key] = ["8080:8080"] - if extra_key == "devices": - services["services"]["reviewer"][extra_key] = ["/dev/null:/dev/null"] - if extra_key in {"secrets", "configs"}: - services["services"]["reviewer"][extra_key] = ["evil"] - if extra_key in {"pid", "ipc", "network_mode"}: - services["services"]["reviewer"][extra_key] = "host" - forged = dict(compose) - # Intentionally inject unauthorized keys into the measured service inventory. - forged["docker_compose_file"] = yaml.safe_dump(services, sort_keys=False) - with pytest.raises(review_compose.ReviewComposeError, match="schema-closed"): - review_compose.validate_review_app_compose(forged) - - -def test_review_build_definition_is_separate_and_contains_only_review_runtime() -> None: - definition = review_compose.review_build_definition() - assert definition.dockerfile.is_file() - assert definition.requirements.is_file() - assert definition.dockerfile != review_compose.eval_build_definition().dockerfile - - dockerfile = definition.dockerfile.read_text(encoding="utf-8") - assert "@sha256:" in dockerfile - assert "review_runtime.py" in dockerfile - assert "review/openrouter.py" in dockerfile - assert "review/policy.py" in dockerfile - for forbidden in ( - "COPY golden", - "COPY src/agent_challenge/evaluation", - "own_runner", - "Dockerfile", - ): - assert forbidden not in dockerfile - assert validate_review_build_definition().digest_pinned - - -def test_review_runtime_only_calls_bounded_get_quote_and_never_executes_input() -> None: - runtime_path = review_compose.review_build_definition().dockerfile.parent / "review_runtime.py" - spec = importlib.util.spec_from_file_location("review_runtime", runtime_path) - assert spec is not None and spec.loader is not None - runtime = importlib.util.module_from_spec(spec) - spec.loader.exec_module(runtime) - - class QuoteClient: - def __init__(self) -> None: - self.report_data: list[bytes] = [] - - def get_quote(self, report_data: bytes) -> object: - self.report_data.append(report_data) - return type("Quote", (), {"quote": "beef", "event_log": [], "vm_config": {}})() - - client = QuoteClient() - assert runtime._quote("ab" * 64, client=client)["quote"] == "beef" - assert client.report_data == [bytes.fromhex("ab" * 64)] - - source = runtime_path.read_text(encoding="utf-8") - for forbidden in ("subprocess", "exec(", "eval(", "get_key(", "extend_rtmr"): - assert forbidden not in source - - -def test_validator_configuration_binds_allowlist_and_rejects_shared_identities() -> None: - private_key = X25519PrivateKey.generate() - public_key_hex = ( - private_key.public_key() - .public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) - .hex() - ) - config = review_input_config_from_settings(_test_settings(public_key_hex)) - - assert config.image_ref == REVIEW_IMAGE - assert config.app_identity == "agent-challenge-review-v1" - assert config.kms_public_key_hex == public_key_hex - assert config.resolved_measurement() == MEASUREMENT - assert config.measurement_allowlist - assert config.measurement_allowlist_sha256 == canonical_sha256( - {"entries": list(config.measurement_allowlist)} - ) - assignment, _ = _assignment(public_key_hex=public_key_hex) - review_app = assignment["assignment_core"]["review_app"] - assert review_app["measurement_allowlist_sha256"] == config.measurement_allowlist_sha256 - assert review_app["measurement_allowlist"] == list(config.measurement_allowlist) - - shared_allowlist = _allowlisted() - with pytest.raises(ReviewDeploymentError, match="disjoint"): - review_input_config_from_settings( - _test_settings(public_key_hex).model_copy( - update={"eval_app_measurement_allowlist": (shared_allowlist,)} - ) - ) - with pytest.raises(ReviewDeploymentError, match="app identities must be disjoint"): - review_input_config_from_settings( - _test_settings(public_key_hex).model_copy( - update={"eval_app_identity": "agent-challenge-review-v1"} - ) - ) - with pytest.raises(ReviewDeploymentError, match="image refs must be disjoint"): - review_input_config_from_settings( - _test_settings(public_key_hex).model_copy(update={"eval_app_image_ref": REVIEW_IMAGE}) - ) - with pytest.raises(ReviewDeploymentError, match="compose hashes must be disjoint"): - review_input_config_from_settings( - _test_settings(public_key_hex).model_copy( - update={"eval_app_compose_hash": config.compose_hash} - ) - ) - with pytest.raises(ReviewDeploymentError, match="KMS public keys must be disjoint"): - review_input_config_from_settings( - _test_settings(public_key_hex).model_copy( - update={"eval_app_kms_public_key_hex": public_key_hex} - ) - ) - - -def test_review_deployment_encrypts_and_transmits_only_exact_secret_names() -> None: - private_key = X25519PrivateKey.generate() - public_key_hex = ( - private_key.public_key() - .public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) - .hex() - ) - assignment, token = _assignment(public_key_hex=public_key_hex) - plan = build_review_deployment_plan( - { - "assignment": assignment, - "review_session_token": token, - } - ) - sentinel_key = "review-openrouter-secret-sentinel" - encrypted = encrypt_review_secrets( - plan, - { - "OPENROUTER_API_KEY": sentinel_key, - "REVIEW_API_BASE_URL": "https://chain.joinbase.ai/challenges/agent-challenge", - "REVIEW_SESSION_TOKEN": token, - }, - ) - payload = _decrypt_ciphertext(encrypted.ciphertext, private_key) - - assert payload == { - "env": [ - {"key": "OPENROUTER_API_KEY", "value": sentinel_key}, - { - "key": "REVIEW_API_BASE_URL", - "value": "https://chain.joinbase.ai/challenges/agent-challenge", - }, - {"key": "REVIEW_SESSION_TOKEN", "value": token}, - ] - } - assert encrypted.env_keys == REVIEW_ALLOWED_ENVS - assert ( - encrypted.measurement_allowlist_sha256 - == assignment["assignment_core"]["review_app"]["measurement_allowlist_sha256"] - ) - assert sentinel_key not in repr(plan) - assert sentinel_key not in repr(encrypted) - - deployment = ReviewPhalaDeployment( - provision_response={ - "app_id": "agent-challenge-review-v1", - "compose_hash": plan.compose_hash, - "app_env_encrypt_pubkey": public_key_hex, - "os_image_hash": MEASUREMENT["os_image_hash"], - }, - create_response={ - "id": "cvm-review-1", - "request_id": "req-review-1", - "created_at_ms": 1000, - "receipt": "created", - }, - ) - acknowledgement = deployment.deploy(plan, encrypted) - assert deployment.provision_requests == [ - { - "app_id": "agent-challenge-review-v1", - "name": "agent-challenge-review-v1", - "instance_type": "tdx.small", - "region": "us-west-1", - "compose_file": plan.compose, - "env_keys": ["OPENROUTER_API_KEY", "REVIEW_API_BASE_URL", "REVIEW_SESSION_TOKEN"], - "image": "dstack-0.5.9", - } - ] - create_request = deployment.create_requests[0] - assert create_request["app_id"] == "agent-challenge-review-v1" - assert create_request["compose_hash"] == plan.compose_hash - assert create_request["env_keys"] == [ - "OPENROUTER_API_KEY", - "REVIEW_API_BASE_URL", - "REVIEW_SESSION_TOKEN", - ] - assert create_request["encrypted_env"] == encrypted.ciphertext - assert not {"env", "environment", "args", "files"} & set(create_request) - assert sentinel_key not in json.dumps(create_request) - assert set(acknowledgement) == { - "schema_version", - "assignment_id", - "cvm_id", - "phala_create_receipt", - "compose_identity", - } - assert acknowledgement["schema_version"] == 1 - assert acknowledgement["compose_identity"] == { - "image_ref": assignment["assignment_core"]["review_app"]["image_ref"], - "compose_hash": assignment["assignment_core"]["review_app"]["compose_hash"], - "app_kms_public_key_sha256": assignment["assignment_core"]["review_app"][ - "kms_public_key_sha256" - ], - } - assert acknowledgement["phala_create_receipt"]["cvm_id"] == "cvm-review-1" - assert acknowledgement["phala_create_receipt"]["app_id"] == "agent-challenge-review-v1" - - second = ReviewPhalaDeployment( - provision_response={ - "app_id": "agent-challenge-review-v1", - "compose_hash": plan.compose_hash, - "app_env_encrypt_pubkey": public_key_hex, - }, - create_response={"id": "cvm-review-2", "request_id": "req-2", "created_at_ms": 1}, - ) - with pytest.raises(ReviewDeploymentError, match="bound"): - second.deploy(plan, replace(encrypted, assignment_id="ra-other")) - assert second.provision_requests == [] - assert second.create_requests == [] - - -@pytest.mark.parametrize( - "mutate", - [ - lambda assignment: assignment["assignment_core"]["review_app"].update( - {"image_ref": EVAL_IMAGE} - ), - lambda assignment: assignment["assignment_core"]["review_app"].update( - {"compose_hash": "f" * 64} - ), - lambda assignment: assignment["assignment_core"]["review_app"].update( - {"app_identity": "evil-review-app"} - ), - lambda assignment: assignment["assignment_core"]["review_app"].update( - {"kms_public_key_hex": "e" * 64} - ), - lambda assignment: assignment["assignment_core"]["review_app"].update( - {"measurement_allowlist_sha256": "a" * 64} - ), - ], -) -def test_bad_assignment_identity_or_missing_or_extra_secret_fails_before_create(mutate) -> None: - private_key = X25519PrivateKey.generate() - public_key_hex = ( - private_key.public_key() - .public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) - .hex() - ) - assignment, token = _assignment(public_key_hex=public_key_hex) - tampered = copy.deepcopy(assignment) - mutate(tampered) - - with pytest.raises(ReviewDeploymentError): - build_review_deployment_plan({"assignment": tampered, "review_session_token": token}) - - plan = build_review_deployment_plan({"assignment": assignment, "review_session_token": token}) - with pytest.raises(ReviewDeploymentError, match="non-empty"): - encrypt_review_secrets( - plan, - { - "OPENROUTER_API_KEY": "", - "REVIEW_API_BASE_URL": "https://chain.joinbase.ai/challenges/agent-challenge", - "REVIEW_SESSION_TOKEN": token, - }, - ) - with pytest.raises(ReviewDeploymentError, match="exactly"): - encrypt_review_secrets( - plan, - { - "OPENROUTER_API_KEY": "key", - "REVIEW_API_BASE_URL": "https://chain.joinbase.ai/challenges/agent-challenge", - "REVIEW_SESSION_TOKEN": token, - "UNTRUSTED_EXTRA": "no", - }, - ) - - -async def test_nested_deployed_acknowledgement_is_bound_before_running_transition( - database_session, -) -> None: - private_key = X25519PrivateKey.generate() - public_key_hex = ( - private_key.public_key() - .public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) - .hex() - ) - config = _review_config(public_key_hex) - submission_bytes = b"review-deployment-artifact" - submission = AgentSubmission( - miner_hotkey="review-miner", - name="review-agent", - agent_hash=hashlib.sha256(submission_bytes).hexdigest(), - artifact_uri="/tmp/review-deployment.zip", - artifact_path="/tmp/review-deployment.zip", - zip_sha256=hashlib.sha256(submission_bytes).hexdigest(), - zip_size_bytes=len(submission_bytes), - raw_status="review_queued", - effective_status="queued", - ) - async with database_session() as session: - session.add(submission) - await session.flush() - created = await create_review_session( - session, - submission=submission, - artifact_bytes=submission_bytes, - rules_files={".rules/policy.md": b"review"}, - rules_revision_id="rules-v1", - settings=ChallengeSettings(shared_token="review-token"), - input_config=config, - now=datetime(2026, 7, 10, tzinfo=UTC), - ) - assignment = json.loads(created.assignment.assignment_bytes) - acknowledgement = build_review_deployed_acknowledgement( - assignment=assignment, - cvm_id="cvm-review-1", - request_id="req-review-1", - receipt_sha256="6" * 64, - created_at_ms=1_000, - ) - validate_review_deployed_acknowledgement(assignment, acknowledgement) - deployed = await mark_review_deployed( - session, - session_row=created.session, - expected_assignment_id=created.assignment.assignment_id, - deployed_receipt=acknowledgement, - now=datetime(2026, 7, 10, 0, 0, 1, tzinfo=UTC), - ) - assert deployed.phase == "review_cvm_running" - - flat_legacy = { - "assignment_id": created.assignment.assignment_id, - "phala_create_receipt_sha256": "6" * 64, - "cvm_id": "cvm-review-1", - "app_identity": assignment["assignment_core"]["review_app"]["app_identity"], - "image_ref": assignment["assignment_core"]["review_app"]["image_ref"], - "compose_hash": assignment["assignment_core"]["review_app"]["compose_hash"], - "kms_public_key_sha256": assignment["assignment_core"]["review_app"][ - "kms_public_key_sha256" - ], - } - with pytest.raises(ReviewDeploymentError, match="schema-closed"): - validate_review_deployed_acknowledgement(assignment, flat_legacy) - - changed = copy.deepcopy(acknowledgement) - changed["compose_identity"]["compose_hash"] = "7" * 64 - with pytest.raises(ReviewConflict): - await mark_review_deployed( - session, - session_row=created.session, - expected_assignment_id=created.assignment.assignment_id, - deployed_receipt=changed, - now=datetime(2026, 7, 10, 0, 0, 2, tzinfo=UTC), - ) diff --git a/packages/challenges/agent-challenge/tests/test_runtime_image_dcap_qvl_bake.py b/packages/challenges/agent-challenge/tests/test_runtime_image_dcap_qvl_bake.py deleted file mode 100644 index 1cab39873..000000000 --- a/packages/challenges/agent-challenge/tests/test_runtime_image_dcap_qvl_bake.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Shipping AC runtime image must bake dcap-qvl onto PATH. - -Product residual after ops interim (ac-dcap-pcs-ready): - live dual-flag AC used a host bind ``/var/lib/base/tools/dcap-qvl`` -> - ``/usr/local/bin/dcap-qvl``. Recreates without that bind surface - ``review_verifier_unavailable``. Runtime Dockerfile now multi-stage builds - Phala ``dcap-qvl-cli`` and installs the binary at ``/usr/local/bin/dcap-qvl``. - -No invent trust roots / offline collateral in the image. PCS egress is still -ops/network and is not asserted here. -""" - -from __future__ import annotations - -import os -import re -import shutil -from pathlib import Path -from unittest.mock import patch - -import pytest - -from agent_challenge.sdk.config import ChallengeSettings - -REPO_ROOT = Path(__file__).resolve().parents[1] -DOCKERFILE = REPO_ROOT / "Dockerfile" -EXPECTED_BINARY_PATH = "/usr/local/bin/dcap-qvl" -PINNED_CRATE_VERSION = "0.5.2" - - -def _runtime_stage_text(dockerfile: str) -> str: - """Extract the ``runtime`` stage (before ``terminal-bench-runner``).""" - - m = re.search( - r"FROM\s+\S+\s+AS\s+runtime\b(.*?)(?=\nFROM\s|\Z)", - dockerfile, - flags=re.DOTALL | re.IGNORECASE, - ) - assert m is not None, "Dockerfile missing runtime stage" - return m.group(1) - - -def test_dockerfile_has_dcap_qvl_builder_stage() -> None: - assert DOCKERFILE.is_file() - body = DOCKERFILE.read_text(encoding="utf-8") - assert re.search( - r"FROM\s+\S+\s+AS\s+dcap-qvl-builder\b", - body, - flags=re.IGNORECASE, - ), "Dockerfile must include a dcap-qvl-builder multi-stage" - assert re.search( - rf"cargo\s+install\s+dcap-qvl-cli[^\n]*{re.escape(PINNED_CRATE_VERSION)}", - body, - ) or re.search( - rf"DCAP_QVL_CLI_VERSION\s*=\s*{re.escape(PINNED_CRATE_VERSION)}", - body, - ), f"dcap-qvl-cli must be pinned to {PINNED_CRATE_VERSION}" - assert "cargo install dcap-qvl-cli" in body - - -def test_runtime_dockerfile_copies_dcap_qvl_to_path() -> None: - """Fail closed if the published runtime image omits dcap-qvl.""" - - body = DOCKERFILE.read_text(encoding="utf-8") - runtime = _runtime_stage_text(body) - - copy_match = re.search( - r"^\s*COPY\s+--from=dcap-qvl-builder\s+\S+\s+/usr/local/bin/dcap-qvl\s*$", - runtime, - flags=re.MULTILINE, - ) - assert copy_match is not None, ( - f"runtime Dockerfile must COPY dcap-qvl from builder stage into {EXPECTED_BINARY_PATH}" - ) - assert re.search( - r"chmod\s+0?755\s+/usr/local/bin/dcap-qvl", - runtime, - ), "dcap-qvl must be world-executable for uid 10001" - # Ensure install happens before USER drops privileges is handled in stage - # (COPY may be root-owned absolute path; chmod + test as root). - user_idx = re.search(r"^\s*USER\s+10001", runtime, flags=re.MULTILINE) - assert user_idx is not None, "runtime stage must run as challenge uid 10001" - assert copy_match.start() < user_idx.start(), ( - "COPY dcap-qvl must precede USER 10001 so root can place /usr/local/bin" - ) - - -def test_require_dcap_qvl_binary_fail_closed_when_attested_and_missing() -> None: - settings = ChallengeSettings( - database_url="sqlite+aiosqlite:///:memory:", - shared_token="test-shared-token-not-evidence", - attested_review_enabled=True, - phala_attestation_enabled=True, - review_evidence_encryption_key="dedicated-evidence-key-material", - eval_result_signer_mnemonic="test mnemonic phrase for offline only", - ) - with patch("shutil.which", return_value=None): - with pytest.raises(ValueError, match="dcap-qvl on PATH"): - settings.require_dcap_qvl_binary_for_production() - - -def test_require_dcap_qvl_binary_ok_when_present() -> None: - settings = ChallengeSettings( - database_url="sqlite+aiosqlite:///:memory:", - shared_token="test-shared-token-not-evidence", - attested_review_enabled=True, - phala_attestation_enabled=True, - review_evidence_encryption_key="dedicated-evidence-key-material", - eval_result_signer_mnemonic="test mnemonic phrase for offline only", - ) - host_bin = shutil.which("dcap-qvl") - if host_bin is None: - # Local workers without cargo install still pass via mock; image smoke - # is covered by Dockerfile assertions above. - with ( - patch("shutil.which", return_value="/usr/local/bin/dcap-qvl"), - patch("pathlib.Path.is_file", return_value=True), - patch("os.access", return_value=True), - ): - settings.require_dcap_qvl_binary_for_production() - return - settings.require_dcap_qvl_binary_for_production() # no raise - assert Path(host_bin).is_file() - assert os.access(host_bin, os.X_OK) - - -def test_require_dcap_qvl_binary_skipped_when_legacy_flags_off() -> None: - legacy = ChallengeSettings( - database_url="sqlite+aiosqlite:///:memory:", - shared_token="test-shared-token-not-evidence", - attested_review_enabled=False, - phala_attestation_enabled=False, - ) - with patch("shutil.which", return_value=None): - legacy.require_dcap_qvl_binary_for_production() # no raise when flags off - - -def test_require_dcap_qvl_error_leaks_no_secrets() -> None: - settings = ChallengeSettings( - database_url="sqlite+aiosqlite:///:memory:", - shared_token="secret-token-must-not-leak", - attested_review_enabled=True, - phala_attestation_enabled=True, - review_evidence_encryption_key="dedicated-evidence-key-material", - eval_result_signer_mnemonic="test mnemonic phrase for offline only", - ) - with patch("shutil.which", return_value=None): - try: - settings.require_dcap_qvl_binary_for_production() - except ValueError as exc: - msg = str(exc).lower() - assert "secret-token" not in msg - assert "dedicated-evidence" not in msg - assert "mnemonic" not in msg - else: # pragma: no cover - raise AssertionError("expected ValueError when dcap-qvl missing") diff --git a/packages/challenges/agent-challenge/tests/test_runtime_image_phala_pre_launch_package.py b/packages/challenges/agent-challenge/tests/test_runtime_image_phala_pre_launch_package.py deleted file mode 100644 index c37c3df6d..000000000 --- a/packages/challenges/agent-challenge/tests/test_runtime_image_phala_pre_launch_package.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Shipping AC runtime image must package Phala pre-launch helper. - -Residual class (live dual flags, pin pack complete): - ``image_package_prelaunch_script_missing`` - -``create_review_session`` / offline review+eval compose generators resolve -``REPO_ROOT / docker/review/phala_pre_launch.sh``. With WORKDIR ``/app`` and -``PYTHONPATH=/app/src``, ``REPO_ROOT`` inherits as ``/app``, so the shipping -runtime image must contain ``/app/docker/review/phala_pre_launch.sh`` (the -checked-in vendor helper, not a reinvented trust root). -""" - -from __future__ import annotations - -import re -from pathlib import Path - -from agent_challenge.canonical import compose as eval_compose -from agent_challenge.review import compose as review_compose - -REPO_ROOT = Path(__file__).resolve().parents[1] -DOCKERFILE = REPO_ROOT / "Dockerfile" -VENDOR_SCRIPT = REPO_ROOT / "docker" / "review" / "phala_pre_launch.sh" -EXPECTED_IMAGE_PATH = "/app/docker/review/phala_pre_launch.sh" - - -def _runtime_stage_text(dockerfile: str) -> str: - """Extract the ``runtime`` stage (before ``terminal-bench-runner``).""" - - m = re.search( - r"FROM\s+\S+\s+AS\s+runtime\b(.*?)(?=\nFROM\s|\Z)", - dockerfile, - flags=re.DOTALL | re.IGNORECASE, - ) - assert m is not None, "Dockerfile missing runtime stage" - return m.group(1) - - -def test_vendor_phala_pre_launch_script_is_checked_in(): - assert VENDOR_SCRIPT.is_file() - text = VENDOR_SCRIPT.read_text(encoding="utf-8") - assert text.startswith("#!/bin/bash") - assert "Phala Cloud Pre-Launch Script" in text - - -def test_repo_root_paths_align_for_image_layout(): - """Compose generators and the server runtime share the same relative path.""" - - assert review_compose.PHALA_PRE_LAUNCH_SCRIPT_PATH == ( - review_compose.REPO_ROOT / "docker" / "review" / "phala_pre_launch.sh" - ) - assert eval_compose.PHALA_PRE_LAUNCH_SCRIPT_PATH == ( - eval_compose.REPO_ROOT / "docker" / "review" / "phala_pre_launch.sh" - ) - # Under the shipping container layout REPO_ROOT must resolve to /app, so - # relative docker/review/phala_pre_launch.sh becomes EXPECTED_IMAGE_PATH. - rel = Path("docker") / "review" / "phala_pre_launch.sh" - assert (review_compose.REPO_ROOT / rel) == review_compose.PHALA_PRE_LAUNCH_SCRIPT_PATH - assert str(rel).replace("\\", "/") == "docker/review/phala_pre_launch.sh" - assert f"/app/{rel.as_posix()}" == EXPECTED_IMAGE_PATH - - -def test_runtime_dockerfile_copies_phala_pre_launch_to_expected_path(): - """Fail closed if the published runtime image omits the pre-launch helper.""" - - assert DOCKERFILE.is_file() - body = DOCKERFILE.read_text(encoding="utf-8") - runtime = _runtime_stage_text(body) - - # Exact destination used on prod: /app/docker/review/phala_pre_launch.sh - # Accept either absolute dest or relative dest under WORKDIR /app. - copy_to_expected = ( - re.search( - r"^\s*COPY\s+(?:--chown=\S+\s+)?docker/review/phala_pre_launch\.sh\s+" - r"(?:/app/)?docker/review/phala_pre_launch\.sh\s*$", - runtime, - flags=re.MULTILINE, - ) - is not None - ) - # Also accept a recursive copy of docker/review if destination keeps layout. - recursive_review = ( - re.search( - r"^\s*COPY\s+(?:--chown=\S+\s+)?docker/review(?:/\s+|\s+)" - r"(?:/app/)?docker/review(?:/)?\s*$", - runtime, - flags=re.MULTILINE, - ) - is not None - ) - assert copy_to_expected or recursive_review, ( - "runtime Dockerfile must COPY docker/review/phala_pre_launch.sh into " - f"{EXPECTED_IMAGE_PATH} (source of residual_class=" - "image_package_prelaunch_script_missing)" - ) - - -def test_runtime_dockerfile_chowns_app_after_prelaunch_copy(): - """Challenge user must own the script after packaging.""" - - body = DOCKERFILE.read_text(encoding="utf-8") - runtime = _runtime_stage_text(body) - copy_idx = runtime.find("phala_pre_launch.sh") - chown_idx = runtime.find("chown -R challenge:challenge /app") - assert copy_idx != -1, "pre-launch COPY missing from runtime stage" - assert chown_idx != -1, "chown of /app missing from runtime stage" - assert copy_idx < chown_idx, "COPY pre-launch script must precede chown of /app" diff --git a/packages/challenges/agent-challenge/tests/test_score_event_log_event_id_project.py b/packages/challenges/agent-challenge/tests/test_score_event_log_event_id_project.py deleted file mode 100644 index 188e23f37..000000000 --- a/packages/challenges/agent-challenge/tests/test_score_event_log_event_id_project.py +++ /dev/null @@ -1,388 +0,0 @@ -"""Score-path event_log[].event projection: live dstack → 1-128 visible ASCII. - -Live residual (image@sha256:ffbb60a9 / compose bb68f5e1): after KR grant, -decrypt_ok, job_done trials=3, emit_start → AttestationEmissionError -'schema-v2 Eval emission is invalid: event_log[].event must be a 1-128 character -visible ASCII id' → phala_attestation_failed. - -Root cause: schema-v2 wire (eval_wire._id) requires event_log[].event to be a -1–128 character visible ASCII id. Live dstack GetQuote often yields IMR0–2 (and -occasionally incomplete) entries with empty/missing/non-string ``event`` after -KR coerce + empty-IMR3 fill. Identity IMR3 events already have proper names; -early boot entries may not. Port a closed projection after coerce+IMR3 fill: -strip control bytes, derive visible id from event_type when empty and RTMR3 is -unaffected (IMR!=3 or non-runtime), else fail closed with a typed emit detail. - -Discriminators (would fail a wrong implementation): - * empty / missing / non-string ``event`` on IMR0–2 entries pass wire after - obtain_quote / schema-v2 emit projection - * IMR3 runtime empty event stays fail-closed (RTMR3-bound; no invent) - * identity event names preserved; RTMR3 self-check still green -No Phala create; targeted offline only. -""" - -from __future__ import annotations - -import json -from typing import Any - -import pytest - -from agent_challenge.canonical import attested_result as ar -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.keyrelease.quote import ( - APP_IMR, - COMPOSE_HASH_EVENT, - DSTACK_RUNTIME_EVENT_TYPE, - KEY_PROVIDER_EVENT, - build_rtmr3_event_log, - build_tdx_quote, - replay_rtmr3, -) - -MEASUREMENT = { - "mrtd": "a" * 96, - "rtmr0": "b0" * 48, - "rtmr1": "b1" * 48, - "rtmr2": "b2" * 48, - "compose_hash": "c" * 64, - "os_image_hash": "e" * 64, -} - - -def _identity_event_log() -> tuple[list[dict[str, Any]], str]: - compose_payload = bytes.fromhex(MEASUREMENT["compose_hash"]) - provider_payload = b'{"name":"phala"}' - bootstrap = bytes.fromhex("11" * 8) - filled, rtmr3 = build_rtmr3_event_log( - [ - ("instance-id", bootstrap), - (COMPOSE_HASH_EVENT, compose_payload), - ("boot-mr-done", bootstrap), - (KEY_PROVIDER_EVENT, provider_payload), - ] - ) - return filled, rtmr3 - - -def _live_shaped_event_log_with_empty_imr_events() -> tuple[list[dict[str, Any]], str]: - """Live residual: IMR0–2 empty/missing event + filled IMR3 identities. - - KR-style closed keys with empty ``digest`` on IMR3 (fill path) not needed - here; events already have digests from ``build_rtmr3_event_log``. The score - residual is the empty early-boot ``event`` field the wire rejects. - """ - - filled, rtmr3 = _identity_event_log() - # Prepend live-shaped IMR0/1 entries (empty / missing / non-string event). - early: list[dict[str, Any]] = [ - { - "imr": 0, - "event_type": 1, - "digest": "aa" * 48, - "event": "", # live residual: empty string - "event_payload": "", - }, - { - "imr": 1, - "event_type": 2, - "digest": "bb" * 48, - # Missing event key — event_type-only residual surface - "event_payload": "cc" * 4, - "digest_extra_drop": True, # coerce path also drops extras - }, - { - "imr": 2, - "event_type": 3, - "digest": "dd" * 48, - "event": None, # non-string / null residual - "event_payload": "", - }, - { - "imr": 0, - "event_type": 4, - "digest": "ee" * 48, - "event": "acpi\x00\x01", # control bytes in non-IMR3 name - "event_payload": "", - }, - ] - # Keep identities exactly so RTMR3 is unchanged - return early + [dict(e) for e in filled], rtmr3 - - -class _LiveEventLogQuoteResponse: - def __init__( - self, - quote_hex: str, - event_log: list[dict[str, Any]], - vm_config: dict[str, Any] | None = None, - ) -> None: - self.quote = quote_hex - self.event_log = json.dumps(event_log) - self.vm_config = json.dumps( - vm_config - or { - "vcpu": 1, - "memory_mb": 2048, - "os_image_hash": MEASUREMENT["os_image_hash"], - } - ) - self.report_data = "" - - -class _Provider: - def __init__(self, response: _LiveEventLogQuoteResponse) -> None: - self._response = response - self.calls: list[bytes] = [] - - def get_quote(self, report_data: bytes) -> _LiveEventLogQuoteResponse: - self.calls.append(report_data) - return self._response - - -def test_unprojected_empty_event_fails_wire() -> None: - """Discriminator: raw empty/missing event fails schema-v2 wire before projection.""" - - filled, rtmr3 = _identity_event_log() - # Explicit closed 5-key entry with empty event string (the live residual - # after KR coerce keeps the key; value is "") — wire rejects via _id. - events = [ - { - "imr": 0, - "event_type": 1, - "digest": "aa" * 48, - "event": "", - "event_payload": "", - }, - *[dict(e) for e in filled], - ] - with pytest.raises(ew.EvalWireError, match=r"event_log\[\]\.event"): - ew.validate_eval_phala_attestation( - { - "tdx_quote": "ab" * 600, - "event_log": events, - "report_data": "00" * 64, - "measurement": {**MEASUREMENT, "rtmr3": rtmr3}, - "vm_config": { - "vcpu": 1, - "memory_mb": 2048, - "os_image_hash": MEASUREMENT["os_image_hash"], - }, - } - ) - - -def test_project_empty_missing_nonstring_event_to_visible_ascii() -> None: - """Empty/missing/null/control-stripped event on non-IMR3 → 1–128 visible ASCII.""" - - raw, _ = _live_shaped_event_log_with_empty_imr_events() - projected = ar._project_eval_event_log(raw) - assert projected, "projected log must be non-empty" - for entry in projected: - event = entry["event"] - assert isinstance(event, str) - assert 1 <= len(event) <= 128 - assert all("!" <= ch <= "~" for ch in event) - assert set(entry) == { - "imr", - "event_type", - "digest", - "event", - "event_payload", - } - - # Control bytes stripped from non-IMR3 name - control_entries = [e for e in projected if e["event"].startswith("acpi")] - assert control_entries - assert control_entries[0]["event"] == "acpi" - - # Identity names preserved (RTMR3uring names) - names = {e["event"] for e in projected if e["imr"] == APP_IMR} - assert COMPOSE_HASH_EVENT in names - assert KEY_PROVIDER_EVENT in names - - -def test_project_imr3_runtime_empty_event_fails_closed() -> None: - """IMR3 runtime empty event is RTMR3-bound: do not invent a name.""" - - events, _ = _identity_event_log() - broken = [dict(e) for e in events] - # Break the compose-hash name (runtime IMR3) to empty — fail closed. - for entry in broken: - if entry["event"] == COMPOSE_HASH_EVENT: - entry["event"] = "" - break - with pytest.raises(ar.AttestationEmissionError, match=r"(?i)event|RTMR3|imr3"): - ar._project_eval_event_log(broken) - - -def test_obtain_quote_projects_event_ids_and_passes_wire() -> None: - """obtain_quote applies event projection after coerce; wire accepts the log.""" - - raw_log, expected_rtmr3 = _live_shaped_event_log_with_empty_imr_events() - quote_hex = build_tdx_quote( - mrtd=MEASUREMENT["mrtd"], - rtmr0=MEASUREMENT["rtmr0"], - rtmr1=MEASUREMENT["rtmr1"], - rtmr2=MEASUREMENT["rtmr2"], - rtmr3=expected_rtmr3, - report_data=b"z" * 32, - ) - provider = _Provider(_LiveEventLogQuoteResponse(quote_hex, raw_log)) - result = ar.obtain_quote(provider, b"z" * 32) - - assert result.event_log - for entry in result.event_log: - assert ew._id(entry["event"], "event_log[].event") == entry["event"] - - # RTMR3 still matches (only non-IMRT3 names projected) - assert replay_rtmr3(result.event_log).rtmr3 == expected_rtmr3 - - # Full attestation shape with projected log is wire-valid - att = ew.validate_eval_phala_attestation( - { - "tdx_quote": result.quote, - "event_log": result.event_log, - "report_data": "00" * 64, - "measurement": {**MEASUREMENT, "rtmr3": expected_rtmr3}, - "vm_config": result.vm_config, - } - ) - assert att["event_log"] - - -def test_schema_v2_emit_with_live_shaped_empty_event_log(monkeypatch) -> None: - """schema-v2 emit green with live-shaped empty/missing event residuals.""" - - del monkeypatch - raw_log, expected_rtmr3 = _live_shaped_event_log_with_empty_imr_events() - - class _DstackProvider: - def get_quote(self, report_data: bytes) -> _LiveEventLogQuoteResponse: - quote_hex = build_tdx_quote( - mrtd=MEASUREMENT["mrtd"], - rtmr0=MEASUREMENT["rtmr0"], - rtmr1=MEASUREMENT["rtmr1"], - rtmr2=MEASUREMENT["rtmr2"], - rtmr3=expected_rtmr3, - report_data=report_data, - ) - return _LiveEventLogQuoteResponse(quote_hex, raw_log) - - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - task_ids = ["task-a", "task-b"] - record = ew.build_canonical_score_record( - eval_run_id="eval-event-id-project", - policy=policy, - trial_scores_by_task={"task-a": [1.0], "task-b": [0.0]}, - ) - line = ar.emit_attested_benchmark_result( - benchmark_result={ - "status": "completed", - "score": 0.5, - "resolved": 1, - "total": 2, - "reason_code": None, - }, - canonical_measurement=dict(MEASUREMENT), - rtmr3="0" * 96, - agent_hash="f" * 64, - task_ids=task_ids, - scores={}, - quote_provider=_DstackProvider(), - manifest_sha256="1" * 64, - eval_run_id="eval-event-id-project", - submission_id="submission-event-id-project", - score_nonce="score-nonce-event-id", - score_record=record, - image_digest="registry.example/eval@sha256:" + "d" * 64, - vm_config=None, - ) - payload = json.loads(line.split("=", 1)[1]) - assert ew.validate_eval_result_request(payload) == payload - for entry in payload["execution_proof"]["attestation"]["event_log"]: - event = entry["event"] - assert isinstance(event, str) - assert 1 <= len(event) <= 128 - assert all("!" <= ch <= "~" for ch in event) - - -def test_schema_v2_emit_fails_closed_imr3_runtime_empty_event() -> None: - """Emit path fails closed with typed attach when RTMR3-bound name is empty.""" - - events, expected_rtmr3 = _identity_event_log() - broken = [dict(e) for e in events] - for entry in broken: - if entry["event"] == COMPOSE_HASH_EVENT: - entry["event"] = "" - break - - class _BrokenProvider: - def get_quote(self, report_data: bytes) -> _LiveEventLogQuoteResponse: - quote_hex = build_tdx_quote( - mrtd=MEASUREMENT["mrtd"], - rtmr0=MEASUREMENT["rtmr0"], - rtmr1=MEASUREMENT["rtmr1"], - rtmr2=MEASUREMENT["rtmr2"], - rtmr3=expected_rtmr3, - report_data=report_data, - ) - return _LiveEventLogQuoteResponse(quote_hex, broken) - - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - record = ew.build_canonical_score_record( - eval_run_id="eval-event-id-imr3-empty", - policy=policy, - trial_scores_by_task={"task-a": [1.0]}, - ) - with pytest.raises(ar.AttestationEmissionError, match=r"(?i)event|RTMR3|imr3|schema-v2"): - ar.emit_attested_benchmark_result( - benchmark_result={ - "status": "completed", - "score": 1.0, - "resolved": 1, - "total": 1, - "reason_code": None, - }, - canonical_measurement=dict(MEASUREMENT), - rtmr3="0" * 96, - agent_hash="f" * 64, - task_ids=["task-a"], - scores={}, - quote_provider=_BrokenProvider(), - manifest_sha256="1" * 64, - eval_run_id="eval-event-id-imr3-empty", - submission_id="submission-event-id-imr3-empty", - score_nonce="score-nonce-imr3-empty", - score_record=record, - image_digest="registry.example/eval@sha256:" + "d" * 64, - vm_config=None, - ) - - -def test_project_preserves_rtmr3_for_identity_only_log() -> None: - """Identity-only log projection is a no-op that keeps RTMR3 green.""" - - events, expected = _identity_event_log() - projected = ar._project_eval_event_log(events) - assert replay_rtmr3(projected).rtmr3 == expected - names = [e["event"] for e in projected] - assert COMPOSE_HASH_EVENT in names - assert KEY_PROVIDER_EVENT in names - # DSTACK runtime type marker still present for identities - for entry in projected: - if entry["event"] in {COMPOSE_HASH_EVENT, KEY_PROVIDER_EVENT}: - assert entry["imr"] == APP_IMR - assert entry["event_type"] == DSTACK_RUNTIME_EVENT_TYPE diff --git a/packages/challenges/agent-challenge/tests/test_score_gate_key_provider_and_canonical_emit.py b/packages/challenges/agent-challenge/tests/test_score_gate_key_provider_and_canonical_emit.py deleted file mode 100644 index 80f389011..000000000 --- a/packages/challenges/agent-challenge/tests/test_score_gate_key_provider_and_canonical_emit.py +++ /dev/null @@ -1,418 +0,0 @@ -"""Score AttestationGate key-provider decode + schema-v2 canonical emit. - -Live residual after image@sha256:5b190365 / compose 36e5c63e: - * Full path preflight→KR grant→decrypt_ok→job_done→emit_start→**score_quote_ok** - with schema-v1 attested result; dcap-qvl UpToDate; measurements match offline. - * ``AttestationGate.decide_eval_result`` still rejects: ``replay.key_provider`` is - raw dstack JSON hex ``{"name":"kms","id":...}`` while the plan pin is ``phala`` - (hex ``7068616c61``). KR already decodes via ``decode_key_provider`` (32ed505b); - the score gate does not. - * Secondary: guest ``BASE_BENCHMARK_RESULT`` / result body may use spaced - ``json.dumps`` (default separators). Host ``process_direct_eval_result`` requires - ``raw_body == eval_wire.canonical_json_v1(validated)`` so non-compact bodies are - ``result_noncanonical``. - -Offline fix (no Phala create): - (1) ``AttestationGate.decide_eval_result``: ``decode_key_provider(replay.key_provider)`` - before pin compare. - (2) schema-v2 / attested emit writes compact ``canonical_json_v1`` bytes. - -Discriminators (would fail a wrong implementation): - * Live-shaped JSON-hex key-provider + pin ``phala`` is VERIFIED (not VERIFICATION_FAILED). - * Pin ``phala`` still rejects a non-phala provider payload. - * Emission suffix bytes equal ``canonical_json_v1`` of the validated request - (no spaced separators; revalidate equals raw). -""" - -from __future__ import annotations - -import hashlib -import io -import json -from typing import Any - -from agent_challenge.canonical import attested_result as ar -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.evaluation.attestation import ( - AttestationGate, - AttestationOutcome, - ResultMeasurementAllowlist, -) -from agent_challenge.keyrelease.quote import ( - StaticQuoteVerifier, - build_rtmr3_event_log, - build_tdx_quote, - decode_key_provider, - os_image_hash_from_registers, - replay_rtmr3, -) - -REGS = { - "mrtd": "11" * 48, - "rtmr0": "22" * 48, - "rtmr1": "33" * 48, - "rtmr2": "44" * 48, -} -COMPOSE_HASH = "ab" * 32 -OS_IMAGE_HASH = os_image_hash_from_registers(REGS["mrtd"], REGS["rtmr1"], REGS["rtmr2"]) -AGENT_HASH = "55" * 32 -LIVE_KEY_PROVIDER_JSON = b'{"name":"kms","id":"kms-live-score-1"}' - - -def _plan(*, key_provider: str = "phala") -> dict[str, Any]: - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - return ew.validate_eval_plan( - { - "schema_version": 1, - "eval_run_id": "eval-score-gate-kp-1", - "submission_id": "submission-score-gate-kp-1", - "submission_version": 1, - "authorizing_review_digest": "66" * 32, - "agent_hash": AGENT_HASH, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "task-a", - "image_ref": "registry.example/task@sha256:" + "77" * 32, - "task_config_sha256": "88" * 32, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "99" * 32, - "compose_hash": COMPOSE_HASH, - "app_identity": "agent-challenge-eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "aa" * 32, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), - "measurement": { - **REGS, - "os_image_hash": OS_IMAGE_HASH, - "key_provider": key_provider, - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-score-gate-kp-1/result", - "key_release_nonce": "key-release-score-gate-1", - "score_nonce": "score-score-gate-1", - "run_token_sha256": "bb" * 32, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - ) - - -def _request( - plan: dict[str, Any], - *, - provider_payload: bytes = LIVE_KEY_PROVIDER_JSON, -) -> dict[str, Any]: - from agent_challenge.evaluation.plan_scoring import build_score_record_from_eval_plan - - event_log, rtmr3 = build_rtmr3_event_log( - [ - ("compose-hash", bytes.fromhex(COMPOSE_HASH)), - ("key-provider", provider_payload), - ] - ) - # Discriminator: raw RTMR3 payload stays JSON hex; not the pin. - assert replay_rtmr3(event_log).key_provider == provider_payload.hex() - record = build_score_record_from_eval_plan(plan, {"task-a": [1.0]}) - scores_digest = ew.score_record_digest(record) - binding = ew.build_score_binding( - canonical_measurement={ - "mrtd": REGS["mrtd"], - "rtmr0": REGS["rtmr0"], - "rtmr1": REGS["rtmr1"], - "rtmr2": REGS["rtmr2"], - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, - }, - agent_hash=AGENT_HASH, - eval_run_id=plan["eval_run_id"], - score_nonce=plan["score_nonce"], - scores_digest=scores_digest, - task_ids=["task-a"], - ) - report_data = ew.score_report_data_hex(binding) - quote = build_tdx_quote( - mrtd=REGS["mrtd"], - rtmr0=REGS["rtmr0"], - rtmr1=REGS["rtmr1"], - rtmr2=REGS["rtmr2"], - rtmr3=rtmr3, - report_data=report_data, - ) - return { - "schema_version": 1, - "eval_run_id": plan["eval_run_id"], - "submission_id": plan["submission_id"], - "agent_hash": AGENT_HASH, - "score_record": record, - "scores_digest": scores_digest, - "execution_proof": { - "version": 1, - "tier": "phala-tdx", - "manifest_sha256": "cc" * 32, - "image_digest": plan["eval_app"]["image_ref"], - "provider": None, - "worker_signature": {"worker_pubkey": "", "sig": ""}, - "attestation": { - "tdx_quote": quote, - "event_log": event_log, - "report_data": report_data, - "measurement": { - **REGS, - "rtmr3": rtmr3, - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, - }, - "vm_config": { - "vcpu": 1, - "memory_mb": 2048, - "os_image_hash": OS_IMAGE_HASH, - }, - }, - }, - } - - -def _gate() -> AttestationGate: - return AttestationGate( - quote_verifier=StaticQuoteVerifier(), - allowlist=ResultMeasurementAllowlist.from_measurements( - [ - { - "mrtd": REGS["mrtd"], - "rtmr0": REGS["rtmr0"], - "rtmr1": REGS["rtmr1"], - "rtmr2": REGS["rtmr2"], - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, - } - ] - ), - ) - - -# --------------------------------------------------------------------------- # -# (1) Score gate decodes live-shaped key-provider JSON hex like KR -# --------------------------------------------------------------------------- # - - -def test_decode_key_provider_maps_live_json_to_phala() -> None: - """Shared KR helper (precondition) maps live dstack JSON hex → ``phala``.""" - - assert decode_key_provider(LIVE_KEY_PROVIDER_JSON.hex()) == "phala" - assert LIVE_KEY_PROVIDER_JSON.hex() != b"phala".hex() - - -def test_score_gate_accepts_live_shaped_key_provider_json_hex_with_pin_phala() -> None: - """Live residual: JSON-hex key-provider + plan pin ``phala`` must VERIFIED. - - Without ``decode_key_provider`` the gate compares raw RTMR3 hex to - ``phala``.encode().hex() and fails closed even though KR/allowlist pin is - correct. Discriminator against that missed decode. - """ - - plan = _plan(key_provider="phala") - request = _request(plan, provider_payload=LIVE_KEY_PROVIDER_JSON) - # Document the residual comparison that used to fail. - raw_hex = LIVE_KEY_PROVIDER_JSON.hex() - pin = str(plan["eval_app"]["measurement"]["key_provider"]) - assert raw_hex != pin.encode("ascii").hex() - assert decode_key_provider(raw_hex) == pin - - decision = _gate().decide_eval_result( - request, - eval_plan=plan, - expected_agent_hash=AGENT_HASH, - nonce_outstanding=True, - key_granted=True, - ) - assert decision.outcome is AttestationOutcome.VERIFIED - assert decision.accepted is True - - -def test_score_gate_still_rejects_mismatched_key_provider_pin() -> None: - """Pin ``phala`` still rejects non-phala provider after decode.""" - - plan = _plan(key_provider="phala") - request = _request(plan, provider_payload=b"evil-kms") - decision = _gate().decide_eval_result( - request, - eval_plan=plan, - expected_agent_hash=AGENT_HASH, - nonce_outstanding=True, - key_granted=True, - ) - assert decision.outcome is AttestationOutcome.VERIFICATION_FAILED - assert decision.accepted is False - - -def test_score_gate_accepts_plain_pin_hex_unchanged() -> None: - """Offline fixtures that already emit plain ``phala`` hex still verify.""" - - plan = _plan(key_provider="phala") - request = _request(plan, provider_payload=b"phala") - decision = _gate().decide_eval_result( - request, - eval_plan=plan, - expected_agent_hash=AGENT_HASH, - nonce_outstanding=True, - key_granted=True, - ) - assert decision.outcome is AttestationOutcome.VERIFIED - - -# --------------------------------------------------------------------------- # -# (2) Attested schema-v2 emit: compact canonical_json_v1 body bytes -# --------------------------------------------------------------------------- # - - -class _QuoteProvider: - def __init__(self, quote_hex: str, event_log: list[dict[str, Any]]) -> None: - self._quote = quote_hex - self._event_log = event_log - - def get_quote(self, report_data: bytes) -> Any: - del report_data - return type( - "Q", - (), - { - "quote": self._quote, - "event_log": self._event_log, - "vm_config": { - "vcpu": 1, - "memory_mb": 2048, - "os_image_hash": OS_IMAGE_HASH, - }, - }, - )() - - -def test_schema_v2_emit_bytes_equal_canonical_json_v1_of_validated() -> None: - """Emission body after BASE_BENCHMARK_RESULT= must be canonical (no spaces). - - Spaced ``json.dumps`` (default separators) would fail - ``process_direct_eval_result``'s exact ``raw_body == canonical_json_v1`` check. - This is the live-shaped non-canonical residual for the score POST path. - """ - - plan = _plan(key_provider="phala") - event_log, rtmr3 = build_rtmr3_event_log( - [ - ("compose-hash", bytes.fromhex(COMPOSE_HASH)), - ("key-provider", LIVE_KEY_PROVIDER_JSON), - ] - ) - # Precomputed score record via plan helper so emission validates. - from agent_challenge.evaluation.plan_scoring import build_score_record_from_eval_plan - - record = build_score_record_from_eval_plan(plan, {"task-a": [1.0]}) - # Binding that matches what emit will recompute. - scores_digest = ew.score_record_digest(record) - binding = ew.build_score_binding( - canonical_measurement={ - "mrtd": REGS["mrtd"], - "rtmr0": REGS["rtmr0"], - "rtmr1": REGS["rtmr1"], - "rtmr2": REGS["rtmr2"], - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, - }, - agent_hash=AGENT_HASH, - eval_run_id=plan["eval_run_id"], - score_nonce=plan["score_nonce"], - scores_digest=scores_digest, - task_ids=["task-a"], - ) - report_data = ew.score_report_data_hex(binding) - quote = build_tdx_quote( - mrtd=REGS["mrtd"], - rtmr0=REGS["rtmr0"], - rtmr1=REGS["rtmr1"], - rtmr2=REGS["rtmr2"], - rtmr3=rtmr3, - report_data=report_data, - ) - provider = _QuoteProvider(quote, event_log) - buf = io.StringIO() - line = ar.emit_attested_eval_result_from_plan( - eval_plan=plan, - score_record=record, - rtmr3=rtmr3, - quote_provider=provider, - manifest_sha256="cc" * 32, - stream=buf, - ) - assert line.startswith(ar.RESULT_LINE_PREFIX) - body = line[len(ar.RESULT_LINE_PREFIX) :].encode("utf-8") - # Discriminator vs spaced dumps: default separators insert spaces. - spaced = json.dumps(json.loads(body), ensure_ascii=False, sort_keys=True).encode("utf-8") - assert b", " in spaced or b": " in spaced - assert body != spaced - - validated = ew.validate_eval_result_request(json.loads(body)) - canonical = ew.canonical_json_v1(validated) - assert body == canonical - # Host process_direct_eval_result exact-byte gate. - assert ew.canonical_json_v1(validated) == body - - -def test_schema_v2_emit_stream_output_matches_return_line() -> None: - plan = _plan(key_provider="phala") - event_log, rtmr3 = build_rtmr3_event_log( - [ - ("compose-hash", bytes.fromhex(COMPOSE_HASH)), - ("key-provider", b"phala"), - ] - ) - from agent_challenge.evaluation.plan_scoring import build_score_record_from_eval_plan - - record = build_score_record_from_eval_plan(plan, {"task-a": [1.0]}) - scores_digest = ew.score_record_digest(record) - binding = ew.build_score_binding( - canonical_measurement={ - "mrtd": REGS["mrtd"], - "rtmr0": REGS["rtmr0"], - "rtmr1": REGS["rtmr1"], - "rtmr2": REGS["rtmr2"], - "compose_hash": COMPOSE_HASH, - "os_image_hash": OS_IMAGE_HASH, - }, - agent_hash=AGENT_HASH, - eval_run_id=plan["eval_run_id"], - score_nonce=plan["score_nonce"], - scores_digest=scores_digest, - task_ids=["task-a"], - ) - report_data = ew.score_report_data_hex(binding) - quote = build_tdx_quote( - mrtd=REGS["mrtd"], - rtmr0=REGS["rtmr0"], - rtmr1=REGS["rtmr1"], - rtmr2=REGS["rtmr2"], - rtmr3=rtmr3, - report_data=report_data, - ) - buf = io.StringIO() - line = ar.emit_attested_eval_result_from_plan( - eval_plan=plan, - score_record=record, - rtmr3=rtmr3, - quote_provider=_QuoteProvider(quote, event_log), - manifest_sha256="cc" * 32, - stream=buf, - ) - assert buf.getvalue() == line + "\n" diff --git a/packages/challenges/agent-challenge/tests/test_score_vm_config_project.py b/packages/challenges/agent-challenge/tests/test_score_vm_config_project.py deleted file mode 100644 index 366814879..000000000 --- a/packages/challenges/agent-challenge/tests/test_score_vm_config_project.py +++ /dev/null @@ -1,346 +0,0 @@ -"""Score-path vm_config projection: dstack → exact schema-v2 keys (offline). - -Live residual (image@sha256:a27bdadd / compose 09a44f24): after KR grant, -decrypt_ok, job_done trials=3, emit_start → AttestationEmissionError -'schema-v2 Eval emission is invalid: vm_config has invalid fields' → -phala_attestation_failed. - -Root cause: schema-v2 wire (eval_wire.py:734) requires exact set -{vcpu, memory_mb, os_image_hash}. Score obtain_quote/_coerce_vm_config -forwarded raw dstack vm_config (cpu_count/memory_size/extras). Review already -projects via _normalize_vm_config (cpu_count→vcpu, memory_size bytes→memory_mb, -drop extras). Port that projection into the score emit path. - -Discriminators (would fail a wrong implementation): - * dstack-shaped {cpu_count, memory_size, extras} must pass - validate_eval_phala_attestation after obtain_quote / schema-v2 emit project - * extras filtered so set(vm_config) == {vcpu, memory_mb, os_image_hash} - * memory_size (bytes) maps to memory_mb via integer // 1MiB - * missing vcpu/cpu_count or memory_mb/memory_size fails closed -No Phala create; targeted offline only. -""" - -from __future__ import annotations - -import json -from typing import Any - -import pytest - -from agent_challenge.canonical import attested_result as ar -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.keyrelease.quote import ( - COMPOSE_HASH_EVENT, - KEY_PROVIDER_EVENT, - build_rtmr3_event_log, - build_tdx_quote, -) - -MEASUREMENT = { - "mrtd": "a" * 96, - "rtmr0": "b0" * 48, - "rtmr1": "b1" * 48, - "rtmr2": "b2" * 48, - "compose_hash": "c" * 64, - "os_image_hash": "e" * 64, -} - -# Live-shaped dstack fixture (extras + native field names). -DSTACK_VM_CONFIG_RAW: dict[str, Any] = { - "cpu_count": 2, - "memory_size": 4 * 1024 * 1024 * 1024, # 4 GiB in bytes - "os_image_hash": MEASUREMENT["os_image_hash"], - "qemu_single_pass_add_pages": True, - "num_gpus": 0, - "hugepages": False, -} - - -def _identity_event_log() -> tuple[list[dict[str, Any]], str]: - compose_payload = bytes.fromhex(MEASUREMENT["compose_hash"]) - provider_payload = b'{"name":"phala"}' - bootstrap = bytes.fromhex("11" * 8) - filled, rtmr3 = build_rtmr3_event_log( - [ - ("instance-id", bootstrap), - (COMPOSE_HASH_EVENT, compose_payload), - ("boot-mr-done", bootstrap), - (KEY_PROVIDER_EVENT, provider_payload), - ] - ) - return filled, rtmr3 - - -class _DstackVmConfigQuoteResponse: - """GetQuote response carrying dstack-native vm_config (JSON string).""" - - def __init__( - self, - quote_hex: str, - event_log: list[dict[str, Any]], - vm_config: dict[str, Any], - ) -> None: - self.quote = quote_hex - self.event_log = json.dumps(event_log) - self.vm_config = json.dumps(vm_config) - self.report_data = "" - - -class _Provider: - def __init__(self, response: _DstackVmConfigQuoteResponse) -> None: - self._response = response - self.calls: list[bytes] = [] - - def get_quote(self, report_data: bytes) -> _DstackVmConfigQuoteResponse: - self.calls.append(report_data) - return self._response - - -def test_project_dstack_vm_config_maps_cpu_memory_drops_extras() -> None: - """cpu_count→vcpu, memory_size bytes→memory_mb, extras dropped, int coerce.""" - - projected = ar._project_eval_vm_config( - DSTACK_VM_CONFIG_RAW, - os_image_hash=MEASUREMENT["os_image_hash"], - ) - assert set(projected) == {"vcpu", "memory_mb", "os_image_hash"} - assert projected["vcpu"] == 2 - assert projected["memory_mb"] == 4096 - assert projected["os_image_hash"] == MEASUREMENT["os_image_hash"] - assert isinstance(projected["vcpu"], int) - assert isinstance(projected["memory_mb"], int) - - -def test_project_dstack_vm_config_accepts_schema_keys_unchanged() -> None: - """Already-schema keys stay as-is (int-coerced) with extras dropped.""" - - projected = ar._project_eval_vm_config( - { - "vcpu": "1", - "memory_mb": "2048", - "os_image_hash": MEASUREMENT["os_image_hash"], - "stray": True, - }, - os_image_hash=MEASUREMENT["os_image_hash"], - ) - assert projected == { - "vcpu": 1, - "memory_mb": 2048, - "os_image_hash": MEASUREMENT["os_image_hash"], - } - - -def test_project_fills_os_image_hash_from_measurement_when_raw_omits() -> None: - """os_image_hash may come from measurement when absente from dstack surface.""" - - projected = ar._project_eval_vm_config( - {"cpu_count": 1, "memory_size": 2 * 1024 * 1024 * 1024}, - os_image_hash=MEASUREMENT["os_image_hash"], - ) - assert projected["os_image_hash"] == MEASUREMENT["os_image_hash"] - assert projected["vcpu"] == 1 - assert projected["memory_mb"] == 2048 - - -def test_project_missing_vcpu_fails_closed() -> None: - """Neither vcpu nor cpu_count → fail closed (no silent default).""" - - with pytest.raises(ar.AttestationEmissionError, match="vcpu"): - ar._project_eval_vm_config( - { - "memory_size": 2 * 1024 * 1024 * 1024, - "os_image_hash": MEASUREMENT["os_image_hash"], - }, - os_image_hash=MEASUREMENT["os_image_hash"], - ) - - -def test_project_missing_memory_fails_closed() -> None: - """Neither memory_mb nor memory_size → fail closed (no silent default).""" - - with pytest.raises(ar.AttestationEmissionError, match="memory"): - ar._project_eval_vm_config( - {"cpu_count": 1, "os_image_hash": MEASUREMENT["os_image_hash"]}, - os_image_hash=MEASUREMENT["os_image_hash"], - ) - - -def test_project_invalid_memory_size_fails_closed() -> None: - with pytest.raises(ar.AttestationEmissionError, match="memory"): - ar._project_eval_vm_config( - { - "cpu_count": 1, - "memory_size": 0, - "os_image_hash": MEASUREMENT["os_image_hash"], - }, - os_image_hash=MEASUREMENT["os_image_hash"], - ) - - -def test_project_non_object_fails_closed() -> None: - with pytest.raises(ar.AttestationEmissionError, match="object"): - ar._project_eval_vm_config( - ["not", "an", "object"], - os_image_hash=MEASUREMENT["os_image_hash"], - ) - - -def test_raw_dstack_vm_config_fails_wire_without_projection() -> None: - """Discriminator: unprojected dstack shape is rejected by schema-v2 wire.""" - - events, rtmr3 = _identity_event_log() - with pytest.raises(ew.EvalWireError, match="vm_config has invalid fields"): - ew.validate_eval_phala_attestation( - { - "tdx_quote": "ab" * 600, - "event_log": events, - "report_data": "00" * 64, - "measurement": {**MEASUREMENT, "rtmr3": rtmr3}, - "vm_config": dict(DSTACK_VM_CONFIG_RAW), - } - ) - - -def test_obtain_quote_projects_dstack_vm_config() -> None: - """Score-path obtain_quote projects dstack vm_config to the three schema keys.""" - - events, rtmr3 = _identity_event_log() - quote_hex = build_tdx_quote( - mrtd=MEASUREMENT["mrtd"], - rtmr0=MEASUREMENT["rtmr0"], - rtmr1=MEASUREMENT["rtmr1"], - rtmr2=MEASUREMENT["rtmr2"], - rtmr3=rtmr3, - report_data=b"z" * 32, - ) - provider = _Provider(_DstackVmConfigQuoteResponse(quote_hex, events, DSTACK_VM_CONFIG_RAW)) - result = ar.obtain_quote(provider, b"z" * 32) - assert set(result.vm_config) == {"vcpu", "memory_mb", "os_image_hash"} - assert result.vm_config["vcpu"] == 2 - assert result.vm_config["memory_mb"] == 4096 - assert result.vm_config["os_image_hash"] == MEASUREMENT["os_image_hash"] - - -def test_schema_v2_emit_with_dstack_shaped_quote_vm_config(monkeypatch) -> None: - """schema-v2 emit projects quote.vm_config so wire validate accepts emission.""" - - del monkeypatch # kept for future env-based residual harness - events, expected_rtmr3 = _identity_event_log() - - class _DstackProvider: - def get_quote(self, report_data: bytes) -> _DstackVmConfigQuoteResponse: - quote_hex = build_tdx_quote( - mrtd=MEASUREMENT["mrtd"], - rtmr0=MEASUREMENT["rtmr0"], - rtmr1=MEASUREMENT["rtmr1"], - rtmr2=MEASUREMENT["rtmr2"], - rtmr3=expected_rtmr3, - report_data=report_data, - ) - # Live residual: raw dstack fields + extras, no env override. - return _DstackVmConfigQuoteResponse(quote_hex, events, dict(DSTACK_VM_CONFIG_RAW)) - - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - task_ids = ["task-a", "task-b"] - record = ew.build_canonical_score_record( - eval_run_id="eval-vm-config-project", - policy=policy, - trial_scores_by_task={"task-a": [1.0], "task-b": [0.0]}, - ) - line = ar.emit_attested_benchmark_result( - benchmark_result={ - "status": "completed", - "score": 0.5, - "resolved": 1, - "total": 2, - "reason_code": None, - }, - canonical_measurement=dict(MEASUREMENT), - rtmr3="0" * 96, - agent_hash="f" * 64, - task_ids=task_ids, - scores={}, - quote_provider=_DstackProvider(), - manifest_sha256="1" * 64, - eval_run_id="eval-vm-config-project", - submission_id="submission-vm-config-project", - score_nonce="score-nonce-vm-config", - score_record=record, - image_digest="registry.example/eval@sha256:" + "d" * 64, - # Critical: no schema-shaped env override — force quote.vm_config path. - vm_config=None, - ) - payload = json.loads(line.split("=", 1)[1]) - assert ew.validate_eval_result_request(payload) == payload - vm = payload["execution_proof"]["attestation"]["vm_config"] - assert set(vm) == {"vcpu", "memory_mb", "os_image_hash"} - assert vm["vcpu"] == 2 - assert vm["memory_mb"] == 4096 - assert vm["os_image_hash"] == MEASUREMENT["os_image_hash"] - - -def test_schema_v2_emit_fails_closed_when_vm_config_missing_cpu() -> None: - """Emit fails closed (typed AttestationEmissionError) when projection cannot derive vcpu.""" - - events, expected_rtmr3 = _identity_event_log() - - class _ProviderMissingCpu: - def get_quote(self, report_data: bytes) -> _DstackVmConfigQuoteResponse: - quote_hex = build_tdx_quote( - mrtd=MEASUREMENT["mrtd"], - rtmr0=MEASUREMENT["rtmr0"], - rtmr1=MEASUREMENT["rtmr1"], - rtmr2=MEASUREMENT["rtmr2"], - rtmr3=expected_rtmr3, - report_data=report_data, - ) - return _DstackVmConfigQuoteResponse( - quote_hex, - events, - { - "memory_size": 2 * 1024 * 1024 * 1024, - "os_image_hash": MEASUREMENT["os_image_hash"], - }, - ) - - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - record = ew.build_canonical_score_record( - eval_run_id="eval-vm-config-missing-cpu", - policy=policy, - trial_scores_by_task={"task-a": [1.0]}, - ) - with pytest.raises(ar.AttestationEmissionError, match="(?i)vm_config|vcpu"): - ar.emit_attested_benchmark_result( - benchmark_result={ - "status": "completed", - "score": 1.0, - "resolved": 1, - "total": 1, - "reason_code": None, - }, - canonical_measurement=dict(MEASUREMENT), - rtmr3="0" * 96, - agent_hash="f" * 64, - task_ids=["task-a"], - scores={}, - quote_provider=_ProviderMissingCpu(), - manifest_sha256="1" * 64, - eval_run_id="eval-vm-config-missing-cpu", - submission_id="submission-vm-config-missing-cpu", - score_nonce="score-nonce-missing-cpu", - score_record=record, - image_digest="registry.example/eval@sha256:" + "d" * 64, - vm_config=None, - ) diff --git a/packages/challenges/agent-challenge/tests/test_selfdeploy_cli.py b/packages/challenges/agent-challenge/tests/test_selfdeploy_cli.py deleted file mode 100644 index 672ab3bc9..000000000 --- a/packages/challenges/agent-challenge/tests/test_selfdeploy_cli.py +++ /dev/null @@ -1,89 +0,0 @@ -"""VAL-DEPLOY-001: the miner self-deploy CLI surface exists and self-documents. - -The top-level CLI and every subcommand expose ``--help``; the CLI's subcommands -cover the full documented flow (fetch/prepare, publish/reproduce measurements, -deploy, run/eval, show attested result, teardown) and match the miner docs, with -no undocumented spend-capable subcommand. -""" - -from __future__ import annotations - -import contextlib -import io -import re -from pathlib import Path - -import pytest - -from agent_challenge.selfdeploy import cli - -DOC = Path(__file__).resolve().parents[1] / "docs" / "miner" / "self-deploy.md" - -# The full-flow operations the CLI must cover (architecture §4 C7 / VAL-DEPLOY-001). -REQUIRED_SUBCOMMANDS = { - "prepare", # fetch/prepare canonical image + generated compose - "measurements", # publish/reproduce measurements - "deploy", # deploy the CVM - "run", # run/eval - "result", # show attested result - "teardown", # teardown -} - - -def _documented_subcommands() -> set[str]: - text = DOC.read_text(encoding="utf-8") - return set(re.findall(r"^###\s+`([a-z-]+)`", text, flags=re.MULTILINE)) - - -def _help_text(argv: list[str]) -> tuple[int, str]: - buffer = io.StringIO() - code = 0 - with contextlib.redirect_stdout(buffer): - with pytest.raises(SystemExit) as exc: - cli.main(argv) - code = int(exc.value.code or 0) - return code, buffer.getvalue() - - -def test_top_level_help_lists_every_subcommand(): - code, text = _help_text(["--help"]) - assert code == 0 - for name in cli.SUBCOMMANDS: - assert name in text, name - - -def test_cli_covers_the_full_documented_flow(): - assert REQUIRED_SUBCOMMANDS.issubset(set(cli.SUBCOMMANDS)) - - -def test_every_subcommand_self_documents_via_help(): - for name in cli.SUBCOMMANDS: - code, text = _help_text([name, "--help"]) - assert code == 0, name - # Help is non-trivial: shows usage for the subcommand. - assert name in text - assert "usage:" in text.lower() - - -def test_cli_subcommands_match_the_miner_docs(): - documented = _documented_subcommands() - cli_set = set(cli.SUBCOMMANDS) - # Every documented subcommand exists in the CLI (docs describe only real ones). - assert documented <= cli_set, documented - cli_set - # Every CLI subcommand is documented (no undocumented surface at all). - assert cli_set <= documented, cli_set - documented - - -def test_no_undocumented_spend_capable_subcommand(): - documented = _documented_subcommands() - # Every spend-capable subcommand must be documented. - assert cli.SPEND_CAPABLE_SUBCOMMANDS <= documented - # And the spend-capable set is a subset of the real CLI surface. - assert cli.SPEND_CAPABLE_SUBCOMMANDS <= set(cli.SUBCOMMANDS) - - -def test_missing_subcommand_is_rejected(): - # A bare invocation (no subcommand) is a usage error, not a silent no-op. - with pytest.raises(SystemExit) as exc: - cli.main([]) - assert int(exc.value.code or 0) != 0 diff --git a/packages/challenges/agent-challenge/tests/test_selfdeploy_client_auth_hardening.py b/packages/challenges/agent-challenge/tests/test_selfdeploy_client_auth_hardening.py deleted file mode 100644 index d63b74678..000000000 --- a/packages/challenges/agent-challenge/tests/test_selfdeploy_client_auth_hardening.py +++ /dev/null @@ -1,710 +0,0 @@ -"""VAL-REVIEW-009 / VAL-DEPLOY-026 / VAL-CROSS-030 client-auth executability. - -These are discriminator tests for the self-deploy route client and ordered CLI: - -1. Review retry must accept / send a scoped ``approval_id`` so rejected/escalated - attempts can consume one-use operator approval. -2. Non-auto-sign deploy must accept the documented explicit header identity - (``--signature`` / ``--nonce`` / ``--timestamp``) instead of forcing auto-sign. -3. Auto-sign canonicalization must cover the exact query string on cursor- - bearing history / report / status requests so signatures match the server - ``canonical_request_string`` rules. -4. Status / rejection surfaces remain executable and report only bounded - retained state (no success, score, or secret bytes on rejected status). -""" - -from __future__ import annotations - -import io -import json -from argparse import Namespace -from contextlib import redirect_stderr, redirect_stdout -from types import SimpleNamespace -from unittest.mock import MagicMock -from urllib.request import Request - -import pytest - -from agent_challenge.auth.security import body_sha256, canonical_request_string -from agent_challenge.selfdeploy import cli -from agent_challenge.selfdeploy.client import ( - RouteClientError, - SelfDeployRouteClient, - SignedIdentity, - build_signed_identity, - sign_request_identity, -) - - -class _CapturingOpener: - """Capture outbound Request objects and return a fixed JSON body.""" - - def __init__(self, payload: dict | None = None, status: int = 200) -> None: - self.requests: list[Request] = [] - self.payload = payload if payload is not None else {"ok": True} - self.status = status - - def __call__(self, request: Request, timeout: float = 0.0): # noqa: ARG002 - self.requests.append(request) - - class _Resp: - def __init__(self, body: bytes) -> None: - self._body = body - - def read(self, n: int = -1) -> bytes: # noqa: ARG002 - return self._body - - return _Resp(json.dumps(self.payload).encode()) - - -class _RecordingCapture: - def __init__(self) -> None: - self.payloads: list[object] = [] - - def __call__(self, payload: object) -> None: - self.payloads.append(payload) - - -# --------------------------------------------------------------------------- # -# 1) approval-backed review retry -# --------------------------------------------------------------------------- # - - -def test_review_retry_sends_scoped_approval_id_in_signed_body(): - opener = _CapturingOpener({"assignment": {"assignment_id": "a-2"}, "attempt": 2}) - identity = SignedIdentity( - hotkey="hk", - signature="0xsig", - nonce="n1", - timestamp="1710000000.0", - ) - client = SelfDeployRouteClient( - "https://challenge.example", - identity=identity, - auto_sign=False, - opener=opener, - ) - - payload = client.review_retry( - 7, - "assignment-1", - approval_id="ra-operator-1", - ) - - assert payload["attempt"] == 2 - assert len(opener.requests) == 1 - request = opener.requests[0] - assert request.get_method() == "POST" - assert request.full_url.endswith("/submissions/7/review/retry") - body = json.loads(request.data.decode()) - assert body == { - "expected_assignment_id": "assignment-1", - "approval_id": "ra-operator-1", - } - # Without approval_id the key must be omitted so ordinary retries stay lean. - opener2 = _CapturingOpener({}) - client2 = SelfDeployRouteClient( - "https://challenge.example", - identity=identity, - auto_sign=False, - opener=opener2, - ) - client2.review_retry(7, "assignment-1") - body2 = json.loads(opener2.requests[0].data.decode()) - assert body2 == {"expected_assignment_id": "assignment-1"} - assert "approval_id" not in body2 - - -def test_cli_review_retry_forwards_approval_id(monkeypatch): - fake_client = MagicMock() - fake_client.review_retry.return_value = { - "assignment": {"assignment_id": "a-2"}, - "review_session_token": "MUST-REDACT", - } - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - capture = _RecordingCapture() - monkeypatch.setattr(cli, "_print", capture) - - args = Namespace( - review_command="retry", - submission_id=3, - assignment_id="assignment-1", - approval_id="ra-xyz", - base_url="https://challenge.example", - hotkey="hk", - signature="sig", - nonce="n", - timestamp=None, - auto_sign=False, - ) - assert cli._ordered_review_command(args) == 0 - fake_client.review_retry.assert_called_once_with(3, "assignment-1", approval_id="ra-xyz") - # Capability bytes never leave the CLI surface. - assert "MUST-REDACT" not in json.dumps(capture.payloads[-1]) - - -def test_cli_review_retry_parses_approval_id_flag(): - parser = cli.build_parser() - args = parser.parse_args( - [ - "review", - "retry", - "--base-url", - "https://challenge.example", - "--submission-id", - "9", - "--hotkey", - "hk", - "--signature", - "sig", - "--nonce", - "n1", - "--assignment-id", - "assignment-1", - "--approval-id", - "ra-operator-9", - ] - ) - assert args.review_command == "retry" - assert args.approval_id == "ra-operator-9" - assert args.assignment_id == "assignment-1" - - -# --------------------------------------------------------------------------- # -# 2) Non-auto-sign deploy accepts explicit signature headers -# --------------------------------------------------------------------------- # - - -def test_route_client_accepts_explicit_signed_headers_without_auto_sign(): - opener = _CapturingOpener({"assignment": {"assignment_id": "a-1"}}) - identity = build_signed_identity( - hotkey="miner-hotkey", - signature="0xexplicit", - nonce="nonce-explicit", - timestamp="1710000001.5", - ) - client = SelfDeployRouteClient( - "https://challenge.example", - identity=identity, - auto_sign=False, - opener=opener, - ) - client.review_prepare(1) - headers = {k.lower(): v for k, v in opener.requests[0].header_items()} - assert headers["x-hotkey"] == "miner-hotkey" - assert headers["x-signature"] == "0xexplicit" - assert headers["x-nonce"] == "nonce-explicit" - assert headers["x-timestamp"] == "1710000001.5" - - -def _sample_review_assignment() -> dict[str, object]: - return { - "assignment_core": { - "assignment_id": "assignment-1", - "review_app": { - "image_ref": "registry.example/review@sha256:" + "a" * 64, - "compose_hash": "c" * 64, - "app_identity": "review-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "ab" * 32, - "kms_public_key_sha256": "d" * 64, - "measurement": { - "mrtd": "01" * 48, - "rtmr0": "02" * 48, - "rtmr1": "03" * 48, - "rtmr2": "04" * 48, - "compose_hash": "c" * 64, - "os_image_hash": "05" * 32, - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - }, - "measurement_allowlist": [ - { - "mrtd": "01" * 48, - "rtmr0": "02" * 48, - "rtmr1": "03" * 48, - "rtmr2": "04" * 48, - "compose_hash": "c" * 64, - "os_image_hash": "05" * 32, - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - } - ], - "instance_type": "tdx.small", - }, - } - } - - -def test_review_deploy_accepts_explicit_signature_headers(monkeypatch): - """Without --auto-sign, complete --signature/--nonce must not hard-fail. - - Discriminator: a plausible wrong implementation that forces auto-sign for - every live review deploy rejects this path even when the documented - explicit headers are complete. - """ - fake_client = MagicMock() - assignment = _sample_review_assignment() - # Minimal prepare payload: use dry-run so we don't need Phala/creds. - fake_client.review_prepare.return_value = { - "assignment": assignment, - "review_session_token": "token-never-persist", - } - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - - # Force plan builder / budget to accept a lightweight dry-run path. - from agent_challenge.selfdeploy import review as review_deploy - - class _Plan: - def __init__(self) -> None: - self.instance_type = "tdx.small" - self.assignment = assignment - self.image_ref = assignment["assignment_core"]["review_app"]["image_ref"] # type: ignore[index] - self.compose_hash = assignment["assignment_core"]["review_app"]["compose_hash"] # type: ignore[index] - self.measurement = assignment["assignment_core"]["review_app"]["measurement"] # type: ignore[index] - self.review_session_token = "token-never-persist" - - monkeypatch.setattr(review_deploy, "build_review_deployment_plan", lambda _r: _Plan()) - monkeypatch.setattr(cli.lifecycle, "validate_lifecycle_budget", lambda **_k: None) - monkeypatch.setattr(cli, "_review_allowlist_verdict", lambda _p: "IN-LIST") - capture = _RecordingCapture() - monkeypatch.setattr(cli, "_print", capture) - - args = SimpleNamespace( - review_command="deploy", - submission_id=1, - base_url="https://challenge.example", - hotkey="hk", - signature="0xexplicit-sig", - nonce="explicit-nonce", - timestamp="1710000002", - auto_sign=False, # documented non-auto-sign path - prepare_response=None, - openrouter_key_env="OPENROUTER_API_KEY", - phala_api=None, - review_instance_type="tdx.small", - eval_instance_type="tdx.small", - review_runtime_hours=6.0, - eval_runtime_hours=6.0, - money_cap_usd=20.0, - dry_run=True, - ) - code = cli._ordered_review_command(args) - assert code == 0 - fake_client.review_prepare.assert_called_once_with(1) - assert capture.payloads[-1]["dry_run"] is True - - -def test_review_deploy_live_requires_signable_identity_either_path(): - """Live review deploy must accept either auto-sign OR complete explicit headers. - - Discriminator: old check ``not auto_sign ⇒ hard error`` fails this matrix. - """ - # Missing both auto-sign and signature still fails. - with pytest.raises(RouteClientError, match="signature|auto-sign|nonce"): - cli._route_client( - SimpleNamespace( - base_url="https://challenge.example", - hotkey="hk", - signature=None, - nonce=None, - timestamp=None, - auto_sign=False, - ) - ) - - # Explicit headers alone must produce a non-auto-sign client. - client = cli._route_client( - SimpleNamespace( - base_url="https://challenge.example", - hotkey="hk", - signature="0xsig", - nonce="n1", - timestamp="1", - auto_sign=False, - ) - ) - assert client._auto_sign is False - assert client._identity is not None - assert client._identity.signature == "0xsig" - - # Auto-sign alone still works. - auto_client = cli._route_client( - SimpleNamespace( - base_url="https://challenge.example", - hotkey="hk", - signature=None, - nonce=None, - timestamp=None, - auto_sign=True, - ) - ) - assert auto_client._auto_sign is True - - -def test_live_review_deploy_no_longer_hard_requires_auto_sign(monkeypatch): - """Remove the auto-sign-only gate on live review deploy.""" - fake_client = MagicMock() - assignment_token = "sentinel-review-token-not-to-leak" - assignment = _sample_review_assignment() - fake_client.review_prepare.return_value = { - "assignment": assignment, - "review_session_token": assignment_token, - } - fake_client.review_deployed.return_value = {"phase": "review_cvm_running"} - - from agent_challenge.selfdeploy import review as review_deploy - - class _Plan: - def __init__(self) -> None: - self.instance_type = "tdx.small" - self.assignment = assignment - self.image_ref = assignment["assignment_core"]["review_app"]["image_ref"] # type: ignore[index] - self.compose_hash = "c" * 64 - self.measurement = assignment["assignment_core"]["review_app"]["measurement"] # type: ignore[index] - self.review_session_token = assignment_token - - class _Encrypted: - env_keys = ["OPENROUTER_API_KEY", "REVIEW_API_BASE_URL", "REVIEW_SESSION_TOKEN"] - - class _Deployer: - def deploy(self, plan, encrypted): # noqa: ARG002 - return { - "schema_version": 1, - "assignment_id": "assignment-1", - "cvm_id": "cvm-1", - "phala_create_receipt": { - "request_id": "r1", - "app_id": "a1", - "cvm_id": "cvm-1", - "receipt_sha256": "e" * 64, - "created_at_ms": 1, - }, - "compose_identity": { - "image_ref": plan.image_ref, - "compose_hash": plan.compose_hash, - "app_identity": "review-v1", - }, - } - - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - monkeypatch.setattr(review_deploy, "build_review_deployment_plan", lambda _r: _Plan()) - monkeypatch.setattr( - review_deploy, - "encrypt_review_secrets", - lambda _plan, _values: _Encrypted(), - ) - monkeypatch.setattr( - review_deploy, - "HttpReviewPhalaDeployment", - lambda _client: _Deployer(), - ) - # Avoid real Phala credential / network construction during the offline test. - monkeypatch.setattr(cli, "PhalaCloudClient", lambda **_k: object()) - monkeypatch.setattr(cli.lifecycle, "validate_lifecycle_budget", lambda **_k: None) - monkeypatch.setenv("OPENROUTER_API_KEY", "sk-test-not-to-leak") - capture = _RecordingCapture() - monkeypatch.setattr(cli, "_print", capture) - - args = SimpleNamespace( - review_command="deploy", - submission_id=1, - base_url="https://challenge.example", - hotkey="hk", - signature="0xexplicit", - nonce="n-explicit", - timestamp="1710000003", - auto_sign=False, - prepare_response=None, - openrouter_key_env="OPENROUTER_API_KEY", - phala_api="https://cloud-api.phala.com/api/v1", - review_instance_type="tdx.small", - eval_instance_type="tdx.small", - review_runtime_hours=6.0, - eval_runtime_hours=6.0, - money_cap_usd=20.0, - dry_run=False, - ) - code = cli._ordered_review_command(args) - assert code == 0, "live review deploy with explicit signed headers must succeed" - assert fake_client.review_prepare.called - assert fake_client.review_deployed.called - printed = json.dumps(capture.payloads[-1]) - assert assignment_token not in printed - assert "sk-test-not-to-leak" not in printed - - -# --------------------------------------------------------------------------- # -# 3) Auto-sign canonicalization covers exact query string -# --------------------------------------------------------------------------- # - - -def test_sign_request_identity_includes_exact_query_string(monkeypatch): - class _Keypair: - ss58_address = "hk" - - def sign(self, message: str) -> bytes: - # Return length-stable bytes so the hex encoding is deterministic. - return body_sha256(message.encode()).encode()[:32] - - monkeypatch.setattr( - "agent_challenge.selfdeploy.client._load_signing_keypair", - lambda: _Keypair(), - ) - - query = "cursor=page-2&limit=10" - identity = sign_request_identity( - hotkey="hk", - method="GET", - path="/submissions/1/review/history", - query_string=query, - timestamp="1710000100", - nonce="auto-nonce", - raw_body=b"", - ) - expected_canonical = canonical_request_string( - method="GET", - path="/submissions/1/review/history", - query_string=query, - timestamp="1710000100", - nonce="auto-nonce", - raw_body=b"", - ) - # Discriminator: empty-query signature must not match the real query. - wrong = canonical_request_string( - method="GET", - path="/submissions/1/review/history", - query_string="", - timestamp="1710000100", - nonce="auto-nonce", - raw_body=b"", - ) - assert expected_canonical != wrong - signed_message = body_sha256(expected_canonical.encode()).encode()[:32] - assert identity.signature == "0x" + signed_message.hex() - assert identity.signature != "0x" + body_sha256(wrong.encode()).encode()[:32].hex() - - -@pytest.mark.parametrize( - ("method_name", "path_suffix", "cursor"), - [ - ("review_history", "review/history", "hist-cursor"), - ("review_report", "review/report", "report-cursor"), - ("eval_status", "eval/status", "status-cursor"), - ], -) -def test_auto_sign_query_bearing_requests_sign_exact_cursor( - monkeypatch, - method_name: str, - path_suffix: str, - cursor: str, -): - messages: list[str] = [] - - class _Keypair: - ss58_address = "hk" - - def sign(self, message: str) -> bytes: - messages.append(message) - return b"\x11" * 32 - - monkeypatch.setattr( - "agent_challenge.selfdeploy.client._load_signing_keypair", - lambda: _Keypair(), - ) - # Freeze time so the canonical timestamp is predictable. - monkeypatch.setattr( - "agent_challenge.selfdeploy.client.time.time", - lambda: 1710000200.5, - ) - monkeypatch.setattr( - "agent_challenge.selfdeploy.client.secrets.token_urlsafe", - lambda _n: "fresh-nonce", - ) - - opener = _CapturingOpener({"items": [], "next_cursor": None, "total_count": 0}) - identity = SignedIdentity( - hotkey="hk", - signature="auto", - nonce="auto", - timestamp="unused", - ) - client = SelfDeployRouteClient( - "https://challenge.example", - identity=identity, - auto_sign=True, - opener=opener, - ) - - getattr(client, method_name)(42, cursor=cursor) - - assert len(messages) == 1 - expected_path = f"/submissions/42/{path_suffix}" - expected_query = f"cursor={cursor}" - expected = canonical_request_string( - method="GET", - path=expected_path, - query_string=expected_query, - timestamp="1710000200.5", - nonce="fresh-nonce", - raw_body=b"", - ) - assert messages[0] == expected - # Explicit discriminator against the empty-query residual bug. - empty_query = canonical_request_string( - method="GET", - path=expected_path, - query_string="", - timestamp="1710000200.5", - nonce="fresh-nonce", - raw_body=b"", - ) - assert messages[0] != empty_query - request = opener.requests[0] - assert f"?cursor={cursor}" in request.full_url - headers = {k.lower(): v for k, v in request.header_items()} - assert headers["x-signature"] == "0x" + (b"\x11" * 32).hex() - assert headers["x-nonce"] == "fresh-nonce" - - -# --------------------------------------------------------------------------- # -# 4) Status / rejection paths remain executable, bounded retained state -# --------------------------------------------------------------------------- # - - -def test_cli_eval_status_and_review_history_report_bounded_retained_state(monkeypatch): - forbidden = { - "OPENROUTER_API_KEY": "sk-openrouter-sentinel", - "EVAL_RUN_TOKEN": "eval-run-token-sentinel", - "REVIEW_SESSION_TOKEN": "review-session-token-sentinel", - "golden_plaintext": "GOLDEN-PLAINTEXT-SENTINEL", - "score": 0.91, - } - history = { - "items": [ - { - "assignment_id": "a-1", - "attempt": 1, - "phase": "review_rejected", - "terminal": True, - "retryable": True, - "reason_code": "policy_reject", - "report_available": True, - } - ], - "next_cursor": "c2", - "total_count": 2, - } - status = { - "items": [ - { - "eval_run_id": "eval-1", - "attempt": 1, - "phase": "eval_rejected", - "terminal": True, - "verified": False, - "retryable": False, - "reason_code": "measurement_mismatch", - "key_grant_state": "none", - "key_release_nonce_state": "issued", - "score_nonce_state": "issued", - "result_available": False, - } - ], - "next_cursor": None, - "total_count": 1, - } - fake = MagicMock() - fake.review_history.return_value = { - **history, - **{k: v for k, v in forbidden.items() if k != "score"}, - } - fake.eval_status.return_value = {**status, "EVAL_RUN_TOKEN": forbidden["EVAL_RUN_TOKEN"]} - monkeypatch.setattr(cli, "_route_client", lambda _args: fake) - capture = _RecordingCapture() - monkeypatch.setattr(cli, "_print", capture) - - review_args = Namespace( - review_command="history", - submission_id=1, - cursor="c1", - base_url="https://challenge.example", - hotkey="hk", - signature="sig", - nonce="n", - timestamp=None, - auto_sign=False, - ) - assert cli._ordered_review_command(review_args) == 0 - fake.review_history.assert_called_once_with(1, cursor="c1") - review_out = capture.payloads[-1] - assert review_out["items"][0]["phase"] == "review_rejected" - assert review_out["items"][0]["retryable"] is True - assert review_out["total_count"] == 2 - rendered_review = json.dumps(review_out) - for secret in ( - "sk-openrouter-sentinel", - "eval-run-token-sentinel", - "review-session-token-sentinel", - "GOLDEN-PLAINTEXT-SENTINEL", - ): - assert secret not in rendered_review - - eval_args = Namespace( - eval_command="status", - submission_id=1, - cursor=None, - base_url="https://challenge.example", - hotkey="hk", - signature="sig", - nonce="n", - timestamp=None, - auto_sign=False, - ) - assert cli._ordered_eval_command(eval_args) == 0 - fake.eval_status.assert_called_once_with(1, cursor=None) - eval_out = capture.payloads[-1] - item = eval_out["items"][0] - assert item["phase"] == "eval_rejected" - assert item["verified"] is False - assert item.get("score") is None - assert "success" not in json.dumps(eval_out).lower() or item["verified"] is False - rendered_eval = json.dumps(eval_out) - assert "eval-run-token-sentinel" not in rendered_eval - # Coarse retained identifiers stay visible. - assert item["eval_run_id"] == "eval-1" - assert item["reason_code"] == "measurement_mismatch" - - -def test_cli_rejection_result_path_remains_executable(): - """Keep VAL-DEPLOY-026 surface executable: rejected result entry is callable.""" - # Build a minimal argv parse for top-level ``result`` (legacy surface). - parser = cli.build_parser() - # Ensure the top-level result subcommand still exists for rejection paths. - args = parser.parse_args( - [ - "result", - "--from", - "/tmp/does-not-need-to-exist-for-parse", - "--allowlist", - "NOT-IN-LIST", - "--quote-verified", - "false", - "--nonce-state", - "stale", - ] - ) - assert args.command == "result" - assert args.quote_verified == "false" - assert args.nonce_state == "stale" - # Missing required signed fields on the ordered CLI must cleanly exit, not crash. - out = io.StringIO() - err = io.StringIO() - with redirect_stdout(out), redirect_stderr(err): - try: - code = cli.main(["review", "retry"]) # missing required args - except SystemExit as exc: - code = int(exc.code) if exc.code is not None else 1 - assert code != 0 - assert "Traceback" not in err.getvalue() - assert "Traceback" not in out.getvalue() diff --git a/packages/challenges/agent-challenge/tests/test_selfdeploy_deploy.py b/packages/challenges/agent-challenge/tests/test_selfdeploy_deploy.py deleted file mode 100644 index 5e9b50da4..000000000 --- a/packages/challenges/agent-challenge/tests/test_selfdeploy_deploy.py +++ /dev/null @@ -1,276 +0,0 @@ -"""Deploy-path guards + wiring for the miner self-deploy CLI. - -Covers VAL-DEPLOY-002 (fetch/prepare → digest-pinned image + valid compose), -VAL-DEPLOY-006 (missing credentials fail clearly with no spend / no key leak), -VAL-DEPLOY-007 (GPU refused before any provisioning), VAL-DEPLOY-008 (smallest -CPU default + over-cap refusal), VAL-DEPLOY-009 (dry-run surfaces the full plan -with zero CVM-creating calls), and VAL-DEPLOY-010 (the operator key-release -endpoint is wired into the run configuration). All offline; no Phala calls. -""" - -from __future__ import annotations - -import io -import json -from contextlib import redirect_stderr, redirect_stdout - -import pytest -import yaml - -from agent_challenge.keyrelease.client import KEY_RELEASE_URL_ENV -from agent_challenge.selfdeploy import cli -from agent_challenge.selfdeploy.plan import ( - PHALA_API_KEY_ENV, - PrepareError, - build_deploy_plan, - prepare_deployment, -) -from agent_challenge.selfdeploy.shapes import ( - DEFAULT_INSTANCE_TYPE, - SMALLEST_CPU_SHAPES, - GpuRefusedError, - OverCapError, -) - -DIGEST = "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" + ("a" * 64) -URL = "https://validator.example/keyrelease" - - -class SpyDeployer: - """Records deploy calls so tests can assert zero (or exactly one) Phala calls.""" - - def __init__(self) -> None: - self.calls: list[tuple] = [] - - def __call__(self, plan, out_dir): - self.calls.append((plan, out_dir)) - return {"ok": True} - - -def _run_cli(argv, *, deployer=None): - out, err = io.StringIO(), io.StringIO() - with redirect_stdout(out), redirect_stderr(err): - code = cli.main(argv, deployer=deployer) - return code, out.getvalue(), err.getvalue() - - -def _docker_compose(prepared) -> dict: - return yaml.safe_load(prepared.compose["docker_compose_file"]) - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-002: fetch/prepare → digest-pinned image + valid generated compose -# --------------------------------------------------------------------------- # -def test_prepare_pins_image_by_digest_not_floating_tag(): - prepared = prepare_deployment(image=DIGEST, key_release_url=URL) - assert prepared.image == DIGEST - assert "@sha256:" in prepared.compose["docker_compose_file"] - service = _docker_compose(prepared)["services"]["orchestrator"] - assert service["image"] == DIGEST - - -def test_prepare_refuses_floating_tag(): - with pytest.raises(PrepareError): - prepare_deployment( - image="ghcr.io/baseintelligence/agent-challenge-canonical:latest", - key_release_url=URL, - ) - - -def test_prepare_requires_key_release_endpoint(): - with pytest.raises(PrepareError): - prepare_deployment(image=DIGEST, key_release_url="") - - -def test_prepared_compose_mounts_both_sockets(): - prepared = prepare_deployment(image=DIGEST, key_release_url=URL) - volumes = _docker_compose(prepared)["services"]["orchestrator"]["volumes"] - assert any(v.startswith("/var/run/dstack.sock:") for v in volumes) - assert any(v.startswith("/var/run/docker.sock:") for v in volumes) - - -def test_prepared_compose_carries_operator_key_release_endpoint(): - prepared = prepare_deployment(image=DIGEST, key_release_url=URL) - env = prepared.orchestrator_environment() - # The compose carries the operator-supplied endpoint as a single static env, - # under exactly the name the in-CVM backend reads. - kr = [e for e in env if e.startswith(f"{KEY_RELEASE_URL_ENV}")] - assert kr == [f"{KEY_RELEASE_URL_ENV}={URL}"], kr - - -def test_prepare_cli_writes_deployable_compose(tmp_path): - code, out, _ = _run_cli( - ["prepare", "--image", DIGEST, "--key-release-url", URL, "--out", str(tmp_path)] - ) - assert code == 0 - payload = json.loads(out) - compose_path = tmp_path / "app-compose.json" - assert compose_path.is_file() - # The digest and both sockets are present in the written bytes. - written = compose_path.read_text(encoding="utf-8") - assert "@sha256:" in written - assert "/var/run/docker.sock" in written and "/var/run/dstack.sock" in written - assert payload["image"] == DIGEST - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-007: GPU-targeted deploys refused before any provisioning -# --------------------------------------------------------------------------- # -def test_gpu_instance_type_refused_with_zero_deployer_calls(): - spy = SpyDeployer() - code, _out, err = _run_cli( - ["deploy", "--image", DIGEST, "--key-release-url", URL, "--instance-type", "h200.small"], - deployer=spy, - ) - assert code != 0 - assert spy.calls == [] - assert "cpu-only" in err.lower() - - -def test_gpu_os_image_refused_with_zero_deployer_calls(): - spy = SpyDeployer() - code, _out, err = _run_cli( - [ - "deploy", - "--image", - DIGEST, - "--key-release-url", - URL, - "--os-image", - "dstack-nvidia-0.5.10", - ], - deployer=spy, - ) - assert code != 0 - assert spy.calls == [] - assert "cpu-only" in err.lower() - - -def test_cpu_tdx_shape_passes_the_same_validation(): - # A CPU Intel TDX shape builds a plan without raising (no GPU/cap refusal). - plan = build_deploy_plan(image=DIGEST, key_release_url=URL, instance_type="tdx.small") - assert plan.instance_type == "tdx.small" - - -def test_gpu_and_overcap_raise_before_prepare_or_phala(): - with pytest.raises(GpuRefusedError): - build_deploy_plan(image=DIGEST, key_release_url=URL, instance_type="a100.large") - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-008: default smallest CPU shape + over-cap refusal -# --------------------------------------------------------------------------- # -def test_default_shape_is_a_smallest_cpu_shape(): - plan = build_deploy_plan(image=DIGEST, key_release_url=URL) - assert plan.instance_type == DEFAULT_INSTANCE_TYPE - assert plan.instance_type in SMALLEST_CPU_SHAPES - - -def test_over_cap_shape_refused_before_provisioning(): - spy = SpyDeployer() - # Force the projected cost over the cap via a large runtime budget. - code, _out, err = _run_cli( - [ - "deploy", - "--image", - DIGEST, - "--key-release-url", - URL, - "--instance-type", - "tdx.xlarge", - "--max-runtime-hours", - "1000", - ], - deployer=spy, - ) - assert code != 0 - assert spy.calls == [] - assert "cap" in err.lower() - - -def test_over_cap_raises_overcap_error(): - with pytest.raises(OverCapError): - build_deploy_plan( - image=DIGEST, - key_release_url=URL, - instance_type="tdx.xlarge", - money_cap_usd=1.0, - max_runtime_hours=100, - ) - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-006: missing credentials fail clearly, no spend, no key leak -# --------------------------------------------------------------------------- # -def test_missing_credentials_fail_clearly_with_no_deployer_call(monkeypatch): - monkeypatch.delenv(PHALA_API_KEY_ENV, raising=False) - spy = SpyDeployer() - code, _out, err = _run_cli( - ["deploy", "--image", DIGEST, "--key-release-url", URL], - deployer=spy, - ) - assert code != 0 - assert spy.calls == [] # never reached provisioning - assert PHALA_API_KEY_ENV in err # names the missing credential - - -def test_credential_value_is_never_printed(monkeypatch, capsys): - sentinel = "phak_" + "s" * 32 - monkeypatch.setenv(PHALA_API_KEY_ENV, sentinel) - spy = SpyDeployer() - code, out, err = _run_cli( - ["deploy", "--image", DIGEST, "--key-release-url", URL], - deployer=spy, - ) - assert code == 0 - assert len(spy.calls) == 1 # credential present → deploy proceeds - assert sentinel not in out and sentinel not in err # key value never echoed - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-009: dry-run surfaces the full plan with zero CVM-creating calls -# --------------------------------------------------------------------------- # -def test_dry_run_surfaces_full_plan_and_makes_no_deployer_call(): - spy = SpyDeployer() - code, out, _err = _run_cli( - ["deploy", "--image", DIGEST, "--key-release-url", URL, "--dry-run"], - deployer=spy, - ) - assert code == 0 - assert spy.calls == [] # dry-run creates nothing - plan = json.loads(out) - assert plan["dry_run"] is True - # Every plan field a miner needs to review is present. - assert plan["image"] == DIGEST - assert plan["instance_type"] == DEFAULT_INSTANCE_TYPE - assert plan["region"] - assert plan["key_release_url"] == URL - assert "@sha256:" in plan["compose"] - assert plan["compose_hash"] - assert plan["projected_cost_usd"] <= plan["money_cap_usd"] - - -def test_dry_run_needs_no_credentials(monkeypatch): - monkeypatch.delenv(PHALA_API_KEY_ENV, raising=False) - code, out, _err = _run_cli(["deploy", "--image", DIGEST, "--key-release-url", URL, "--dry-run"]) - assert code == 0 - assert json.loads(out)["dry_run"] is True - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-010: the operator key-release endpoint is wired into the run -# --------------------------------------------------------------------------- # -def test_deploy_plan_wires_operator_endpoint_under_backend_env_name(): - plan = build_deploy_plan(image=DIGEST, key_release_url=URL) - # The deploy config carries the operator URL under EXACTLY the env name the - # in-CVM backend reads to reach the key-release endpoint. - assert plan.encrypted_env[KEY_RELEASE_URL_ENV] == URL - - -def test_endpoint_is_not_hardcoded_or_defaulted(): - other = "https://other-validator.example/kr" - plan = build_deploy_plan(image=DIGEST, key_release_url=other) - assert plan.key_release_url == other - assert plan.encrypted_env[KEY_RELEASE_URL_ENV] == other - env = plan.prepared.orchestrator_environment() - assert f"{KEY_RELEASE_URL_ENV}={other}" in env diff --git a/packages/challenges/agent-challenge/tests/test_selfdeploy_docs_accuracy.py b/packages/challenges/agent-challenge/tests/test_selfdeploy_docs_accuracy.py deleted file mode 100644 index 3723c7b79..000000000 --- a/packages/challenges/agent-challenge/tests/test_selfdeploy_docs_accuracy.py +++ /dev/null @@ -1,551 +0,0 @@ -"""Docs-accuracy contract for the M6 miner + validator self-deploy docs. - -Covers VAL-DEPLOY-017 (every documented command/flag/endpoint exists and behaves -as documented -- no documented-but-absent feature), VAL-DEPLOY-018 (zero -"watchtower" phrasing), VAL-DEPLOY-019 (cross-repo links to the base repo are -labeled "available after PR merge"), VAL-DEPLOY-020 (mandatory teardown + -money-cap guidance with valid `phala cvms ...` commands), and VAL-DEPLOY-021 (no -leaked secrets; credentials referenced only as the `PHALA_CLOUD_API_KEY` env var, -never written to a committed file). - -The M6 self-deploy docs under test are the miner CLI doc and the validator -operations doc; both are walked for documented-command accuracy. -""" - -from __future__ import annotations - -import importlib -import re -import shlex -import shutil -import subprocess -from pathlib import Path - -import pytest -from _routing import iter_api_routes - -from agent_challenge.app import app -from agent_challenge.selfdeploy import cli - -_REPO_ROOT = Path(__file__).resolve().parents[1] -MINER_DOC = _REPO_ROOT / "docs" / "miner" / "self-deploy.md" -VALIDATOR_DOC = _REPO_ROOT / "docs" / "validator" / "self-deploy.md" -M6_DOCS = (MINER_DOC, VALIDATOR_DOC) - -SELF_DEPLOY_PREFIX = "python -m agent_challenge.selfdeploy" - - -def _fenced_blocks(text: str) -> list[str]: - return re.findall(r"```(?:[a-zA-Z0-9]*)\n(.*?)```", text, flags=re.DOTALL) - - -def _logical_commands(text: str) -> list[str]: - """Return shell commands from fenced blocks, joining backslash continuations.""" - - commands: list[str] = [] - for block in _fenced_blocks(text): - joined = block.replace("\\\n", " ") - for line in joined.splitlines(): - stripped = line.strip() - if stripped and not stripped.startswith("#"): - commands.append(re.sub(r"\s+", " ", stripped)) - return commands - - -def _self_deploy_commands(text: str) -> list[list[str]]: - out: list[list[str]] = [] - prefix_len = len(shlex.split(SELF_DEPLOY_PREFIX)) - for command in _logical_commands(text): - if not command.startswith(SELF_DEPLOY_PREFIX): - continue - rest = shlex.split(command)[prefix_len:] - # Skip meta/placeholder invocations (usage lines, `--help`). - if not rest or rest[0].startswith(("<", "-")): - continue - out.append(rest) - return out - - -# --------------------------------------------------------------------------- # -# The docs exist. -# --------------------------------------------------------------------------- # -def test_m6_docs_exist(): - for doc in M6_DOCS: - assert doc.is_file(), doc - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-017: every documented self-deploy command/flag is real. -# --------------------------------------------------------------------------- # -def test_documented_self_deploy_commands_parse_against_the_real_cli(): - parser = cli.build_parser() - seen_subcommands: set[str] = set() - total = 0 - for doc in M6_DOCS: - for argv in _self_deploy_commands(doc.read_text(encoding="utf-8")): - total += 1 - assert argv, "empty self-deploy invocation in the docs" - seen_subcommands.add(argv[0]) - # parse_args validates the subcommand + every flag; an unknown flag - # or subcommand raises SystemExit (documented-but-absent feature). - namespace = parser.parse_args(argv) - assert namespace.command == argv[0] - assert total > 0, "no documented self-deploy commands found" - # Every documented subcommand is a real CLI subcommand. - assert seen_subcommands <= (set(cli.SUBCOMMANDS) | set(cli.ORDERED_SUBCOMMANDS)), ( - seen_subcommands - (set(cli.SUBCOMMANDS) | set(cli.ORDERED_SUBCOMMANDS)) - ) - - -def test_ordered_review_and_eval_stages_are_documented_and_parseable(): - parser = cli.build_parser() - expected = { - ("review", "prepare"), - ("review", "deploy"), - ("review", "deployed"), - ("review", "result"), - ("review", "history"), - ("review", "cancel"), - ("review", "retry"), - ("review", "teardown"), - ("eval", "prepare"), - ("eval", "deploy"), - ("eval", "result"), - ("eval", "status"), - ("eval", "cancel"), - ("eval", "retry"), - ("eval", "failure"), - ("eval", "teardown"), - } - observed: set[tuple[str, str]] = set() - for doc in M6_DOCS: - text = doc.read_text(encoding="utf-8") - for argv in _self_deploy_commands(text): - if len(argv) >= 2 and argv[0] in cli.ORDERED_SUBCOMMANDS: - parser.parse_args(argv) - observed.add((argv[0], argv[1])) - assert observed == expected - - -def test_documented_module_entrypoints_exist(): - # The validator doc references module entrypoints; each must be importable - # and expose a main() (a documented-but-absent module would fail here). - referenced = set() - for doc in M6_DOCS: - text = doc.read_text(encoding="utf-8") - referenced.update(re.findall(r"python -m ([a-zA-Z0-9_.]+)", text)) - # Drop the selfdeploy package invocation (its subcommands are validated above). - module_targets = {m for m in referenced if m != "agent_challenge.selfdeploy"} - for module_name in module_targets: - module = importlib.import_module(module_name) - assert hasattr(module, "main"), module_name - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-018: no "watchtower" phrasing anywhere in the M6 docs. -# --------------------------------------------------------------------------- # -def test_no_watchtower_phrasing(): - for doc in M6_DOCS: - assert "watchtower" not in doc.read_text(encoding="utf-8").lower(), doc - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-019: cross-repo (base) links carry an "available after PR merge" tag. -# --------------------------------------------------------------------------- # -def test_cross_repo_base_links_are_labeled_available_after_pr_merge(): - # Any reference to the base validator/master repo (NOT the published - # `baseagent` template) must be labeled as not-yet-merged. - base_ref = re.compile(r"BaseIntelligence/base(?![a-zA-Z])") - label = "available after pr merge" - found_any = False - for doc in M6_DOCS: - for raw_line in doc.read_text(encoding="utf-8").splitlines(): - if base_ref.search(raw_line): - found_any = True - assert label in raw_line.lower(), raw_line - assert found_any, "expected at least one labeled cross-repo base reference in the M6 docs" - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-020: teardown + money-cap guidance with valid phala commands. -# --------------------------------------------------------------------------- # -def test_teardown_and_cap_guidance_present_with_valid_commands(): - miner = MINER_DOC.read_text(encoding="utf-8") - validator = VALIDATOR_DOC.read_text(encoding="utf-8") - for doc_text in (miner, validator): - # Mandatory teardown commands are shown verbatim. - assert "phala cvms list" in doc_text - assert "phala cvms delete -f" in doc_text - # Money cap is stated. - assert "$20" in doc_text - # total: 0 teardown-confirmation guidance is present. - assert "total: 0" in doc_text - - -def test_documented_teardown_command_matches_the_cli_implementation(): - # Teardown must use the HTTP client DELETE path — never a phala binary. - import inspect - - source = inspect.getsource(cli.default_phala_teardown) - assert "delete_cvm" in source - assert "subprocess" not in source - assert '"phala", "cvms", "delete"' not in source - client_source = inspect.getsource(cli.PhalaCloudClient.delete_cvm) - assert "DELETE" in client_source - assert "/cvms/" in client_source - - -def test_documented_phala_commands_are_valid_when_cli_available(): - if shutil.which("phala") is None: - pytest.skip("phala CLI not installed") - help_text = subprocess.run( - ["phala", "cvms", "--help"], capture_output=True, text=True, timeout=30 - ).stdout.lower() - assert "delete" in help_text - assert "list" in help_text - delete_help = subprocess.run( - ["phala", "cvms", "delete", "--help"], capture_output=True, text=True, timeout=30 - ).stdout.lower() - assert "-f" in delete_help or "--force" in delete_help - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-021: no leaked secrets; env-var-only credential handling. -# --------------------------------------------------------------------------- # -def test_docs_leak_no_secrets(): - secret_shapes = [ - re.compile(r"phak_[A-Za-z0-9]{16,}"), # a real Phala key - re.compile(r"\bsk-[A-Za-z0-9]{16,}"), # a provider secret key - ] - for doc in M6_DOCS: - text = doc.read_text(encoding="utf-8") - for pattern in secret_shapes: - assert pattern.search(text) is None, (doc, pattern.pattern) - - -def test_credentials_referenced_as_env_var_only(): - # The credential is referenced as the env-var name, never assigned a literal - # secret value in the docs (env-var-only handling). - assignment = re.compile(r"PHALA_CLOUD_API_KEY\s*=\s*(\S+)") - referenced = False - for doc in M6_DOCS: - text = doc.read_text(encoding="utf-8") - if "PHALA_CLOUD_API_KEY" in text: - referenced = True - for match in assignment.finditer(text): - value = match.group(1).strip("\"'`") - # An assignment, if shown at all, must not carry a real key value. - assert not value.startswith(("phak_", "sk-")), value - assert referenced, "the Phala credential env var must be referenced in the docs" - - -def test_ordered_routes_match_the_implemented_route_table_and_no_aliases_are_claimed(): - docs = "\n".join(doc.read_text(encoding="utf-8") for doc in M6_DOCS) - expected = { - ("POST", "/submissions/{submission_id}/review/prepare"), - ("POST", "/submissions/{submission_id}/review/retry"), - ("POST", "/submissions/{submission_id}/review/cancel"), - ("POST", "/submissions/{submission_id}/review/deployed"), - ("GET", "/submissions/{submission_id}/review/history"), - ("GET", "/submissions/{submission_id}/review/report"), - ("POST", "/submissions/{submission_id}/eval/prepare"), - ("POST", "/submissions/{submission_id}/eval/retry"), - ("POST", "/submissions/{submission_id}/eval/cancel"), - ("POST", "/submissions/{submission_id}/eval/failure"), - ("GET", "/submissions/{submission_id}/eval/status"), - ("POST", "/evaluation/v1/runs/{eval_run_id}/result"), - } - available = { - (method, route.path) - for route in iter_api_routes(app) - for method in (route.methods or set()) - if method not in {"HEAD", "OPTIONS"} - } - for method, path in expected: - assert f"{method} {path}" in docs - assert (method, path) in available - for forbidden in ( - "/internal/v1/reviews/{session_id}/approvals", - "/internal/v1/reviews/{session_id}/evidence/{object_ref}", - "/review/v1/assignments/{assignment_id}/report", - "/evaluation/v1/runs/{eval_run_id}/result", - ): - if forbidden == "/evaluation/v1/runs/{eval_run_id}/result": - continue - assert f"BASE public alias: {forbidden}" not in docs - assert "/internal/v1/" in docs - assert "BASE-blocked" in docs - - -def test_docs_pin_safe_ordered_observability_and_rejection_contract(): - docs = "\n".join(doc.read_text(encoding="utf-8") for doc in M6_DOCS) - required = ( - "review_queued", - "review_cvm_running", - "review_provider_standby", - "review_verifying", - "review_allowed", - "review_rejected", - "review_escalated", - "review_expired", - "review_cancelled", - "review_error", - "eval_prepared", - "eval_running", - "eval_verifying", - "eval_expired", - "eval_cancelled", - "eval_error", - "eval_rejected", - "eval_accepted", - "attempt", - "retryable", - "reason_code", - "report_available", - "key_grant_state", - "key_release_nonce_state", - "score_nonce_state", - "receipt_id", - "body_sha256", - "result_available", - "next_cursor", - "total_count", - "review_allow_required", - "eval_deploy_failed", - "eval_tunnel_failed", - "eval_key_release_unavailable", - "eval_no_result", - "no score", - "no benchmark work", - ) - for term in required: - assert term in docs, term - forbidden_claims = ( - "exposes unrestricted source", - "returns raw OpenRouter response", - "sends plaintext OPENROUTER_API_KEY", - "returns the OPENROUTER_API_KEY value", - "returns a raw quote", - "exposes the full task list before allow", - ) - for term in forbidden_claims: - assert term.lower() not in docs.lower(), term - assert "only digests" in docs.lower() - assert "redacted" in docs.lower() - - -def test_docs_pin_encrypted_env_boundaries_and_raw_tcp_ratls_contract(): - miner = MINER_DOC.read_text(encoding="utf-8") - validator = VALIDATOR_DOC.read_text(encoding="utf-8") - docs = miner + "\n" + validator - for term in ( - "OPENROUTER_API_KEY", - "REVIEW_SESSION_TOKEN", - "EVAL_RUN_TOKEN", - # Named as forbidden residual (not required eval secrets). - "BASE_GATEWAY_TOKEN", - "BASE_LLM_GATEWAY_URL", - "encrypted_env", - "env_keys", - "POST /cvms/provision", - "POST /cvms", - "KEY_RELEASE_RA_TLS_HOST", - "KEY_RELEASE_RA_TLS_PORT", - "KEY_RELEASE_RA_TLS_CERT_FILE", - "KEY_RELEASE_RA_TLS_KEY_FILE", - "KEY_RELEASE_RA_TLS_CA_FILE", - "8701", - "TLS 1.3", - "4-byte big-endian", - "schema_version", - "quote_hex", - "event_log", - "no HTTP status framing", - ): - assert term in docs, term - # Must not still document gateway flags as required eval deploy options. - assert "--gateway-token-env BASE_GATEWAY_TOKEN" not in docs - assert "--gateway-url-env BASE_LLM_GATEWAY_URL" not in docs - assert "OPENROUTER_API_KEY" in miner - assert "REVIEW_SESSION_TOKEN" in miner - assert "EVAL_RUN_TOKEN" in miner - assert "OPENROUTER_API_KEY" in validator - assert "eval CVM" in validator - assert "encrypted_env" in validator - - -def test_docs_secret_scan_rejects_values_and_unrestricted_evidence(): - secret_shapes = ( - re.compile(r"(?:PHALA_CLOUD_API_KEY|OPENROUTER_API_KEY)\s*=\s*(?!<)[^`\s]+"), - re.compile(r"\b(?:phak_|sk-)[A-Za-z0-9_-]{16,}"), - re.compile(r"Bearer\s+(?!<)[A-Za-z0-9._~+/=-]{20,}"), - ) - for doc in M6_DOCS: - text = doc.read_text(encoding="utf-8") - for pattern in secret_shapes: - assert pattern.search(text) is None, (doc, pattern.pattern) - assert "never" in text.lower() - assert "secret" in text.lower() - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-017 residual: approval retry, explicit signing, cursor signing, -# RA-TLS setup, teardown, and failure behavior match the hardened CLI. -# These modules are discriminators: a docs revision that omitted any of the -# post-harden flags or behavior would fail even if CLI-parse tests still pass. -# --------------------------------------------------------------------------- # -def _first_matching_selfdeploy(doc_text: str, *must_contain: str) -> list[str]: - """Return argv for the first documented command containing every token.""" - - parser = cli.build_parser() - for argv in _self_deploy_commands(doc_text): - cont = set(argv) - if all(token in cont for token in must_contain): - namespace = parser.parse_args(argv) - assert namespace.command == argv[0] - return argv - raise AssertionError(f"no documented command containing {must_contain!r}") - - -def test_docs_include_approval_backed_retry_example_with_required_args(): - """Approval-backed review retry docs must show --approval-id and full identity.""" - - miner = MINER_DOC.read_text(encoding="utf-8") - assert "--approval-id" in miner - assert "approval_id" in miner - # The documented approval-backed form must be a complete, parseable example. - argv = _first_matching_selfdeploy( - miner, - "review", - "retry", - "--approval-id", - "--assignment-id", - "--base-url", - "--submission-id", - "--hotkey", - ) - # Signing path is either --auto-sign or the explicit header pair. - has_auto = "--auto-sign" in argv - has_explicit = "--signature" in argv and "--nonce" in argv - assert has_auto or has_explicit, argv - # surroundings also describe ordinary retries (omit approval_id) so readers - # do not believe the flag is always required. - assert "omit" in miner.lower() or "optional" in miner.lower() - assert "reject" in miner.lower() and "escalate" in miner.lower() - - -def test_docs_include_explicit_signature_examples_for_deploy_and_signed_routes(): - """Non-auto-sign deploy / prepare examples must include every required arg.""" - - miner = MINER_DOC.read_text(encoding="utf-8") - parser = cli.build_parser() - - # review prepare already uses explicit signature headers; they must remain - # complete (signature + nonce; timestamp optional but documented). - prepare_argv = _first_matching_selfdeploy( - miner, - "review", - "prepare", - "--signature", - "--nonce", - "--timestamp", - "--hotkey", - "--base-url", - "--submission-id", - ) - assert prepare_argv # for type checkers / for clarity - - # At least one live review deploy example must use the explicit-header path - # (no --auto-sign) so the docs no longer look like auto-sign is forced. - explicit_deploys = [ - argv - for argv in _self_deploy_commands(miner) - if argv[:2] == ["review", "deploy"] - and "--signature" in argv - and "--nonce" in argv - and "--auto-sign" not in argv - ] - assert explicit_deploys, "docs must show non-auto-sign review deploy with explicit headers" - for argv in explicit_deploys: - ns = parser.parse_args(argv) - assert ns.command == "review" - assert ns.review_command == "deploy" - assert ns.signature is not None and ns.nonce is not None - assert not ns.auto_sign - - # Wording must document that either path is accepted. - assert "--signature" in miner and "--nonce" in miner - assert "auto-sign" in miner.lower() - assert "explicit" in miner.lower() - - -def test_docs_document_cursor_query_signing_behavior(): - """Cursor-bearing commands must show --cursor and note exact query signing.""" - - miner = MINER_DOC.read_text(encoding="utf-8") - parser = cli.build_parser() - - cursor_commands = { - ("review", "history"): False, - ("review", "result"): False, - ("eval", "status"): False, - } - for argv in _self_deploy_commands(miner): - key = (argv[0], argv[1]) if len(argv) >= 2 else None - if key in cursor_commands and "--cursor" in argv: - parser.parse_args(argv) - cursor_commands[key] = True - missing = [k for k, ok in cursor_commands.items() if not ok] - assert not missing, f"cursor examples missing for {missing}" - - # Exact query-string signing (history/report/status) is the hardened rule. - lowered = miner.lower() - assert "query string" in lowered or "exact query" in lowered - assert "cursor" in lowered - # Server seal wording must not claim body-only (pre-hardening) signing. - assert "canonical_request_string" in miner or "canonical request" in lowered - - -def test_docs_describe_ratls_setup_failure_and_teardown_behavior(): - """Docs must pin RA-TLS client envs, post-create teardown, and failure exits.""" - - miner = MINER_DOC.read_text(encoding="utf-8") - validator = VALIDATOR_DOC.read_text(encoding="utf-8") - docs = miner + "\n" + validator - lowered = docs.lower() - - # Client-side mTLS credential paths used by the measured eval compose. - for env_name in ( - "CHALLENGE_PHALA_RA_TLS_CERT_FILE", - "CHALLENGE_PHALA_RA_TLS_KEY_FILE", - "CHALLENGE_PHALA_RA_TLS_CA_FILE", - "KEY_RELEASE_RA_TLS_HOST", - "KEY_RELEASE_RA_TLS_PORT", - ): - assert env_name in docs, env_name - - # Production does not fall back to HTTP /release for the live path. - assert "no http status framing" in lowered - assert "http `/release` is not the production transport" in lowered or ( - "not the production transport" in lowered and "/release" in docs - ) - - # Post-create failure deletes the attributable CVM; teardown is fail-closed. - assert "attributable" in lowered - assert "post-create" in lowered or "after create" in lowered - assert "non-zero" in lowered or "nonzero" in lowered or "non zero" in lowered - assert "bounded" in lowered and "diagnostic" in lowered - - # Acceptance remains a conjunction (no "quoted-only" claim). - for term in ("binding", "quote", "measurement", "nonce", "key-grant"): - assert term in lowered, term - - # Dry-run never fabricates IN-LIST membership. - assert "unknown" in lowered - assert "in-list" in lowered or "allowlist" in lowered - - # Documented teardown commands continue to parse and use delete -f. - tear_argv = _first_matching_selfdeploy(miner, "teardown", "--cvm-id") - assert tear_argv[0] in {"teardown", "review", "eval"} - assert "phala cvms delete -f" in miner - assert "phala cvms delete -f" in validator diff --git a/packages/challenges/agent-challenge/tests/test_selfdeploy_live_registry_wiring.py b/packages/challenges/agent-challenge/tests/test_selfdeploy_live_registry_wiring.py deleted file mode 100644 index 989495fc3..000000000 --- a/packages/challenges/agent-challenge/tests/test_selfdeploy_live_registry_wiring.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Compose + deploy-plan wiring for the live-registry orchestrator/task refs. - -* The pullable canonical digest threads through - ``prepare_deployment``/``build_deploy_plan``/``generate_app_compose`` and the - compose pins it (assert_digest_pinned passes; no bare tag). -* Optionally the compose points the in-CVM orchestrator at the live-registry side - manifest via the ``CHALLENGE_OWN_RUNNER_LIVE_REGISTRY`` static env; with no live - manifest configured the compose is byte-identical (compose-hash stable). -""" - -from __future__ import annotations - -import yaml - -from agent_challenge.canonical import compose as c -from agent_challenge.canonical import live_registry as lr -from agent_challenge.selfdeploy import plan as p - -PULLABLE = "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" + ("a" * 64) -KEY_URL = "https://validator.example/key-release" - - -def _orchestrator_env(compose) -> list[str]: - parsed = yaml.safe_load(compose["docker_compose_file"]) - return parsed["services"][c.ORCHESTRATOR_SERVICE]["environment"] - - -# --------------------------------------------------------------------------- # -# compose: default byte-identical; live env only when configured -# --------------------------------------------------------------------------- # -def test_compose_default_has_no_live_registry_env(): - compose = c.generate_app_compose(orchestrator_image=PULLABLE) - assert not any(e.startswith(f"{lr.LIVE_REGISTRY_ENV}=") for e in _orchestrator_env(compose)) - - -def test_compose_default_is_byte_identical_to_before(): - # Adding the optional param must not perturb the default compose bytes/hash. - base = c.generate_app_compose(orchestrator_image=PULLABLE) - also = c.generate_app_compose(orchestrator_image=PULLABLE, live_registry_manifest_path=None) - assert c.render_app_compose(base) == c.render_app_compose(also) - assert c.app_compose_hash(base) == c.app_compose_hash(also) - - -def test_compose_live_registry_env_present_when_configured(): - manifest_path = "/opt/agent-challenge/golden/live-registry-refs.json" - compose = c.generate_app_compose( - orchestrator_image=PULLABLE, live_registry_manifest_path=manifest_path - ) - env = _orchestrator_env(compose) - assert f"{lr.LIVE_REGISTRY_ENV}={manifest_path}" in env - # Different from the default compose-hash (a distinct, still-deterministic run). - assert c.app_compose_hash(compose) != c.app_compose_hash( - c.generate_app_compose(orchestrator_image=PULLABLE) - ) - - -def test_compose_with_live_registry_is_deterministic(): - manifest_path = "/opt/agent-challenge/golden/live-registry-refs.json" - a = c.generate_app_compose( - orchestrator_image=PULLABLE, live_registry_manifest_path=manifest_path - ) - b = c.generate_app_compose( - orchestrator_image=PULLABLE, live_registry_manifest_path=manifest_path - ) - assert c.render_app_compose(a) == c.render_app_compose(b) - # render == normalize verbatim invariant preserved. - from agent_challenge.canonical import measurement as m - - assert c.render_app_compose(a) == m.normalize_app_compose(a) - - -# --------------------------------------------------------------------------- # -# plan: thread the pullable orchestrator digest + resolve recorded image -# --------------------------------------------------------------------------- # -def test_prepare_deployment_pins_pushed_digest(): - prepared = p.prepare_deployment(image=PULLABLE, key_release_url=KEY_URL) - assert prepared.image == PULLABLE - assert c.assert_digest_pinned(prepared.image) == PULLABLE - assert PULLABLE in prepared.compose_text - - -def test_resolve_orchestrator_image_prefers_explicit(): - assert p.resolve_orchestrator_image(image=PULLABLE) == PULLABLE - - -def test_resolve_orchestrator_image_from_recorded_manifest(): - reg_path = lr.DEFAULT_LIVE_REGISTRY_PATH - resolved = p.resolve_orchestrator_image(image=None, live_registry_path=reg_path) - reg = lr.load_live_registry(reg_path) - assert resolved == reg.orchestrator_image - assert c.assert_digest_pinned(resolved) == resolved - - -def test_resolve_orchestrator_image_requires_a_source(): - import pytest - - with pytest.raises(p.PrepareError): - p.resolve_orchestrator_image(image=None, live_registry_path=None) - - -def test_build_deploy_plan_pins_pushed_digest_and_live_env(): - plan = p.build_deploy_plan( - image=PULLABLE, - key_release_url=KEY_URL, - live_registry_manifest_path="/opt/agent-challenge/golden/live-registry-refs.json", - ) - assert plan.image == PULLABLE - env = _orchestrator_env(plan.prepared.compose) - assert any(e.startswith(f"{lr.LIVE_REGISTRY_ENV}=") for e in env) diff --git a/packages/challenges/agent-challenge/tests/test_selfdeploy_measurements.py b/packages/challenges/agent-challenge/tests/test_selfdeploy_measurements.py deleted file mode 100644 index 334133c66..000000000 --- a/packages/challenges/agent-challenge/tests/test_selfdeploy_measurements.py +++ /dev/null @@ -1,200 +0,0 @@ -"""Measurement reproduction + allowlist verdict for the miner self-deploy CLI. - -Covers VAL-DEPLOY-003 (measurement publish/reproduce is deterministic), -VAL-DEPLOY-004 (miner and validator agree on the canonical measurement set), and -VAL-DEPLOY-012 (the CLI reports a measurement and a correct allowlist verdict). - -``dstack-mr`` needs the multi-hundred-MB dstack OS image files (only feasible -live), so these offline tests drive the wrapper via a faithful stub binary that -derives deterministic registers from its inputs (matching the M1 measurement-tool -tests); real-tool equivalence is a live (M6) concern. -""" - -from __future__ import annotations - -import json -import stat -import sys -from pathlib import Path - -import pytest - -from agent_challenge.selfdeploy import measurements as measure - -_STUB = """#!{python} -import hashlib, json, sys -args = sys.argv[1:] -opts = {{}} -i = 0 -while i < len(args): - tok = args[i] - if tok == "-json": - opts["json"] = True; i += 1 - elif tok.startswith("-"): - opts[tok[1:]] = args[i + 1] if i + 1 < len(args) else ""; i += 2 - else: - i += 1 -meta = open(opts.get("metadata", ""), "rb").read() -seed = meta + opts.get("cpu", "").encode() + opts.get("memory", "").encode() -reg = lambda t: hashlib.sha384(t + seed).hexdigest() -img = lambda t: hashlib.sha256(t + seed).hexdigest() -print(json.dumps({{"mrtd": reg(b"mrtd"), "rtmr0": reg(b"rtmr0"), "rtmr1": reg(b"rtmr1"), - "rtmr2": reg(b"rtmr2"), "mr_image": img(b"img")}})) -""" - - -@pytest.fixture -def stub_mr(tmp_path) -> str: - path = tmp_path / "dstack-mr-stub" - path.write_text(_STUB.format(python=sys.executable)) - path.chmod(path.stat().st_mode | stat.S_IEXEC | stat.S_IRUSR) - return str(path) - - -@pytest.fixture -def metadata(tmp_path) -> Path: - meta = tmp_path / "metadata.json" - meta.write_text(json.dumps({"bios": "ovmf.fd", "kernel": "bzImage"})) - return meta - - -@pytest.fixture -def compose() -> dict: - return { - "manifest_version": 2, - "name": "agent-challenge-canonical", - "runner": "docker-compose", - "docker_compose_file": "services:\n orchestrator:\n image: repo@sha256:" - + ("a" * 64) - + "\n", - "allowed_envs": ["BASE_GATEWAY_TOKEN"], - } - - -CANONICAL_FIELDS = ("mrtd", "rtmr0", "rtmr1", "rtmr2", "compose_hash", "os_image_hash") - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-003: measurement publish/reproduce is deterministic -# --------------------------------------------------------------------------- # -def test_reproduce_is_byte_identical_across_runs(stub_mr, metadata, compose): - first = measure.reproduce_measurement( - metadata_path=metadata, cpu=1, memory="2G", compose=compose, dstack_mr_bin=stub_mr - ) - second = measure.reproduce_measurement( - metadata_path=metadata, cpu=1, memory="2G", compose=compose, dstack_mr_bin=stub_mr - ) - assert first.to_json() == second.to_json() # byte-identical, empty diff - - -def test_reproduce_covers_every_field_non_empty(stub_mr, metadata, compose): - record = measure.reproduce_measurement( - metadata_path=metadata, cpu=1, memory="2G", compose=compose, dstack_mr_bin=stub_mr - ).as_dict() - for field in CANONICAL_FIELDS: - assert record.get(field), field - - -def test_reproduce_via_cli_is_deterministic(tmp_path, stub_mr, metadata, compose, capsys): - from agent_challenge.selfdeploy import cli - - compose_path = tmp_path / "app-compose.json" - compose_path.write_text(json.dumps(compose)) - argv = [ - "measurements", - "--metadata", - str(metadata), - "--cpu", - "1", - "--memory", - "2G", - "--compose", - str(compose_path), - "--dstack-mr", - stub_mr, - ] - assert cli.main(argv) == 0 - first = capsys.readouterr().out.strip() - assert cli.main(argv) == 0 - second = capsys.readouterr().out.strip() - assert first == second - assert set(json.loads(first)) == set(CANONICAL_FIELDS) - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-004: miner and validator agree on the canonical measurement set -# --------------------------------------------------------------------------- # -def test_miner_record_equals_validator_allowlist_entry(stub_mr, metadata, compose): - # Both sides recompute with the same tooling/inputs → identical record. - miner = measure.reproduce_measurement( - metadata_path=metadata, cpu=1, memory="2G", compose=compose, dstack_mr_bin=stub_mr - ) - validator = measure.reproduce_measurement( - metadata_path=metadata, cpu=1, memory="2G", compose=compose, dstack_mr_bin=stub_mr - ) - # The validator pins the canonical subset (+ its key_provider) into an allowlist. - entry = {**validator.as_dict(), "key_provider": "kms"} - assert measure.measurements_agree(miner.as_dict(), entry) - - -def test_miner_disagrees_when_validator_entry_differs(stub_mr, metadata, compose): - miner = measure.reproduce_measurement( - metadata_path=metadata, cpu=1, memory="2G", compose=compose, dstack_mr_bin=stub_mr - ).as_dict() - tampered = {**miner, "compose_hash": "f" * 64} - assert not measure.measurements_agree(miner, tampered) - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-012: CLI reports a measurement and a correct allowlist verdict -# --------------------------------------------------------------------------- # -def _record(stub_mr, metadata, compose) -> dict: - return measure.reproduce_measurement( - metadata_path=metadata, cpu=1, memory="2G", compose=compose, dstack_mr_bin=stub_mr - ).as_dict() - - -def test_matching_measurement_reports_in_list(stub_mr, metadata, compose): - record = _record(stub_mr, metadata, compose) - allowlist = [{**record, "key_provider": "kms"}] - verdict = measure.allowlist_verdict(record, allowlist) - assert verdict.in_allowlist is True - assert verdict.as_dict()["verdict"] == "IN-LIST" - # The full canonical set is reported. - assert set(verdict.measurement) == set(CANONICAL_FIELDS) - - -def test_single_field_tampered_measurement_reports_not_in_list(stub_mr, metadata, compose): - record = _record(stub_mr, metadata, compose) - allowlist = [{**record, "key_provider": "kms"}] - tampered = {**record, "rtmr0": "0" * 96} - verdict = measure.allowlist_verdict(tampered, allowlist) - assert verdict.in_allowlist is False - assert verdict.as_dict()["verdict"] == "NOT-IN-LIST" - - -def test_empty_allowlist_fails_closed(stub_mr, metadata, compose): - record = _record(stub_mr, metadata, compose) - assert measure.allowlist_verdict(record, []).in_allowlist is False - - -def test_verdict_cli_reports_measurement_and_verdict(tmp_path, stub_mr, metadata, compose, capsys): - from agent_challenge.selfdeploy import cli - - record = _record(stub_mr, metadata, compose) - allow_path = tmp_path / "allowlist.json" - allow_path.write_text(json.dumps([{**record, "key_provider": "kms"}])) - meas_path = tmp_path / "measurement.json" - meas_path.write_text(json.dumps(record)) - - code = cli.main(["verdict", "--measurement", str(meas_path), "--allowlist", str(allow_path)]) - assert code == 0 - payload = json.loads(capsys.readouterr().out) - assert payload["verdict"] == "IN-LIST" - assert set(payload["measurement"]) == set(CANONICAL_FIELDS) - - # A one-field-tampered measurement → NOT-IN-LIST and a non-zero exit. - meas_path.write_text(json.dumps({**record, "mrtd": "1" * 96})) - code = cli.main(["verdict", "--measurement", str(meas_path), "--allowlist", str(allow_path)]) - assert code == 1 - assert json.loads(capsys.readouterr().out)["verdict"] == "NOT-IN-LIST" diff --git a/packages/challenges/agent-challenge/tests/test_selfdeploy_optin.py b/packages/challenges/agent-challenge/tests/test_selfdeploy_optin.py deleted file mode 100644 index 546d8de12..000000000 --- a/packages/challenges/agent-challenge/tests/test_selfdeploy_optin.py +++ /dev/null @@ -1,140 +0,0 @@ -"""VAL-DEPLOY-022: self-deploy is opt-in and never runs automatically. - -With the Phala backend feature flag OFF (the default), no code path auto-triggers -a CVM deploy: the default validator-run eval behavior is unchanged and a legacy -run creates zero CVMs. The self-deploy CLI touches Phala ONLY on an explicit -`deploy`/`teardown` invocation -- every other subcommand (and a dry-run deploy) -makes zero CVM-creating calls. - -Offline discriminators: a SpyDeployer/SpyTeardowner assert zero calls for the -non-provisioning surface, the config default is off, and the evaluation path does -not import the self-deploy deployer (so a legacy run cannot reach it). -""" - -from __future__ import annotations - -import io -from contextlib import redirect_stderr, redirect_stdout -from pathlib import Path - -from agent_challenge.sdk.config import ChallengeSettings -from agent_challenge.selfdeploy import cli -from agent_challenge.selfdeploy.plan import PHALA_API_KEY_ENV - -_REPO_ROOT = Path(__file__).resolve().parents[1] -_EVAL_PKG = _REPO_ROOT / "src" / "agent_challenge" / "evaluation" - -DIGEST = "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" + ("a" * 64) -URL = "https://validator.example/keyrelease" - - -class _Spy: - def __init__(self, result=None) -> None: - self.calls: list[tuple] = [] - self._result = result if result is not None else {"ok": True} - - def __call__(self, *args): - self.calls.append(args) - return self._result - - -def _run_cli(argv, *, deployer=None, teardowner=None): - out, err = io.StringIO(), io.StringIO() - with redirect_stdout(out), redirect_stderr(err): - code = cli.main(argv, deployer=deployer, teardowner=teardowner) - return code, out.getvalue(), err.getvalue() - - -# --------------------------------------------------------------------------- # -# Config default: the Phala path is OFF unless explicitly enabled. -# --------------------------------------------------------------------------- # -def test_config_default_has_phala_path_off(monkeypatch): - monkeypatch.delenv("CHALLENGE_PHALA_ATTESTATION_ENABLED", raising=False) - monkeypatch.delenv("CHALLENGE_TERMINAL_BENCH_EXECUTION_BACKEND", raising=False) - settings = ChallengeSettings() - assert settings.phala_attestation_enabled is False - assert settings.terminal_bench_execution_backend == "own_runner" - - -# --------------------------------------------------------------------------- # -# A legacy eval run cannot reach the self-deploy deployer at all. -# --------------------------------------------------------------------------- # -def test_evaluation_path_does_not_import_self_deploy(): - """Evaluation code must not import the self-deploy deployer surface. - - Fixture strings may mention the shipping miner CLI name; only real - ``import`` / ``from ... import`` edges against ``agent_challenge.selfdeploy`` - (or a bare ``import selfdeploy``) are residualed as auto-trigger risk. - """ - - offenders = [] - import re - - # Match active import statements only; ignore comments and string literals that - # mention the miner self-deploy CLI as documentation/fixtures. - import_pat = re.compile( - r"^\s*(?:from\s+agent_challenge\.selfdeploy(?:\.[\w.]+)?\s+import\b" - r"|import\s+agent_challenge\.selfdeploy\b" - r"|import\s+selfdeploy\b)", - re.MULTILINE, - ) - for path in _EVAL_PKG.rglob("*.py"): - text = path.read_text(encoding="utf-8") - if import_pat.search(text): - offenders.append(path.relative_to(_REPO_ROOT)) - assert offenders == [], offenders - - -# --------------------------------------------------------------------------- # -# Non-provisioning subcommands make ZERO CVM-creating / teardown calls. -# --------------------------------------------------------------------------- # -def test_prepare_makes_no_phala_calls(tmp_path): - deployer, teardowner = _Spy(), _Spy() - code, _out, _err = _run_cli( - ["prepare", "--image", DIGEST, "--key-release-url", URL, "--out", str(tmp_path)], - deployer=deployer, - teardowner=teardowner, - ) - assert code == 0 - assert deployer.calls == [] - assert teardowner.calls == [] - - -def test_dry_run_deploy_makes_no_phala_calls(): - deployer, teardowner = _Spy(), _Spy() - code, _out, _err = _run_cli( - ["deploy", "--image", DIGEST, "--key-release-url", URL, "--dry-run"], - deployer=deployer, - teardowner=teardowner, - ) - assert code == 0 - assert deployer.calls == [] - assert teardowner.calls == [] - - -# --------------------------------------------------------------------------- # -# Only an EXPLICIT deploy / teardown reaches Phala. -# --------------------------------------------------------------------------- # -def test_explicit_deploy_is_the_only_thing_that_provisions(monkeypatch): - monkeypatch.setenv(PHALA_API_KEY_ENV, "phak_" + "s" * 32) - deployer, teardowner = _Spy(), _Spy() - code, _out, _err = _run_cli( - ["deploy", "--image", DIGEST, "--key-release-url", URL], - deployer=deployer, - teardowner=teardowner, - ) - assert code == 0 - assert len(deployer.calls) == 1 # exactly one deploy, on explicit invocation - assert teardowner.calls == [] # deploy never tears down - - -def test_explicit_teardown_is_the_only_thing_that_deletes(): - deployer, teardowner = _Spy(), _Spy() - code, _out, _err = _run_cli( - ["teardown", "--cvm-id", "cvm-123"], - deployer=deployer, - teardowner=teardowner, - ) - assert code == 0 - assert teardowner.calls == [("cvm-123",)] - assert deployer.calls == [] # teardown never provisions diff --git a/packages/challenges/agent-challenge/tests/test_selfdeploy_ordered_trust_hardening.py b/packages/challenges/agent-challenge/tests/test_selfdeploy_ordered_trust_hardening.py deleted file mode 100644 index 54cf31dc3..000000000 --- a/packages/challenges/agent-challenge/tests/test_selfdeploy_ordered_trust_hardening.py +++ /dev/null @@ -1,729 +0,0 @@ -"""Discriminators for ordered self-deploy trust/lifecycle hardening. - -These encode the scrutiny remediation contract for -``selfdeploy-ordered-trust-lifecycle-hardening``: - -* dry-run never fabricates allowlist IN-LIST -* acceptance is a conjunction of binding + quote + measurement + nonce + key-grant -* Eval compose provisions raw RA-TLS host/port and client mTLS path envs (no HTTP) -* pre-create budget counts both immutable review and Eval shapes -* post-create failures delete attributable CVMs -* teardown failures return nonzero with bounded diagnostics -""" - -from __future__ import annotations - -import copy -import hashlib -import io -import json -from types import SimpleNamespace -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from agent_challenge.canonical import eval_wire -from agent_challenge.canonical.attested_result import emit_attested_benchmark_result -from agent_challenge.canonical.compose import ( - RA_TLS_HOST_ENV, - RA_TLS_PORT_ENV, - generate_app_compose, - render_app_compose, -) -from agent_challenge.evaluation.own_runner.result_schema import build_benchmark_result -from agent_challenge.keyrelease.client import ( - KEY_RELEASE_TLS_CA_ENV, - KEY_RELEASE_TLS_CERT_ENV, - KEY_RELEASE_TLS_KEY_ENV, - GoldenKeyReleaseClient, - KeyReleaseProtocolError, -) -from agent_challenge.keyrelease.nonce import NonceState -from agent_challenge.review.canonical import canonical_sha256 -from agent_challenge.review.compose import ( - generate_review_app_compose, - review_app_compose_hash, -) -from agent_challenge.review.schemas import ReviewInputConfig, build_review_assignment -from agent_challenge.selfdeploy import cli, lifecycle -from agent_challenge.selfdeploy import eval as eval_deploy -from agent_challenge.selfdeploy import result as result_mod -from agent_challenge.selfdeploy import review as review_deploy -from agent_challenge.selfdeploy.measurements import domain_allowlist_verdict - -REVIEW_IMAGE = "registry.example/review@sha256:" + "a" * 64 -EVAL_IMAGE = "registry.example/eval@sha256:" + "b" * 64 -PUBLIC_KEY = "c" * 64 -MEASUREMENT = { - "mrtd": "01" * 48, - "rtmr0": "02" * 48, - "rtmr1": "03" * 48, - "rtmr2": "04" * 48, - "os_image_hash": "05" * 32, - "key_provider": "validator-kms", - "vm_shape": "tdx-small", -} - - -def _canonical_from_measurement(measurement: dict[str, str], compose_hash: str) -> dict[str, str]: - return { - "mrtd": measurement["mrtd"], - "rtmr0": measurement["rtmr0"], - "rtmr1": measurement["rtmr1"], - "rtmr2": measurement["rtmr2"], - "compose_hash": compose_hash, - "os_image_hash": measurement["os_image_hash"], - } - - -def _review_assignment() -> tuple[dict[str, object], str]: - compose = generate_review_app_compose( - review_image=REVIEW_IMAGE, - app_identity="review-v1", - ) - compose_hash = review_app_compose_hash(compose) - allowlist_entry = _canonical_from_measurement(MEASUREMENT, compose_hash) - config = ReviewInputConfig( - image_ref=REVIEW_IMAGE, - compose_hash=compose_hash, - app_identity="review-v1", - kms_public_key_hex=PUBLIC_KEY, - measurement=MEASUREMENT, - measurement_allowlist=(allowlist_entry,), - measurement_allowlist_sha256=canonical_sha256({"entries": [allowlist_entry]}), - ) - token = "review-token-sentinel" - assignment, _, _ = build_review_assignment( - session_id="session-1", - assignment_id="assignment-1", - attempt=1, - submission_id="1", - artifact={ - "agent_hash": "10" * 32, - "zip_sha256": "20" * 32, - "zip_size_bytes": 1, - "manifest_sha256": "30" * 32, - "manifest_entries_sha256": "40" * 32, - "fetch_path": "/review/v1/assignments/assignment-1/artifact", - }, - rules_snapshot_sha256_value="50" * 32, - rules_revision_id="rules-v1", - review_nonce="nonce-review", - issued_at_ms=1, - expires_at_ms=2, - session_token_sha256=hashlib.sha256(token.encode()).hexdigest(), - config=config, - ) - return assignment, token - - -def _eval_plan(*, include_compose: bool = True) -> dict[str, object]: - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - compose = generate_app_compose( - orchestrator_image=EVAL_IMAGE, - name="eval-v1", - key_release_url="validator.example:8701", - allowed_envs=eval_deploy.EVAL_ALLOWED_ENVS, - ) - compose_hash = hashlib.sha256(render_app_compose(compose).encode()).hexdigest() - plan = { - "schema_version": 1, - "eval_run_id": "eval-1", - "submission_id": "1", - "submission_version": 1, - "authorizing_review_digest": "d" * 64, - "agent_hash": "e" * 64, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "task-1", - "image_ref": "registry.example/task@sha256:" + "f" * 64, - "task_config_sha256": "1" * 64, - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": eval_wire.scoring_policy_digest(policy), - "eval_app": { - "image_ref": EVAL_IMAGE, - "compose_hash": compose_hash, - "app_identity": "eval-v1", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": PUBLIC_KEY, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex(PUBLIC_KEY)).hexdigest(), - "measurement": MEASUREMENT, - }, - "key_release_endpoint": "validator.example:8701", - "result_endpoint": "/evaluation/v1/runs/eval-1/result", - "key_release_nonce": "key-release-nonce", - "score_nonce": "score-nonce", - "run_token_sha256": "3" * 64, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - if not include_compose: - plan["eval_app"]["compose_hash"] = "0" * 64 - return eval_wire.validate_eval_plan(plan) - - -class _RecordingCapture: - def __init__(self) -> None: - self.payloads: list[Any] = [] - - def __call__(self, payload: Any) -> None: - self.payloads.append(payload) - - -# --------------------------------------------------------------------------- # -# Dry-run allowlist verdict: verified or explicit unknown, never fabricated. -# --------------------------------------------------------------------------- # -def test_dry_run_reports_verified_allowlist_or_unknown_never_fabricated(monkeypatch): - assignment, token = _review_assignment() - prepare_response = { - "assignment": assignment, - "review_session_token": token, - } - fake_client = MagicMock() - fake_client.review_prepare.return_value = prepare_response - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - capture = _RecordingCapture() - monkeypatch.setattr(cli, "_print", capture) - - args = SimpleNamespace( - review_command="deploy", - submission_id=1, - base_url="https://challenge.example", - hotkey="hk", - signature="sig", - nonce="n", - timestamp=None, - auto_sign=True, - prepare_response=None, - openrouter_key_env="OPENROUTER_API_KEY", - phala_api=None, - review_instance_type="tdx.small", - eval_instance_type="tdx.small", - review_runtime_hours=6.0, - eval_runtime_hours=6.0, - money_cap_usd=20.0, - dry_run=True, - ) - code = cli._ordered_review_command(args) - assert code == 0 - payload = capture.payloads[-1] - assert payload["dry_run"] is True - # Must not hard-code membership. Valid assignment is actually on allowlist. - assert payload["allowlist_verdict"] in {"IN-LIST", "NOT-IN-LIST", "UNKNOWN"} - assert payload["allowlist_verdict"] == "IN-LIST" - - # One-field measurement mutation must never report fabricated IN-LIST. - mutated = copy.deepcopy(assignment) - mutated["assignment_core"]["review_app"]["measurement"] = { - **MEASUREMENT, - "mrtd": "ff" * 48, - } - # Keep compose hash as-is -> measurement off allowlist. - # Review plan builder rejects diverge, so use the public helper. - measurement = { - "mrtd": "ff" * 48, - "rtmr0": MEASUREMENT["rtmr0"], - "rtmr1": MEASUREMENT["rtmr1"], - "rtmr2": MEASUREMENT["rtmr2"], - "compose_hash": assignment["assignment_core"]["review_app"]["compose_hash"], - "os_image_hash": MEASUREMENT["os_image_hash"], - } - allowlist = assignment["assignment_core"]["review_app"]["measurement_allowlist"] - verdict = domain_allowlist_verdict( - domain="review", - measurement=measurement, - review_allowlist=allowlist, - ) - assert verdict.as_dict()["verdict"] == "NOT-IN-LIST" - - # Eval dry-run without a bindable allowlist must report UNKNOWN, never IN-LIST. - plan = _eval_plan() - token_value = "run-token" - plan["run_token_sha256"] = hashlib.sha256(token_value.encode()).hexdigest() - prepare = { - "schema_version": 1, - "plan": plan, - "plan_sha256": hashlib.sha256(eval_wire.canonical_json_v1(plan)).hexdigest(), - "secret_delivery": {"env_key": "EVAL_RUN_TOKEN", "token": token_value}, - } - eval_client = MagicMock() - eval_client.eval_prepare.return_value = prepare - monkeypatch.setattr(cli, "_route_client", lambda _args: eval_client) - capture = _RecordingCapture() - monkeypatch.setattr(cli, "_print", capture) - eval_args = SimpleNamespace( - eval_command="deploy", - submission_id=1, - base_url="https://challenge.example", - hotkey="hk", - signature="sig", - nonce="n", - timestamp=None, - auto_sign=True, - prepare_response=None, - gateway_token_env="BASE_GATEWAY_TOKEN", - gateway_url_env="BASE_LLM_GATEWAY_URL", - llm_cost_limit_env="LLM_COST_LIMIT", - phala_api=None, - review_instance_type="tdx.small", - eval_instance_type="tdx.small", - review_runtime_hours=6.0, - eval_runtime_hours=6.0, - money_cap_usd=20.0, - dry_run=True, - token_env="EVAL_RUN_TOKEN", - ) - assert cli._ordered_eval_command(eval_args) == 0 - eval_payload = capture.payloads[-1] - assert eval_payload["allowlist_verdict"] in {"IN-LIST", "NOT-IN-LIST", "UNKNOWN"} - # Without a validator-owned eval allowlist in the plan, do not fabricate membership. - assert eval_payload["allowlist_verdict"] == "UNKNOWN" - - -# --------------------------------------------------------------------------- # -# Acceptance conjunction of all signals (binding, quote, measurement, nonce, grant). -# --------------------------------------------------------------------------- # -class _FakeQuote: - quote = "deadbeef" * 16 - event_log = [{"event": "compose-hash", "payload": "c" * 64}] - # schema-v2 Eval keys (or dstack aliases cpu_count/memory_size) - vm_config = {"vcpu": 1, "memory_mb": 2048} - - -class _FakeProvider: - def get_quote(self, report_data): # noqa: ARG002 - return _FakeQuote() - - -def _measurement() -> dict[str, str]: - return { - "mrtd": "a" * 96, - "rtmr0": "a" * 96, - "rtmr1": "a" * 96, - "rtmr2": "a" * 96, - "compose_hash": "b" * 64, - "os_image_hash": "b" * 64, - } - - -def _attested_stdout() -> str: - br = build_benchmark_result( - status="completed", score=0.5, resolved=1, total=2, reason_code="ok" - ) - buffer = io.StringIO() - emit_attested_benchmark_result( - benchmark_result=br, - canonical_measurement=_measurement(), - rtmr3="d" * 96, - agent_hash="agent-abc", - task_ids=["t1", "t2"], - scores={"t1": 1.0, "t2": 0.0}, - validator_nonce="nonce-123", - quote_provider=_FakeProvider(), - manifest_sha256="m" * 64, - stream=buffer, - ) - return buffer.getvalue() - - -def test_acceptance_requires_binding_quote_measurement_nonce_and_key_grant(): - surfaced = result_mod.surface_result(_attested_stdout(), quote_verifier=lambda _q: True) - # A single positive signal must never alone produce acceptance. - for kwargs in ( - {"quote_verified": True}, - {"measurement_allowlisted": True}, - {"nonce_state": NonceState.OK}, - {"key_grant_ok": True}, - {"quote_verified": True, "measurement_allowlisted": True}, - { - "quote_verified": True, - "measurement_allowlisted": True, - "nonce_state": NonceState.OK, - }, - ): - verdict = result_mod.evaluate_acceptance(surfaced, **kwargs) - assert verdict.accepted is not True, kwargs - - # Conjunction of all required checks accepts. - accepted = result_mod.evaluate_acceptance( - surfaced, - quote_verified=True, - measurement_allowlisted=True, - nonce_state=NonceState.OK, - key_grant_ok=True, - ) - assert accepted.accepted is True - assert accepted.reason is None - - # Failed key grant alone rejects a fully otherwise-valid result. - rejected = result_mod.evaluate_acceptance( - surfaced, - quote_verified=True, - measurement_allowlisted=True, - nonce_state=NonceState.OK, - key_grant_ok=False, - ) - assert rejected.accepted is False - assert rejected.reason == result_mod.ACCEPTANCE_KEY_GRANT_MISSING - - -# --------------------------------------------------------------------------- # -# Eval compose provisions RA-TLS host/port and client mTLS paths (no HTTP URL). -# --------------------------------------------------------------------------- # -def test_eval_compose_provisions_ratls_host_port_and_client_mtls_paths(): - compose = generate_app_compose( - orchestrator_image=EVAL_IMAGE, - name="eval-v1", - key_release_url="validator.example:8701", - allowed_envs=eval_deploy.EVAL_ALLOWED_ENVS, - ) - text = render_app_compose(compose) - docker = compose["docker_compose_file"] - assert RA_TLS_HOST_ENV in docker - assert RA_TLS_PORT_ENV in docker - assert f"{RA_TLS_HOST_ENV}=validator.example" in docker - assert f"{RA_TLS_PORT_ENV}=8701" in docker - # No HTTP helper URL for production deploy wiring. - assert "CHALLENGE_PHALA_KEY_RELEASE_URL=" not in docker - assert "http://" not in docker - assert "https://" not in docker - for env_name in ( - KEY_RELEASE_TLS_CERT_ENV, - KEY_RELEASE_TLS_KEY_ENV, - KEY_RELEASE_TLS_CA_ENV, - ): - assert env_name in docker or env_name in compose["allowed_envs"] - assert env_name in text - - -def test_key_release_client_rejects_http_fallback_for_production_ratls_endpoint(monkeypatch): - # Production authority on 8701 must use raw RA-TLS and refuse silent HTTP. - client = GoldenKeyReleaseClient( - "validator.example:8701", - quote_provider=MagicMock(), - urlopen=lambda *a, **k: (_ for _ in ()).throw(AssertionError("HTTP used")), - ) - # Without mTLS files the raw path must fail closed, not fall back to HTTP. - monkeypatch.delenv(KEY_RELEASE_TLS_CERT_ENV, raising=False) - monkeypatch.delenv(KEY_RELEASE_TLS_KEY_ENV, raising=False) - monkeypatch.delenv(KEY_RELEASE_TLS_CA_ENV, raising=False) - with pytest.raises(Exception) as excinfo: - client.release( - nonce="nonce", - quote="ab" * 32, - event_log=[], - eval_run_id="eval-1", - ) - assert "HTTP" not in str(excinfo.value).upper() or "fallback" not in str(excinfo.value).lower() - # Confirms raw path was selected (missing mTLS files), not an HTTP call. - assert ( - "mTLS" in str(excinfo.value) - or "raw" in str(excinfo.value).lower() - or isinstance(excinfo.value, (KeyReleaseProtocolError, Exception)) - ) - - -# --------------------------------------------------------------------------- # -# Pre-create budget includes both immutable review and Eval shapes. -# --------------------------------------------------------------------------- # -def test_pre_create_budget_includes_both_review_and_eval_shapes(monkeypatch): - assignment, token = _review_assignment() - prepare_response = { - "assignment": assignment, - "review_session_token": token, - } - fake_client = MagicMock() - fake_client.review_prepare.return_value = prepare_response - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - - args = SimpleNamespace( - review_command="deploy", - submission_id=1, - base_url="https://challenge.example", - hotkey="hk", - signature="sig", - nonce="n", - timestamp=None, - auto_sign=True, - prepare_response=None, - openrouter_key_env="OPENROUTER_API_KEY", - phala_api=None, - # Review assignment is tdx.small; request a larger distinct eval stage. - review_instance_type="tdx.small", - eval_instance_type="tdx.xlarge", - review_runtime_hours=100.0, - eval_runtime_hours=100.0, - money_cap_usd=20.0, - dry_run=True, - ) - code = cli._ordered_review_command(args) - # Combined projection of tdx.small*100 + tdx.xlarge*100 = 5.8 + 46.4 = 52.2 > 20. - assert code == 2 - - -def test_lifecycle_budget_refuses_when_either_stage_exceeds_even_if_current_is_small(): - with pytest.raises(lifecycle.LifecycleBudgetError): - lifecycle.validate_lifecycle_budget( - review_instance_type="tdx.small", - eval_instance_type="tdx.xlarge", - review_runtime_hours=100, - eval_runtime_hours=100, - money_cap_usd=20, - ) - - -# --------------------------------------------------------------------------- # -# Nested review acknowledgement schema already expected by production route. -# --------------------------------------------------------------------------- # -def test_review_deploy_ack_uses_exact_nested_schema(): - assignment, token = _review_assignment() - plan = review_deploy.build_review_deployment_plan( - {"assignment": assignment, "review_session_token": token} - ) - encrypted = review_deploy.encrypt_review_secrets( - plan, - { - "OPENROUTER_API_KEY": "openrouter-secret", - "REVIEW_API_BASE_URL": "https://chain.joinbase.ai/challenges/agent-challenge", - "REVIEW_SESSION_TOKEN": token, - }, - ) - deployment = review_deploy.ReviewPhalaDeployment( - provision_response={ - "app_id": plan.app_identity, - "compose_hash": plan.compose_hash, - "app_env_encrypt_pubkey": plan.kms_public_key_hex, - "os_image_hash": plan.measurement["os_image_hash"], - }, - create_response={ - "id": "cvm-review-1", - "request_id": "req-1", - "created_at_ms": 1000, - }, - ) - ack = deployment.deploy(plan, encrypted) - assert set(ack) == { - "schema_version", - "assignment_id", - "cvm_id", - "phala_create_receipt", - "compose_identity", - } - assert set(ack["phala_create_receipt"]) == { - "request_id", - "app_id", - "cvm_id", - "receipt_sha256", - "created_at_ms", - } - assert set(ack["compose_identity"]) == { - "image_ref", - "compose_hash", - "app_kms_public_key_sha256", - } - - -# --------------------------------------------------------------------------- # -# Post-create failure deletes every attributable CVM. -# --------------------------------------------------------------------------- # -def test_review_post_create_failure_deletes_attributable_cvm(monkeypatch): - assignment, token = _review_assignment() - prepare_response = { - "assignment": assignment, - "review_session_token": token, - } - fake_client = MagicMock() - fake_client.review_prepare.return_value = prepare_response - fake_client.review_deployed.side_effect = cli.RouteClientError("signed ack failed") - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - monkeypatch.setenv("OPENROUTER_API_KEY", "or-secret") - # Offline test double uses a non-joinbase base_url; allow pin override. - monkeypatch.setenv("CHALLENGE_ALLOW_DEV_URLS", "1") - - deleted: list[str] = [] - - def _teardown(cvm_id: str) -> dict[str, Any]: - deleted.append(cvm_id) - return {"returncode": 0, "stdout": "", "stderr": "", "ok": True} - - monkeypatch.setattr(cli, "default_phala_teardown", _teardown) - - class _Deployer: - def __init__(self, _api): - pass - - def deploy(self, plan, encrypted): # noqa: ARG002 - from agent_challenge.review.deployment import build_review_deployed_acknowledgement - - return build_review_deployed_acknowledgement( - assignment=plan.assignment, - cvm_id="cvm-attr-1", - request_id="req-attr-1", - receipt_sha256="a" * 64, - created_at_ms=1, - ) - - monkeypatch.setattr(review_deploy, "HttpReviewPhalaDeployment", _Deployer) - monkeypatch.setattr(cli, "PhalaCloudClient", lambda **_k: object()) - capture = _RecordingCapture() - monkeypatch.setattr(cli, "_print", capture) - - args = SimpleNamespace( - review_command="deploy", - submission_id=1, - base_url="https://challenge.example", - hotkey="hk", - signature="sig", - nonce="n", - timestamp=None, - auto_sign=True, - prepare_response=None, - openrouter_key_env="OPENROUTER_API_KEY", - phala_api=None, - review_instance_type="tdx.small", - eval_instance_type="tdx.small", - review_runtime_hours=1.0, - eval_runtime_hours=1.0, - money_cap_usd=20.0, - dry_run=False, - ) - code = cli._ordered_review_command(args) - assert code == 2 - assert deleted == ["cvm-attr-1"] - - -def test_eval_post_create_failure_deletes_attributable_cvm(monkeypatch): - plan = _eval_plan() - token_value = "run-token" - plan["run_token_sha256"] = hashlib.sha256(token_value.encode()).hexdigest() - prepare = { - "schema_version": 1, - "plan": plan, - "plan_sha256": hashlib.sha256(eval_wire.canonical_json_v1(plan)).hexdigest(), - "secret_delivery": {"env_key": "EVAL_RUN_TOKEN", "token": token_value}, - } - fake_client = MagicMock() - fake_client.eval_prepare.return_value = prepare - monkeypatch.setattr(cli, "_route_client", lambda _args: fake_client) - # VAL-ACAT-013: Base gateway env is not required for eval deploy. - monkeypatch.setenv("LLM_COST_LIMIT", "1.00") - # Validator server CA is required before any Phala create so the guest can - # verify the raw RA-TLS listener (fail closed without fabrications). Must be - # an OpenSSL-loadable PEM (normalize_server_ca_pem preloads it). - monkeypatch.setenv( - "CHALLENGE_PHALA_RA_TLS_SERVER_CA_PEM", - ( - "-----BEGIN CERTIFICATE-----\n" - "MIICxTCCAa2gAwIBAgIUIOBn+Iz4ZK61F3pcFJGHjx995acwDQYJKoZIhvcNAQEL\n" - "BQAwEjEQMA4GA1UEAwwHdGVzdC1jYTAeFw0yNjA3MTIxMzAzNDRaFw0zNjA3MTAx\n" - "MzAzNDRaMBIxEDAOBgNVBAMMB3Rlc3QtY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IB\n" - "DwAwggEKAoIBAQDWxZ5PVNf+JlSNkpDlJdqP/WWwZL4fxpJZegSJE7gipUIUH8l6\n" - "SsDhVBiE0eD2GJzGnjx7+I6Q5+36oqoVDBgukVERFkfEZ0d4MtwQ5+rU2pdBx24B\n" - "VeBkNQLFu8qNLzPQuKlU0uIDrGvK157kvMlFQl2cvaJKLGwxRd/j5x+xVRynEfuA\n" - "RSJvt6pvv2Md1Na8ES9QR8pv6q9U4DMnanc4hMjlGMKuF8xKz/ls05e8KTEkDJJP\n" - "7FiZNi0vvlMJQxch9cfzjjnK7mjQm2nrebaFMr/nJNccdq5fcEaIaJhNMU65V0LI\n" - "B2IKwLO/GhcgiFNZ43nfe93WWVaKl8vx382nAgMBAAGjEzARMA8GA1UdEwEB/wQF\n" - "MAMBAf8wDQYJKoZIhvcNAQELBQADggEBAAmfmX6/kAciNHTdvE2mrK7KUDDiDhT7\n" - "kMRWOqiBaYxxiOiz3h1vrzEo81NQqc2dZF4+MrlODcnXUMgT62ijw0O/71IYl33E\n" - "nZBV+MBry5w5vlNw1El2aO3ERtWwjxrN0sLKkqht0h7hU/+wc7+5aBV4URFoNx2E\n" - "EkcZZVknVD9EMvNlWnVVQoLnOIIW4e5F4yHqHQTdxM1TD4F0gKjfNwGK6xZNpObG\n" - "QbDfN3wSkU7DIxeNJCMB+Uc5GDHMKNiEg0yb59SEvypiDuU6cD7OuhLQM0gbjXlC\n" - "81hvjyhx/T/mRQhf6MOu8RbVdp5CDp7IqhouLwEHvHjS4bA/AZIuIP8=\n" - "-----END CERTIFICATE-----\n" - ), - ) - - deleted: list[str] = [] - - def _teardown(cvm_id: str) -> dict[str, Any]: - deleted.append(cvm_id) - return {"returncode": 0, "ok": True, "stdout": "", "stderr": ""} - - monkeypatch.setattr(cli, "default_phala_teardown", _teardown) - - class _Boom: - def __init__(self, _api): - pass - - def deploy(self, plan, encrypted): # noqa: ARG002 - # Simulate create succeeded then acknowledgement failed mid-path by - # raising after recording a cvm_id on the exception attribute. - error = eval_deploy.EvalDeploymentError("post-create bind failed") - error.attributable_cvm_id = "cvm-eval-attr-1" # type: ignore[attr-defined] - raise error - - monkeypatch.setattr(eval_deploy, "HttpEvalPhalaDeployment", _Boom) - monkeypatch.setattr(cli, "PhalaCloudClient", lambda **_k: object()) - - args = SimpleNamespace( - eval_command="deploy", - submission_id=1, - base_url="https://challenge.example", - hotkey="hk", - signature="sig", - nonce="n", - timestamp=None, - auto_sign=True, - prepare_response=None, - gateway_token_env="BASE_GATEWAY_TOKEN", - gateway_url_env="BASE_LLM_GATEWAY_URL", - llm_cost_limit_env="LLM_COST_LIMIT", - phala_api=None, - review_instance_type="tdx.small", - eval_instance_type="tdx.small", - review_runtime_hours=1.0, - eval_runtime_hours=1.0, - money_cap_usd=20.0, - dry_run=False, - token_env="EVAL_RUN_TOKEN", - # Live deploy requires an explicit one-time token handoff path. - emit_run_token=True, - token_output=None, - output=None, - ) - code = cli._ordered_eval_command(args) - assert code == 2 - assert deleted == ["cvm-eval-attr-1"] - - -# --------------------------------------------------------------------------- # -# Teardown failure returns nonzero with bounded diagnostics. -# --------------------------------------------------------------------------- # -def test_teardown_failure_returns_nonzero_with_bounded_diagnostics(monkeypatch): - long_stderr = "x" * 10_000 - - def _boom(_cvm_id: str) -> dict[str, Any]: - return { - "returncode": 1, - "ok": False, - "stdout": "", - "stderr": long_stderr, - "error": "phala cvms delete failed", - } - - monkeypatch.setattr(cli, "default_phala_teardown", _boom) - capture = _RecordingCapture() - monkeypatch.setattr(cli, "_print", capture) - - args = SimpleNamespace(review_command="teardown", cvm_id="cvm-leftover") - code = cli._ordered_review_command(args) - assert code != 0 - payload = capture.payloads[-1] - assert payload["torn_down"] == "cvm-leftover" - assert payload["ok"] is False - # Diagnostics are present and size-bounded. - diagnostics = payload.get("diagnostics") or payload.get("result") or {} - text = json.dumps(diagnostics) - assert len(text) < 2048 - assert "phala cvms delete failed" in text or diagnostics.get("returncode") == 1 diff --git a/packages/challenges/agent-challenge/tests/test_selfdeploy_rejection_observability.py b/packages/challenges/agent-challenge/tests/test_selfdeploy_rejection_observability.py deleted file mode 100644 index ce0e4e40f..000000000 --- a/packages/challenges/agent-challenge/tests/test_selfdeploy_rejection_observability.py +++ /dev/null @@ -1,254 +0,0 @@ -"""VAL-DEPLOY-026: a rejected/unaccepted attested result is surfaced to the miner. - -When the validator rejects or parks a submission's attested result (attestation -absent, quote fails verification, measurement not on the allowlist, nonce stale), -the miner-facing self-deploy ``result`` surface must report a clear, non-sensitive -NOT-ACCEPTED verdict with a coarse reason -- never reporting success, never -reporting a (fabricated) score, and never leaking golden material, the golden key, -or any quote-embedded secret. - -These are discriminators: each rejection cause is driven independently over a -genuinely valid attested envelope (so a naive "attested ⇒ accepted" surface would -FAIL), and the accepted control confirms the surface does not reject a good run. -""" - -from __future__ import annotations - -import io -import json -from contextlib import redirect_stderr, redirect_stdout - -from agent_challenge.canonical.attested_result import emit_attested_benchmark_result -from agent_challenge.evaluation.own_runner.result_schema import build_benchmark_result -from agent_challenge.keyrelease.client import KEY_RELEASE_FAILED_REASON -from agent_challenge.keyrelease.nonce import NonceState -from agent_challenge.selfdeploy import cli -from agent_challenge.selfdeploy import result as result_mod - -# A sentinel that stands in for any secret/golden material. It never appears in -# the surfaced envelope, so it must never appear in any surfaced verdict either. -GOLDEN_SENTINEL = "GOLDEN-PLAINTEXT-SENTINEL-do-not-leak" - - -def _measurement() -> dict: - return { - "mrtd": "a" * 96, - "rtmr0": "a" * 96, - "rtmr1": "a" * 96, - "rtmr2": "a" * 96, - "compose_hash": "b" * 64, - "os_image_hash": "b" * 64, - } - - -class _FakeQuote: - quote = "deadbeef" * 16 - event_log = [{"event": "compose-hash", "payload": "c" * 64}] - # schema-v2 Eval keys (or dstack aliases cpu_count/memory_size) - vm_config = {"vcpu": 1, "memory_mb": 2048} - - -class _FakeProvider: - def get_quote(self, report_data): # noqa: ARG002 - return _FakeQuote() - - -def _attested_stdout(scores=None) -> str: - scores = scores or {"t1": 1.0, "t2": 0.0} - br = build_benchmark_result( - status="completed", score=0.5, resolved=1, total=2, reason_code="ok" - ) - buffer = io.StringIO() - emit_attested_benchmark_result( - benchmark_result=br, - canonical_measurement=_measurement(), - rtmr3="d" * 96, - agent_hash="agent-abc", - task_ids=sorted(scores), - scores=scores, - validator_nonce="nonce-123", - quote_provider=_FakeProvider(), - manifest_sha256="m" * 64, - stream=buffer, - ) - return buffer.getvalue() - - -def _failclosed_line(reason: str = KEY_RELEASE_FAILED_REASON) -> str: - return "BASE_BENCHMARK_RESULT=" + json.dumps( - {"reason_code": reason, "resolved": 0, "score": 0.0, "status": "failed", "total": 1}, - sort_keys=True, - ) - - -def _run_result_cli(argv): - out, err = io.StringIO(), io.StringIO() - with redirect_stdout(out), redirect_stderr(err): - code = cli.main(argv) - return code, out.getvalue(), err.getvalue() - - -# --------------------------------------------------------------------------- # -# Function-level: evaluate_acceptance yields a coarse NOT-ACCEPTED verdict per -# rejection cause and does not reject a fully valid, allowlisted, verified run. -# --------------------------------------------------------------------------- # -def test_attestation_absent_is_not_accepted(): - surfaced = result_mod.surface_result(_failclosed_line()) - verdict = result_mod.evaluate_acceptance(surfaced) - assert verdict.accepted is False - assert verdict.reason == result_mod.ACCEPTANCE_ATTESTATION_ABSENT - - -def test_quote_not_verified_is_not_accepted(): - surfaced = result_mod.surface_result(_attested_stdout(), quote_verifier=lambda _q: False) - verdict = result_mod.evaluate_acceptance(surfaced) - assert verdict.accepted is False - assert verdict.reason == result_mod.ACCEPTANCE_ATTESTATION_NOT_VERIFIED - - -def test_measurement_not_allowlisted_is_not_accepted(): - surfaced = result_mod.surface_result(_attested_stdout()) - verdict = result_mod.evaluate_acceptance(surfaced, measurement_allowlisted=False) - assert verdict.accepted is False - assert verdict.reason == result_mod.ACCEPTANCE_MEASUREMENT_NOT_ALLOWLISTED - - -def test_stale_nonce_is_not_accepted(): - surfaced = result_mod.surface_result(_attested_stdout()) - verdict = result_mod.evaluate_acceptance(surfaced, nonce_state=NonceState.EXPIRED) - assert verdict.accepted is False - assert verdict.reason == result_mod.ACCEPTANCE_NONCE_STALE - - -def test_fully_valid_verified_allowlisted_fresh_run_is_accepted(): - surfaced = result_mod.surface_result(_attested_stdout(), quote_verifier=lambda _q: True) - verdict = result_mod.evaluate_acceptance( - surfaced, - measurement_allowlisted=True, - nonce_state=NonceState.OK, - key_grant_ok=True, - ) - assert verdict.accepted is True - assert verdict.reason is None - - -def test_tampered_binding_is_not_accepted(): - # A surfaced result whose scores were altered breaks the report_data binding. - stdout = _attested_stdout() - execution_proof, binding = result_mod.extract_envelope(stdout) - binding["scores"]["t1"] = 0.0 - check = result_mod.verify_report_data_binding(execution_proof, binding) - assert check.valid is False - # A surfaced result carrying an invalid binding is never accepted. - surfaced = result_mod.surface_result(stdout) - assert surfaced.binding_check.valid is True # sanity: the on-wire binding is intact - forced = result_mod.SurfacedResult( - attested=True, - benchmark_result=surfaced.benchmark_result, - status=surfaced.status, - reason_code=surfaced.reason_code, - execution_proof=surfaced.execution_proof, - binding=surfaced.binding, - binding_check=check, - quote_verified=None, - ) - verdict = result_mod.evaluate_acceptance(forced) - assert verdict.accepted is False - assert verdict.reason == result_mod.ACCEPTANCE_BINDING_MISMATCH - - -def test_no_signal_is_pending_not_a_false_accept(): - surfaced = result_mod.surface_result(_attested_stdout()) - verdict = result_mod.evaluate_acceptance(surfaced) - # Without any validator signal the verdict is PENDING (None), never a - # fabricated acceptance. - assert verdict.accepted is None - assert verdict.reason is None - - -# --------------------------------------------------------------------------- # -# CLI-level: each rejection cause surfaces a clear, non-sensitive verdict with a -# non-zero exit, NO score, and no secret material. -# --------------------------------------------------------------------------- # -def _assert_rejection_output(code: int, out: str, expected_reason: str): - assert code == 1, out - payload = json.loads(out) - assert payload["accepted"] is False - assert payload["reason"] == expected_reason - # No fabricated score and no quote/secret material in the rejection surface. - assert "score" not in payload - assert "scores" not in payload - assert "attestation" not in payload - assert "tdx_quote" not in out - assert _FakeQuote.quote not in out - assert GOLDEN_SENTINEL not in out - # The reason is a coarse, non-sensitive label (whitelist). - assert payload["reason"] in { - result_mod.ACCEPTANCE_ATTESTATION_ABSENT, - result_mod.ACCEPTANCE_ATTESTATION_NOT_VERIFIED, - result_mod.ACCEPTANCE_MEASUREMENT_NOT_ALLOWLISTED, - result_mod.ACCEPTANCE_NONCE_STALE, - result_mod.ACCEPTANCE_NONCE_CONSUMED, - result_mod.ACCEPTANCE_NONCE_UNKNOWN, - result_mod.ACCEPTANCE_BINDING_MISMATCH, - result_mod.ACCEPTANCE_KEY_GRANT_MISSING, - } - - -def test_cli_attestation_absent_rejection(tmp_path): - path = tmp_path / "run.txt" - path.write_text(_failclosed_line()) - code, out, _err = _run_result_cli(["result", "--from", str(path)]) - _assert_rejection_output(code, out, result_mod.ACCEPTANCE_ATTESTATION_ABSENT) - - -def test_cli_quote_not_verified_rejection(tmp_path): - path = tmp_path / "run.txt" - path.write_text(_attested_stdout()) - code, out, _err = _run_result_cli(["result", "--from", str(path), "--quote-verified", "false"]) - _assert_rejection_output(code, out, result_mod.ACCEPTANCE_ATTESTATION_NOT_VERIFIED) - - -def test_cli_measurement_not_allowlisted_rejection(tmp_path): - path = tmp_path / "run.txt" - path.write_text(_attested_stdout()) - allow = tmp_path / "allow.json" - # An allowlist that does NOT contain the run's measurement. - other = {**_measurement(), "mrtd": "f" * 96} - allow.write_text(json.dumps([other])) - code, out, _err = _run_result_cli(["result", "--from", str(path), "--allowlist", str(allow)]) - _assert_rejection_output(code, out, result_mod.ACCEPTANCE_MEASUREMENT_NOT_ALLOWLISTED) - - -def test_cli_stale_nonce_rejection(tmp_path): - path = tmp_path / "run.txt" - path.write_text(_attested_stdout()) - code, out, _err = _run_result_cli(["result", "--from", str(path), "--nonce-state", "stale"]) - _assert_rejection_output(code, out, result_mod.ACCEPTANCE_NONCE_STALE) - - -def test_cli_accepted_run_reports_acceptance_and_scores(tmp_path): - path = tmp_path / "run.txt" - path.write_text(_attested_stdout()) - allow = tmp_path / "allow.json" - allow.write_text(json.dumps([_measurement()])) - code, out, _err = _run_result_cli( - [ - "result", - "--from", - str(path), - "--allowlist", - str(allow), - "--quote-verified", - "true", - "--nonce-state", - "ok", - ] - ) - assert code == 0, out - summary = json.loads(out) - assert summary["acceptance"] == {"accepted": True, "reason": None} - assert summary["allowlist_verdict"]["verdict"] == "IN-LIST" - # An accepted run DOES surface its scores/attestation (contrast with reject). - assert summary["scores"] == {"t1": 1.0, "t2": 0.0} - assert summary["attested"] is True diff --git a/packages/challenges/agent-challenge/tests/test_selfdeploy_run_result.py b/packages/challenges/agent-challenge/tests/test_selfdeploy_run_result.py deleted file mode 100644 index ce65449bd..000000000 --- a/packages/challenges/agent-challenge/tests/test_selfdeploy_run_result.py +++ /dev/null @@ -1,271 +0,0 @@ -"""Run fail-closed + attested-result surfacing/verification for the miner CLI. - -Covers VAL-DEPLOY-011 (wrong/unreachable key-release endpoint → clear failure, -no golden key, NO fabricated attested result), VAL-DEPLOY-013 (a completed run -surfaces a well-formed attested-result envelope), and VAL-DEPLOY-014 (a surfaced -quote's ``report_data`` recomputes to the documented §6 binding for the run). - -Run assertions drive the REAL in-CVM backend (`own_runner_backend.main`) with the -endpoint wired in for the unreachable case (it fails closed before touching docker -or golden), and an injected fail-closed backend for the deny case. Live TDX quote -verification (Phala verify / dcap-qvl) is a M6 concern; here the quote-verify hook -is injected. -""" - -from __future__ import annotations - -import io -import json -from contextlib import redirect_stderr, redirect_stdout - -from agent_challenge.canonical.attested_result import emit_attested_benchmark_result -from agent_challenge.evaluation.own_runner.result_schema import build_benchmark_result -from agent_challenge.keyrelease.client import KEY_RELEASE_FAILED_REASON -from agent_challenge.selfdeploy import cli -from agent_challenge.selfdeploy import result as result_mod -from agent_challenge.selfdeploy import run as run_mod - -URL = "https://validator.example/keyrelease" - -# A definitely-closed local port (fail fast, no external network, no spend). -UNREACHABLE_URL = "http://127.0.0.1:9/" - - -def _failclosed_line(reason: str = KEY_RELEASE_FAILED_REASON) -> str: - """A fail-closed BASE_BENCHMARK_RESULT= line (score 0, no attestation).""" - - return "BASE_BENCHMARK_RESULT=" + json.dumps( - {"reason_code": reason, "resolved": 0, "score": 0.0, "status": "failed", "total": 1}, - sort_keys=True, - ) - - -# --------------------------------------------------------------------------- # -# Fixtures: build a genuine attested-result stdout line -# --------------------------------------------------------------------------- # -class _FakeQuote: - quote = "deadbeef" * 16 - event_log = [{"event": "compose-hash", "payload": "c" * 64}] - # schema-v2 Eval keys (or dstack aliases cpu_count/memory_size) - vm_config = {"vcpu": 1, "memory_mb": 2048} - - -class _FakeProvider: - def get_quote(self, report_data): # noqa: ARG002 - return _FakeQuote() - - -def _measurement() -> dict: - return { - "mrtd": "a" * 96, - "rtmr0": "a" * 96, - "rtmr1": "a" * 96, - "rtmr2": "a" * 96, - "compose_hash": "b" * 64, - "os_image_hash": "b" * 64, - } - - -def _attested_stdout(scores=None) -> str: - scores = scores or {"t1": 1.0, "t2": 0.0} - br = build_benchmark_result( - status="completed", score=0.5, resolved=1, total=2, reason_code="ok" - ) - buffer = io.StringIO() - emit_attested_benchmark_result( - benchmark_result=br, - canonical_measurement=_measurement(), - rtmr3="d" * 96, - agent_hash="agent-abc", - task_ids=sorted(scores), - scores=scores, - validator_nonce="nonce-123", - quote_provider=_FakeProvider(), - manifest_sha256="m" * 64, - stream=buffer, - ) - return buffer.getvalue() - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-011: wrong/unreachable endpoint → clear failure, no fabricated result -# --------------------------------------------------------------------------- # -def test_unreachable_endpoint_fails_closed_no_attested_result(tmp_path): - # A standalone self-deploy invocation has no validator-issued immutable Eval - # plan, so it fails closed before docker, dstack, or golden handling. - outcome = run_mod.run_eval( - job_dir=str(tmp_path), - task_ids=["hello-world"], - key_release_url=UNREACHABLE_URL, - ) - assert outcome.succeeded is False - assert outcome.attested is False - assert outcome.exit_code != 0 - assert outcome.surfaced is not None - assert outcome.surfaced.reason_code == "terminal_bench_failed" - assert outcome.surfaced.attestation is None # NO fabricated attestation - assert outcome.clear_error and "failed closed" in outcome.clear_error.lower() - - -def test_denying_endpoint_fails_closed_no_attested_result(): - # A wrong endpoint that DENIES the quote: model the backend's fail-closed - # emission (score 0, key-release reason, no envelope). - def denying_backend(argv): # noqa: ARG001 - print(_failclosed_line()) - return 1 - - outcome = run_mod.run_eval( - job_dir="/tmp/job", - task_ids=["t1"], - key_release_url="https://wrong.example/kr", - backend_main=denying_backend, - ) - assert outcome.succeeded is False - assert outcome.attested is False - assert outcome.surfaced is not None - assert outcome.surfaced.attestation is None - assert outcome.clear_error - - -def test_run_cli_reports_clear_failure_and_no_attested_result(tmp_path): - out, err = io.StringIO(), io.StringIO() - with redirect_stdout(out), redirect_stderr(err): - code = cli.main( - [ - "run", - "--job-dir", - str(tmp_path), - "--task", - "hello-world", - "--key-release-url", - UNREACHABLE_URL, - ] - ) - assert code != 0 - # A clear miner-facing error, and no attested envelope in stdout. - assert "failed closed" in err.getvalue().lower() - assert "tdx_quote" not in out.getvalue() - assert "execution_proof" not in out.getvalue() - - -def test_run_endpoint_wired_into_backend_env(): - # The run wires the operator-supplied endpoint into the backend via the - # key-release env var (the backend requests golden from exactly that URL). - seen = {} - - def capturing_backend(argv): # noqa: ARG001 - import os - - from agent_challenge.keyrelease.client import KEY_RELEASE_URL_ENV - - seen["url"] = os.environ.get(KEY_RELEASE_URL_ENV) - print(_failclosed_line()) - return 1 - - run_mod.run_eval( - job_dir="/tmp/job", - task_ids=["t1"], - key_release_url="https://exact-endpoint.example/kr", - backend_main=capturing_backend, - ) - assert seen["url"] == "https://exact-endpoint.example/kr" - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-013: a completed run surfaces a well-formed attested envelope -# --------------------------------------------------------------------------- # -def test_completed_run_surfaces_wellformed_envelope(): - surfaced = result_mod.surface_result(_attested_stdout()) - assert surfaced.attested is True - att = surfaced.attestation - assert isinstance(att["tdx_quote"], str) and att["tdx_quote"] - assert isinstance(att["event_log"], list) and att["event_log"] - assert isinstance(att["report_data"], str) and len(att["report_data"]) == 128 - measurement = att["measurement"] - for field in ("mrtd", "rtmr0", "rtmr1", "rtmr2", "rtmr3", "compose_hash", "os_image_hash"): - assert isinstance(measurement[field], str) and measurement[field], field - # Per-task scores are surfaced and well-typed. - assert surfaced.scores == {"t1": 1.0, "t2": 0.0} - assert all(isinstance(v, (int, float)) for v in surfaced.scores.values()) - - -def test_result_cli_surfaces_envelope_fields(tmp_path, capsys): - path = tmp_path / "run.txt" - path.write_text(_attested_stdout()) - code = cli.main(["result", "--from", str(path)]) - assert code == 0 - summary = json.loads(capsys.readouterr().out) - assert summary["attested"] is True - assert set(summary["attestation"]) >= { - "tdx_quote", - "event_log", - "report_data", - "measurement", - "vm_config", - } - assert summary["scores"] == {"t1": 1.0, "t2": 0.0} - - -def test_fail_closed_run_surfaces_no_attestation(): - surfaced = result_mod.surface_result(_failclosed_line()) - assert surfaced.attested is False - assert surfaced.attestation is None - assert surfaced.reason_code == KEY_RELEASE_FAILED_REASON - - -# --------------------------------------------------------------------------- # -# VAL-DEPLOY-014: surfaced quote's report_data recomputes to the §6 binding -# --------------------------------------------------------------------------- # -def test_surfaced_report_data_recomputes_to_binding(): - surfaced = result_mod.surface_result(_attested_stdout()) - check = surfaced.binding_check - assert check is not None - assert check.valid is True - assert check.report_data_matches is True - assert check.scores_digest_matches is True - assert check.measurement_consistent is True - # The recomputed report_data equals the quote's report_data exactly. - assert check.recomputed_report_data == surfaced.attestation["report_data"] - - -def test_tampered_scores_break_the_binding(): - stdout = _attested_stdout() - envelope = result_mod.extract_envelope(stdout) - assert envelope is not None - execution_proof, binding = envelope - binding["scores"]["t1"] = 0.0 # alter a reported score - check = result_mod.verify_report_data_binding(execution_proof, binding) - assert check.valid is False - assert check.scores_digest_matches is False - - -def test_tampered_nonce_breaks_the_binding(): - stdout = _attested_stdout() - execution_proof, binding = result_mod.extract_envelope(stdout) - binding["validator_nonce"] = "different-nonce" - check = result_mod.verify_report_data_binding(execution_proof, binding) - assert check.valid is False - assert check.report_data_matches is False - - -def test_quote_verifier_hook_is_surfaced(): - # An injected quote verifier (Phala verify / dcap-qvl at M6) verdict rides - # along with the surfaced result. - surfaced = result_mod.surface_result(_attested_stdout(), quote_verifier=lambda q: True) - assert surfaced.quote_verified is True - summary = surfaced.summary() - assert summary["quote_verified"] is True - - tampered = result_mod.surface_result(_attested_stdout(), quote_verifier=lambda q: False) - assert tampered.quote_verified is False - - -def test_result_cli_verdict_with_allowlist(tmp_path, capsys): - path = tmp_path / "run.txt" - path.write_text(_attested_stdout()) - allow = tmp_path / "allow.json" - allow.write_text(json.dumps([{**_measurement(), "key_provider": "kms"}])) - code = cli.main(["result", "--from", str(path), "--allowlist", str(allow)]) - assert code == 0 - summary = json.loads(capsys.readouterr().out) - assert summary["allowlist_verdict"]["verdict"] == "IN-LIST" diff --git a/packages/challenges/agent-challenge/tests/test_selfdeploy_shapes_disk.py b/packages/challenges/agent-challenge/tests/test_selfdeploy_shapes_disk.py deleted file mode 100644 index d8aa6aa64..000000000 --- a/packages/challenges/agent-challenge/tests/test_selfdeploy_shapes_disk.py +++ /dev/null @@ -1,297 +0,0 @@ -"""Disk-aware CPU TDX sizing + stage defaults (offline, no Phala spend). - -Covers: - * stage defaults: review tdx.small/20GB, eval tdx.xlarge/100GB - * disk validator bounds [20, 500] - * disk billing in projected cost - * lifecycle budget includes disk for both stages under the $20 cap - * provision bodies emit disk_size as a sibling of compose_file -""" - -from __future__ import annotations - -from types import SimpleNamespace -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from agent_challenge.selfdeploy import lifecycle, shapes -from agent_challenge.selfdeploy.eval import EvalDeploymentPlan, HttpEvalPhalaDeployment -from agent_challenge.selfdeploy.review import HttpReviewPhalaDeployment, ReviewDeploymentPlan - - -def test_stage_instance_defaults_are_split(): - assert shapes.DEFAULT_REVIEW_INSTANCE_TYPE == "tdx.small" - assert shapes.DEFAULT_EVAL_INSTANCE_TYPE == "tdx.xlarge" - # Backward-compat alias remains the review default. - assert shapes.DEFAULT_INSTANCE_TYPE == shapes.DEFAULT_REVIEW_INSTANCE_TYPE - assert shapes.DEFAULT_REVIEW_DISK_SIZE_GB == 20 - assert shapes.DEFAULT_EVAL_DISK_SIZE_GB == 100 - - -def test_disk_constants_match_decided_billing(): - assert shapes.DISK_USD_PER_GB_HOUR == 0.000139 - assert shapes.MIN_DISK_SIZE_GB == 20 - assert shapes.MAX_DISK_SIZE_GB == 500 - - -@pytest.mark.parametrize("gb", [20, 100, 500]) -def test_validate_disk_size_accepts_bounds(gb: int): - assert shapes.validate_disk_size(gb) == gb - - -@pytest.mark.parametrize("gb", [0, 19, 501, -1, 20.5, "20", None]) -def test_validate_disk_size_refuses_out_of_range(gb: object): - with pytest.raises(shapes.ShapeError): - shapes.validate_disk_size(gb) # type: ignore[arg-type] - - -def test_projected_cost_includes_disk(): - # tdx.xlarge @ 0.464/h * 6h = 2.784; disk 100GB * 0.000139 * 6 = 0.0834 - cpu_only = shapes.projected_cost_usd("tdx.xlarge", max_runtime_hours=6.0) - with_disk = shapes.projected_cost_usd( - "tdx.xlarge", - max_runtime_hours=6.0, - disk_size_gb=100, - ) - assert cpu_only == pytest.approx(0.464 * 6.0) - assert with_disk == pytest.approx(0.464 * 6.0 + 0.000139 * 100 * 6.0) - assert with_disk > cpu_only - - -def test_validate_within_cap_counts_disk(): - # Force over-cap via huge runtime + large disk on xlarge. - with pytest.raises(shapes.OverCapError): - shapes.validate_within_cap( - "tdx.xlarge", - money_cap_usd=1.0, - max_runtime_hours=6.0, - disk_size_gb=500, - ) - - -def test_default_lifecycle_budget_fits_money_cap(): - cost = lifecycle.validate_lifecycle_budget( - review_instance_type=shapes.DEFAULT_REVIEW_INSTANCE_TYPE, - eval_instance_type=shapes.DEFAULT_EVAL_INSTANCE_TYPE, - review_runtime_hours=shapes.DEFAULT_MAX_RUNTIME_HOURS, - eval_runtime_hours=shapes.DEFAULT_MAX_RUNTIME_HOURS, - review_disk_size_gb=shapes.DEFAULT_REVIEW_DISK_SIZE_GB, - eval_disk_size_gb=shapes.DEFAULT_EVAL_DISK_SIZE_GB, - money_cap_usd=shapes.DEFAULT_MONEY_CAP_USD, - ) - assert cost.total_usd <= shapes.DEFAULT_MONEY_CAP_USD - assert cost.total_usd == pytest.approx( - lifecycle.projected_lifecycle_cost_usd( - review_instance_type=shapes.DEFAULT_REVIEW_INSTANCE_TYPE, - eval_instance_type=shapes.DEFAULT_EVAL_INSTANCE_TYPE, - review_runtime_hours=shapes.DEFAULT_MAX_RUNTIME_HOURS, - eval_runtime_hours=shapes.DEFAULT_MAX_RUNTIME_HOURS, - review_disk_size_gb=shapes.DEFAULT_REVIEW_DISK_SIZE_GB, - eval_disk_size_gb=shapes.DEFAULT_EVAL_DISK_SIZE_GB, - ) - ) - # Disk contribution is strictly positive vs CPU-only projection. - cpu_only = ( - shapes.CPU_TDX_SHAPES[shapes.DEFAULT_REVIEW_INSTANCE_TYPE].usd_per_hour - * shapes.DEFAULT_MAX_RUNTIME_HOURS - + shapes.CPU_TDX_SHAPES[shapes.DEFAULT_EVAL_INSTANCE_TYPE].usd_per_hour - * shapes.DEFAULT_MAX_RUNTIME_HOURS - ) - assert cost.total_usd > cpu_only - - -def test_lifecycle_budget_refuses_over_cap_with_disk(): - with pytest.raises(lifecycle.LifecycleBudgetError): - lifecycle.validate_lifecycle_budget( - review_instance_type="tdx.small", - eval_instance_type="tdx.xlarge", - review_runtime_hours=100.0, - eval_runtime_hours=100.0, - review_disk_size_gb=20, - eval_disk_size_gb=100, - money_cap_usd=20.0, - ) - - -def test_lifecycle_budget_refuses_invalid_disk(): - with pytest.raises(lifecycle.LifecycleBudgetError): - lifecycle.validate_lifecycle_budget( - review_instance_type="tdx.small", - eval_instance_type="tdx.xlarge", - review_runtime_hours=1.0, - eval_runtime_hours=1.0, - review_disk_size_gb=10, - eval_disk_size_gb=100, - ) - - -def _minimal_eval_plan(*, disk_size_gb: int = 100) -> EvalDeploymentPlan: - return EvalDeploymentPlan( - plan={"eval_run_id": "run-1"}, - plan_sha256="a" * 64, - compose={"manifest_version": 2, "docker_compose_file": "services: {}\n"}, - compose_text="services: {}\n", - compose_hash="b" * 64, - app_identity="c" * 40, - image_ref="ghcr.io/example/eval@sha256:" + ("d" * 64), - kms_public_key_hex="ab" * 32, - kms_public_key_sha256="e" * 64, - measurement={"vm_shape": "tdx.xlarge"}, - eval_run_id="run-1", - eval_run_token="token-secret", - instance_type="tdx.xlarge", - disk_size_gb=disk_size_gb, - ) - - -def _minimal_review_plan(*, disk_size_gb: int = 20) -> ReviewDeploymentPlan: - return ReviewDeploymentPlan( - assignment={"assignment_core": {"assignment_id": "asg-1"}}, - compose={"manifest_version": 2, "docker_compose_file": "services: {}\n"}, - compose_text="services: {}\n", - compose_hash="b" * 64, - app_identity="c" * 40, - image_ref="ghcr.io/example/review@sha256:" + ("d" * 64), - kms_public_key_hex="ab" * 32, - kms_public_key_sha256="e" * 64, - measurement={"vm_shape": "tdx.small"}, - measurement_allowlist_sha256="f" * 64, - review_session_token="token-secret", - instance_type="tdx.small", - disk_size_gb=disk_size_gb, - ) - - -def test_eval_provision_emits_disk_size_sibling_of_compose(): - captured: list[dict[str, Any]] = [] - - class Api: - def post(self, path: str, payload: dict[str, Any]) -> dict[str, Any]: - if path == "/cvms/provision": - captured.append(dict(payload)) - return { - "compose_hash": "b" * 64, - "app_id": "c" * 40, - "app_env_encrypt_pubkey": "ab" * 32, - "os_image_hash": "0" * 64, - } - return {"id": "cvm-1", "request_id": "req-1", "created_at_ms": 1} - - plan = _minimal_eval_plan(disk_size_gb=100) - encrypted = SimpleNamespace( - eval_run_id=plan.eval_run_id, - app_identity=plan.app_identity, - kms_public_key_sha256=plan.kms_public_key_sha256, - env_keys=("EVAL_RUN_TOKEN",), - ciphertext="cipher", - ) - # Avoid OS-identity side checks by stubbing the private verifier. - dep = HttpEvalPhalaDeployment(Api()) # type: ignore[arg-type] - dep._verify_provision_os_identity = MagicMock() # type: ignore[method-assign] - try: - dep.deploy(plan, encrypted) # type: ignore[arg-type] - except Exception: - # Create path may still fail closed; provision capture is what we need. - pass - assert captured, "provision was never called" - body = captured[0] - assert "disk_size" in body - assert body["disk_size"] == 100 - assert "compose_file" in body - assert body["compose_file"] is plan.compose - # Must not mutate compose document. - assert "disk_size" not in plan.compose - - -def test_review_provision_emits_disk_size_sibling_of_compose(): - captured: list[dict[str, Any]] = [] - - class Api: - def post(self, path: str, payload: dict[str, Any]) -> dict[str, Any]: - if path == "/cvms/provision": - captured.append(dict(payload)) - return { - "compose_hash": "b" * 64, - "app_id": "c" * 40, - "app_env_encrypt_pubkey": "ab" * 32, - "os_image_hash": "0" * 64, - } - return {"id": "cvm-1", "request_id": "req-1", "created_at_ms": 1} - - plan = _minimal_review_plan(disk_size_gb=20) - encrypted = SimpleNamespace( - assignment_id="asg-1", - app_identity=plan.app_identity, - kms_public_key_sha256=plan.kms_public_key_sha256, - measurement_allowlist_sha256=plan.measurement_allowlist_sha256, - env_keys=plan # placeholder replaced below - ) - from agent_challenge.selfdeploy import review as review_mod - - encrypted = SimpleNamespace( - assignment_id="asg-1", - app_identity=plan.app_identity, - kms_public_key_sha256=plan.kms_public_key_sha256, - measurement_allowlist_sha256=plan.measurement_allowlist_sha256, - env_keys=review_mod.REVIEW_ALLOWED_ENVS, - ciphertext="cipher", - ) - dep = HttpReviewPhalaDeployment(Api()) # type: ignore[arg-type] - dep._verify_provision_response = MagicMock() # type: ignore[method-assign] - dep._resolve_created_cvm_id = MagicMock(return_value="cvm-1") # type: ignore[method-assign] - try: - dep.deploy(plan, encrypted) # type: ignore[arg-type] - except Exception: - pass - assert captured, "provision was never called" - body = captured[0] - assert body["disk_size"] == 20 - assert body["compose_file"] is plan.compose - assert "disk_size" not in plan.compose - - -def test_cli_stage_defaults_and_disk_flags(): - from agent_challenge.selfdeploy import cli - - parser = cli.build_parser() - review_ns = parser.parse_args( - [ - "review", - "deploy", - "--base-url", - "https://example.test", - "--submission-id", - "1", - "--hotkey", - "5FakeHotkey", - "--auto-sign", - "--dry-run", - ] - ) - assert review_ns.review_instance_type == "tdx.small" - assert review_ns.eval_instance_type == "tdx.xlarge" - assert review_ns.review_disk_size_gb == 20 - assert review_ns.eval_disk_size_gb == 100 - - eval_ns = parser.parse_args( - [ - "eval", - "deploy", - "--base-url", - "https://example.test", - "--submission-id", - "1", - "--hotkey", - "5FakeHotkey", - "--dry-run", - "--eval-disk-size-gb", - "200", - "--review-disk-size-gb", - "40", - ] - ) - assert eval_ns.eval_instance_type == "tdx.xlarge" - assert eval_ns.eval_disk_size_gb == 200 - assert eval_ns.review_disk_size_gb == 40 diff --git a/packages/challenges/agent-challenge/tests/test_selfdeploy_teardown_http.py b/packages/challenges/agent-challenge/tests/test_selfdeploy_teardown_http.py deleted file mode 100644 index 7cdb959ae..000000000 --- a/packages/challenges/agent-challenge/tests/test_selfdeploy_teardown_http.py +++ /dev/null @@ -1,239 +0,0 @@ -"""HTTP teardown for Phala CVMs (no external ``phala`` binary). - -Covers: - * DELETE /cvms/{id} allowlisted on PhalaCloudClient - * 204 success, 404 idempotent success, other status → PhalaApiError - * default_phala_teardown never shells out to a ``phala`` binary - * optional --cvm-id resolved via GET /cvms + unique app_id match - * ambiguous multi-match refused -""" - -from __future__ import annotations - -import io -import json -from types import SimpleNamespace -from typing import Any -from unittest.mock import MagicMock -from urllib.error import HTTPError -from urllib.request import Request - -import pytest - -from agent_challenge.selfdeploy import cli -from agent_challenge.selfdeploy.phala import ( - PhalaApiError, - PhalaCloudClient, - resolve_cvm_id_from_list, -) - - -class _FakeResponse: - def __init__(self, body: bytes = b"", *, status: int = 204) -> None: - self._body = body - self.status = status - - def read(self) -> bytes: - return self._body - - -def test_delete_cvm_issues_delete_to_cvms_id_path() -> None: - seen: list[Request] = [] - - def opener(request: Request, timeout: float = 0) -> _FakeResponse: # noqa: ARG001 - seen.append(request) - return _FakeResponse(b"", status=204) - - client = PhalaCloudClient(api_key="k" * 32, opener=opener) - client.delete_cvm("cvm-abc-1") - assert len(seen) == 1 - assert seen[0].get_method() == "DELETE" - assert seen[0].full_url.endswith("/cvms/cvm-abc-1") - - -def test_delete_cvm_treats_204_as_success() -> None: - client = PhalaCloudClient( - api_key="k" * 32, - opener=lambda request, timeout=0: _FakeResponse(b"", status=204), # noqa: ARG005 - ) - client.delete_cvm("42") # must not raise - - -def test_delete_cvm_treats_404_as_idempotent_success() -> None: - def opener(request: Request, timeout: float = 0) -> _FakeResponse: # noqa: ARG001 - raise HTTPError( - url=request.full_url, - code=404, - msg="Not Found", - hdrs=None, # type: ignore[arg-type] - fp=io.BytesIO(b""), - ) - - client = PhalaCloudClient(api_key="k" * 32, opener=opener) - client.delete_cvm("already-gone") # must not raise - - -@pytest.mark.parametrize("code", [400, 401, 403, 500]) -def test_delete_cvm_raises_on_non_success_status(code: int) -> None: - def opener(request: Request, timeout: float = 0) -> _FakeResponse: # noqa: ARG001 - raise HTTPError( - url=request.full_url, - code=code, - msg="err", - hdrs=None, # type: ignore[arg-type] - fp=io.BytesIO(b"{}"), - ) - - client = PhalaCloudClient(api_key="k" * 32, opener=opener) - with pytest.raises(PhalaApiError, match=f"HTTP {code}"): - client.delete_cvm("cvm-1") - - -def test_delete_cvm_refuses_non_allowlisted_path_shape() -> None: - client = PhalaCloudClient( - api_key="k" * 32, - opener=lambda *_a, **_k: _FakeResponse(), # pragma: no cover - ) - with pytest.raises(PhalaApiError, match="unsupported|invalid"): - client.delete_cvm("../escape") - with pytest.raises(PhalaApiError, match="unsupported|invalid"): - client.delete_cvm("id/with/slash") - - -def test_delete_is_not_available_via_post_allowlist() -> None: - client = PhalaCloudClient( - api_key="k" * 32, - opener=lambda *_a, **_k: _FakeResponse(b"{}"), # pragma: no cover - ) - with pytest.raises(PhalaApiError, match="unsupported"): - client.post("/cvms/cvm-1", {}) - - -def test_resolve_cvm_id_require_unique_refuses_ambiguous_match() -> None: - listing = { - "items": [ - {"id": 1, "app_id": "same-app"}, - {"id": 2, "app_id": "same-app"}, - ] - } - with pytest.raises(PhalaApiError, match="multiple|ambiguous"): - resolve_cvm_id_from_list(listing, app_id="same-app", require_unique=True) - - -def test_resolve_cvm_id_require_unique_returns_single_match() -> None: - listing = { - "items": [ - {"id": 9, "app_id": "other"}, - {"id": 77, "app_id": "target-app"}, - ] - } - assert resolve_cvm_id_from_list(listing, app_id="target-app", require_unique=True) == "77" - - -def test_default_phala_teardown_uses_http_delete_not_subprocess( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr( - cli.subprocess, - "run", - MagicMock(side_effect=AssertionError("no subprocess")), - ) - deleted: list[str] = [] - - class _Client: - def delete_cvm(self, cvm_id: str) -> None: - deleted.append(cvm_id) - - result = cli.default_phala_teardown("cvm-live-1", client=_Client()) # type: ignore[arg-type] - assert result["ok"] is True - assert result["returncode"] == 0 - assert deleted == ["cvm-live-1"] - assert "phala" not in json.dumps(result).lower() or result.get("error") is None - - -def test_default_phala_teardown_maps_api_error() -> None: - class _Client: - def delete_cvm(self, cvm_id: str) -> None: - raise PhalaApiError("Phala delete returned HTTP 500") - - result = cli.default_phala_teardown("cvm-x", client=_Client()) # type: ignore[arg-type] - assert result["ok"] is False - assert result["returncode"] != 0 - assert "500" in str(result.get("error") or "") - - -def test_cli_teardown_resolves_cvm_id_from_app_id(monkeypatch: pytest.MonkeyPatch) -> None: - listing = {"items": [{"id": "resolved-9", "app_id": "app-hex-1"}]} - deleted: list[str] = [] - - class _Client: - def __init__(self, **_kwargs: Any) -> None: - pass - - def get(self, path: str) -> dict[str, Any]: - assert path == "/cvms" - return listing - - def delete_cvm(self, cvm_id: str) -> None: - deleted.append(cvm_id) - - monkeypatch.setattr(cli, "PhalaCloudClient", _Client) - monkeypatch.setenv("PHALA_CLOUD_API_KEY", "k" * 32) - capture: list[Any] = [] - monkeypatch.setattr(cli, "_print", lambda payload: capture.append(payload)) - - parser = cli.build_parser() - args = parser.parse_args(["review", "teardown", "--app-id", "app-hex-1"]) - code = cli._ordered_review_command(args) - assert code == 0 - assert deleted == ["resolved-9"] - assert capture and capture[0].get("ok") is True - assert capture[0].get("torn_down") == "resolved-9" - - -def test_cli_teardown_refuses_ambiguous_app_id(monkeypatch: pytest.MonkeyPatch) -> None: - listing = { - "items": [ - {"id": "a", "app_id": "dup"}, - {"id": "b", "app_id": "dup"}, - ] - } - - class _Client: - def __init__(self, **_kwargs: Any) -> None: - pass - - def get(self, path: str) -> dict[str, Any]: - return listing - - def delete_cvm(self, cvm_id: str) -> None: # pragma: no cover - raise AssertionError(f"must not delete on ambiguity: {cvm_id}") - - monkeypatch.setattr(cli, "PhalaCloudClient", _Client) - monkeypatch.setenv("PHALA_CLOUD_API_KEY", "k" * 32) - args = SimpleNamespace(review_command="teardown", cvm_id=None, app_id="dup", phala_api=None) - capture: list = [] - monkeypatch.setattr(cli, "_print", lambda payload: capture.append(payload)) - code = cli._ordered_review_command(args) - assert code != 0 - assert capture and "multiple" in str(capture[0].get("diagnostics", {}).get("error", "")).lower() - - -def test_cli_teardown_requires_cvm_id_or_app_id() -> None: - parser = cli.build_parser() - # --cvm-id no longer required at parse time; runtime refuses empty identity. - args = parser.parse_args(["teardown"]) - assert getattr(args, "cvm_id", None) in (None, "") - # Runtime path - import sys - from io import StringIO - - buf = StringIO() - old = sys.stderr - try: - sys.stderr = buf - code = cli.main(["teardown"], teardowner=lambda *_a, **_k: {"ok": True, "returncode": 0}) - finally: - sys.stderr = old - # Without cvm-id/app-id should fail closed (non-zero) rather than call teardowner with None - assert code != 0 diff --git a/packages/challenges/agent-challenge/tests/test_sizing_compose_hash_freeze.py b/packages/challenges/agent-challenge/tests/test_sizing_compose_hash_freeze.py deleted file mode 100644 index 5a2eb587e..000000000 --- a/packages/challenges/agent-challenge/tests/test_sizing_compose_hash_freeze.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Freeze compose hashes so sizing changes cannot move measured documents. - -Sizing (instance_type / disk_size) must stay OUTSIDE the measured app-compose -documents. These constants were measured from the generators BEFORE any sizing -edit on branch feat/agent-challenge-cvm-sizing. -""" - -from __future__ import annotations - -from agent_challenge.canonical.compose import app_compose_hash, generate_app_compose -from agent_challenge.review.compose import generate_review_app_compose, review_app_compose_hash - -# Measured on clean HEAD 263eeb1b before sizing edits (same fixtures as -# tests/test_canonical_compose.py and default review moniker). -_CANONICAL_IMAGE = "ghcr.io/baseintelligence/agent-challenge-canonical@sha256:" + ("a" * 64) -_REVIEW_IMAGE = "ghcr.io/baseintelligence/agent-challenge-review@sha256:" + ("b" * 64) - -FREEZE_EVAL_APP_COMPOSE_HASH = ( - "f8f05273959469a2b8eb3e599863cdb2ddc7c741055d1e6830f101fc2e79d334" -) -FREEZE_REVIEW_APP_COMPOSE_HASH = ( - "9ef4435f4bd3e938f371c93c5ee8076fabf16b75a1ea18f3bdb9c0e24176325f" -) - - -def test_eval_app_compose_hash_frozen_against_sizing_work(): - compose = generate_app_compose(orchestrator_image=_CANONICAL_IMAGE) - assert app_compose_hash(compose) == FREEZE_EVAL_APP_COMPOSE_HASH - # Sizing keys must never appear inside the measured document. - blob = str(compose) - assert "disk_size" not in blob - assert "instance_type" not in blob - assert "tdx.xlarge" not in blob - - -def test_review_app_compose_hash_frozen_against_sizing_work(): - compose = generate_review_app_compose(review_image=_REVIEW_IMAGE) - assert review_app_compose_hash(compose) == FREEZE_REVIEW_APP_COMPOSE_HASH - blob = str(compose) - assert "disk_size" not in blob - assert "instance_type" not in blob - assert "tdx.xlarge" not in blob diff --git a/packages/challenges/agent-challenge/tests/test_tbench_integrity_policy.py b/packages/challenges/agent-challenge/tests/test_tbench_integrity_policy.py deleted file mode 100644 index 65bd45baa..000000000 --- a/packages/challenges/agent-challenge/tests/test_tbench_integrity_policy.py +++ /dev/null @@ -1,340 +0,0 @@ -"""VAL-ACLOCK-011..016: TB 2.1 integrity contracts + allow_internet policy.""" - -from __future__ import annotations - -import hashlib -import inspect -import json -from pathlib import Path - -import pytest - -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.evaluation.benchmarks import ( - TERMINAL_BENCH_2_1_DIGEST_SHA256, - TERMINAL_BENCH_2_1_FALLBACK_TASK_IDS, -) -from agent_challenge.evaluation.own_runner.container_builder import network_arg -from agent_challenge.evaluation.own_runner.taskdefs import ( - DigestMismatch, - ResourceLimits, - bare_task_name, - compute_task_digest, - load_dataset_digest, - load_task, - load_task_from_manifest, -) -from agent_challenge.evaluation.tbench_integrity import ( - ALLOW_INTERNET_POLICY_ID, - ANSWER_HARDCODING_IS_CHEAT, - FORBIDDEN_TASK_SOURCE_KEYS, - REQUIRED_HARNESS_PINS, - TbenchIntegrityError, - allow_internet_policy_snapshot, - assert_fallback_ids_subset_of_frozen, - assert_no_miner_task_source_fields, - assert_selected_task_ids_in_frozen, - assert_taskdefs_loader_is_local_only, - effective_network_arg, - frozen_digest_path, - inventory_allow_internet, - load_frozen_task_ids, - selected_task_item_allowed_keys, -) -from agent_challenge.selfdeploy import eval as eval_deploy - -_REPO = Path(__file__).resolve().parents[1] -_DIGEST = _REPO / "golden" / "dataset-digest.json" -_LIVE_CACHE = _REPO / "docker" / "canonical" / "live-task-cache" -_HARDCODING_RULES = _REPO / ".rules" / "hardcoding.md" -_ANTI_CHEAT_RULES = _REPO / ".rules" / "anti-cheat.md" -_SECURITY_DOC = _REPO / "docs" / "security.md" -_EVAL_DOC = _REPO / "docs" / "evaluation.md" - - -# --------------------------------------------------------------------------- # -# VAL-ACLOCK-011 — no network at eval for task defs -# --------------------------------------------------------------------------- # -def test_taskdefs_loader_has_no_network_clients() -> None: - assert_taskdefs_loader_is_local_only() - - -def test_taskdefs_loader_contract_language_present() -> None: - text = (_REPO / "src/agent_challenge/evaluation/own_runner/taskdefs.py").read_text( - encoding="utf-8" - ) - assert "No network at eval time" in text or "never network-fetches" in text - assert "DigestMismatch" in text - assert "dataset-digest.json" in text - - -def test_digest_mismatch_fail_closed(tmp_path: Path) -> None: - root = tmp_path / "toy" - root.mkdir() - (root / "task.toml").write_text( - '[task]\nname="toy"\n[environment]\nallow_internet=false\n', - encoding="utf-8", - ) - (root / "instruction.md").write_text("do the thing\n", encoding="utf-8") - env = root / "environment" - env.mkdir() - (env / "Dockerfile").write_text("FROM ubuntu:24.04\n", encoding="utf-8") - tests = root / "tests" - tests.mkdir() - (tests / "test.sh").write_text("#!/bin/bash\necho ok\n", encoding="utf-8") - actual = compute_task_digest(root) - with pytest.raises(DigestMismatch): - load_task(root, task_id="toy", expected_digest="0" * 64) - with pytest.raises(DigestMismatch): - load_task_from_manifest( - root, - task_id="toy", - digest_manifest={"tasks": {"toy": {"content_digest_sha256": "1" * 64}}}, - ) - # Matching digest still loads. - loaded = load_task(root, task_id="toy", expected_digest=actual) - assert loaded.content_digest_sha256 == actual - - -def test_eval_docs_state_no_network_fetch() -> None: - text = _EVAL_DOC.read_text(encoding="utf-8") - assert "no network" in text.lower() or "does not network-fetch" in text.lower() - assert "dataset-digest.json" in text or "task-cache" in text - - -# --------------------------------------------------------------------------- # -# VAL-ACLOCK-012 — miner cannot supply alternate task URL/git -# --------------------------------------------------------------------------- # -def test_forbidden_task_source_keys_rejected() -> None: - for key in sorted(FORBIDDEN_TASK_SOURCE_KEYS): - with pytest.raises(TbenchIntegrityError, match="alternate task"): - assert_no_miner_task_source_fields({key: "https://evil.example/tasks.git"}) - - -def test_honest_plan_fields_have_no_task_url() -> None: - assert_no_miner_task_source_fields( - { - "task_id": "terminal-bench/bn-fit-modify", - "image_ref": "registry.example/t@sha256:" + "a" * 64, - "task_config_sha256": "b" * 64, - } - ) - - -def test_selected_tasks_schema_rejects_task_url_extra() -> None: - """Plan selected_tasks[] is schema-closed; task_url is unknown → reject.""" - policy = { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": "off", - "drop_lowest_n": 0, - "threshold_f64be": None, - } - base = { - "schema_version": 1, - "eval_run_id": "eval-run-tbench", - "submission_id": "submission-001", - "submission_version": 1, - "authorizing_review_digest": "1" * 64, - "agent_hash": "a" * 64, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": "task-a", - "image_ref": "registry.example/task@sha256:" + "d" * 64, - "task_config_sha256": "2" * 64, - "task_url": "https://evil.example/task.git", - } - ], - "k": 1, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "d" * 64, - "compose_hash": "c" * 64, - "app_identity": "agent-challenge-eval", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "3" * 64, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("3" * 64)).hexdigest(), - "measurement": { - "mrtd": "a1" * 48, - "rtmr0": "a2" * 48, - "rtmr1": "a3" * 48, - "rtmr2": "a4" * 48, - "os_image_hash": "a5" * 32, - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "ratls://kr.internal:8701", - "result_endpoint": "/evaluation/v1/runs/eval-run-tbench/result", - "key_release_nonce": "key-nonce-001", - "score_nonce": "score-nonce-001", - "run_token_sha256": "5" * 64, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - with pytest.raises(ew.EvalWireError, match="invalid fields|unknown"): - ew.validate_eval_plan(base) - - -def test_prepare_response_has_no_miner_task_body_fields() -> None: - """eval/prepare deploy helper only accepts signed plan wrapper (validator only).""" - src = inspect.getsource(eval_deploy.build_eval_deployment_plan) - assert "task_url" not in src - assert "plan" in src - - -# --------------------------------------------------------------------------- # -# VAL-ACLOCK-013 — frozen set + fallback IDs -# --------------------------------------------------------------------------- # -def test_digest_pin_matches_constant() -> None: - actual = hashlib.sha256(_DIGEST.read_bytes()).hexdigest() - assert actual == TERMINAL_BENCH_2_1_DIGEST_SHA256 - assert frozen_digest_path() == _DIGEST - - -def test_fallback_ids_subset_of_frozen() -> None: - assert_fallback_ids_subset_of_frozen() - frozen = load_frozen_task_ids() - for task_id in TERMINAL_BENCH_2_1_FALLBACK_TASK_IDS: - assert task_id in frozen - assert bare_task_name(task_id) in frozen - - -def test_manifest_task_count_and_load() -> None: - manifest = load_dataset_digest(_DIGEST) - assert len(manifest["tasks"]) == 89 - assert set(TERMINAL_BENCH_2_1_FALLBACK_TASK_IDS).issubset( - {f"terminal-bench/{n}" for n in manifest["tasks"]} - ) - - -def test_unknown_selected_id_fails() -> None: - with pytest.raises(TbenchIntegrityError, match="not in frozen"): - assert_selected_task_ids_in_frozen(["terminal-bench/this-is-not-a-real-task"]) - - -# --------------------------------------------------------------------------- # -# VAL-ACLOCK-014 — selected_tasks validator-authored only -# --------------------------------------------------------------------------- # -def test_selected_tasks_allowed_keys_only() -> None: - assert selected_task_item_allowed_keys() == frozenset( - {"task_id", "image_ref", "task_config_sha256"} - ) - - -def test_authorization_build_plan_uses_validator_selection() -> None: - """_build_plan pulls tasks from load_benchmark_tasks + settings, not miner body.""" - from agent_challenge.evaluation import authorization as auth - - src = inspect.getsource(auth._build_plan) - assert "select_benchmark_tasks" in src - assert "load_benchmark_tasks" in src - assert "task_url" not in src - assert "selected_tasks" in src - - -def test_prepare_route_has_no_selected_tasks_request_model() -> None: - routes_src = (_REPO / "src/agent_challenge/api/routes.py").read_text(encoding="utf-8") - # prepare_submission_eval takes submission_id + auth only (no body tasks). - assert "async def prepare_submission_eval" in routes_src - start = routes_src.index("async def prepare_submission_eval") - chunk = routes_src[start : start + 500] - assert "selected_tasks" not in chunk - assert "task_url" not in chunk - - -# --------------------------------------------------------------------------- # -# VAL-ACLOCK-015 — hardcode answers cheat; harness pins required -# --------------------------------------------------------------------------- # -def test_hardcoding_rules_flag_answer_cheat() -> None: - text = _HARDCODING_RULES.read_text(encoding="utf-8") - assert "branches_on_task_identity" in text - assert "tailors_to_tests" in text or "static output" in text.lower() - anti = _ANTI_CHEAT_RULES.read_text(encoding="utf-8") - assert "branches_on_task_identity" in anti - assert "hardcode" in anti.lower() or "expected result" in anti.lower() - - -def test_harness_pins_documented_and_listed() -> None: - assert len(REQUIRED_HARNESS_PINS) >= 6 - snap = allow_internet_policy_snapshot() - joined = " ".join(snap["required_harness_pins"]).lower() - assert "dataset-digest" in joined or "digest" in joined - assert "hardcoding" in ANSWER_HARDCODING_IS_CHEAT.lower() - assert "cheat" in ANSWER_HARDCODING_IS_CHEAT.lower() - sec = _SECURITY_DOC.read_text(encoding="utf-8") - # Product docs distinguish harness pins vs answer hardcoding. - assert "hardcod" in sec.lower() or "anti-cheat" in sec.lower() or "digest" in sec.lower() - - -def test_security_doc_states_hardcode_vs_pin_policy() -> None: - text = _SECURITY_DOC.read_text(encoding="utf-8") - assert "allow_internet" in text or "Terminal-Bench" in text - assert "dataset-digest" in text or "task-cache" in text or "content-address" in text.lower() - - -# --------------------------------------------------------------------------- # -# VAL-ACLOCK-016 — allow_internet policy documented + gated -# --------------------------------------------------------------------------- # -def test_allow_internet_policy_id_locked() -> None: - snap = allow_internet_policy_snapshot() - assert snap["policy_id"] == ALLOW_INTERNET_POLICY_ID - assert snap["default_scored_behavior"] == "honor_task_toml_allow_internet" - assert snap["opt_in_restrict_default"] is False - assert snap["task_def_network_at_eval"] is False - assert snap["selected_tasks_author"] == "validator_prepare_only" - - -def test_inventory_live_cache_all_allow_true() -> None: - if not _LIVE_CACHE.is_dir(): - pytest.skip("live-task-cache not present in checkout") - inv = inventory_allow_internet(_LIVE_CACHE) - assert inv["counts"]["scanned"] == 89 - assert inv["counts"]["true"] == 89 - assert inv["counts"]["false"] == 0 - assert inv["policy_id"] == ALLOW_INTERNET_POLICY_ID - - -def test_default_network_arg_honors_task_authored() -> None: - assert network_arg(ResourceLimits(allow_internet=True)) is None - assert network_arg(ResourceLimits(allow_internet=False)) == "none" - assert network_arg(ResourceLimits(allow_internet=None)) == "none" - assert effective_network_arg(ResourceLimits(allow_internet=True), scored_run=True) is None - - -def test_opt_in_restrict_forces_none(monkeypatch: pytest.MonkeyPatch) -> None: - env = {"CHALLENGE_SCORED_TASK_NETWORK_RESTRICT": "1"} - assert ( - effective_network_arg( - ResourceLimits(allow_internet=True), - scored_run=True, - environ=env, - ) - == "none" - ) - # Unscored / non-restrict path still honors task when env not set. - assert ( - effective_network_arg( - ResourceLimits(allow_internet=True), - scored_run=True, - environ={}, - ) - is None - ) - monkeypatch.setenv("CHALLENGE_SCORED_TASK_NETWORK_RESTRICT", "true") - assert network_arg(ResourceLimits(allow_internet=True)) == "none" - monkeypatch.delenv("CHALLENGE_SCORED_TASK_NETWORK_RESTRICT", raising=False) - assert network_arg(ResourceLimits(allow_internet=True)) is None - - -def test_security_doc_documents_allow_internet_choice() -> None: - text = _SECURITY_DOC.read_text(encoding="utf-8") - assert "allow_internet" in text - assert "retain" in text.lower() or "review-class" in text.lower() or "residual" in text.lower() - - -def test_policy_json_serializable() -> None: - payload = json.dumps(allow_internet_policy_snapshot()) - assert ALLOW_INTERNET_POLICY_ID in payload diff --git a/packages/challenges/agent-challenge/tests/test_weights_canonical_population.py b/packages/challenges/agent-challenge/tests/test_weights_canonical_population.py deleted file mode 100644 index 5b3733291..000000000 --- a/packages/challenges/agent-challenge/tests/test_weights_canonical_population.py +++ /dev/null @@ -1,634 +0,0 @@ -"""Direct-weight path: one canonical eligible EvalRun population. - -VAL-SCORE-012 / VAL-VERIFY-019 / VAL-CROSS-006: - -* Weights and winner-take-all must consume exactly the same filtered - population reconstructed from immutable plan-bound score bytes plus exact - submission/review state. -* Mutable ``EvalRun.score`` / ``passed_tasks`` / ``total_tasks`` columns cannot - alter the output. -* Invalid, suspicious, overridden-invalid, stale-version, mismatched-review, - incomplete, or malformed runs earn no weight. -* Canonical ``final.total_tasks`` remains the complete selected set even when a - keep-policy excludes tasks from the scoring mean. -* Fully legacy (flag-off) weight calculation remains unchanged. -""" - -from __future__ import annotations - -import hashlib -import json -from datetime import UTC, datetime, timedelta - -import pytest -from sqlalchemy import select - -from agent_challenge.canonical import eval_wire as ew -from agent_challenge.core.config import settings -from agent_challenge.core.models import ( - AgentSubmission, - EvalRun, - EvaluationJob, - ReviewAssignment, - ReviewSession, -) -from agent_challenge.evaluation.plan_scoring import ( - build_score_record_from_eval_plan, - canonical_eval_plan_json, -) -from agent_challenge.evaluation.weights import get_weights -from agent_challenge.sdk.config import effective_evaluation_task_count - -NOW = datetime(2026, 6, 1, 12, 0, tzinfo=UTC) -REQUIRED_TASKS = effective_evaluation_task_count(settings.evaluation_task_count) -MEASUREMENT = { - "mrtd": "a" * 96, - "rtmr0": "b" * 96, - "rtmr1": "c" * 96, - "rtmr2": "d" * 96, - "compose_hash": "e" * 64, - "os_image_hash": "f" * 64, -} -AGENT_HASH = "1" * 64 -REVIEW_DIGEST = "2" * 64 - - -def _policy( - *, - keep_policy: str = "off", - drop_lowest_n: int = 0, - threshold_f64be: str | None = None, -) -> dict[str, object]: - return { - "schema_version": 1, - "per_task_aggregation": "mean", - "keep_policy": keep_policy, - "drop_lowest_n": drop_lowest_n, - "threshold_f64be": threshold_f64be, - } - - -def _plan( - *, - eval_run_id: str, - policy: dict[str, object] | None = None, - k: int = 1, - task_count: int | None = None, - authorizing_review_digest: str = REVIEW_DIGEST, - submission_version: int = 1, -) -> dict[str, object]: - count = task_count if task_count is not None else REQUIRED_TASKS - policy = policy or _policy() - task_ids = [f"task-{index:03d}" for index in range(count)] - return { - "schema_version": 1, - "eval_run_id": eval_run_id, - "submission_id": f"submission-{eval_run_id}", - "submission_version": submission_version, - "authorizing_review_digest": authorizing_review_digest, - "agent_hash": AGENT_HASH, - "package_tree_sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "selected_tasks": [ - { - "task_id": task_id, - "image_ref": "registry.example/task@sha256:" + "3" * 64, - "task_config_sha256": "4" * 64, - } - for task_id in task_ids - ], - "k": k, - "scoring_policy": policy, - "scoring_policy_digest": ew.scoring_policy_digest(policy), - "eval_app": { - "image_ref": "registry.example/eval@sha256:" + "5" * 64, - "compose_hash": MEASUREMENT["compose_hash"], - "app_identity": "agent-challenge-eval", - "kms_key_algorithm": "x25519", - "kms_public_key_hex": "6" * 64, - "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("6" * 64)).hexdigest(), - "measurement": { - "mrtd": MEASUREMENT["mrtd"], - "rtmr0": MEASUREMENT["rtmr0"], - "rtmr1": MEASUREMENT["rtmr1"], - "rtmr2": MEASUREMENT["rtmr2"], - "os_image_hash": MEASUREMENT["os_image_hash"], - "key_provider": "validator-kms", - "vm_shape": "tdx-small", - }, - }, - "key_release_endpoint": "keyrelease.example:8701", - "result_endpoint": f"/evaluation/v1/runs/{eval_run_id}/result", - "key_release_nonce": f"key-nonce-{eval_run_id}", - "score_nonce": f"score-nonce-{eval_run_id}", - "run_token_sha256": "7" * 64, - "issued_at_ms": 1, - "expires_at_ms": 2, - } - - -def _trials_for( - plan: dict[str, object], - *, - perfect_count: int, -) -> dict[str, list[float]]: - task_ids = [item["task_id"] for item in plan["selected_tasks"]] - trials: dict[str, list[float]] = {} - for index, task_id in enumerate(task_ids): - value = 1.0 if index < perfect_count else 0.0 - trials[task_id] = [value] * int(plan["k"]) - return trials - - -async def _attach_verified_review( - session, submission: AgentSubmission, *, review_digest: str -) -> None: - assignment_id = f"ra-{submission.id}-{review_digest[:8]}" - artifact_sha = submission.agent_hash or ("8" * 64) - review_session = ReviewSession( - session_id=f"review-session-{submission.id}", - submission_id=submission.id, - artifact_sha256=artifact_sha, - artifact_size_bytes=1, - manifest_sha256="11" * 32, - manifest_entries_sha256="12" * 32, - current_assignment_id=assignment_id, - authorizing_assignment_id=assignment_id, - ) - session.add(review_session) - await session.flush() - assignment = ReviewAssignment( - session_id=review_session.id, - assignment_id=assignment_id, - attempt=1, - assignment_bytes="{}", - assignment_digest="13" * 32, - artifact_sha256=artifact_sha, - rules_snapshot_sha256="14" * 32, - rules_revision_id="rules-v1", - review_nonce=f"review-nonce-{submission.id}", - session_token_sha256="15" * 32, - capability_state="revoked", - phase="review_allowed", - issued_at=NOW, - expires_at=NOW + timedelta(hours=1), - review_report_envelope_json='{"schema_version":1}', - review_digest=review_digest, - review_verification_outcome_json=json.dumps( - { - "status": "verified_allow", - "terminal": True, - "retryable": False, - "reason_code": "policy_allowed", - "nonce_consumed": True, - "measurement_allowlisted": True, - "report_data_matched": True, - "verified_at_ms": 1, - }, - sort_keys=True, - separators=(",", ":"), - ), - ) - session.add(assignment) - await session.flush() - - -async def _add_direct_run( - session, - *, - hotkey: str, - eval_run_id: str, - plan: dict[str, object], - trial_scores_by_task: dict[str, list[float]], - created_at: datetime = NOW, - raw_status: str = "tb_completed", - effective_status: str = "valid", - submission_version: int = 1, - run_submission_version: int | None = None, - authorizing_review_digest: str = REVIEW_DIGEST, - attach_matching_review: bool = True, - phase: str = "eval_accepted", - verified: bool = True, - reward_eligible: bool = True, - result_available: bool = True, - mutate_score: float | None = None, - mutate_passed: int | None = None, - mutate_total: int | None = None, - corrupt_score_record: bool = False, - omit_score_record: bool = False, -) -> EvalRun: - unique_agent_hash = hashlib.sha256(eval_run_id.encode("utf-8")).hexdigest() - submission = AgentSubmission( - miner_hotkey=hotkey, - name=f"agent-{eval_run_id}", - agent_hash=unique_agent_hash, - artifact_uri=f"/tmp/{eval_run_id}.zip", - status="tb_completed", - raw_status=raw_status, - effective_status=effective_status, - version_number=submission_version, - submitted_at=created_at, - created_at=created_at, - ) - session.add(submission) - await session.flush() - if attach_matching_review: - await _attach_verified_review(session, submission, review_digest=authorizing_review_digest) - - record = build_score_record_from_eval_plan(plan, trial_scores_by_task) - plan_json = canonical_eval_plan_json(plan) - score_json = ew.canonical_json_v1(record).decode("utf-8") - if corrupt_score_record: - # Structural malformed JSON for the score record surface. - score_json = '{"schema_version":1,"not":"a-score-record"}' - run = EvalRun( - eval_run_id=eval_run_id, - submission_id=submission.id, - submission_version=( - run_submission_version if run_submission_version is not None else submission_version - ), - authorizing_review_digest=authorizing_review_digest, - plan_json=plan_json, - plan_sha256=hashlib.sha256(plan_json.encode("utf-8")).hexdigest(), - token_sha256=hashlib.sha256(f"token-{eval_run_id}".encode()).hexdigest(), - phase=phase, - verified=verified, - reward_eligible=reward_eligible, - result_available=result_available, - score=( - mutate_score - if mutate_score is not None - else ew.decode_score_f64be(record["final"]["job_score_f64be"]) - ), - passed_tasks=( - mutate_passed if mutate_passed is not None else record["final"]["passed_tasks"] - ), - total_tasks=(mutate_total if mutate_total is not None else record["final"]["total_tasks"]), - canonical_score_record_json=None if omit_score_record else score_json, - canonical_score_record_sha256=( - None if omit_score_record else ew.score_record_digest(record) - ), - issued_at=created_at, - expires_at=created_at + timedelta(hours=6), - finalized_at=created_at + timedelta(minutes=5), - ) - session.add(run) - await session.flush() - return run - - -@pytest.fixture -def enable_attestation(monkeypatch): - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.phala_attestation_enabled", - True, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.attested_review_enabled", - True, - ) - - -async def test_mutable_score_columns_cannot_change_weights( - database_session, enable_attestation, monkeypatch -): - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.weights_winner_take_all", - False, - ) - plan = _plan(eval_run_id="eval-mutable") - trials = _trials_for(plan, perfect_count=max(1, REQUIRED_TASKS // 2)) - true_record = build_score_record_from_eval_plan(plan, trials) - true_score = ew.decode_score_f64be(true_record["final"]["job_score_f64be"]) - - async with database_session() as session: - await _add_direct_run( - session, - hotkey="hk-mutable", - eval_run_id="eval-mutable", - plan=plan, - trial_scores_by_task=trials, - mutate_score=0.999, - mutate_passed=REQUIRED_TASKS, - mutate_total=1, # under-count on the mutable column - ) - await session.commit() - - assert await get_weights() == {"hk-mutable": true_score} - - -async def test_keep_policy_exclusion_preserves_full_selected_total_tasks( - database_session, enable_attestation, monkeypatch -): - """VAL-SCORE-012: eligibility gates on full selected set, not kept-set size.""" - - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.weights_winner_take_all", - False, - ) - # One perfect, rest zeros; drop-lowest keeps only the perfect task for the mean - # but final.total_tasks / eligibility still cover the FULL selected set. - policy = _policy(keep_policy="drop_lowest_n", drop_lowest_n=REQUIRED_TASKS - 1) - plan = _plan(eval_run_id="eval-keep", policy=policy) - trials = _trials_for(plan, perfect_count=1) - record = build_score_record_from_eval_plan(plan, trials) - assert record["final"]["total_tasks"] == REQUIRED_TASKS - assert record["final"]["passed_tasks"] == 1 - true_score = ew.decode_score_f64be(record["final"]["job_score_f64be"]) - - async with database_session() as session: - await _add_direct_run( - session, - hotkey="hk-keep", - eval_run_id="eval-keep", - plan=plan, - trial_scores_by_task=trials, - # Attacker shrinks the mutable total_tasks gate. - mutate_total=1, - mutate_passed=1, - mutate_score=1.0, - ) - await session.commit() - - assert await get_weights() == {"hk-keep": true_score} - - -@pytest.mark.parametrize( - "effective_status", - ["invalid", "suspicious", "overridden_invalid"], -) -async def test_non_valid_effective_status_earns_no_weight( - database_session, enable_attestation, effective_status -): - plan = _plan(eval_run_id=f"eval-{effective_status}") - trials = _trials_for(plan, perfect_count=REQUIRED_TASKS) - async with database_session() as session: - await _add_direct_run( - session, - hotkey=f"hk-{effective_status}", - eval_run_id=f"eval-{effective_status}", - plan=plan, - trial_scores_by_task=trials, - effective_status=effective_status, - ) - await session.commit() - - assert await get_weights() == {} - - -async def test_stale_submission_version_earns_no_weight(database_session, enable_attestation): - plan = _plan(eval_run_id="eval-stale-version", submission_version=1) - trials = _trials_for(plan, perfect_count=REQUIRED_TASKS) - async with database_session() as session: - await _add_direct_run( - session, - hotkey="hk-stale", - eval_run_id="eval-stale-version", - plan=plan, - trial_scores_by_task=trials, - submission_version=2, - run_submission_version=1, - ) - await session.commit() - - assert await get_weights() == {} - - -async def test_mismatched_review_digest_earns_no_weight(database_session, enable_attestation): - plan = _plan(eval_run_id="eval-mismatch", authorizing_review_digest=REVIEW_DIGEST) - trials = _trials_for(plan, perfect_count=REQUIRED_TASKS) - async with database_session() as session: - await _add_direct_run( - session, - hotkey="hk-mismatch", - eval_run_id="eval-mismatch", - plan=plan, - trial_scores_by_task=trials, - authorizing_review_digest=REVIEW_DIGEST, - attach_matching_review=True, - ) - # Point the run at a different digest than the authorizing assignment. - run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == "eval-mismatch")) - assert run is not None - run.authorizing_review_digest = "c" * 64 - await session.commit() - - assert await get_weights() == {} - - -async def test_incomplete_and_malformed_runs_earn_no_weight( - database_session, enable_attestation, monkeypatch -): - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.weights_winner_take_all", - False, - ) - good_plan = _plan(eval_run_id="eval-good") - incomplete_plan = _plan(eval_run_id="eval-incomplete") - malformed_plan = _plan(eval_run_id="eval-malformed") - good_trials = _trials_for(good_plan, perfect_count=REQUIRED_TASKS) - incomplete_trials = _trials_for(incomplete_plan, perfect_count=REQUIRED_TASKS) - malformed_trials = _trials_for(malformed_plan, perfect_count=REQUIRED_TASKS) - async with database_session() as session: - await _add_direct_run( - session, - hotkey="hk-good", - eval_run_id="eval-good", - plan=good_plan, - trial_scores_by_task=good_trials, - ) - await _add_direct_run( - session, - hotkey="hk-incomplete", - eval_run_id="eval-incomplete", - plan=incomplete_plan, - trial_scores_by_task=incomplete_trials, - omit_score_record=True, - ) - await _add_direct_run( - session, - hotkey="hk-malformed", - eval_run_id="eval-malformed", - plan=malformed_plan, - trial_scores_by_task=malformed_trials, - corrupt_score_record=True, - ) - await session.commit() - - assert await get_weights() == {"hk-good": 1.0} - - -async def test_per_hotkey_and_wta_use_identical_filtered_population( - database_session, enable_attestation, monkeypatch -): - good_plan_a = _plan(eval_run_id="eval-a") - good_plan_b = _plan(eval_run_id="eval-b") - bad_plan = _plan(eval_run_id="eval-bad-status") - trials_pass = _trials_for(good_plan_a, perfect_count=REQUIRED_TASKS) - trials_half = _trials_for(good_plan_b, perfect_count=max(1, REQUIRED_TASKS // 2)) - trials_bad = _trials_for(bad_plan, perfect_count=REQUIRED_TASKS) - score_half = ew.decode_score_f64be( - build_score_record_from_eval_plan(good_plan_b, trials_half)["final"]["job_score_f64be"] - ) - - async with database_session() as session: - await _add_direct_run( - session, - hotkey="hk-a", - eval_run_id="eval-a", - plan=good_plan_a, - trial_scores_by_task=trials_pass, - created_at=NOW, - ) - await _add_direct_run( - session, - hotkey="hk-b", - eval_run_id="eval-b", - plan=good_plan_b, - trial_scores_by_task=trials_half, - created_at=NOW + timedelta(minutes=1), - ) - await _add_direct_run( - session, - hotkey="hk-bad", - eval_run_id="eval-bad-status", - plan=bad_plan, - trial_scores_by_task=trials_bad, - effective_status="suspicious", - # Mutable columns claim a perfect score that must still be excluded. - mutate_score=1.0, - mutate_passed=REQUIRED_TASKS, - mutate_total=REQUIRED_TASKS, - ) - await session.commit() - - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.weights_winner_take_all", - False, - ) - per_hotkey = await get_weights() - assert per_hotkey == {"hk-a": 1.0, "hk-b": score_half} - assert "hk-bad" not in per_hotkey - - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.weights_winner_take_all", - True, - ) - wta = await get_weights() - assert wta == {"hk-a": 1.0} - assert "hk-bad" not in wta - - -async def test_wta_tie_break_uses_filtered_population_only( - database_session, enable_attestation, monkeypatch -): - """Winner-take-all must not consult toxic unfiltered EvalRun rows.""" - - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.weights_winner_take_all", - True, - ) - eligible_plan = _plan(eval_run_id="eval-eligible-tie") - ineligible_plan = _plan(eval_run_id="eval-ineligible-higher") - trials = _trials_for(eligible_plan, perfect_count=max(1, REQUIRED_TASKS // 3)) - high_trials = _trials_for(ineligible_plan, perfect_count=REQUIRED_TASKS) - eligible_score = ew.decode_score_f64be( - build_score_record_from_eval_plan(eligible_plan, trials)["final"]["job_score_f64be"] - ) - - async with database_session() as session: - await _add_direct_run( - session, - hotkey="hk-eligible", - eval_run_id="eval-eligible-tie", - plan=eligible_plan, - trial_scores_by_task=trials, - created_at=NOW, - ) - # Higher mutable score + earlier arrival, but invalid status: must not win. - await _add_direct_run( - session, - hotkey="hk-toxic", - eval_run_id="eval-ineligible-higher", - plan=ineligible_plan, - trial_scores_by_task=high_trials, - created_at=NOW - timedelta(hours=1), - effective_status="invalid", - mutate_score=1.0, - ) - await session.commit() - - assert await get_weights() == {"hk-eligible": eligible_score} - - -async def test_flag_off_legacy_weights_ignore_eval_runs(database_session, monkeypatch): - """Fully legacy mode is byte-identical: only EvaluationJob rows feed weights.""" - - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.phala_attestation_enabled", - False, - ) - monkeypatch.setattr( - "agent_challenge.evaluation.weights.settings.weights_winner_take_all", - False, - ) - plan = _plan(eval_run_id="eval-legacy-ignore") - trials = _trials_for(plan, perfect_count=REQUIRED_TASKS) - async with database_session() as session: - await _add_direct_run( - session, - hotkey="hk-eval-only", - eval_run_id="eval-legacy-ignore", - plan=plan, - trial_scores_by_task=trials, - ) - submission = AgentSubmission( - miner_hotkey="hk-legacy-job", - name="agent-legacy-job", - agent_hash="legacy-hash", - artifact_uri="/tmp/legacy.zip", - status="tb_completed", - raw_status="tb_completed", - effective_status="valid", - submitted_at=NOW, - created_at=NOW, - ) - session.add(submission) - await session.flush() - job = EvaluationJob( - job_id="job-legacy", - submission_id=submission.id, - status="completed", - selected_tasks_json="[]", - score=0.42, - passed_tasks=1, - total_tasks=REQUIRED_TASKS, - verdict="valid", - ) - session.add(job) - await session.flush() - submission.latest_evaluation_job_id = job.id - await session.commit() - - assert await get_weights() == {"hk-legacy-job": 0.42} - - -async def test_reward_eligible_flag_alone_cannot_admit_ineligible_run( - database_session, enable_attestation -): - """Mutable reward_eligible=True cannot override a missing verified review.""" - - plan = _plan(eval_run_id="eval-no-review") - trials = _trials_for(plan, perfect_count=REQUIRED_TASKS) - async with database_session() as session: - await _add_direct_run( - session, - hotkey="hk-no-review", - eval_run_id="eval-no-review", - plan=plan, - trial_scores_by_task=trials, - attach_matching_review=False, - reward_eligible=True, - ) - await session.commit() - - assert await get_weights() == {} diff --git a/src/base/config/settings.py b/src/base/config/settings.py index c05b80c81..ecbc6ec26 100644 --- a/src/base/config/settings.py +++ b/src/base/config/settings.py @@ -72,7 +72,9 @@ class MasterSettings(BaseModel): upload_extra_registered_hotkeys: list[str] = Field(default_factory=list) # Opt-in canonical Agent Challenge review/eval proxy topology. OFF preserves # the legacy signed submission/env/launch proxy behavior byte-for-byte. - agent_challenge_attested_routes_enabled: bool = False + agent_challenge_attested_routes_enabled: bool = ( + False # T40: Phala routes removed; keep False + ) # Validator coordination plane (architecture.md sec 4). The proxy serves the # hotkey-signed register/heartbeat/pull/progress/result routes, returns # ``validator_heartbeat_interval_seconds`` to validators, marks a validator diff --git a/tests/unit/test_master_embed_env_file.py b/tests/unit/test_master_embed_env_file.py index 6028e4c35..f4af4f6dd 100644 --- a/tests/unit/test_master_embed_env_file.py +++ b/tests/unit/test_master_embed_env_file.py @@ -3,14 +3,16 @@ ``docker/master-entrypoint.sh`` launches each embedded challenge under ``env -i`` so Prism never inherits ``CHALLENGE_*`` and agent-challenge never inherits ``PRISM_*``. That isolation is deliberate, but it also dropped every -operator-supplied setting -- notably the Phala attestation switches -(``CHALLENGE_PHALA_ATTESTATION_ENABLED``, ``CHALLENGE_ATTESTED_REVIEW_ENABLED``) -and ``PHALA_CLOUD_API_KEY`` -- because the allowlist was hardcoded with no +operator-supplied setting because the allowlist was hardcoded with no extension point. These tests lock the supported extension point: a per-challenge env file whose keys are merged into the isolated child environment, without breaking cross-challenge isolation. + +T40/T41: product path is host-trust / unattested only. Dual Phala flags must +not be true in templates; tests use host-trust keys (NO_PHALA / UNATTESTED / +DOCKER_BACKEND) to prove the merge mechanism. """ from __future__ import annotations @@ -91,17 +93,17 @@ def _run_entrypoint(tmp_path: Path, ac_env_file_body: str | None) -> dict[str, s return dumps -def test_ac_env_file_supplies_phala_settings_to_isolated_child(tmp_path: Path) -> None: +def test_ac_env_file_supplies_host_trust_settings(tmp_path: Path) -> None: """Operator embed.env must reach the agent-challenge child under env -i.""" dumps = _run_entrypoint( tmp_path, "\n".join( [ - "# durable AC embed overrides", - "CHALLENGE_PHALA_ATTESTATION_ENABLED=true", - "CHALLENGE_ATTESTED_REVIEW_ENABLED=true", - "PHALA_CLOUD_API_KEY=phala-secret", + "# durable AC embed overrides (host-trust / T40)", + "CHALLENGE_NO_PHALA=true", + "CHALLENGE_UNATTESTED_EXECUTION=true", + "CHALLENGE_DOCKER_BACKEND=broker", "BASE_CHALLENGE_SLUG=agent-challenge", "", ] @@ -109,9 +111,9 @@ def test_ac_env_file_supplies_phala_settings_to_isolated_child(tmp_path: Path) - ) ac_env = dumps["ac"] - assert "CHALLENGE_PHALA_ATTESTATION_ENABLED=true" in ac_env - assert "CHALLENGE_ATTESTED_REVIEW_ENABLED=true" in ac_env - assert "PHALA_CLOUD_API_KEY=phala-secret" in ac_env + assert "CHALLENGE_NO_PHALA=true" in ac_env + assert "CHALLENGE_UNATTESTED_EXECUTION=true" in ac_env + assert "CHALLENGE_DOCKER_BACKEND=broker" in ac_env assert "BASE_CHALLENGE_SLUG=agent-challenge" in ac_env @@ -120,12 +122,12 @@ def test_ac_env_file_does_not_leak_into_prism_child(tmp_path: Path) -> None: dumps = _run_entrypoint( tmp_path, - "CHALLENGE_PHALA_ATTESTATION_ENABLED=true\nPHALA_CLOUD_API_KEY=phala-secret\n", + "CHALLENGE_NO_PHALA=true\nCHALLENGE_DOCKER_BACKEND=broker\n", ) prism_env = dumps["prism"] - assert "CHALLENGE_PHALA_ATTESTATION_ENABLED" not in prism_env - assert "PHALA_CLOUD_API_KEY" not in prism_env + assert "CHALLENGE_NO_PHALA" not in prism_env + assert "CHALLENGE_DOCKER_BACKEND=broker" not in prism_env def test_ac_env_file_overrides_builtin_default(tmp_path: Path) -> None: @@ -143,7 +145,7 @@ def test_missing_env_file_is_not_fatal(tmp_path: Path) -> None: dumps = _run_entrypoint(tmp_path, None) assert "CHALLENGE_DOCKER_ENABLED=false" in dumps["ac"] - assert "PHALA_CLOUD_API_KEY" not in dumps["ac"] + assert "CHALLENGE_NO_PHALA" not in dumps["ac"] def test_env_file_ignores_comments_blanks_and_malformed_keys(tmp_path: Path) -> None: