Skip to content

evaluate: 48 evaluation(s) (run 20261007T150255Z) #2290

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fedcourtsai-dev merged 1 commit into from
Oct 7, 2026
Merged

evaluate: 48 evaluation(s) (run 20261007T150255Z) #2290

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
1 change: 1 addition & 0 deletions ...evt-petition-disposition/evaluations/gemini-judge/20261007T150255Z/retrieval.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
No retrieval beyond the provisioned inputs.
343 changes: 343 additions & 0 deletions ...nts/evt-petition-disposition/evaluations/gemini-judge/20261007T150255Z/retrieval_log.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,343 @@
{
"actor_id": "gemini-judge",
"calls": [
{
"call_source": "transcript_item",
"params_digest": "97e321e682ed2a18",
"query": "{\"strategic_intent\": \"Read instructions and requirements for the evaluator role.\", \"summary\": \"I am starting the evaluation task. I need to read .github/prompts/evaluate.md and AGENTS.md to understand the prompt contract and the outputs expected for the evaluator cell.\", \"title\": \"Read Evaluator Instructions\"}",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:43:59.522Z",
"tool": "update_topic"
},
{
"call_source": "transcript_item",
"params_digest": "4f333312b1459284",
"query": "{\"file_path\": \".github/prompts/evaluate.md\"}",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:02.192Z",
"tool": "read_file"
},
{
"call_source": "transcript_item",
"params_digest": "d5466b59ea22872b",
"query": "{\"file_path\": \"AGENTS.md\"}",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:02.193Z",
"tool": "read_file"
},
{
"call_source": "transcript_item",
"params_digest": "f31f156e4808b0eb",
"query": "{\"strategic_intent\": \"Read input data for the evaluation task.\", \"summary\": \"I have read the evaluator instructions. I will now gather the prediction data for each candidate, the outcome, the schema, and the statpack. This will provide the necessary inputs for scoring the predictions.\", \"title\": \"Gather Evaluation Inputs\"}",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:10.173Z",
"tool": "update_topic"
},
{
"call_source": "transcript_item",
"params_digest": "604c5990e8fb51e0",
"query": "ls -la data/cases/scotus/73287447/record/blinded/",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:16.505Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "84e0ca50a6ea1c44",
"query": "{\"file_path\": \"data/cases/scotus/73287447/events/evt-petition-disposition/outcome.json\"}",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:16.506Z",
"tool": "read_file"
},
{
"call_source": "transcript_item",
"params_digest": "c426da729cbe7f3c",
"query": "{\"file_path\": \"schemas/evaluation.schema.json\"}",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:16.506Z",
"tool": "read_file"
},
{
"call_source": "transcript_item",
"params_digest": "40880455d2ef5a77",
"query": "{\"file_path\": \"data/cases/scotus/73287447/events/evt-petition-disposition/event.yaml\"}",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:16.507Z",
"tool": "read_file"
},
{
"call_source": "transcript_item",
"params_digest": "623ec4092a36c2e3",
"query": "grep -n \"Segment base rate by salience band\" metrics/statpack.md",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:20.302Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "311138c21a85b755",
"query": "{\"file_path\": \"metrics/statpack.md\"}",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:20.303Z",
"tool": "read_file"
},
{
"call_source": "transcript_item",
"params_digest": "31b025715a13f84f",
"query": "{\"file_path\": \"data/cases/scotus/73287447/record/blinded/claude-baseline/retrieval_log.json\"}",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:27.152Z",
"tool": "read_file"
},
{
"call_source": "transcript_item",
"params_digest": "743536fbfee32fae",
"query": "{\"file_path\": \"data/cases/scotus/73287447/record/blinded/claude-baseline/reasoning.md\"}",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:27.152Z",
"tool": "read_file"
},
{
"call_source": "transcript_item",
"params_digest": "a41285018c67e0c8",
"query": "{\"file_path\": \"data/cases/scotus/73287447/record/blinded/claude-baseline/prediction.json\"}",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:27.152Z",
"tool": "read_file"
},
{
"call_source": "transcript_item",
"params_digest": "5bf1f07059c23d13",
"query": "cat data/cases/scotus/73287447/record/blinded/claude-baseline/prediction.json",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:30.917Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "0a69e36b6f476383",
"query": "cat data/cases/scotus/73287447/record/blinded/gemini-baseline/prediction.json",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:30.918Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "c31542bcafcd428a",
"query": "cat data/cases/scotus/73287447/record/blinded/codex-baseline/prediction.json",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:30.918Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "c02b9cec823d837f",
"query": "cat data/cases/scotus/73287447/record/blinded/claude-baseline/retrieval_log.json",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:43.603Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "f7c4fb015e20a0e7",
"query": "cat data/cases/scotus/73287447/record/blinded/codex-baseline/retrieval_log.json",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:43.604Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "5030a1d295b943f0",
"query": "cat data/cases/scotus/73287447/record/blinded/gemini-baseline/retrieval_log.json",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:43.605Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "e6b102a4e9744252",
"query": "cat data/cases/scotus/73287447/record/blinded/claude-baseline/reasoning.md",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:53.502Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "0a171fd573cd0490",
"query": "cat data/cases/scotus/73287447/record/blinded/gemini-baseline/reasoning.md",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:53.503Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "2353b5a7a78d8493",
"query": "cat data/cases/scotus/73287447/record/blinded/codex-baseline/reasoning.md",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:44:53.503Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "c11f223d0f0affbc",
"query": "{\"strategic_intent\": \"Write evaluation outputs.\", \"summary\": \"I have extracted all the predictions, reasoned about their quality, and calculated the necessary scores (Brier, Brier skill score, segment base rate). I am now going to write the `evaluation.json` and `evaluation.md` files for each candidate, as well as the global `tooling.json` and `retrieval.md` for this evaluation run.\", \"title\": \"Write Evaluation Files\"}",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:45:07.292Z",
"tool": "update_topic"
},
{
"call_source": "transcript_item",
"params_digest": "c3908978e01a9578",
"query": "cat << 'EOF' > evaluate_outputs.py\nimport json\nimport os\nfrom datetime import datetime, timezone\n\ncourt_id = \"scotus\"\ndocket_id = \"73287447\"\nevent_id = \"evt-petition-disposition\"\nevaluator_id = \"gemini-judge\"\nrun_id = \"20261007T150255Z\"\nmodel_id = \"gemini-3.1-pro-preview\"\nengine = \"gemini\"\n\nbase_dir = f\"data/cases/{court_id}/{docket_id}/events/{event_id}/evaluations/{evaluator_id}\"\n\n# 1. tooling.json and retrieval.md\nrun_dir = os.path.join(base_dir, run_id)\nos.makedirs(run_dir, exist_ok=True)\n\nw",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:45:29.457Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "14b1cd450d258038",
"query": "{\"content\": \"import json\\nimport os\\nfrom datetime import datetime, timezone\\n\\ncourt_id = \\\"scotus\\\"\\ndocket_id = \\\"73287447\\\"\\nevent_id = \\\"evt-petition-disposition\\\"\\nevaluator_id = \\\"gemini-judge\\\"\\nrun_id = \\\"20261007T150255Z\\\"\\nmodel_id = \\\"gemini-3.1-pro-preview\\\"\\nengine = \\\"gemini\\\"\\n\\nbase_dir = f\\\"data/cases/{court_id}/{docket_id}/events/{event_id}/evaluations/{evaluator_id}\\\"\\n\\n# 1. tooling.json and retrieval.md\\nrun_dir = os.path.join(base_dir, run_id)\\nos.makedirs(run_dir, exist_o",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:45:41.371Z",
"tool": "write_file"
},
{
"call_source": "transcript_item",
"params_digest": "5872bb37a5352e89",
"query": "python3 evaluate_outputs.py && uv run fedcourts validate data 2>&1 | tail -n 20",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:46:17.944Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "e865e5422c620616",
"query": "cat data/cases/scotus/73287447/events/evt-petition-disposition/evaluations/gemini-judge/claude-baseline/20261007T150255Z/evaluation.json",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:46:30.113Z",
"tool": "run_shell_command"
},
{
"call_source": "transcript_item",
"params_digest": "8a1391125edc8b98",
"query": "{\"strategic_intent\": \"Finish task.\", \"summary\": \"I have successfully evaluated all predictions against the provided schema and guidelines. The expected outputs (`evaluation.json`, `evaluation.md`, `retrieval.md`, `tooling.json`) were generated for the target paths. No git commit or push is required.\", \"title\": \"Finish\"}",
"result_capture": "unobserved",
"result_digest": null,
"result_status": "unobserved",
"retrieved_doc_date": null,
"timestamp": "2026-10-07T16:46:35.324Z",
"tool": "update_topic"
}
],
"case_id": "scotus/73287447",
"engine": "gemini",
"mcp_servers": [
"courtlistener=courtlistener-api-client[mcp]==1.1.0"
],
"mcp_tools": [
"courtlistener.analyze_citations",
"courtlistener.call_endpoint",
"courtlistener.create_search_alert",
"courtlistener.delete_search_alert",
"courtlistener.extract_citations",
"courtlistener.get_choices",
"courtlistener.get_counts",
"courtlistener.get_endpoint_item",
"courtlistener.get_endpoint_schema",
"courtlistener.get_more_results",
"courtlistener.read_document",
"courtlistener.resume_citation_analysis",
"courtlistener.search",
"courtlistener.search_document",
"courtlistener.subscribe_to_docket_alert",
"courtlistener.unsubscribe_from_docket_alert"
],
"mode": null,
"result_capture_coverage": 0.0,
"role": "evaluator",
"run_id": "20261007T150255Z",
"schema_version": "1.0",
"self_provisioned_fetches": 0,
"throttled_calls": null
}
12 changes: 12 additions & 0 deletions ...47/events/evt-petition-disposition/evaluations/gemini-judge/20261007T150255Z/tooling.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"case_id": "scotus/73287447",
"run_id": "20261007T150255Z",
"role": "evaluator",
"actor_id": "gemini-judge",
"used_corpus_query": false,
"used_base_rates": true,
"tools_used": [],
"helpful": [],
"gaps": [],
"notes": ""
}
17 changes: 17 additions & 0 deletions ...7447/events/evt-petition-disposition/evaluations/gemini-judge/20261007T150255Z/usage.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"actor_id": "gemini-judge",
"cache_creation_input_tokens": 0,
"cache_read_input_tokens": 941620,
"case_id": "scotus/73287447",
"created_at": "2026-10-07T15:02:55Z",
"engine": "gemini",
"estimated_cost_usd": 0.591726,
"event_id": "evt-petition-disposition",
"input_tokens": 138989,
"model": "gemini-3.1-pro-preview",
"output_tokens": 10452,
"pipeline_sha": "6282fde457410ebd1be77945b31e764c61a18b84",
"role": "evaluator",
"run_id": "20261007T150255Z",
"schema_version": "1.0"
}
Loading
Loading