From 4aeaf46ceedd3acb730ecc75bef1e737f3a88da5 Mon Sep 17 00:00:00 2001 From: Shuxin Lin Date: Thu, 20 Aug 2026 20:39:06 +0000 Subject: [PATCH] feat(benchmarks): add model matrix run script Signed-off-by: Shuxin Lin --- benchmarks/run.sh | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 benchmarks/run.sh diff --git a/benchmarks/run.sh b/benchmarks/run.sh new file mode 100755 index 00000000..f9d42819 --- /dev/null +++ b/benchmarks/run.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -euo pipefail + +scenario_dir="${1:-${SCENARIO_DIR:-}}" +leaderboard_dir="${2:-${LEADERBOARD_DIR:-}}" + +if [[ -z "$scenario_dir" || -z "$leaderboard_dir" ]]; then + printf 'Usage: %s SCENARIO_DIR LEADERBOARD_DIR\n' "$0" >&2 + printf ' or set SCENARIO_DIR and LEADERBOARD_DIR\n' >&2 + exit 2 +fi + +agent_name=stirrup_agent +scenario_ids=lite + +model_configs=( + "litellm_proxy/gcp/gemini-3.6-flash high" + "litellm_proxy/azure/gpt-5.6-sol max" + "tokenrouter/z-ai/glm-5.2 max" + "tokenrouter/MiniMax-M3 high" + "litellm_proxy/aws/claude-opus-5 high" + "litellm_proxy/azure/DeepSeek-V4-Flash max" + "litellm_proxy/aws/gpt-oss-120b high" + "litellm_proxy/aws/claude-sonnet-5 max" +) + +for model_config in "${model_configs[@]}"; do + read -r model_id reasoning_effort <<< "$model_config" + + echo "Running $model_id with reasoning effort $reasoning_effort" + + uv run python -m benchmark.scenario_suite_runner \ + --scenario-ids "$scenario_ids" \ + --scenario-root "$scenario_dir" \ + --agent_name "$agent_name" \ + --model-id "$model_id" \ + --reasoning-effort "$reasoning_effort" \ + --trajectory-root "$leaderboard_dir/assetopsbench-trajectories" \ + --reports-root "$leaderboard_dir/assetopsbench-reports" \ + --stirrup-workspace-root "$leaderboard_dir/assetopsbench-stirrup-workspaces" \ + --skip-existing \ + --continue-on-error +done