diff --git a/.gitignore b/.gitignore index 37fb92d5..08344c13 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,18 @@ analysis_out/op/* # e.g. crop_window_eval_richmond.json. !analysis_out/crop_window_eval.json !analysis_out/crop_window_eval_*.json +# The machine-readable scoreboard behind docs/model_scoreboard.md: every (leg, split) cell -- +# P/R/F1/AP, TP/FP/FN, panorama and GT counts -- for the aggregates that page tabulates. +# ~80 KB. It regenerates from committed data alone, but committing it is what lets a +# reviewer diff two runs instead of re-deriving both, and the doc points at it by name. +# Written with LF on every platform (scoreboard_render.write_json), so a Windows re-run +# is a no-op diff rather than whole-file churn. +# +# The PR-curve POINT ARRAYS are deliberately not in it -- see scoreboard_render. +# json_payload. They are ~120k points, which is 7.7 MB of committed artifact nobody +# diffs; the AP and the marked thresholds, which the page does cite, are kept. If this +# file is ever megabytes again, that stripping regressed. +!analysis_out/scoreboard.json !analysis_out/farfield_forensics.json !analysis_out/silent_activation.json diff --git a/analysis_out/scoreboard.json b/analysis_out/scoreboard.json new file mode 100644 index 00000000..64ef5d8a --- /dev/null +++ b/analysis_out/scoreboard.json @@ -0,0 +1,2864 @@ +{ + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville" + ], + "city_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville", + "budapest_district5", + "sao_paulo" + ], + "all_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville", + "budapest_district5", + "sao_paulo", + "manual_gold" + ], + "held_out": { + "budapest_district5": "single-rater GT at low reviewer confidence (docs/model_comparison.md: do not pool)", + "sao_paulo": "non-US city \u2014 the pooled recommendation is a US-deployment basis (GT is HIGH reviewer confidence; held out for geography, not GT quality)", + "manual_gold": "in-distribution GSV + independently-labelled GT (in-domain reference, not a deployment city)" + }, + "in_distribution_split": "manual_gold", + "unregistered_exports": [], + "curves": { + "rampnet": { + "ap": 0.844306, + "n_points": 2882, + "marks": { + "0.55": { + "recall": 0.72233, + "precision": 0.963731, + "f1": 0.825749 + }, + "0.30": { + "recall": 0.792718, + "precision": 0.900221, + "f1": 0.843056 + } + } + }, + "google/owlv2-large-patch14-ensemble": { + "ap": 0.092832, + "n_points": 57480, + "marks": {} + }, + "IDEA-Research/grounding-dino-base": { + "ap": 0.035978, + "n_points": 63344, + "marks": {} + }, + "y11l_pano": { + "ap": 0.726311, + "n_points": 2868, + "marks": {} + }, + "y11x_pano_h200": { + "ap": 0.734334, + "n_points": 2507, + "marks": {} + }, + "y26_pano": { + "ap": 0.608235, + "n_points": 5594, + "marks": {} + } + }, + "splits": { + "richmond": { + "n_panos": 124, + "n_gt": 310, + "pooled": true + }, + "bend": { + "n_panos": 110, + "n_gt": 327, + "pooled": true + }, + "clovis": { + "n_panos": 125, + "n_gt": 195, + "pooled": true + }, + "morgantown": { + "n_panos": 125, + "n_gt": 267, + "pooled": true + }, + "annapolis": { + "n_panos": 125, + "n_gt": 294, + "pooled": true + }, + "paterson": { + "n_panos": 125, + "n_gt": 395, + "pooled": true + }, + "gainesville": { + "n_panos": 125, + "n_gt": 272, + "pooled": true + }, + "budapest_district5": { + "n_panos": 125, + "n_gt": 300, + "pooled": false + }, + "sao_paulo": { + "n_panos": 125, + "n_gt": 281, + "pooled": false + }, + "manual_gold": { + "n_panos": 1000, + "n_gt": 3919, + "pooled": false + } + }, + "models": [ + { + "model": "rampnet", + "label": "rampnet", + "spec": "rampnet", + "provider": "rampnet", + "standing": true, + "display": "RampNet", + "class": "purpose-trained", + "operating_point": 0.55, + "operating_point_note": "0.55", + "coverage": "7/7", + "complete": true, + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville" + ], + "precision": 0.958231, + "recall": 0.727867, + "f1": 0.826981, + "ap": 0.848699, + "ap_bundle": 0.719941, + "ap_is_substituted": true, + "fp_per_pano": 0.071356, + "micro_precision": 0.960897, + "micro_recall": 0.72767, + "f1_min": 0.801153, + "f1_max": 0.854578, + "f1_min_split": "clovis", + "f1_max_split": "richmond", + "n_splits_run": 10, + "budapest_district5_f1": 0.644211, + "budapest_district5_precision": 0.874286, + "budapest_district5_recall": 0.51, + "budapest_district5_ap": 0.647705, + "sao_paulo_f1": 0.77666, + "sao_paulo_precision": 0.893519, + "sao_paulo_recall": 0.686833, + "sao_paulo_ap": 0.811637, + "manual_gold_f1": 0.908487, + "manual_gold_precision": 0.947368, + "manual_gold_recall": 0.872672, + "manual_gold_ap": 0.917292 + }, + { + "model": "y11l_pano", + "label": "y11l_pano", + "spec": "yolo:yolo_ckpts/y11l_pano.pt", + "provider": "yolo", + "standing": false, + "display": "YOLO11l (pano)", + "class": "supervised", + "operating_point": 0.25, + "operating_point_note": "0.25", + "coverage": "7/7", + "complete": true, + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville" + ], + "precision": 0.939038, + "recall": 0.449446, + "f1": 0.603724, + "ap": 0.722441, + "ap_bundle": 0.722441, + "ap_is_substituted": false, + "fp_per_pano": 0.067478, + "micro_precision": 0.941532, + "micro_recall": 0.453398, + "f1_min": 0.480818, + "f1_max": 0.712909, + "f1_min_split": "annapolis", + "f1_max_split": "bend", + "n_splits_run": 10, + "budapest_district5_f1": 0.247191, + "budapest_district5_precision": 0.785714, + "budapest_district5_recall": 0.146667, + "budapest_district5_ap": 0.443481, + "sao_paulo_f1": 0.662162, + "sao_paulo_precision": 0.90184, + "sao_paulo_recall": 0.523132, + "sao_paulo_ap": 0.783949, + "manual_gold_f1": 0.838918, + "manual_gold_precision": 0.930927, + "manual_gold_recall": 0.76346, + "manual_gold_ap": 0.913984 + }, + { + "model": "y11x_pano_h200", + "label": "y11x_pano_h200", + "spec": "yolo:yolo_ckpts/y11x_pano_h200.pt", + "provider": "yolo", + "standing": false, + "display": "YOLO11x (pano)", + "class": "supervised", + "operating_point": 0.25, + "operating_point_note": "0.25", + "coverage": "7/7", + "complete": true, + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville" + ], + "precision": 0.968716, + "recall": 0.415557, + "f1": 0.574875, + "ap": 0.729641, + "ap_bundle": 0.729641, + "ap_is_substituted": false, + "fp_per_pano": 0.028938, + "micro_precision": 0.971973, + "micro_recall": 0.420874, + "f1_min": 0.396739, + "f1_max": 0.709804, + "f1_min_split": "annapolis", + "f1_max_split": "bend", + "n_splits_run": 10, + "budapest_district5_f1": 0.22093, + "budapest_district5_precision": 0.863636, + "budapest_district5_recall": 0.126667, + "budapest_district5_ap": 0.42687, + "sao_paulo_f1": 0.659039, + "sao_paulo_precision": 0.923077, + "sao_paulo_recall": 0.512456, + "sao_paulo_ap": 0.782801, + "manual_gold_f1": 0.850913, + "manual_gold_precision": 0.955789, + "manual_gold_recall": 0.766777, + "manual_gold_ap": 0.930925 + }, + { + "model": "y26_pano", + "label": "y26_pano", + "spec": "yolo:yolo_ckpts/y26_pano.pt", + "provider": "yolo", + "standing": false, + "display": "YOLO26 (pano)", + "class": "supervised", + "operating_point": 0.25, + "operating_point_note": "0.25", + "coverage": "7/7", + "complete": true, + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville" + ], + "precision": 0.736366, + "recall": 0.44627, + "f1": 0.55033, + "ap": 0.6029, + "ap_bundle": 0.6029, + "ap_is_substituted": false, + "fp_per_pano": 0.377815, + "micro_precision": 0.741754, + "micro_recall": 0.447573, + "f1_min": 0.449761, + "f1_max": 0.681034, + "f1_min_split": "annapolis", + "f1_max_split": "morgantown", + "n_splits_run": 10, + "budapest_district5_f1": 0.276762, + "budapest_district5_precision": 0.638554, + "budapest_district5_recall": 0.176667, + "budapest_district5_ap": 0.349591, + "sao_paulo_f1": 0.604555, + "sao_paulo_precision": 0.722772, + "sao_paulo_recall": 0.519573, + "sao_paulo_ap": 0.658092, + "manual_gold_f1": 0.739247, + "manual_gold_precision": 0.73953, + "manual_gold_recall": 0.738964, + "manual_gold_ap": 0.843378 + }, + { + "model": "gemini-3.1-pro-preview", + "label": "gemini-3.1-pro-preview", + "spec": "gemini:gemini-3.1-pro-preview", + "provider": "gemini", + "standing": true, + "display": "Gemini 3.1 Pro", + "class": "chat-vlm", + "operating_point": 0.0, + "operating_point_note": "no score", + "coverage": "7/7", + "complete": true, + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville" + ], + "precision": 0.655804, + "recall": 0.57276, + "f1": 0.608078, + "ap": null, + "ap_bundle": null, + "ap_is_substituted": false, + "fp_per_pano": 0.711759, + "micro_precision": 0.660744, + "micro_recall": 0.57767, + "f1_min": 0.513514, + "f1_max": 0.680851, + "f1_min_split": "clovis", + "f1_max_split": "paterson", + "n_splits_run": 9, + "budapest_district5_f1": 0.381308, + "budapest_district5_precision": 0.434043, + "budapest_district5_recall": 0.34, + "budapest_district5_ap": null, + "sao_paulo_f1": 0.453721, + "sao_paulo_precision": 0.462963, + "sao_paulo_recall": 0.44484, + "sao_paulo_ap": null, + "manual_gold_f1": null, + "manual_gold_precision": null, + "manual_gold_recall": null, + "manual_gold_ap": null + }, + { + "model": "gemini-3.7-flash", + "label": "gemini-3.7-flash", + "spec": "gemini:gemini-3.7-flash", + "provider": "gemini", + "standing": false, + "display": "Gemini 3.7 Flash", + "class": "chat-vlm", + "operating_point": 0.0, + "operating_point_note": "no score", + "coverage": "7/7", + "complete": true, + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville" + ], + "precision": 0.697686, + "recall": 0.496471, + "f1": 0.57606, + "ap": null, + "ap_bundle": null, + "ap_is_substituted": false, + "fp_per_pano": 0.514148, + "micro_precision": 0.700954, + "micro_recall": 0.499515, + "f1_min": 0.456212, + "f1_max": 0.664286, + "f1_min_split": "gainesville", + "f1_max_split": "richmond", + "n_splits_run": 10, + "budapest_district5_f1": 0.338395, + "budapest_district5_precision": 0.484472, + "budapest_district5_recall": 0.26, + "budapest_district5_ap": null, + "sao_paulo_f1": 0.358354, + "sao_paulo_precision": 0.560606, + "sao_paulo_recall": 0.263345, + "sao_paulo_ap": null, + "manual_gold_f1": 0.527401, + "manual_gold_precision": 0.734731, + "manual_gold_recall": 0.411329, + "manual_gold_ap": null + }, + { + "model": "gemini-3.6-flash", + "label": "gemini-3.6-flash", + "spec": "gemini:gemini-3.6-flash", + "provider": "gemini", + "standing": true, + "display": "Gemini 3.6 Flash", + "class": "chat-vlm", + "operating_point": 0.0, + "operating_point_note": "no score", + "coverage": "7/7", + "complete": true, + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville" + ], + "precision": 0.588383, + "recall": 0.548933, + "f1": 0.56369, + "ap": null, + "ap_bundle": null, + "ap_is_substituted": false, + "fp_per_pano": 0.930136, + "micro_precision": 0.587565, + "micro_recall": 0.550485, + "f1_min": 0.43771, + "f1_max": 0.633758, + "f1_min_split": "gainesville", + "f1_max_split": "richmond", + "n_splits_run": 9, + "budapest_district5_f1": 0.335664, + "budapest_district5_precision": 0.352941, + "budapest_district5_recall": 0.32, + "budapest_district5_ap": null, + "sao_paulo_f1": 0.34589, + "sao_paulo_precision": 0.333333, + "sao_paulo_recall": 0.359431, + "sao_paulo_ap": null, + "manual_gold_f1": null, + "manual_gold_precision": null, + "manual_gold_recall": null, + "manual_gold_ap": null + }, + { + "model": "Qwen/Qwen3-VL-32B-Instruct", + "label": "Qwen/Qwen3-VL-32B-Instruct", + "spec": "qwen:Qwen/Qwen3-VL-32B-Instruct", + "provider": "qwen", + "standing": true, + "display": "Qwen3-VL-32B", + "class": "chat-vlm", + "operating_point": 0.0, + "operating_point_note": "no score", + "coverage": "7/7", + "complete": true, + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville" + ], + "precision": 0.664401, + "recall": 0.246286, + "f1": 0.355773, + "ap": null, + "ap_bundle": null, + "ap_is_substituted": false, + "fp_per_pano": 0.288787, + "micro_precision": 0.675, + "micro_recall": 0.249029, + "f1_min": 0.16763, + "f1_max": 0.426914, + "f1_min_split": "gainesville", + "f1_max_split": "richmond", + "n_splits_run": 10, + "budapest_district5_f1": 0.078788, + "budapest_district5_precision": 0.433333, + "budapest_district5_recall": 0.043333, + "budapest_district5_ap": null, + "sao_paulo_f1": 0.217877, + "sao_paulo_precision": 0.506494, + "sao_paulo_recall": 0.13879, + "sao_paulo_ap": null, + "manual_gold_f1": 0.285361, + "manual_gold_precision": 0.738806, + "manual_gold_recall": 0.176831, + "manual_gold_ap": null + }, + { + "model": "Qwen/Qwen3-VL-8B-Instruct", + "label": "Qwen/Qwen3-VL-8B-Instruct", + "spec": "qwen:Qwen/Qwen3-VL-8B-Instruct", + "provider": "qwen", + "standing": true, + "display": "Qwen3-VL-8B", + "class": "chat-vlm", + "operating_point": 0.0, + "operating_point_note": "no score", + "coverage": "7/7", + "complete": true, + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville" + ], + "precision": 0.325138, + "recall": 0.360131, + "f1": 0.338115, + "ap": null, + "ap_bundle": null, + "ap_is_substituted": false, + "fp_per_pano": 1.799895, + "micro_precision": 0.326077, + "micro_recall": 0.363592, + "f1_min": 0.256637, + "f1_max": 0.405099, + "f1_min_split": "clovis", + "f1_max_split": "paterson", + "n_splits_run": 10, + "budapest_district5_f1": 0.169065, + "budapest_district5_precision": 0.183594, + "budapest_district5_recall": 0.156667, + "budapest_district5_ap": null, + "sao_paulo_f1": 0.218924, + "sao_paulo_precision": 0.228682, + "sao_paulo_recall": 0.209964, + "sao_paulo_ap": null, + "manual_gold_f1": 0.386482, + "manual_gold_precision": 0.445258, + "manual_gold_recall": 0.341414, + "manual_gold_ap": null + }, + { + "model": "allenai/Molmo2-8B", + "label": "allenai/Molmo2-8B", + "spec": "molmo:allenai/Molmo2-8B", + "provider": "molmo", + "standing": true, + "display": "Molmo2-8B", + "class": "pointing", + "operating_point": 0.0, + "operating_point_note": "no score", + "coverage": "7/7", + "complete": true, + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville" + ], + "precision": 0.431711, + "recall": 0.439993, + "f1": 0.430479, + "ap": null, + "ap_bundle": null, + "ap_is_substituted": false, + "fp_per_pano": 1.422899, + "micro_precision": 0.425492, + "micro_recall": 0.440777, + "f1_min": 0.328725, + "f1_max": 0.510699, + "f1_min_split": "gainesville", + "f1_max_split": "paterson", + "n_splits_run": 10, + "budapest_district5_f1": 0.274448, + "budapest_district5_precision": 0.260479, + "budapest_district5_recall": 0.29, + "budapest_district5_ap": null, + "sao_paulo_f1": 0.326389, + "sao_paulo_precision": 0.318644, + "sao_paulo_recall": 0.33452, + "sao_paulo_ap": null, + "manual_gold_f1": 0.422236, + "manual_gold_precision": 0.511434, + "manual_gold_recall": 0.35953, + "manual_gold_ap": null + }, + { + "model": "google/owlv2-large-patch14-ensemble", + "label": "google/owlv2-large-patch14-ensemble", + "spec": "owlv2", + "provider": "owlv2", + "standing": true, + "display": "OWLv2-large", + "class": "open-vocab", + "operating_point": 0.0, + "operating_point_note": "0.05 floor", + "coverage": "7/7", + "complete": true, + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville" + ], + "precision": 0.033648, + "recall": 0.943607, + "f1": 0.064934, + "ap": 0.097632, + "ap_bundle": 0.097632, + "ap_is_substituted": false, + "fp_per_pano": 64.826707, + "micro_precision": 0.033787, + "micro_recall": 0.942718, + "f1_min": 0.048881, + "f1_max": 0.077192, + "f1_min_split": "clovis", + "f1_max_split": "paterson", + "n_splits_run": 10, + "budapest_district5_f1": 0.061685, + "budapest_district5_precision": 0.0319, + "budapest_district5_recall": 0.93, + "budapest_district5_ap": 0.088843, + "sao_paulo_f1": 0.05194, + "sao_paulo_precision": 0.026723, + "sao_paulo_recall": 0.921708, + "sao_paulo_ap": 0.049839, + "manual_gold_f1": 0.087846, + "manual_gold_precision": 0.046159, + "manual_gold_recall": 0.906864, + "manual_gold_ap": 0.097078 + }, + { + "model": "IDEA-Research/grounding-dino-base", + "label": "IDEA-Research/grounding-dino-base", + "spec": "gdino", + "provider": "gdino", + "standing": true, + "display": "Grounding DINO", + "class": "open-vocab", + "operating_point": 0.0, + "operating_point_note": "0.05 floor", + "coverage": "7/7", + "complete": true, + "pooled_splits": [ + "richmond", + "bend", + "clovis", + "morgantown", + "annapolis", + "paterson", + "gainesville" + ], + "precision": 0.028281, + "recall": 0.856985, + "f1": 0.054647, + "ap": 0.037518, + "ap_bundle": 0.037518, + "ap_is_substituted": false, + "fp_per_pano": 71.5537, + "micro_precision": 0.027754, + "micro_recall": 0.853398, + "f1_min": 0.034705, + "f1_max": 0.073409, + "f1_min_split": "clovis", + "f1_max_split": "bend", + "n_splits_run": 10, + "budapest_district5_f1": 0.041803, + "budapest_district5_precision": 0.021472, + "budapest_district5_recall": 0.786667, + "budapest_district5_ap": 0.025415, + "sao_paulo_f1": 0.048796, + "sao_paulo_precision": 0.025169, + "sao_paulo_recall": 0.797153, + "sao_paulo_ap": 0.033603, + "manual_gold_f1": 0.081559, + "manual_gold_precision": 0.04282, + "manual_gold_recall": 0.855575, + "manual_gold_ap": 0.06732 + }, + { + "model": "mask2former-vistas-curb-cut", + "label": "mask2former-vistas-curb-cut", + "spec": "vistas:curb-cut", + "provider": "vistas", + "standing": false, + "display": "Mask2Former Vistas (curb cut)", + "class": "supervised-transfer", + "operating_point": 0.0, + "operating_point_note": "export floor", + "coverage": "1/7", + "complete": false, + "pooled_splits": [ + "richmond" + ], + "precision": 0.411429, + "recall": 0.696774, + "f1": 0.517365, + "ap": 0.512564, + "ap_bundle": 0.512564, + "ap_is_substituted": false, + "fp_per_pano": 2.491935, + "micro_precision": 0.411429, + "micro_recall": 0.696774, + "f1_min": 0.517365, + "f1_max": 0.517365, + "f1_min_split": "richmond", + "f1_max_split": "richmond", + "n_splits_run": 1, + "budapest_district5_f1": null, + "budapest_district5_precision": null, + "budapest_district5_recall": null, + "budapest_district5_ap": null, + "sao_paulo_f1": null, + "sao_paulo_precision": null, + "sao_paulo_recall": null, + "sao_paulo_ap": null, + "manual_gold_f1": null, + "manual_gold_precision": null, + "manual_gold_recall": null, + "manual_gold_ap": null + }, + { + "model": "mask2former-vistas-curb-cut+curb", + "label": "mask2former-vistas-curb-cut+curb", + "spec": "vistas:curb-cut+curb", + "provider": "vistas", + "standing": false, + "display": "Mask2Former Vistas (+curb)", + "class": "supervised-transfer", + "operating_point": 0.0, + "operating_point_note": "export floor", + "coverage": "1/7", + "complete": false, + "pooled_splits": [ + "richmond" + ], + "precision": 0.125625, + "recall": 0.648387, + "f1": 0.210471, + "ap": 0.089131, + "ap_bundle": 0.089131, + "ap_is_substituted": false, + "fp_per_pano": 11.282258, + "micro_precision": 0.125625, + "micro_recall": 0.648387, + "f1_min": 0.210471, + "f1_max": 0.210471, + "f1_min_split": "richmond", + "f1_max_split": "richmond", + "n_splits_run": 1, + "budapest_district5_f1": null, + "budapest_district5_precision": null, + "budapest_district5_recall": null, + "budapest_district5_ap": null, + "sao_paulo_f1": null, + "sao_paulo_precision": null, + "sao_paulo_recall": null, + "sao_paulo_ap": null, + "manual_gold_f1": null, + "manual_gold_precision": null, + "manual_gold_recall": null, + "manual_gold_ap": null + }, + { + "model": "claude-opus-5-effort-low", + "label": "claude-opus-5", + "spec": "claude:claude-opus-5", + "provider": "claude", + "standing": false, + "display": "Claude Opus 5 (low)", + "class": "chat-vlm", + "operating_point": 0.0, + "operating_point_note": "no score", + "coverage": "1/7", + "complete": false, + "pooled_splits": [ + "annapolis" + ], + "precision": 0.572347, + "recall": 0.605442, + "f1": 0.58843, + "ap": null, + "ap_bundle": null, + "ap_is_substituted": false, + "fp_per_pano": 1.064, + "micro_precision": 0.572347, + "micro_recall": 0.605442, + "f1_min": 0.58843, + "f1_max": 0.58843, + "f1_min_split": "annapolis", + "f1_max_split": "annapolis", + "n_splits_run": 1, + "budapest_district5_f1": null, + "budapest_district5_precision": null, + "budapest_district5_recall": null, + "budapest_district5_ap": null, + "sao_paulo_f1": null, + "sao_paulo_precision": null, + "sao_paulo_recall": null, + "sao_paulo_ap": null, + "manual_gold_f1": null, + "manual_gold_precision": null, + "manual_gold_recall": null, + "manual_gold_ap": null + }, + { + "model": "claude-opus-5-effort-high", + "label": "claude-opus-5", + "spec": "claude:claude-opus-5", + "provider": "claude", + "standing": false, + "display": "Claude Opus 5 (high)", + "class": "chat-vlm", + "operating_point": 0.0, + "operating_point_note": "no score", + "coverage": "1/7", + "complete": false, + "pooled_splits": [ + "annapolis" + ], + "precision": 0.429844, + "recall": 0.656463, + "f1": 0.519515, + "ap": null, + "ap_bundle": null, + "ap_is_substituted": false, + "fp_per_pano": 2.048, + "micro_precision": 0.429844, + "micro_recall": 0.656463, + "f1_min": 0.519515, + "f1_max": 0.519515, + "f1_min_split": "annapolis", + "f1_max_split": "annapolis", + "n_splits_run": 1, + "budapest_district5_f1": null, + "budapest_district5_precision": null, + "budapest_district5_recall": null, + "budapest_district5_ap": null, + "sao_paulo_f1": null, + "sao_paulo_precision": null, + "sao_paulo_recall": null, + "sao_paulo_ap": null, + "manual_gold_f1": null, + "manual_gold_precision": null, + "manual_gold_recall": null, + "manual_gold_ap": null + }, + { + "model": "claude-sonnet-5-effort-low", + "label": "claude-sonnet-5", + "spec": "claude:claude-sonnet-5", + "provider": "claude", + "standing": false, + "display": "Claude Sonnet 5 (low)", + "class": "chat-vlm", + "operating_point": 0.0, + "operating_point_note": "no score", + "coverage": "1/7", + "complete": false, + "pooled_splits": [ + "annapolis" + ], + "precision": 0.589474, + "recall": 0.380952, + "f1": 0.46281, + "ap": null, + "ap_bundle": null, + "ap_is_substituted": false, + "fp_per_pano": 0.624, + "micro_precision": 0.589474, + "micro_recall": 0.380952, + "f1_min": 0.46281, + "f1_max": 0.46281, + "f1_min_split": "annapolis", + "f1_max_split": "annapolis", + "n_splits_run": 1, + "budapest_district5_f1": null, + "budapest_district5_precision": null, + "budapest_district5_recall": null, + "budapest_district5_ap": null, + "sao_paulo_f1": null, + "sao_paulo_precision": null, + "sao_paulo_recall": null, + "sao_paulo_ap": null, + "manual_gold_f1": null, + "manual_gold_precision": null, + "manual_gold_recall": null, + "manual_gold_ap": null + }, + { + "model": "claude-sonnet-5-effort-high", + "label": "claude-sonnet-5", + "spec": "claude:claude-sonnet-5", + "provider": "claude", + "standing": false, + "display": "Claude Sonnet 5 (high)", + "class": "chat-vlm", + "operating_point": 0.0, + "operating_point_note": "no score", + "coverage": "1/7", + "complete": false, + "pooled_splits": [ + "annapolis" + ], + "precision": 0.506224, + "recall": 0.414966, + "f1": 0.456075, + "ap": null, + "ap_bundle": null, + "ap_is_substituted": false, + "fp_per_pano": 0.952, + "micro_precision": 0.506224, + "micro_recall": 0.414966, + "f1_min": 0.456075, + "f1_max": 0.456075, + "f1_min_split": "annapolis", + "f1_max_split": "annapolis", + "n_splits_run": 1, + "budapest_district5_f1": null, + "budapest_district5_precision": null, + "budapest_district5_recall": null, + "budapest_district5_ap": null, + "sao_paulo_f1": null, + "sao_paulo_precision": null, + "sao_paulo_recall": null, + "sao_paulo_ap": null, + "manual_gold_f1": null, + "manual_gold_precision": null, + "manual_gold_recall": null, + "manual_gold_ap": null + } + ], + "per_split": { + "rampnet": { + "richmond": { + "split": "richmond", + "precision": 0.963563, + "recall": 0.767742, + "f1": 0.854578, + "ap": 0.876075, + "ap_source": "op_cache (0.05 floor)", + "ap_bundle": 0.763464, + "tp": 238, + "fp": 9, + "fn": 72, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 0.072581 + }, + "bend": { + "split": "bend", + "precision": 0.96139, + "recall": 0.761468, + "f1": 0.849829, + "ap": 0.86798, + "ap_source": "op_cache (0.05 floor)", + "ap_bundle": 0.754272, + "tp": 249, + "fp": 10, + "fn": 78, + "n_panos": 110, + "n_gt_recall": 327, + "fp_per_pano": 0.090909 + }, + "clovis": { + "split": "clovis", + "precision": 0.914474, + "recall": 0.712821, + "f1": 0.801153, + "ap": 0.871181, + "ap_source": "op_cache (0.05 floor)", + "ap_bundle": 0.687686, + "tp": 139, + "fp": 13, + "fn": 56, + "n_panos": 125, + "n_gt_recall": 195, + "fp_per_pano": 0.104 + }, + "morgantown": { + "split": "morgantown", + "precision": 0.975, + "recall": 0.730337, + "f1": 0.835118, + "ap": 0.856273, + "ap_source": "op_cache (0.05 floor)", + "ap_bundle": 0.728091, + "tp": 195, + "fp": 5, + "fn": 72, + "n_panos": 125, + "n_gt_recall": 267, + "fp_per_pano": 0.04 + }, + "annapolis": { + "split": "annapolis", + "precision": 0.973094, + "recall": 0.738095, + "f1": 0.839458, + "ap": 0.875444, + "ap_source": "op_cache (0.05 floor)", + "ap_bundle": 0.73384, + "tp": 217, + "fp": 6, + "fn": 77, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 0.048 + }, + "paterson": { + "split": "paterson", + "precision": 0.97482, + "recall": 0.686076, + "f1": 0.805349, + "ap": 0.748379, + "ap_source": "op_cache (0.05 floor)", + "ap_bundle": 0.681419, + "tp": 271, + "fp": 7, + "fn": 124, + "n_panos": 125, + "n_gt_recall": 395, + "fp_per_pano": 0.056 + }, + "gainesville": { + "split": "gainesville", + "precision": 0.945274, + "recall": 0.698529, + "f1": 0.803383, + "ap": 0.845563, + "ap_source": "op_cache (0.05 floor)", + "ap_bundle": 0.690818, + "tp": 190, + "fp": 11, + "fn": 82, + "n_panos": 125, + "n_gt_recall": 272, + "fp_per_pano": 0.088 + }, + "budapest_district5": { + "split": "budapest_district5", + "precision": 0.874286, + "recall": 0.51, + "f1": 0.644211, + "ap": 0.647705, + "ap_source": "op_cache (0.05 floor)", + "ap_bundle": 0.478209, + "tp": 153, + "fp": 22, + "fn": 147, + "n_panos": 125, + "n_gt_recall": 300, + "fp_per_pano": 0.176 + }, + "sao_paulo": { + "split": "sao_paulo", + "precision": 0.893519, + "recall": 0.686833, + "f1": 0.77666, + "ap": 0.811637, + "ap_source": "op_cache (0.05 floor)", + "ap_bundle": 0.665505, + "tp": 193, + "fp": 23, + "fn": 88, + "n_panos": 125, + "n_gt_recall": 281, + "fp_per_pano": 0.184 + }, + "manual_gold": { + "split": "manual_gold", + "precision": 0.947368, + "recall": 0.872672, + "f1": 0.908487, + "ap": 0.917292, + "ap_source": "bundle", + "ap_bundle": 0.917292, + "tp": 3420, + "fp": 190, + "fn": 499, + "n_panos": 1000, + "n_gt_recall": 3919, + "fp_per_pano": 0.19 + } + }, + "y11l_pano": { + "richmond": { + "split": "richmond", + "precision": 0.92517, + "recall": 0.43871, + "f1": 0.595186, + "ap": 0.723784, + "ap_source": "bundle", + "ap_bundle": 0.723784, + "tp": 136, + "fp": 11, + "fn": 174, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 0.08871 + }, + "bend": { + "split": "bend", + "precision": 0.963542, + "recall": 0.565749, + "f1": 0.712909, + "ap": 0.777931, + "ap_source": "bundle", + "ap_bundle": 0.777931, + "tp": 185, + "fp": 7, + "fn": 142, + "n_panos": 110, + "n_gt_recall": 327, + "fp_per_pano": 0.063636 + }, + "clovis": { + "split": "clovis", + "precision": 0.915789, + "recall": 0.446154, + "f1": 0.6, + "ap": 0.70759, + "ap_source": "bundle", + "ap_bundle": 0.70759, + "tp": 87, + "fp": 8, + "fn": 108, + "n_panos": 125, + "n_gt_recall": 195, + "fp_per_pano": 0.064 + }, + "morgantown": { + "split": "morgantown", + "precision": 0.945946, + "recall": 0.524345, + "f1": 0.674699, + "ap": 0.797114, + "ap_source": "bundle", + "ap_bundle": 0.797114, + "tp": 140, + "fp": 8, + "fn": 127, + "n_panos": 125, + "n_gt_recall": 267, + "fp_per_pano": 0.064 + }, + "annapolis": { + "split": "annapolis", + "precision": 0.969072, + "recall": 0.319728, + "f1": 0.480818, + "ap": 0.618997, + "ap_source": "bundle", + "ap_bundle": 0.618997, + "tp": 94, + "fp": 3, + "fn": 200, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 0.024 + }, + "paterson": { + "split": "paterson", + "precision": 0.946341, + "recall": 0.491139, + "f1": 0.646667, + "ap": 0.792867, + "ap_source": "bundle", + "ap_bundle": 0.792867, + "tp": 194, + "fp": 11, + "fn": 201, + "n_panos": 125, + "n_gt_recall": 395, + "fp_per_pano": 0.088 + }, + "gainesville": { + "split": "gainesville", + "precision": 0.907407, + "recall": 0.360294, + "f1": 0.515789, + "ap": 0.638803, + "ap_source": "bundle", + "ap_bundle": 0.638803, + "tp": 98, + "fp": 10, + "fn": 174, + "n_panos": 125, + "n_gt_recall": 272, + "fp_per_pano": 0.08 + }, + "budapest_district5": { + "split": "budapest_district5", + "precision": 0.785714, + "recall": 0.146667, + "f1": 0.247191, + "ap": 0.443481, + "ap_source": "bundle", + "ap_bundle": 0.443481, + "tp": 44, + "fp": 12, + "fn": 256, + "n_panos": 125, + "n_gt_recall": 300, + "fp_per_pano": 0.096 + }, + "sao_paulo": { + "split": "sao_paulo", + "precision": 0.90184, + "recall": 0.523132, + "f1": 0.662162, + "ap": 0.783949, + "ap_source": "bundle", + "ap_bundle": 0.783949, + "tp": 147, + "fp": 16, + "fn": 134, + "n_panos": 125, + "n_gt_recall": 281, + "fp_per_pano": 0.128 + }, + "manual_gold": { + "split": "manual_gold", + "precision": 0.930927, + "recall": 0.76346, + "f1": 0.838918, + "ap": 0.913984, + "ap_source": "bundle", + "ap_bundle": 0.913984, + "tp": 2992, + "fp": 222, + "fn": 927, + "n_panos": 1000, + "n_gt_recall": 3919, + "fp_per_pano": 0.222 + } + }, + "y11x_pano_h200": { + "richmond": { + "split": "richmond", + "precision": 0.952, + "recall": 0.383871, + "f1": 0.547126, + "ap": 0.748061, + "ap_source": "bundle", + "ap_bundle": 0.748061, + "tp": 119, + "fp": 6, + "fn": 191, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 0.048387 + }, + "bend": { + "split": "bend", + "precision": 0.989071, + "recall": 0.553517, + "f1": 0.709804, + "ap": 0.781366, + "ap_source": "bundle", + "ap_bundle": 0.781366, + "tp": 181, + "fp": 2, + "fn": 146, + "n_panos": 110, + "n_gt_recall": 327, + "fp_per_pano": 0.018182 + }, + "clovis": { + "split": "clovis", + "precision": 0.938272, + "recall": 0.389744, + "f1": 0.550725, + "ap": 0.710942, + "ap_source": "bundle", + "ap_bundle": 0.710942, + "tp": 76, + "fp": 5, + "fn": 119, + "n_panos": 125, + "n_gt_recall": 195, + "fp_per_pano": 0.04 + }, + "morgantown": { + "split": "morgantown", + "precision": 0.992908, + "recall": 0.524345, + "f1": 0.686275, + "ap": 0.789853, + "ap_source": "bundle", + "ap_bundle": 0.789853, + "tp": 140, + "fp": 1, + "fn": 127, + "n_panos": 125, + "n_gt_recall": 267, + "fp_per_pano": 0.008 + }, + "annapolis": { + "split": "annapolis", + "precision": 0.986486, + "recall": 0.248299, + "f1": 0.396739, + "ap": 0.661568, + "ap_source": "bundle", + "ap_bundle": 0.661568, + "tp": 73, + "fp": 1, + "fn": 221, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 0.008 + }, + "paterson": { + "split": "paterson", + "precision": 0.973822, + "recall": 0.470886, + "f1": 0.634812, + "ap": 0.800153, + "ap_source": "bundle", + "ap_bundle": 0.800153, + "tp": 186, + "fp": 5, + "fn": 209, + "n_panos": 125, + "n_gt_recall": 395, + "fp_per_pano": 0.04 + }, + "gainesville": { + "split": "gainesville", + "precision": 0.948454, + "recall": 0.338235, + "f1": 0.498645, + "ap": 0.615546, + "ap_source": "bundle", + "ap_bundle": 0.615546, + "tp": 92, + "fp": 5, + "fn": 180, + "n_panos": 125, + "n_gt_recall": 272, + "fp_per_pano": 0.04 + }, + "budapest_district5": { + "split": "budapest_district5", + "precision": 0.863636, + "recall": 0.126667, + "f1": 0.22093, + "ap": 0.42687, + "ap_source": "bundle", + "ap_bundle": 0.42687, + "tp": 38, + "fp": 6, + "fn": 262, + "n_panos": 125, + "n_gt_recall": 300, + "fp_per_pano": 0.048 + }, + "sao_paulo": { + "split": "sao_paulo", + "precision": 0.923077, + "recall": 0.512456, + "f1": 0.659039, + "ap": 0.782801, + "ap_source": "bundle", + "ap_bundle": 0.782801, + "tp": 144, + "fp": 12, + "fn": 137, + "n_panos": 125, + "n_gt_recall": 281, + "fp_per_pano": 0.096 + }, + "manual_gold": { + "split": "manual_gold", + "precision": 0.955789, + "recall": 0.766777, + "f1": 0.850913, + "ap": 0.930925, + "ap_source": "bundle", + "ap_bundle": 0.930925, + "tp": 3005, + "fp": 139, + "fn": 914, + "n_panos": 1000, + "n_gt_recall": 3919, + "fp_per_pano": 0.139 + } + }, + "y26_pano": { + "richmond": { + "split": "richmond", + "precision": 0.68, + "recall": 0.383871, + "f1": 0.490722, + "ap": 0.536502, + "ap_source": "bundle", + "ap_bundle": 0.536502, + "tp": 119, + "fp": 56, + "fn": 191, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 0.451613 + }, + "bend": { + "split": "bend", + "precision": 0.733068, + "recall": 0.562691, + "f1": 0.636678, + "ap": 0.661388, + "ap_source": "bundle", + "ap_bundle": 0.661388, + "tp": 184, + "fp": 67, + "fn": 143, + "n_panos": 110, + "n_gt_recall": 327, + "fp_per_pano": 0.609091 + }, + "clovis": { + "split": "clovis", + "precision": 0.687023, + "recall": 0.461538, + "f1": 0.552147, + "ap": 0.596431, + "ap_source": "bundle", + "ap_bundle": 0.596431, + "tp": 90, + "fp": 41, + "fn": 105, + "n_panos": 125, + "n_gt_recall": 195, + "fp_per_pano": 0.328 + }, + "morgantown": { + "split": "morgantown", + "precision": 0.80203, + "recall": 0.59176, + "f1": 0.681034, + "ap": 0.739571, + "ap_source": "bundle", + "ap_bundle": 0.739571, + "tp": 158, + "fp": 39, + "fn": 109, + "n_panos": 125, + "n_gt_recall": 267, + "fp_per_pano": 0.312 + }, + "annapolis": { + "split": "annapolis", + "precision": 0.758065, + "recall": 0.319728, + "f1": 0.449761, + "ap": 0.493248, + "ap_source": "bundle", + "ap_bundle": 0.493248, + "tp": 94, + "fp": 30, + "fn": 200, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 0.24 + }, + "paterson": { + "split": "paterson", + "precision": 0.785714, + "recall": 0.473418, + "f1": 0.590837, + "ap": 0.711963, + "ap_source": "bundle", + "ap_bundle": 0.711963, + "tp": 187, + "fp": 51, + "fn": 208, + "n_panos": 125, + "n_gt_recall": 395, + "fp_per_pano": 0.408 + }, + "gainesville": { + "split": "gainesville", + "precision": 0.708661, + "recall": 0.330882, + "f1": 0.451128, + "ap": 0.4812, + "ap_source": "bundle", + "ap_bundle": 0.4812, + "tp": 90, + "fp": 37, + "fn": 182, + "n_panos": 125, + "n_gt_recall": 272, + "fp_per_pano": 0.296 + }, + "budapest_district5": { + "split": "budapest_district5", + "precision": 0.638554, + "recall": 0.176667, + "f1": 0.276762, + "ap": 0.349591, + "ap_source": "bundle", + "ap_bundle": 0.349591, + "tp": 53, + "fp": 30, + "fn": 247, + "n_panos": 125, + "n_gt_recall": 300, + "fp_per_pano": 0.24 + }, + "sao_paulo": { + "split": "sao_paulo", + "precision": 0.722772, + "recall": 0.519573, + "f1": 0.604555, + "ap": 0.658092, + "ap_source": "bundle", + "ap_bundle": 0.658092, + "tp": 146, + "fp": 56, + "fn": 135, + "n_panos": 125, + "n_gt_recall": 281, + "fp_per_pano": 0.448 + }, + "manual_gold": { + "split": "manual_gold", + "precision": 0.73953, + "recall": 0.738964, + "f1": 0.739247, + "ap": 0.843378, + "ap_source": "bundle", + "ap_bundle": 0.843378, + "tp": 2896, + "fp": 1020, + "fn": 1023, + "n_panos": 1000, + "n_gt_recall": 3919, + "fp_per_pano": 1.02 + } + }, + "gemini-3.1-pro-preview": { + "richmond": { + "split": "richmond", + "precision": 0.633721, + "recall": 0.703226, + "f1": 0.666667, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 218, + "fp": 126, + "fn": 92, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 1.016129 + }, + "bend": { + "split": "bend", + "precision": 0.70632, + "recall": 0.58104, + "f1": 0.637584, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 190, + "fp": 79, + "fn": 137, + "n_panos": 110, + "n_gt_recall": 327, + "fp_per_pano": 0.718182 + }, + "clovis": { + "split": "clovis", + "precision": 0.542857, + "recall": 0.487179, + "f1": 0.513514, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 95, + "fp": 80, + "fn": 100, + "n_panos": 125, + "n_gt_recall": 195, + "fp_per_pano": 0.64 + }, + "morgantown": { + "split": "morgantown", + "precision": 0.679167, + "recall": 0.610487, + "f1": 0.642998, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 163, + "fp": 77, + "fn": 104, + "n_panos": 125, + "n_gt_recall": 267, + "fp_per_pano": 0.616 + }, + "annapolis": { + "split": "annapolis", + "precision": 0.612648, + "recall": 0.527211, + "f1": 0.566728, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 155, + "fp": 98, + "fn": 139, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 0.784 + }, + "paterson": { + "split": "paterson", + "precision": 0.851711, + "recall": 0.567089, + "f1": 0.680851, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 224, + "fp": 39, + "fn": 171, + "n_panos": 125, + "n_gt_recall": 395, + "fp_per_pano": 0.312 + }, + "gainesville": { + "split": "gainesville", + "precision": 0.564202, + "recall": 0.533088, + "f1": 0.548204, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 145, + "fp": 112, + "fn": 127, + "n_panos": 125, + "n_gt_recall": 272, + "fp_per_pano": 0.896 + }, + "budapest_district5": { + "split": "budapest_district5", + "precision": 0.434043, + "recall": 0.34, + "f1": 0.381308, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 102, + "fp": 133, + "fn": 198, + "n_panos": 125, + "n_gt_recall": 300, + "fp_per_pano": 1.064 + }, + "sao_paulo": { + "split": "sao_paulo", + "precision": 0.462963, + "recall": 0.44484, + "f1": 0.453721, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 125, + "fp": 145, + "fn": 156, + "n_panos": 125, + "n_gt_recall": 281, + "fp_per_pano": 1.16 + } + }, + "gemini-3.7-flash": { + "richmond": { + "split": "richmond", + "precision": 0.744, + "recall": 0.6, + "f1": 0.664286, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 186, + "fp": 64, + "fn": 124, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 0.516129 + }, + "bend": { + "split": "bend", + "precision": 0.713208, + "recall": 0.577982, + "f1": 0.638514, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 189, + "fp": 76, + "fn": 138, + "n_panos": 110, + "n_gt_recall": 327, + "fp_per_pano": 0.690909 + }, + "clovis": { + "split": "clovis", + "precision": 0.571429, + "recall": 0.451282, + "f1": 0.504298, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 88, + "fp": 66, + "fn": 107, + "n_panos": 125, + "n_gt_recall": 195, + "fp_per_pano": 0.528 + }, + "morgantown": { + "split": "morgantown", + "precision": 0.700508, + "recall": 0.516854, + "f1": 0.594828, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 138, + "fp": 59, + "fn": 129, + "n_panos": 125, + "n_gt_recall": 267, + "fp_per_pano": 0.472 + }, + "annapolis": { + "split": "annapolis", + "precision": 0.733696, + "recall": 0.459184, + "f1": 0.564854, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 135, + "fp": 49, + "fn": 159, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 0.392 + }, + "paterson": { + "split": "paterson", + "precision": 0.909548, + "recall": 0.458228, + "f1": 0.609428, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 181, + "fp": 18, + "fn": 214, + "n_panos": 125, + "n_gt_recall": 395, + "fp_per_pano": 0.144 + }, + "gainesville": { + "split": "gainesville", + "precision": 0.511416, + "recall": 0.411765, + "f1": 0.456212, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 112, + "fp": 107, + "fn": 160, + "n_panos": 125, + "n_gt_recall": 272, + "fp_per_pano": 0.856 + }, + "budapest_district5": { + "split": "budapest_district5", + "precision": 0.484472, + "recall": 0.26, + "f1": 0.338395, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 78, + "fp": 83, + "fn": 222, + "n_panos": 125, + "n_gt_recall": 300, + "fp_per_pano": 0.664 + }, + "sao_paulo": { + "split": "sao_paulo", + "precision": 0.560606, + "recall": 0.263345, + "f1": 0.358354, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 74, + "fp": 58, + "fn": 207, + "n_panos": 125, + "n_gt_recall": 281, + "fp_per_pano": 0.464 + }, + "manual_gold": { + "split": "manual_gold", + "precision": 0.734731, + "recall": 0.411329, + "f1": 0.527401, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 1612, + "fp": 582, + "fn": 2307, + "n_panos": 1000, + "n_gt_recall": 3919, + "fp_per_pano": 0.582 + } + }, + "gemini-3.6-flash": { + "richmond": { + "split": "richmond", + "precision": 0.625786, + "recall": 0.641935, + "f1": 0.633758, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 199, + "fp": 119, + "fn": 111, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 0.959677 + }, + "bend": { + "split": "bend", + "precision": 0.607595, + "recall": 0.587156, + "f1": 0.597201, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 192, + "fp": 124, + "fn": 135, + "n_panos": 110, + "n_gt_recall": 327, + "fp_per_pano": 1.127273 + }, + "clovis": { + "split": "clovis", + "precision": 0.464455, + "recall": 0.502564, + "f1": 0.482759, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 98, + "fp": 113, + "fn": 97, + "n_panos": 125, + "n_gt_recall": 195, + "fp_per_pano": 0.904 + }, + "morgantown": { + "split": "morgantown", + "precision": 0.636364, + "recall": 0.629213, + "f1": 0.632768, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 168, + "fp": 96, + "fn": 99, + "n_panos": 125, + "n_gt_recall": 267, + "fp_per_pano": 0.768 + }, + "annapolis": { + "split": "annapolis", + "precision": 0.637168, + "recall": 0.489796, + "f1": 0.553846, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 144, + "fp": 82, + "fn": 150, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 0.656 + }, + "paterson": { + "split": "paterson", + "precision": 0.74359, + "recall": 0.513924, + "f1": 0.607784, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 203, + "fp": 70, + "fn": 192, + "n_panos": 125, + "n_gt_recall": 395, + "fp_per_pano": 0.56 + }, + "gainesville": { + "split": "gainesville", + "precision": 0.403727, + "recall": 0.477941, + "f1": 0.43771, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 130, + "fp": 192, + "fn": 142, + "n_panos": 125, + "n_gt_recall": 272, + "fp_per_pano": 1.536 + }, + "budapest_district5": { + "split": "budapest_district5", + "precision": 0.352941, + "recall": 0.32, + "f1": 0.335664, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 96, + "fp": 176, + "fn": 204, + "n_panos": 125, + "n_gt_recall": 300, + "fp_per_pano": 1.408 + }, + "sao_paulo": { + "split": "sao_paulo", + "precision": 0.333333, + "recall": 0.359431, + "f1": 0.34589, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 101, + "fp": 202, + "fn": 180, + "n_panos": 125, + "n_gt_recall": 281, + "fp_per_pano": 1.616 + } + }, + "Qwen/Qwen3-VL-32B-Instruct": { + "richmond": { + "split": "richmond", + "precision": 0.760331, + "recall": 0.296774, + "f1": 0.426914, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 92, + "fp": 29, + "fn": 218, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 0.233871 + }, + "bend": { + "split": "bend", + "precision": 0.705882, + "recall": 0.293578, + "f1": 0.414687, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 96, + "fp": 40, + "fn": 231, + "n_panos": 110, + "n_gt_recall": 327, + "fp_per_pano": 0.363636 + }, + "clovis": { + "split": "clovis", + "precision": 0.696429, + "recall": 0.2, + "f1": 0.310757, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 39, + "fp": 17, + "fn": 156, + "n_panos": 125, + "n_gt_recall": 195, + "fp_per_pano": 0.136 + }, + "morgantown": { + "split": "morgantown", + "precision": 0.674797, + "recall": 0.310861, + "f1": 0.425641, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 83, + "fp": 40, + "fn": 184, + "n_panos": 125, + "n_gt_recall": 267, + "fp_per_pano": 0.32 + }, + "annapolis": { + "split": "annapolis", + "precision": 0.608392, + "recall": 0.295918, + "f1": 0.398169, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 87, + "fp": 56, + "fn": 207, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 0.448 + }, + "paterson": { + "split": "paterson", + "precision": 0.813084, + "recall": 0.220253, + "f1": 0.346614, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 87, + "fp": 20, + "fn": 308, + "n_panos": 125, + "n_gt_recall": 395, + "fp_per_pano": 0.16 + }, + "gainesville": { + "split": "gainesville", + "precision": 0.391892, + "recall": 0.106618, + "f1": 0.16763, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 29, + "fp": 45, + "fn": 243, + "n_panos": 125, + "n_gt_recall": 272, + "fp_per_pano": 0.36 + }, + "budapest_district5": { + "split": "budapest_district5", + "precision": 0.433333, + "recall": 0.043333, + "f1": 0.078788, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 13, + "fp": 17, + "fn": 287, + "n_panos": 125, + "n_gt_recall": 300, + "fp_per_pano": 0.136 + }, + "sao_paulo": { + "split": "sao_paulo", + "precision": 0.506494, + "recall": 0.13879, + "f1": 0.217877, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 39, + "fp": 38, + "fn": 242, + "n_panos": 125, + "n_gt_recall": 281, + "fp_per_pano": 0.304 + }, + "manual_gold": { + "split": "manual_gold", + "precision": 0.738806, + "recall": 0.176831, + "f1": 0.285361, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 693, + "fp": 245, + "fn": 3226, + "n_panos": 1000, + "n_gt_recall": 3919, + "fp_per_pano": 0.245 + } + }, + "Qwen/Qwen3-VL-8B-Instruct": { + "richmond": { + "split": "richmond", + "precision": 0.323326, + "recall": 0.451613, + "f1": 0.376851, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 140, + "fp": 293, + "fn": 170, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 2.362903 + }, + "bend": { + "split": "bend", + "precision": 0.381443, + "recall": 0.33945, + "f1": 0.359223, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 111, + "fp": 180, + "fn": 216, + "n_panos": 110, + "n_gt_recall": 327, + "fp_per_pano": 1.636364 + }, + "clovis": { + "split": "clovis", + "precision": 0.225681, + "recall": 0.297436, + "f1": 0.256637, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 58, + "fp": 199, + "fn": 137, + "n_panos": 125, + "n_gt_recall": 195, + "fp_per_pano": 1.592 + }, + "morgantown": { + "split": "morgantown", + "precision": 0.303835, + "recall": 0.385768, + "f1": 0.339934, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 103, + "fp": 236, + "fn": 164, + "n_panos": 125, + "n_gt_recall": 267, + "fp_per_pano": 1.888 + }, + "annapolis": { + "split": "annapolis", + "precision": 0.304094, + "recall": 0.353741, + "f1": 0.327044, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 104, + "fp": 238, + "fn": 190, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 1.904 + }, + "paterson": { + "split": "paterson", + "precision": 0.459807, + "recall": 0.362025, + "f1": 0.405099, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 143, + "fp": 168, + "fn": 252, + "n_panos": 125, + "n_gt_recall": 395, + "fp_per_pano": 1.344 + }, + "gainesville": { + "split": "gainesville", + "precision": 0.277778, + "recall": 0.330882, + "f1": 0.302013, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 90, + "fp": 234, + "fn": 182, + "n_panos": 125, + "n_gt_recall": 272, + "fp_per_pano": 1.872 + }, + "budapest_district5": { + "split": "budapest_district5", + "precision": 0.183594, + "recall": 0.156667, + "f1": 0.169065, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 47, + "fp": 209, + "fn": 253, + "n_panos": 125, + "n_gt_recall": 300, + "fp_per_pano": 1.672 + }, + "sao_paulo": { + "split": "sao_paulo", + "precision": 0.228682, + "recall": 0.209964, + "f1": 0.218924, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 59, + "fp": 199, + "fn": 222, + "n_panos": 125, + "n_gt_recall": 281, + "fp_per_pano": 1.592 + }, + "manual_gold": { + "split": "manual_gold", + "precision": 0.445258, + "recall": 0.341414, + "f1": 0.386482, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 1338, + "fp": 1667, + "fn": 2581, + "n_panos": 1000, + "n_gt_recall": 3919, + "fp_per_pano": 1.667 + } + }, + "allenai/Molmo2-8B": { + "richmond": { + "split": "richmond", + "precision": 0.410256, + "recall": 0.516129, + "f1": 0.457143, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 160, + "fp": 230, + "fn": 150, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 1.854839 + }, + "bend": { + "split": "bend", + "precision": 0.509728, + "recall": 0.400612, + "f1": 0.44863, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 131, + "fp": 126, + "fn": 196, + "n_panos": 110, + "n_gt_recall": 327, + "fp_per_pano": 1.145455 + }, + "clovis": { + "split": "clovis", + "precision": 0.33463, + "recall": 0.441026, + "f1": 0.380531, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 86, + "fp": 171, + "fn": 109, + "n_panos": 125, + "n_gt_recall": 195, + "fp_per_pano": 1.368 + }, + "morgantown": { + "split": "morgantown", + "precision": 0.465909, + "recall": 0.460674, + "f1": 0.463277, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 123, + "fp": 141, + "fn": 144, + "n_panos": 125, + "n_gt_recall": 267, + "fp_per_pano": 1.128 + }, + "annapolis": { + "split": "annapolis", + "precision": 0.434164, + "recall": 0.414966, + "f1": 0.424348, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 122, + "fp": 159, + "fn": 172, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 1.272 + }, + "paterson": { + "split": "paterson", + "precision": 0.584967, + "recall": 0.453165, + "f1": 0.510699, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 179, + "fp": 127, + "fn": 216, + "n_panos": 125, + "n_gt_recall": 395, + "fp_per_pano": 1.016 + }, + "gainesville": { + "split": "gainesville", + "precision": 0.282322, + "recall": 0.393382, + "f1": 0.328725, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 107, + "fp": 272, + "fn": 165, + "n_panos": 125, + "n_gt_recall": 272, + "fp_per_pano": 2.176 + }, + "budapest_district5": { + "split": "budapest_district5", + "precision": 0.260479, + "recall": 0.29, + "f1": 0.274448, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 87, + "fp": 247, + "fn": 213, + "n_panos": 125, + "n_gt_recall": 300, + "fp_per_pano": 1.976 + }, + "sao_paulo": { + "split": "sao_paulo", + "precision": 0.318644, + "recall": 0.33452, + "f1": 0.326389, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 94, + "fp": 201, + "fn": 187, + "n_panos": 125, + "n_gt_recall": 281, + "fp_per_pano": 1.608 + }, + "manual_gold": { + "split": "manual_gold", + "precision": 0.511434, + "recall": 0.35953, + "f1": 0.422236, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 1409, + "fp": 1346, + "fn": 2510, + "n_panos": 1000, + "n_gt_recall": 3919, + "fp_per_pano": 1.346 + } + }, + "google/owlv2-large-patch14-ensemble": { + "richmond": { + "split": "richmond", + "precision": 0.033077, + "recall": 0.970968, + "f1": 0.063974, + "ap": 0.10368, + "ap_source": "bundle", + "ap_bundle": 0.10368, + "tp": 301, + "fp": 8799, + "fn": 9, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 70.959677 + }, + "bend": { + "split": "bend", + "precision": 0.03671, + "recall": 0.954128, + "f1": 0.0707, + "ap": 0.093345, + "ap_source": "bundle", + "ap_bundle": 0.093345, + "tp": 312, + "fp": 8187, + "fn": 15, + "n_panos": 110, + "n_gt_recall": 327, + "fp_per_pano": 74.427273 + }, + "clovis": { + "split": "clovis", + "precision": 0.025113, + "recall": 0.912821, + "f1": 0.048881, + "ap": 0.06743, + "ap_source": "bundle", + "ap_bundle": 0.06743, + "tp": 178, + "fp": 6910, + "fn": 17, + "n_panos": 125, + "n_gt_recall": 195, + "fp_per_pano": 55.28 + }, + "morgantown": { + "split": "morgantown", + "precision": 0.036848, + "recall": 0.947566, + "f1": 0.070938, + "ap": 0.113985, + "ap_source": "bundle", + "ap_bundle": 0.113985, + "tp": 253, + "fp": 6613, + "fn": 14, + "n_panos": 125, + "n_gt_recall": 267, + "fp_per_pano": 52.904 + }, + "annapolis": { + "split": "annapolis", + "precision": 0.032317, + "recall": 0.959184, + "f1": 0.062528, + "ap": 0.125787, + "ap_source": "bundle", + "ap_bundle": 0.125787, + "tp": 282, + "fp": 8444, + "fn": 12, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 67.552 + }, + "paterson": { + "split": "paterson", + "precision": 0.040338, + "recall": 0.893671, + "f1": 0.077192, + "ap": 0.116483, + "ap_source": "bundle", + "ap_bundle": 0.116483, + "tp": 353, + "fp": 8398, + "fn": 42, + "n_panos": 125, + "n_gt_recall": 395, + "fp_per_pano": 67.184 + }, + "gainesville": { + "split": "gainesville", + "precision": 0.031132, + "recall": 0.966912, + "f1": 0.060321, + "ap": 0.062715, + "ap_source": "bundle", + "ap_bundle": 0.062715, + "tp": 263, + "fp": 8185, + "fn": 9, + "n_panos": 125, + "n_gt_recall": 272, + "fp_per_pano": 65.48 + }, + "budapest_district5": { + "split": "budapest_district5", + "precision": 0.0319, + "recall": 0.93, + "f1": 0.061685, + "ap": 0.088843, + "ap_source": "bundle", + "ap_bundle": 0.088843, + "tp": 279, + "fp": 8467, + "fn": 21, + "n_panos": 125, + "n_gt_recall": 300, + "fp_per_pano": 67.736 + }, + "sao_paulo": { + "split": "sao_paulo", + "precision": 0.026723, + "recall": 0.921708, + "f1": 0.05194, + "ap": 0.049839, + "ap_source": "bundle", + "ap_bundle": 0.049839, + "tp": 259, + "fp": 9433, + "fn": 22, + "n_panos": 125, + "n_gt_recall": 281, + "fp_per_pano": 75.464 + }, + "manual_gold": { + "split": "manual_gold", + "precision": 0.046159, + "recall": 0.906864, + "f1": 0.087846, + "ap": 0.097078, + "ap_source": "bundle", + "ap_bundle": 0.097078, + "tp": 3554, + "fp": 73441, + "fn": 365, + "n_panos": 1000, + "n_gt_recall": 3919, + "fp_per_pano": 73.441 + } + }, + "IDEA-Research/grounding-dino-base": { + "richmond": { + "split": "richmond", + "precision": 0.027543, + "recall": 0.851613, + "f1": 0.05336, + "ap": 0.032503, + "ap_source": "bundle", + "ap_bundle": 0.032503, + "tp": 264, + "fp": 9321, + "fn": 46, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 75.169355 + }, + "bend": { + "split": "bend", + "precision": 0.038361, + "recall": 0.850153, + "f1": 0.073409, + "ap": 0.049403, + "ap_source": "bundle", + "ap_bundle": 0.049403, + "tp": 278, + "fp": 6969, + "fn": 49, + "n_panos": 110, + "n_gt_recall": 327, + "fp_per_pano": 63.354545 + }, + "clovis": { + "split": "clovis", + "precision": 0.017705, + "recall": 0.871795, + "f1": 0.034705, + "ap": 0.025875, + "ap_source": "bundle", + "ap_bundle": 0.025875, + "tp": 170, + "fp": 9432, + "fn": 25, + "n_panos": 125, + "n_gt_recall": 195, + "fp_per_pano": 75.456 + }, + "morgantown": { + "split": "morgantown", + "precision": 0.021737, + "recall": 0.831461, + "f1": 0.042366, + "ap": 0.028492, + "ap_source": "bundle", + "ap_bundle": 0.028492, + "tp": 222, + "fp": 9991, + "fn": 45, + "n_panos": 125, + "n_gt_recall": 267, + "fp_per_pano": 79.928 + }, + "annapolis": { + "split": "annapolis", + "precision": 0.028522, + "recall": 0.897959, + "f1": 0.055288, + "ap": 0.042235, + "ap_source": "bundle", + "ap_bundle": 0.042235, + "tp": 264, + "fp": 8992, + "fn": 30, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 71.936 + }, + "paterson": { + "split": "paterson", + "precision": 0.035747, + "recall": 0.802532, + "f1": 0.068444, + "ap": 0.043644, + "ap_source": "bundle", + "ap_bundle": 0.043644, + "tp": 317, + "fp": 8551, + "fn": 78, + "n_panos": 125, + "n_gt_recall": 395, + "fp_per_pano": 68.408 + }, + "gainesville": { + "split": "gainesville", + "precision": 0.028351, + "recall": 0.893382, + "f1": 0.054959, + "ap": 0.040476, + "ap_source": "bundle", + "ap_bundle": 0.040476, + "tp": 243, + "fp": 8328, + "fn": 29, + "n_panos": 125, + "n_gt_recall": 272, + "fp_per_pano": 66.624 + }, + "budapest_district5": { + "split": "budapest_district5", + "precision": 0.021472, + "recall": 0.786667, + "f1": 0.041803, + "ap": 0.025415, + "ap_source": "bundle", + "ap_bundle": 0.025415, + "tp": 236, + "fp": 10755, + "fn": 64, + "n_panos": 125, + "n_gt_recall": 300, + "fp_per_pano": 86.04 + }, + "sao_paulo": { + "split": "sao_paulo", + "precision": 0.025169, + "recall": 0.797153, + "f1": 0.048796, + "ap": 0.033603, + "ap_source": "bundle", + "ap_bundle": 0.033603, + "tp": 224, + "fp": 8676, + "fn": 57, + "n_panos": 125, + "n_gt_recall": 281, + "fp_per_pano": 69.408 + }, + "manual_gold": { + "split": "manual_gold", + "precision": 0.04282, + "recall": 0.855575, + "f1": 0.081559, + "ap": 0.06732, + "ap_source": "bundle", + "ap_bundle": 0.06732, + "tp": 3353, + "fp": 74951, + "fn": 566, + "n_panos": 1000, + "n_gt_recall": 3919, + "fp_per_pano": 74.951 + } + }, + "mask2former-vistas-curb-cut": { + "richmond": { + "split": "richmond", + "precision": 0.411429, + "recall": 0.696774, + "f1": 0.517365, + "ap": 0.512564, + "ap_source": "bundle", + "ap_bundle": 0.512564, + "tp": 216, + "fp": 309, + "fn": 94, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 2.491935 + } + }, + "mask2former-vistas-curb-cut+curb": { + "richmond": { + "split": "richmond", + "precision": 0.125625, + "recall": 0.648387, + "f1": 0.210471, + "ap": 0.089131, + "ap_source": "bundle", + "ap_bundle": 0.089131, + "tp": 201, + "fp": 1399, + "fn": 109, + "n_panos": 124, + "n_gt_recall": 310, + "fp_per_pano": 11.282258 + } + }, + "claude-opus-5-effort-low": { + "annapolis": { + "split": "annapolis", + "precision": 0.572347, + "recall": 0.605442, + "f1": 0.58843, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 178, + "fp": 133, + "fn": 116, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 1.064 + } + }, + "claude-opus-5-effort-high": { + "annapolis": { + "split": "annapolis", + "precision": 0.429844, + "recall": 0.656463, + "f1": 0.519515, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 193, + "fp": 256, + "fn": 101, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 2.048 + } + }, + "claude-sonnet-5-effort-low": { + "annapolis": { + "split": "annapolis", + "precision": 0.589474, + "recall": 0.380952, + "f1": 0.46281, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 112, + "fp": 78, + "fn": 182, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 0.624 + } + }, + "claude-sonnet-5-effort-high": { + "annapolis": { + "split": "annapolis", + "precision": 0.506224, + "recall": 0.414966, + "f1": 0.456075, + "ap": null, + "ap_source": "bundle", + "ap_bundle": null, + "tp": 122, + "fp": 119, + "fn": 172, + "n_panos": 125, + "n_gt_recall": 294, + "fp_per_pano": 0.952 + } + } + } +} diff --git a/analysis_out/usage_log.jsonl b/analysis_out/usage_log.jsonl index 3f1dafac..d7c56180 100644 --- a/analysis_out/usage_log.jsonl +++ b/analysis_out/usage_log.jsonl @@ -1 +1,3 @@ {"ts": "2026-08-18T14:54:02+00:00", "bundle": "annapolis", "label": "claude-sonnet-5", "provider": "claude", "model_id": "claude-sonnet-5", "model_versions": {"claude-sonnet-5": 6}, "panos_scored": 125, "signature": {"provider": "claude", "model_id": "claude-sonnet-5", "tile": true, "max_edge": 1568, "source_max_edge": 4096, "views": [[0.0, -30.0, 90.0, 90.0, 1024, 1024], [60.0, -30.0, 90.0, 90.0, 1024, 1024], [120.0, -30.0, 90.0, 90.0, 1024, 1024], [180.0, -30.0, 90.0, 90.0, 1024, 1024], [240.0, -30.0, 90.0, 90.0, 1024, 1024], [300.0, -30.0, 90.0, 90.0, 1024, 1024]], "prompt": "Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.", "effort": "low", "tool_choice": "auto", "box_tool": "{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}"}, "stop_reasons": {"tool_use": 6}, "calls": 6, "input_tokens": 12594, "output_tokens": 480, "thoughts_tokens": 0, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0, "est_cost_usd": 0.03, "pricing": {"input_per_m": 2.0, "output_per_m": 10.0, "cache_read_per_m": 0.2, "cache_write_per_m": 2.5, "as_of": "2026-08-15", "note": "Vertex `global`; promotional launch pricing through 2026-08-31, $3.00/$15.00 after. Regional +10%."}} +{"ts": "2026-08-18T23:29:11+00:00", "bundle": "richmond", "label": "claude-opus-5", "provider": "claude", "model_id": "claude-opus-5", "model_versions": {"claude-opus-5": 6}, "panos_scored": 1, "signature": {"provider": "claude", "model_id": "claude-opus-5", "tile": true, "max_edge": 1568, "source_max_edge": 4096, "views": [[0.0, -30.0, 90.0, 90.0, 1024, 1024], [60.0, -30.0, 90.0, 90.0, 1024, 1024], [120.0, -30.0, 90.0, 90.0, 1024, 1024], [180.0, -30.0, 90.0, 90.0, 1024, 1024], [240.0, -30.0, 90.0, 90.0, 1024, 1024], [300.0, -30.0, 90.0, 90.0, 1024, 1024]], "prompt": "Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.", "effort": "low", "tool_choice": "auto", "box_tool": "{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}"}, "stop_reasons": {"tool_use": 6}, "calls": 6, "input_tokens": 12186, "output_tokens": 780, "thoughts_tokens": 0, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0, "est_cost_usd": 0.0804, "pricing": {"input_per_m": 5.0, "output_per_m": 25.0, "cache_read_per_m": 0.5, "cache_write_per_m": 6.25, "as_of": "2026-08-15", "note": "Vertex `global`; regional +10%."}} +{"ts": "2026-08-18T23:30:52+00:00", "bundle": "richmond", "label": "claude-opus-5", "provider": "claude", "model_id": "claude-opus-5", "model_versions": {"claude-opus-5": 18}, "panos_scored": 4, "signature": {"provider": "claude", "model_id": "claude-opus-5", "tile": true, "max_edge": 1568, "source_max_edge": 4096, "views": [[0.0, -30.0, 90.0, 90.0, 1024, 1024], [60.0, -30.0, 90.0, 90.0, 1024, 1024], [120.0, -30.0, 90.0, 90.0, 1024, 1024], [180.0, -30.0, 90.0, 90.0, 1024, 1024], [240.0, -30.0, 90.0, 90.0, 1024, 1024], [300.0, -30.0, 90.0, 90.0, 1024, 1024]], "prompt": "Detect every curb ramp in this street-level image. A curb ramp (curb cut) is the short sloped ramp cut into a sidewalk curb at a street corner or crossing that lets a wheelchair or stroller roll from sidewalk to street. Return one tight bounding box per curb ramp. Do not box driveways, stairs, or crosswalk paint. If there are no curb ramps, return an empty list.", "effort": "low", "tool_choice": "auto", "box_tool": "{\"description\": \"Report every curb ramp visible in the image, as tight pixel bounding boxes in the image's own coordinate space. Report an empty list if there are none.\", \"input_schema\": {\"additionalProperties\": false, \"properties\": {\"boxes\": {\"items\": {\"additionalProperties\": false, \"properties\": {\"x1\": {\"type\": \"integer\"}, \"x2\": {\"type\": \"integer\"}, \"y1\": {\"type\": \"integer\"}, \"y2\": {\"type\": \"integer\"}}, \"required\": [\"x1\", \"y1\", \"x2\", \"y2\"], \"type\": \"object\"}, \"type\": \"array\"}}, \"required\": [\"boxes\"], \"type\": \"object\"}, \"name\": \"report_curb_ramps\"}"}, "stop_reasons": {"tool_use": 18}, "calls": 18, "input_tokens": 36558, "output_tokens": 1972, "thoughts_tokens": 32, "cache_read_input_tokens": 0, "cache_write_input_tokens": 0, "est_cost_usd": 0.2321, "pricing": {"input_per_m": 5.0, "output_per_m": 25.0, "cache_read_per_m": 0.5, "cache_write_per_m": 6.25, "as_of": "2026-08-15", "note": "Vertex `global`; regional +10%."}} diff --git a/docs/figures/scoreboard_by_split.png b/docs/figures/scoreboard_by_split.png new file mode 100644 index 00000000..3ead555c Binary files /dev/null and b/docs/figures/scoreboard_by_split.png differ diff --git a/docs/figures/scoreboard_f1.png b/docs/figures/scoreboard_f1.png new file mode 100644 index 00000000..87940896 Binary files /dev/null and b/docs/figures/scoreboard_f1.png differ diff --git a/docs/figures/scoreboard_generalization.png b/docs/figures/scoreboard_generalization.png new file mode 100644 index 00000000..680f34a7 Binary files /dev/null and b/docs/figures/scoreboard_generalization.png differ diff --git a/docs/figures/scoreboard_pr.png b/docs/figures/scoreboard_pr.png new file mode 100644 index 00000000..7591f12f Binary files /dev/null and b/docs/figures/scoreboard_pr.png differ diff --git a/docs/figures/scoreboard_pr_curves.png b/docs/figures/scoreboard_pr_curves.png new file mode 100644 index 00000000..b1a256f0 Binary files /dev/null and b/docs/figures/scoreboard_pr_curves.png differ diff --git a/docs/model_comparison.md b/docs/model_comparison.md index e1fe7a91..46c3971b 100644 --- a/docs/model_comparison.md +++ b/docs/model_comparison.md @@ -6,6 +6,15 @@ RampNet against off-the-shelf models. The question: does a general model match o purpose-trained RampNet on real deployment imagery (GSV + Mapillary 360)? The harness is model-agnostic, so new models (issues #20, #39) plug in the same way. +> **Looking for one table with every model on it?** → **[`model_scoreboard.md`](model_scoreboard.md)**. +> This document is the comprehensive log — per-split tables in the order the splits were +> run, the mechanism behind each number, the caveats, the negative results — so "who wins, +> and by how much" is spread across a dozen tables in chronological order. `model_scoreboard.md` +> is the same data pivoted the other way (rows are models, columns are metrics, aggregated +> across splits) plus five summary figures, and it covers every leg in `rampnet/roster.py` +> rather than the standing eight. It is generated from the committed detections by +> `scripts/analysis/scoreboard.py`, and a test fails if it drifts from this log. + ## What has been run where Every split in `benchmark/` appears here, including the ones with gaps — an omission below is @@ -202,18 +211,42 @@ Perspective tiling, match radius 0.022, all models scored against the same deriv Open detectors are shown at their 0.05 cache floor; their tuned operating points are in the sweep below. Run on Hyak (L40S); RampNet and Gemini rows are cache-scored. +**Regenerated 2026-08-18 for the #132 seam wrap.** `score_pano` now wraps the 360° +seam, so a detection at x=0.99 is matched against ground truth at x=0.01 rather than +treated as ~1,010 px away. That recovers a genuine match on each of **19 (model, split) +pairs** and moved **66 cells** in the tables below — every one a challenger, all in the +same direction, none large enough to reorder anything. **RampNet, the three YOLO arms, +both Vistas arms and the four Claude legs are unchanged**, which is why the effect was +missed when #132 landed: it was checked against RampNet. Ranking, the headline claim and +every conclusion in this document are unaffected; the largest single move is +gemini-3.1-pro on clovis, F1 0.503 → 0.514. + +These tables are now checked against the scorer on every CI run +(`tests/test_scoreboard.py::test_every_number_matches_model_comparison`), so a code change +that moves a published number here fails the build instead of going unnoticed. + +**The AP column is computed from each split's bundle**, which for RampNet means a curve +truncated at its deployed 0.55 — the bundles *are* a production run, and that is where +production stops. So RampNet's AP below is not comparable to the arms exported at 0.05, and +pooled it reads 0.720, *below* the YOLO arms. [`model_scoreboard.md`](model_scoreboard.md) +re-reads it from `analysis_out/op_cache/` (the #54 low-floor extraction of the same +panoramas) and reports 0.849; that page carries a per-split table mapping each number here +to the one there. Every other model's AP is identical in both documents, as are all P/R/F1. +`manual_gold` is the exception on this side too: its bundle is already at 0.05, so its 0.917 +is the one untruncated RampNet AP here. + **richmond** (124 reviewed panos, 310 GT ramps) | model | P | R | F1 | AP | tp/fp/fn | |---|---|---|---|---|---| | **rampnet** | **0.964** | 0.768 | **0.855** | 0.763 | 238/9/72 | -| gemini-3.1-pro-preview | 0.631 | 0.700 | 0.664 | – | 217/127/93 | +| gemini-3.1-pro-preview | 0.634 | 0.703 | 0.667 | – | 218/126/92 | | gemini-3.6-flash | 0.626 | 0.642 | 0.634 | – | 199/119/111 | | **molmo2-8B** (points) | 0.410 | 0.516 | **0.457** | – | 160/230/150 | | Qwen3-VL-32B-Instruct | 0.760 | 0.297 | 0.427 | – | 92/29/218 | | Qwen3-VL-8B-Instruct | 0.323 | 0.452 | 0.377 | – | 140/293/170 | | owlv2-large-patch14-ensemble | 0.033 | **0.971** | 0.064 | 0.104 | 301/8799/9 | -| grounding-dino-base | 0.028 | 0.852 | 0.053 | 0.032 | 264/9321/46 | +| grounding-dino-base | 0.028 | 0.852 | 0.053 | 0.033 | 264/9321/46 | **bend** (110 reviewed panos, 327 GT ramps) @@ -224,8 +257,8 @@ sweep below. Run on Hyak (L40S); RampNet and Gemini rows are cache-scored. | gemini-3.6-flash | 0.608 | 0.587 | 0.597 | – | 192/124/135 | | **molmo2-8B** (points) | 0.510 | 0.401 | **0.449** | – | 131/126/196 | | Qwen3-VL-32B-Instruct | 0.706 | 0.294 | 0.415 | – | 96/40/231 | -| Qwen3-VL-8B-Instruct | 0.379 | 0.336 | 0.357 | – | 110/180/217 | -| owlv2-large-patch14-ensemble | 0.037 | 0.951 | 0.070 | 0.093 | 311/8187/16 | +| Qwen3-VL-8B-Instruct | 0.381 | 0.339 | 0.359 | – | 111/180/216 | +| owlv2-large-patch14-ensemble | 0.037 | 0.954 | 0.071 | 0.093 | 312/8187/15 | | grounding-dino-base | 0.038 | 0.850 | 0.073 | 0.049 | 278/6969/49 | **clovis** (125 reviewed panos, 195 GT ramps) — Mapillary GoPro Fusion 360s, the hardest of the @@ -234,18 +267,18 @@ three deployment cities | model | P | R | F1 | AP | tp/fp/fn | |---|---|---|---|---|---| | **rampnet** | **0.914** | 0.713 | **0.801** | 0.688 | 139/13/56 | -| gemini-3.1-pro-preview | 0.531 | 0.477 | 0.503 | – | 93/82/102 | -| gemini-3.6-flash | 0.460 | 0.497 | 0.478 | – | 97/114/98 | -| **molmo2-8B** (points) | 0.331 | 0.436 | **0.376** | – | 85/172/110 | +| gemini-3.1-pro-preview | 0.543 | 0.487 | 0.514 | – | 95/80/100 | +| gemini-3.6-flash | 0.464 | 0.503 | 0.483 | – | 98/113/97 | +| **molmo2-8B** (points) | 0.335 | 0.441 | **0.381** | – | 86/171/109 | | Qwen3-VL-32B-Instruct | 0.696 | 0.200 | 0.311 | – | 39/17/156 | -| Qwen3-VL-8B-Instruct | 0.222 | 0.292 | 0.252 | – | 57/200/138 | -| owlv2-large-patch14-ensemble | 0.025 | **0.908** | 0.049 | 0.067 | 177/6911/18 | -| grounding-dino-base | 0.018 | 0.867 | 0.035 | 0.026 | 169/9433/26 | +| Qwen3-VL-8B-Instruct | 0.226 | 0.297 | 0.257 | – | 58/199/137 | +| owlv2-large-patch14-ensemble | 0.025 | **0.913** | 0.049 | 0.067 | 178/6910/17 | +| grounding-dino-base | 0.018 | 0.872 | 0.035 | 0.026 | 170/9432/25 | Clovis is 100% soft, 2018-era GoPro Fusion 360 imagery, so every model degrades relative to -richmond/bend — RampNet's own P/R slips to 0.914/0.713 (from richmond's 0.960/0.765). But the +richmond/bend — RampNet's own P/R slips to 0.914/0.713 (from richmond's 0.964/0.768). But the **ranking is identical across all three cities**, and RampNet's lead *widens*: the gap to the best -challenger grows from ~0.19 (richmond) to **~0.30** here. These are the all-125 numbers, so every +challenger grows from ~0.19 (richmond) to **~0.29** here. These are the all-125 numbers, so every model is scored on the same panos; clovis's ground-truth quality against the 120-pano *unbiased* subset (P 0.889 / R 0.650) is in `benchmark/README.md`. (`gemini-2.5-flash`, not run on richmond, scores F1 0.278 on clovis — between Qwen-32B and Qwen-8B, tracking its 0.252 on bend.) @@ -275,7 +308,7 @@ Best sweep F1 for the open detectors: OWLv2 **0.208** (thr 0.25), Grounding DINO signature recurs: Molmo best open-weight with the only balanced profile, Qwen-32B cautious (challenger-best precision 0.608 at the worst recall 0.296), Qwen-8B FP-leaky, open-vocab detectors trading a huge nominal recall for ~3% precision. RampNet's lead over the best -challenger is 0.27 F1, between richmond's ~0.19 and clovis's ~0.30. +challenger is 0.27 F1, between richmond's ~0.19 and clovis's ~0.29. Annapolis is also where the open detectors' recall column stops being believable — see "How much of a detector's recall is real?" below, which was measured here first and then @@ -287,17 +320,18 @@ the benchmark and the control for everything below | model | P | R | F1 | AP | tp/fp/fn | |---|---|---|---|---|---| | **rampnet** | **0.975** | 0.730 | **0.835** | 0.728 | 195/5/72 | -| gemini-3.1-pro-preview | 0.675 | 0.607 | 0.639 | – | 162/78/105 | -| gemini-3.6-flash | 0.633 | 0.625 | 0.629 | – | 167/97/100 | -| **molmo2-8B** (points) | 0.462 | 0.457 | **0.460** | – | 122/142/145 | -| Qwen3-VL-32B-Instruct | 0.667 | 0.307 | 0.421 | – | 82/41/185 | -| Qwen3-VL-8B-Instruct | 0.301 | 0.382 | 0.337 | – | 102/237/165 | +| gemini-3.1-pro-preview | 0.679 | 0.610 | 0.643 | – | 163/77/104 | +| gemini-3.6-flash | 0.636 | 0.629 | 0.633 | – | 168/96/99 | +| **molmo2-8B** (points) | 0.466 | 0.461 | **0.463** | – | 123/141/144 | +| Qwen3-VL-32B-Instruct | 0.675 | 0.311 | 0.426 | – | 83/40/184 | +| Qwen3-VL-8B-Instruct | 0.304 | 0.386 | 0.340 | – | 103/236/164 | | owlv2-large-patch14-ensemble | 0.037 | **0.948** | 0.071 | 0.114 | 253/6613/14 | | grounding-dino-base | 0.022 | 0.831 | 0.042 | 0.028 | 222/9991/45 | Best sweep F1: OWLv2 **0.196** (thr 0.25), Grounding DINO **0.068** (thr 0.15). This is the **canonical ordering, position for position**, and every model posts its best or near-best -score of any city — Gemini-3.1-pro's 0.639 and Molmo's 0.460 are their highest anywhere. That +score of any city — Gemini-3.1-pro's 0.643 and Molmo's 0.463 were their highest anywhere +when this split ran; paterson has since beaten both (0.681 and 0.511). That is what you would expect from the sharpest imagery in the benchmark, and it is why morgantown is the right control to read budapest against. @@ -312,8 +346,8 @@ confidence HIGH; the split whose misses are structural (`benchmark/README.md`) | **molmo2-8B** (points) | 0.585 | 0.453 | **0.511** | – | 179/127/216 | | Qwen3-VL-8B-Instruct | 0.460 | 0.362 | 0.405 | – | 143/168/252 | | **Qwen3-VL-32B-Instruct** | 0.813 | **0.220** | **0.347** | – | 87/20/308 | -| owlv2-large-patch14-ensemble | 0.040 | **0.891** | 0.077 | 0.116 | 352/8399/43 | -| grounding-dino-base | 0.036 | 0.800 | 0.068 | 0.043 | 316/8552/79 | +| owlv2-large-patch14-ensemble | 0.040 | **0.894** | 0.077 | 0.116 | 353/8398/42 | +| grounding-dino-base | 0.036 | 0.803 | 0.068 | 0.044 | 317/8551/78 | Best sweep F1: OWLv2 **0.216** (thr 0.25), Grounding DINO **0.100** (thr 0.15). @@ -1234,7 +1268,7 @@ detections; both runs agree to every digit printed here. | model | P | R | F1 | AP | tp/fp/fn | |---|---|---|---|---|---| | **rampnet** | **0.964** | 0.768 | **0.855** | **0.763** | 238/9/72 | -| gemini-3.1-pro-preview | 0.631 | 0.700 | 0.664 | – | 217/127/93 | +| gemini-3.1-pro-preview | 0.634 | 0.703 | 0.667 | – | 218/126/92 | | gemini-3.6-flash | 0.626 | 0.642 | 0.634 | – | 199/119/111 | | **mask2former-vistas-curb-cut** | **0.411** | **0.697** | **0.517** | **0.513** | 216/309/94 | | molmo2-8B (points) | 0.410 | 0.516 | 0.457 | – | 160/230/150 | @@ -1242,7 +1276,7 @@ detections; both runs agree to every digit printed here. | Qwen3-VL-8B-Instruct | 0.323 | 0.452 | 0.377 | – | 140/293/170 | | *mask2former-vistas-curb-cut+curb* | *0.126* | *0.648* | *0.210* | *0.089* | 201/1399/109 | | owlv2-large-patch14-ensemble | 0.033 | **0.971** | 0.064 | 0.104 | 301/8799/9 | -| grounding-dino-base | 0.028 | 0.852 | 0.053 | 0.032 | 264/9321/46 | +| grounding-dino-base | 0.028 | 0.852 | 0.053 | 0.033 | 264/9321/46 | **Every row above is at one operating point — no confidence floor — which is what the rest of this document's roster tables use.** An earlier version of this table scored the two Vistas rows @@ -1837,8 +1871,8 @@ in the loop) | **molmo2-8B** (points) | 0.511 | 0.360 | **0.422** | – | 1409/1346/2510 | | Qwen3-VL-8B-Instruct | 0.445 | 0.341 | 0.386 | – | 1338/1667/2581 | | Qwen3-VL-32B-Instruct | 0.739 | 0.177 | 0.285 | – | 693/245/3226 | -| owlv2-large-patch14-ensemble | 0.046 | **0.906** | 0.088 | 0.097 | 3551/73444/368 | -| grounding-dino-base | 0.043 | 0.855 | 0.082 | 0.067 | 3351/74953/568 | +| owlv2-large-patch14-ensemble | 0.046 | **0.907** | 0.088 | 0.097 | 3554/73441/365 | +| grounding-dino-base | 0.043 | 0.856 | 0.082 | 0.067 | 3353/74951/566 | Best sweep F1 for the open detectors: OWLv2 **0.180** (thr 0.20), Grounding DINO **0.140** (thr 0.20) — the FP flood is not a threshold artifact. diff --git a/docs/model_scoreboard.md b/docs/model_scoreboard.md new file mode 100644 index 00000000..34707dd3 --- /dev/null +++ b/docs/model_scoreboard.md @@ -0,0 +1,422 @@ +# Scoreboard: every model, one table + +Eighteen model legs, ten splits, one page. This is the summary view of the curb-ramp benchmark — +**rows are models, columns are metrics** — for the question "which model is best, and by how +much". + +It is the companion to [`model_comparison.md`](model_comparison.md), not a replacement. +That document is the comprehensive log: per-split tables in the order the splits were run, +the mechanism behind every number, the negative results, the caveats, the harness +self-validation. It is where you go to find out *why* Qwen-32B inverts on budapest. It is a +bad place to find out *who wins*, because that answer is spread across a dozen tables in +chronological order rather than model order. Hence this page. + +Every number here is regenerated from committed data by +[`scripts/analysis/scoreboard.py`](../scripts/analysis/scoreboard.py) — no GPU, no +credentials, no network, no `.model_cache`. The tables below sit inside generated blocks and +are replaced wholesale on each run, so this page cannot quietly drift out of step with the +log it summarizes. `--check` turns that drift into a failure. + +--- + +## The board + +Macro-mean over the seven pooled US city splits, each city weighted equally. +**Read the operating-point column before comparing rows** — it is not the same for every +model, and the reasons are in "How to read this" below. + + + +| model | class | op | P | R | F1 | ΔF1 vs RampNet | AP (macro) | FP/pano | F1 range | +|---|---|--:|--:|--:|--:|--:|--:|--:|:-:| +| **RampNet** | purpose-trained | 0.55 | 0.958 | 0.728 | **0.827** | — | 0.849 † | 0.1 | 0.80–0.85 | +| YOLO11l (pano) | supervised baseline | 0.25 | 0.939 | 0.449 | 0.604 | -0.223 | 0.722 | 0.1 | 0.48–0.71 | +| YOLO11x (pano) | supervised baseline | 0.25 | **0.969** | 0.416 | 0.575 | -0.252 | 0.730 | 0.0 | 0.40–0.71 | +| YOLO26 (pano) | supervised baseline | 0.25 | 0.736 | 0.446 | 0.550 | -0.277 | 0.603 | 0.4 | 0.45–0.68 | +| Gemini 3.1 Pro | chat VLM | no score | 0.656 | 0.573 | 0.608 | -0.219 | – | 0.7 | 0.51–0.68 | +| Gemini 3.7 Flash | chat VLM | no score | 0.698 | 0.496 | 0.576 | -0.251 | – | 0.5 | 0.46–0.66 | +| Gemini 3.6 Flash | chat VLM | no score | 0.588 | 0.549 | 0.564 | -0.263 | – | 0.9 | 0.44–0.63 | +| Qwen3-VL-32B | chat VLM | no score | 0.664 | 0.246 | 0.356 | -0.471 | – | 0.3 | 0.17–0.43 | +| Qwen3-VL-8B | chat VLM | no score | 0.325 | 0.360 | 0.338 | -0.489 | – | 1.8 | 0.26–0.41 | +| Molmo2-8B | pointing model | no score | 0.432 | 0.440 | 0.430 | -0.397 | – | 1.4 | 0.33–0.51 | +| OWLv2-large | open-vocab detector | 0.05 floor | 0.034 | **0.944** | 0.065 | -0.762 | 0.098 | 64.8 | 0.05–0.08 | +| Grounding DINO | open-vocab detector | 0.05 floor | 0.028 | 0.857 | 0.055 | -0.772 | 0.038 | 71.6 | 0.03–0.07 | + + + +† RampNet's AP is read from `analysis_out/op_cache/`, not from its bundle — see "Choosing an operating point" below for why, and what it fixes. + +![Pooled F1 by model](figures/scoreboard_f1.png) + +**RampNet wins by 0.219 F1**, and the gap is not a threshold artifact: it holds at every +operating point anyone has committed to, and on ground truth that never saw a RampNet review +(`manual_gold`). The three findings that only become visible once the splits are pooled: + +1. **The supervised baseline and the best zero-shot VLM are a dead heat.** YOLO11l trained on + the RampNet dataset scores **0.604**; Gemini 3.1 Pro, zero-shot with an untuned prompt, + scores **0.608**. Issue #51 asks whether RampNet's advantage is the data or the + architecture; pooled, the answer is that the *data alone*, handed to a generic detector, + buys you a tie with an off-the-shelf chat model — and the remaining 0.22 F1 is what the + keypoint architecture adds. +2. **RampNet is the only strong model that is also stable.** Its F1 spans 0.80–0.85 across the + seven cities, a range of 0.053. Every challenger scoring above 0.1 swings between 0.148 + (Qwen-8B) and 0.313 (YOLO11x). The two open-vocabulary detectors *are* flatter (0.028, + 0.039) — because they are pinned near zero everywhere, which is consistency of a kind + nobody wants. +3. **Precision is not the differentiator; recall is.** YOLO11x posts the highest precision on + the board (0.969, above RampNet's 0.958) at **0.416 recall against RampNet's 0.728**. Every + model here can be made precise. Finding the ramps is the hard part, which is why the + project's operating-point work optimizes recall-first + ([`operating_point.md`](operating_point.md)). + +![Precision vs recall](figures/scoreboard_pr.png) + +The P/R plane is where the single-number ranking stops being enough. Models sitting on the +same F1 contour fail in opposite directions: Qwen-32B and Qwen-8B score 0.356 and 0.338 — +practically tied — but 32B gets there by firing rarely at high precision and 8B by flooding. +Which one you would deploy depends entirely on whether a miss or a false positive costs more, +and F1 cannot tell you. + +--- + +## Legs that have not run every pooled split + +Six legs have run one split each, so they have no pooled mean to put in the table above — +a one-city average printed beside a seven-city one is exactly the comparison the coverage +column exists to prevent. They are reported per split instead, at the split they ran on: + + + +| model | class | split | P | R | F1 | AP | FP/pano | tp/fp/fn | +|---|---|---|--:|--:|--:|--:|--:|--:| +| Mask2Former Vistas (curb cut) | supervised transfer | `richmond` | 0.411 | 0.697 | 0.517 | 0.513 | 2.5 | 216/309/94 | +| Mask2Former Vistas (+curb) | supervised transfer | `richmond` | 0.126 | 0.648 | 0.210 | 0.089 | 11.3 | 201/1399/109 | +| Claude Opus 5 (low) | chat VLM | `annapolis` | 0.572 | 0.605 | 0.588 | – | 1.1 | 178/133/116 | +| Claude Opus 5 (high) | chat VLM | `annapolis` | 0.430 | 0.656 | 0.520 | – | 2.0 | 193/256/101 | +| Claude Sonnet 5 (low) | chat VLM | `annapolis` | 0.589 | 0.381 | 0.463 | – | 0.6 | 112/78/182 | +| Claude Sonnet 5 (high) | chat VLM | `annapolis` | 0.506 | 0.415 | 0.456 | – | 1.0 | 122/119/172 | + + + +Two things worth carrying out of that table, both from splits where the roster's own +numbers are directly above them in `model_comparison.md`: + +- **Claude Opus 5 at low effort is the strongest challenger measured on annapolis** (F1 + 0.588, against gemini-3.1-pro's 0.567) — and **more thinking makes it worse** (0.520 at + high effort). The same direction holds for Sonnet 5 (0.463 → 0.456). Effort moves the + operating point; it does not raise the ceiling (#122). +- **Supervised transfer fixes most of the precision problem and still loses.** Mask2Former + reading Vistas' `Curb Cut` class scores 0.517 on richmond with **12.4× OWLv2's + precision** and no training at all — but RampNet's 0.855 on that split is 0.337 clear of + it. The union arm (`+curb`) is a committed negative result: adding Vistas' `Curb` class + *loses* recall while precision collapses, because `Curb` fuses adjacent ramps into one + component (#126). + +--- + +## Every model, every split + + + +| model | rich | bend | clovis | morg | annap | pater | gaines | **pooled** | budapest † | sao_paulo † | manual_gold † | +|---|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:| +| **RampNet** | **0.855** | **0.850** | **0.801** | **0.835** | **0.839** | **0.805** | **0.803** | **0.827** | **0.644** | **0.777** | **0.908** | +| YOLO11l (pano) | 0.595 | 0.713 | 0.600 | 0.675 | 0.481 | 0.647 | 0.516 | 0.604 | 0.247 | 0.662 | 0.839 | +| YOLO11x (pano) | 0.547 | 0.710 | 0.551 | 0.686 | 0.397 | 0.635 | 0.499 | 0.575 | 0.221 | 0.659 | 0.851 | +| YOLO26 (pano) | 0.491 | 0.637 | 0.552 | 0.681 | 0.450 | 0.591 | 0.451 | 0.550 | 0.277 | 0.605 | 0.739 | +| Gemini 3.1 Pro | 0.667 | 0.638 | 0.514 | 0.643 | 0.567 | 0.681 | 0.548 | 0.608 | 0.381 | 0.454 | – | +| Gemini 3.7 Flash | 0.664 | 0.639 | 0.504 | 0.595 | 0.565 | 0.609 | 0.456 | 0.576 | 0.338 | 0.358 | 0.527 | +| Gemini 3.6 Flash | 0.634 | 0.597 | 0.483 | 0.633 | 0.554 | 0.608 | 0.438 | 0.564 | 0.336 | 0.346 | – | +| Qwen3-VL-32B | 0.427 | 0.415 | 0.311 | 0.426 | 0.398 | 0.347 | 0.168 | 0.356 | 0.079 | 0.218 | 0.285 | +| Qwen3-VL-8B | 0.377 | 0.359 | 0.257 | 0.340 | 0.327 | 0.405 | 0.302 | 0.338 | 0.169 | 0.219 | 0.386 | +| Molmo2-8B | 0.457 | 0.449 | 0.381 | 0.463 | 0.424 | 0.511 | 0.329 | 0.430 | 0.274 | 0.326 | 0.422 | +| OWLv2-large | 0.064 | 0.071 | 0.049 | 0.071 | 0.063 | 0.077 | 0.060 | 0.065 | 0.062 | 0.052 | 0.088 | +| Grounding DINO | 0.053 | 0.073 | 0.035 | 0.042 | 0.055 | 0.068 | 0.055 | 0.055 | 0.042 | 0.049 | 0.082 | +| Mask2Former Vistas (curb cut) | 0.517 | – | – | – | – | – | – | – | – | – | – | +| Mask2Former Vistas (+curb) | 0.210 | – | – | – | – | – | – | – | – | – | – | +| Claude Opus 5 (low) | – | – | – | – | 0.588 | – | – | – | – | – | – | +| Claude Opus 5 (high) | – | – | – | – | 0.520 | – | – | – | – | – | – | +| Claude Sonnet 5 (low) | – | – | – | – | 0.463 | – | – | – | – | – | – | +| Claude Sonnet 5 (high) | – | – | – | – | 0.456 | – | – | – | – | – | – | + + + +† held out of the pooled column, for the reasons in the split table at the bottom. They are +shown because omitting them would be worse, not because they belong in the headline. + +![F1 by model and split](figures/scoreboard_by_split.png) + +Three things this matrix settles that no single-number ranking can: + +- **RampNet is the top score in all ten splits**, including the two it is weakest on. There is + no city, imagery type, or ground-truth regime in this benchmark where any other model wins. +- **No single city is hardest for everyone.** clovis (2018 GoPro Fusion) is the worst pooled + city for 5 of the 12 models, gainesville for 4, annapolis for 3. "Difficulty" here is not a + property of the imagery alone — it is an interaction between imagery and model. +- **budapest separates the US-trained models from the zero-shot ones, in the wrong direction.** + The three YOLO arms fall to 0.221–0.277, *below all three Gemini legs* (0.336–0.381). A + detector trained on US curb-ramp data loses to an off-the-shelf chat model the moment the + design vocabulary changes. RampNet drops too (0.827 → 0.644) but keeps the lead. Read this + split with `benchmark/README.md`'s budapest caveat in hand — its GT is single-rater at low + reviewer confidence, which is exactly why it is held out of the pooled column. + +--- + +## In-distribution vs deployed + +![Generalization gap](figures/scoreboard_generalization.png) + +`manual_gold` is 1,000 GSV panoramas from RampNet's own training distribution, labelled +independently of any model. Plotting it against the deployed average separates two things +that a single F1 confuses: + +- **A zero-shot model has no training distribution to be inside**, so it lands on the diagonal + — and the seven that have a `manual_gold` cell scatter to *both* sides of it, by small + amounts: Qwen-32B +0.07, Gemini 3.7 Flash +0.05, Molmo +0.01, OWLv2 −0.02, Grounding DINO + −0.03, Qwen-8B −0.05. A two-sided scatter of ±0.07 with no systematic direction is the + un-anchored-GT check from #58 coming out clean, and it is a *stronger* result than a + one-sided one would be: these models neither gain nor lose on a split whose ground truth + RampNet never touched, which is what "the city GT was not tilted toward what RampNet finds" + predicts. +- **A model trained on the RampNet dataset starts above the line and falls.** How far it falls + is the generalization penalty, and it is the whole #51 ablation in one distance: RampNet + **−0.08**, YOLO26 **−0.19**, YOLO11l **−0.24**, YOLO11x **−0.28**. + +The uncomfortable corollary, stated because it is real: **in-distribution, YOLO11x is not +behind.** Its `manual_gold` AP is **0.931** against RampNet's **0.917**, at the same 0.05 +export floor — and RampNet's export used horizontal-flip TTA while YOLO's did not, so that +comparison is if anything generous to RampNet. On home turf a generic detector trained on this +dataset matches the purpose-built one. It is the 0.20 F1 it gives back on unfamiliar cities +that RampNet does not — and out of domain the AP ordering is not close either, **0.849 to +0.730** (macro-mean, the table above; micro-pooled it is 0.844 to 0.734 — see the note on the +two AP families under "Choosing an operating point"). + +--- + +## Choosing an operating point + +Every row above is one point. For the models that emit a calibrated score, that point is a +choice, and the choice is RampNet's to make — it is the subject of +[#54](https://github.com/ProjectSidewalk/RampNet/issues/54) and +[#55](https://github.com/ProjectSidewalk/RampNet/issues/55), written up in +[`operating_point.md`](operating_point.md). This is the surface those points sit on: + +![PR curves, pooled over the seven US splits](figures/scoreboard_pr_curves.png) + +The figure says three things a table of F1 cannot: + +1. **A calibrated score is a dial; a chat VLM is a dot.** RampNet, the three YOLO arms and + the two open detectors can be moved anywhere along their curves for free — no retraining, + no second inference pass. The Gemini/Qwen/Molmo rows are single points because those + models emit no confidence to threshold on. Comparing a tuned model against an untunable + one at one threshold flatters whichever happened to land well. +2. **RampNet's curve dominates over the whole range**, not just at 0.55. At every recall the + YOLO arms reach, RampNet is above them, and the AP ordering (0.844 vs 0.734) is the + integral of that. +3. **The deployed point is not the F1 optimum.** RampNet sits at 0.55 (hollow marker) where + the curve is nearly flat; #54's recommended 0.30 (filled) buys recall at a shallow + precision cost. + +> **Two AP families, and this figure uses the other one.** A PR curve is an integral over +> ranked predictions, so pooling it across splits has to be **micro** — concatenate every +> panorama, integrate once — and the legend above reports that. The headline table's AP +> column is the **macro-mean** of the per-split APs, each city weighted equally, like every +> other column in it. Same detections, same scorer; the two land a few thousandths apart +> (RampNet 0.844 micro / 0.849 macro, YOLO11x 0.734 / 0.730 — note it moves the *other* way). +> Neither is more correct. They are labelled everywhere both appear, and a comparison is only +> meaningful within one family: macro-to-macro the gap is 0.119, micro-to-micro 0.110. + + + +| peak threshold | P | R | F1 | | +|---|--:|--:|--:|---| +| **0.55** | 0.964 | 0.722 | 0.826 | deployed today (`OPERATIONAL_CONFIDENCE`, auto-labeler) | +| **0.30** | 0.900 | 0.793 | 0.843 | recommended by #54; **not yet adopted** (labeler#20 open) | + + + +Those are the **raw** numbers. Applying #55's per-split GT-completeness correction — +27.2% of the incremental false positives in `[0.30, 0.55)` are real ramps the GT missed — +`operating_point.md` reports corrected **P 0.919 / R 0.796 / F1 0.853** at 0.30, against +0.964 / 0.722 / 0.826 deployed. Corrected precision stays ≥0.88 on every US split, and +detection density rises only 1.86 → 2.23 per panorama. + +**Why the scoreboard still reports 0.55.** Because that is what is deployed: +`OPERATIONAL_CONFIDENCE = 0.55` in the auto-labeler's `detectors/__init__.py`, and the +benchmark bundles are a sample of a real run at that setting. #54's recommendation has not +been adopted — [ProjectSidewalk/sidewalk-auto-labeler#20](https://github.com/ProjectSidewalk/sidewalk-auto-labeler/issues/20) +is open, and its 2026-08-04 status update raises a genuine complication: in *world* space, +after multi-view fusion across a whole city, the drop buys only +0.4 to +3.2 recall points +rather than the per-panorama +7.4, because other views were already covering for each other. +That is a deployment question about a different repo's product metric. The per-panorama +choice — which is what this benchmark measures and what this page reports — is settled, and +the table above is what settled it. + +--- + +## How to read this + +**The pool is seven cities, not ten.** The split registry is imported from +`low_floor_sweep.US_SPLITS`, the same one `miss_decomposition.py` and the operating-point +sweep use, so a split cannot be pooled here and held out there. The three held-out splits +carry their documented reasons in the table below. `model_comparison.md` states outright that +budapest's numbers "must not be pooled with the US splits or averaged into a headline"; this +page obeys that. + +**Macro, not micro.** Each city contributes equally. Pooling raw counts would weight paterson +(395 GT ramps) twice as heavily as clovis (195), and folding in `manual_gold` would be far +worse — its 3,919 GT points outnumber all nine cities combined, so a pooled headline would be +59% one split that is in-distribution for exactly one model on the board. + +**Operating points differ by model class, and are inherited rather than chosen here.** + +| class | reported at | why | +|---|---|---| +| RampNet | peak threshold **0.55** | its deployed threshold; the city bundles are extracted at it, so this is RampNet as shipped. Its AP alone comes from the 0.05 low-floor cache — see "Choosing an operating point" | +| YOLO arms | conf **0.25** | pre-registered in the #71 protocol before any benchmark contact; per-split best-F1 sits at 0.10–0.15, which would be tuning on test | +| open-vocab detectors | **0.05 export floor** | as the log reports them; their tuned sweep points are in `model_comparison.md` and roughly triple their F1, to ~0.2 | +| chat VLMs, Molmo | **no score to threshold** | they emit boxes/points with no confidence, so their single row *is* the model — not a low threshold someone picked for them | + +**AP is cross-comparable, but RampNet's comes from a second file.** The city bundles hold +RampNet's detections only down to its deployed 0.55, because they *are* a production run and +that is where production stops — so an AP computed from them integrates a curve cut off at +the operating point. Read that way RampNet's pooled AP is **0.720**, which sits *below* the +YOLO arms' 0.730 and is an artifact of the floor, not a result. `analysis_out/op_cache/` is +the #54 re-extraction of the same panoramas down to 0.05 — the floor every other scored model +is exported at — and RampNet's AP read from it is **0.849**. That is the number in the table, +marked †. + +The substitution is gated on *measured* truncation, not a list of split names: it applies +only where the bundle floor sits more than 0.1 above the cache floor. `manual_gold`'s bundle +is already at 0.05, so it keeps its own AP (0.917) — there is nothing to un-truncate there, +and swapping in the cache would quietly trade that split's flip-TTA export for a no-TTA one. + +**This is the one number on this page that differs from `model_comparison.md`**, so here is +the whole mapping rather than a description of it. The middle column is what the log prints; +the test asserts it against the log's committed tables, so the two documents cannot drift +apart without failing CI: + + + +| split | AP in `model_comparison.md` | AP here | read from | why | +|---|--:|--:|---|---| +| `richmond` | 0.763 | **0.876** | `op_cache` (0.05 floor) | truncated at the deployed 0.55 | +| `bend` | 0.754 | **0.868** | `op_cache` (0.05 floor) | truncated at the deployed 0.55 | +| `clovis` | 0.688 | **0.871** | `op_cache` (0.05 floor) | truncated at the deployed 0.55 | +| `morgantown` | 0.728 | **0.856** | `op_cache` (0.05 floor) | truncated at the deployed 0.55 | +| `annapolis` | 0.734 | **0.875** | `op_cache` (0.05 floor) | truncated at the deployed 0.55 | +| `paterson` | 0.681 | **0.748** | `op_cache` (0.05 floor) | truncated at the deployed 0.55 | +| `gainesville` | 0.691 | **0.846** | `op_cache` (0.05 floor) | truncated at the deployed 0.55 | +| `budapest_district5` | 0.478 | **0.648** | `op_cache` (0.05 floor) | truncated at the deployed 0.55 | +| `sao_paulo` | 0.666 | **0.812** | `op_cache` (0.05 floor) | truncated at the deployed 0.55 | +| `manual_gold` | 0.917 | 0.917 | bundle — already at 0.05 | no truncation to undo; flip-TTA export | + + + +Everything else agrees to three decimals, and that is enforced rather than claimed: +`test_every_number_matches_model_comparison` parses all ten of the log's per-split tables +and checks P, R, F1 and AP on every row it finds. A number edited in either document +without re-running fails CI. + +Two things travel with that number. **P/R/F1 still come from `records.jsonl`**, the published +deployment-faithful operating point, so a single row has two sources; and **the sub-0.55 half +of the curve is a lower bound**, because the GT was assembled from detections at or above 0.55 +and #55 measured that 27.2% of the incremental FPs in [0.30, 0.55) are GT-completeness +artifacts. RampNet's 0.849 is therefore itself conservative. + +**Two known asymmetries in the `manual_gold` column.** RampNet's detections there were +exported with horizontal-flip TTA and at a 0.05 floor (`benchmark/manual_gold/detections_meta.json`); +the city splits used neither. Flip-TTA is worth roughly nothing out of domain but is +measurably positive in domain (#78), so RampNet's `manual_gold` row is its most flattering +number on this page. + +**Everything else about the protocol is shared** — same greedy matcher, same 0.022 normalized +match radius, same derived ground truth, same perspective tiling for the VLM inputs. The +scorer is `rampnet/detection_eval.py` and the thresholding is `compare.operating_report`, both +imported rather than reimplemented, so a row here means exactly what the same row means in the +log. + +--- + +## What is missing + +Omissions are content, so they are named rather than left as blanks: + +- **`gemini-3.1-pro-preview` and `gemini-3.6-flash` have no `manual_gold` row here.** Their + city detections are published; their `manual_gold` detections are not, and are absent from + this workstation's `.model_cache` (probed by reconstructing the cache keys — the same probe + returns 124/124 for richmond, so the method is sound and the split is genuinely not here). + `model_comparison.md` quotes F1 0.568 and 0.540 for them from the original run. **Those two + numbers are currently the only ones in this comparison that cannot be re-derived from a + clean clone.** Fixing it means locating the machine that holds that cache and running + `export_model_cache.py --models gemini:gemini-3.6-flash,gemini:gemini-3.1-pro-preview`, or + re-paying for the run. It is also why both are absent from the generalization figure. +- **`gemini-3.7-flash`, the YOLO arms, the Vistas arms and the Claude legs are all on this + board but `standing=False` in the roster**, so `model_comparison.md`'s roster tables keep + a consistent 8-model set pending each write-up. Every one of those legs has its + detections published and verified against cache, and `gemini-3.7-flash`'s silent-panorama + behaviour specifically was investigated under #120 (10/10 pairs identical to cache). + Including them here is deliberate: this page's job is to show everything that has been + measured, and `standing` governs which roster tables a leg appears in, not whether its + numbers are real. +- **The YOLO tiles arms are absent** — still training. The three pano arms are the completed + half of the #51 ablation; the resolution-controlled half is not done. +- **`manual_gold` has no null-recall pass** (O(n²) in panos), so the open detectors' recall + discount is unmeasured on that split. +- **Six legs have one split each**, so they are in the partial table rather than the + headline: the two Vistas arms (richmond) and the four Claude legs (annapolis). Extending + either to the full pool is a run, not a code change. +- **Nothing else in the registry is missing.** The board is driven by `rampnet/roster.py`, + and `unregistered_exports` is empty — every published detections file is claimed by a + roster entry and scored here. + +--- + +## Splits + + + +| split | role | panos | GT ramps | note | +|---|---|--:|--:|---| +| `richmond` | pooled | 124 | 310 | US deployment city, verdict-grade GT | +| `bend` | pooled | 110 | 327 | US deployment city, verdict-grade GT | +| `clovis` | pooled | 125 | 195 | US deployment city, verdict-grade GT | +| `morgantown` | pooled | 125 | 267 | US deployment city, verdict-grade GT | +| `annapolis` | pooled | 125 | 294 | US deployment city, verdict-grade GT | +| `paterson` | pooled | 125 | 395 | US deployment city, verdict-grade GT | +| `gainesville` | pooled | 125 | 272 | US deployment city, verdict-grade GT | +| `budapest_district5` | held out † | 125 | 300 | single-rater GT at low reviewer confidence (docs/model_comparison.md: do not pool) | +| `sao_paulo` | held out † | 125 | 281 | non-US city — the pooled recommendation is a US-deployment basis (GT is HIGH reviewer confidence; held out for geography, not GT quality) | +| `manual_gold` | held out † | 1000 | 3919 | in-distribution GSV + independently-labelled GT (in-domain reference, not a deployment city) | + + + +--- + +## Reproducing this page + +From a clean clone, with no GPU, no credentials and no network: + +```bash +pip install numpy pillow # the whole scoring path, nothing else +python scripts/analysis/scoreboard.py --no-figures # tables + analysis_out/scoreboard.json +python scripts/analysis/scoreboard.py --check # non-zero if this page or its JSON is stale + +# The figures additionally need matplotlib, which lives in requirements.txt / +# environment.yml rather than requirements-dev.txt — the test suite stays plotting-free, +# the same arrangement plot_operating_point.py uses. +pip install matplotlib && python scripts/analysis/scoreboard.py + +# requirements-dev.txt also works, but it installs torch, timm, transformers, datasets +# and scikit-image for the REST of the test suite — several GB this page does not need. +``` + +Inputs are `benchmark//{records.jsonl,verdicts.json}`, `manual_labels/`, and +`benchmark/model_detections/`. All are committed. `analysis_out/scoreboard.json` carries every +per-(model, split) cell — precision, recall, F1, AP, TP/FP/FN, panorama and GT counts — for +anything this page does not tabulate. + +Files: [`scripts/analysis/scoreboard.py`](../scripts/analysis/scoreboard.py) (scoring), +[`scripts/analysis/scoreboard_render.py`](../scripts/analysis/scoreboard_render.py) (tables +and the splice), [`scripts/analysis/scoreboard_figures.py`](../scripts/analysis/scoreboard_figures.py) +(the five figures). diff --git a/rampnet/detection_eval.py b/rampnet/detection_eval.py index 417f00ed..c16406c5 100644 --- a/rampnet/detection_eval.py +++ b/rampnet/detection_eval.py @@ -195,9 +195,17 @@ def score_pano(pred_points, gt, radius_sq=None, scale_x=PANO_SCALE_X, scale_y=PA ``wrap_x`` defaults to **True** here, unlike the generic matcher: this is the panorama scorer, its default scales are the panorama's, and every production caller is in pano space. Pass ``wrap_x=False`` for a non-cyclic coordinate space — the - synthetic unit-scale spaces in the tests are the only such callers today. Wrapping - moves no metric on any committed split (issue #132 §4), because the duplicate ground - truth in #130 currently masks it; the two fixes have to land together to have effect. + synthetic unit-scale spaces in the tests are the only such callers today. + + **Wrapping moves no RampNet or YOLO metric on any committed split — but it does move + the challengers.** That distinction was missed when #132 landed, because the effect + was checked against RampNet: measured across the whole roster it recovers a genuine + match on 19 (model, split) pairs and moves 66 published cells in + ``docs/model_comparison.md`` (six splits, seven chat-VLM and open-vocab models, all + in the same direction, no reordering). That document has been regenerated for it, and + ``tests/test_scoreboard.py::test_every_number_matches_model_comparison`` now re-derives + every one of its table cells on each CI run, so the next change to this function that + moves a published number fails the build rather than going unnoticed. """ if radius_sq is None: radius_sq = radius_sq_for(scale_x=scale_x) diff --git a/scripts/analysis/scoreboard.py b/scripts/analysis/scoreboard.py new file mode 100644 index 00000000..cffb8d13 --- /dev/null +++ b/scripts/analysis/scoreboard.py @@ -0,0 +1,649 @@ +"""One table with every model on it: the aggregated scoreboard behind docs/model_scoreboard.md. + +``docs/model_comparison.md`` is the comprehensive log — per-split tables, the mechanism +behind each number, the caveats, the negative results. It is the right document for "why +does Qwen-32B invert on budapest". It is the wrong document for "which model is best, and +by how much", because that answer is spread across ten per-split tables in reading order +rather than model order. + +This script produces the other view: **rows are models, columns are metrics**, aggregated +across splits, plus figures for the findings that are easier to see than to read. It is +generated rather than hand-maintained so the summary cannot drift from the log it +summarizes — ``--check`` fails when the committed doc no longer matches the committed +data, which is a failure mode a hand-copied summary table has and a generated one does not. + +Reads **only committed artifacts** — the benchmark bundles, the published detections in +``benchmark/model_detections/``, and ``manual_labels/``. No ``.model_cache``, no GPU, no +credentials, no network, so a fresh clone reproduces every number here:: + + python scripts/analysis/scoreboard.py # JSON + doc tables + figures + python scripts/analysis/scoreboard.py --check # doc AND JSON current? + python scripts/analysis/scoreboard.py --no-figures # tables only (no matplotlib) + +The scoring path needs **numpy and pillow only** -- not ``requirements-dev.txt``, which +pulls the whole training stack for the rest of the suite. + +**Nothing here may disagree with ``docs/model_comparison.md``.** Both documents score the +same committed detections, so a difference is a bug in one of them, not a choice. +``tests/test_scoreboard.py::test_every_number_matches_model_comparison`` parses every +per-split table out of the log and asserts that every (model, split) row in it agrees on +P, R, F1 and AP. The single deliberate exception is RampNet's AP -- see below -- and that +is asserted too, against ``ap_bundle``, so the exception cannot silently widen. + +**Aggregation.** The headline is the **macro-mean over the seven US city splits** — the +pool is ``low_floor_sweep.US_SPLITS``, imported rather than restated, because a third copy +of that registry is how a split ends up silently in one headline and out of another. The +three held-out splits keep their own columns and their documented reasons travel with +them: ``docs/model_comparison.md`` states outright that budapest's numbers "must not be +pooled with the US splits or averaged into a headline", ``sao_paulo`` is held out for +geography rather than GT quality, and ``manual_gold`` is the in-distribution reference, +not a deployment city. + +Macro rather than micro even within the pool: pooling counts would weight paterson (395 GT +ramps) twice as heavily as clovis (195) for no reason anyone would defend, and pooling +``manual_gold`` in would be worse still — its 3,919 GT points outnumber all nine cities +combined, so a pooled headline would be 59% one split that is in-distribution for exactly +one model on the roster. + +**Operating points** are per model class, and are the ones the log already committed to +rather than new choices (see ``OPERATING_POINT``): RampNet at its deployed 0.55, the +supervised YOLO arms at the pre-registered conf 0.25 (#71), the open-vocabulary detectors +at their 0.05 export floor, and the chat VLMs wherever they are — they emit no confidence, +so there is nothing to threshold and the setting is a no-op for them. + +**AP comes from a different file than P/R/F1, and only for RampNet.** The city bundles hold +RampNet's detections only down to its deployed 0.55, because they *are* a production run +and that is where production stops — so an AP computed from them integrates a curve that +has been cut off at the operating point. Read that way RampNet's pooled AP is 0.720 and +sits *below* the YOLO arms' 0.730, which is an artifact of the floor and not a result. +``analysis_out/op_cache/`` (committed, 928 KB, all ten splits) holds the same panoramas +re-extracted down to 0.05, which is the floor every other scored model is exported at, and +scoring RampNet's curve from it gives a pooled AP of **0.849**. So on the city splits: +P/R/F1 stay on the committed ``records.jsonl`` — the published, deployment-faithful +operating point — and AP and the PR curve come from the low-floor cache. Both are stated +wherever the number appears, because two sources for one row is exactly the kind of thing +that reads as an error later if it is not said out loud. + +**This is the one place the scoreboard and the log print different numbers under the same +column heading**, so the truncated value is carried on every row as ``ap_bundle``, printed +in the page's AP-provenance table beside the substituted one, and asserted against the log +in the tests. A reader who finds 0.876 here and 0.763 there can see in one table why. + +The substitution is gated on *measured* truncation (bundle floor more than 0.1 above the +cache floor), not on a list of split names. ``manual_gold``'s bundle is already exported at +0.05, so it keeps its own AP: there is no truncation to undo there, and swapping in the +cache would quietly trade that split's flip-TTA export for a no-TTA one — a different +change, and one that would leave a single row's AP and P/R/F1 describing two different +inference configurations. + +One consequence worth carrying: the sub-0.55 half of that curve is scored against ground +truth assembled from detections at or above 0.55, so a real ramp nobody marked counts as a +false positive. #55 measured that 27.2% of the incremental false positives in [0.30, 0.55) +are ground-truth-completeness artifacts. RampNet's untruncated AP is therefore itself a +lower bound. +""" +import argparse +import functools +import json +import os +import sys + +REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, REPO) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(REPO, "scripts", "model_comparison")) + +import compare as C # noqa: E402 (torch-free: detectors.py imports torch lazily) +from rampnet import roster # noqa: E402 +from rampnet.detection_eval import ( # noqa: E402 + aggregate, prediction_confidence, radius_sq_for, score_pano) +from export_model_cache import PUBLISHED_DIR, load_detections # noqa: E402 +# The split registry, imported not restated. test_registries_agree_with_low_floor_sweep +# already keeps low_floor_sweep and miss_decomposition in step; a third private copy here +# is exactly how a split ends up pooled in one headline and held out of another. +from low_floor_sweep import ( # noqa: E402 + ALL_SPLITS, CACHE_DIR, CITY_SPLITS, DEPLOYED_THRESHOLD, HELD_OUT, US_SPLITS, + load_split as load_low_floor) +from operating_point_curve import pr_curve_and_ap # noqa: E402 + +IN_DISTRIBUTION = "manual_gold" +# Held-out splits in a stable reading order: the two cities, then the reference split. +HELD_OUT_ORDER = tuple(s for s in ALL_SPLITS if s in HELD_OUT) + +RAMPNET = "rampnet" +# #54's recommendation, not yet adopted by the deployment consumer +# (sidewalk-auto-labeler#20 is open). Marked on the PR figure beside the deployed point so +# the choice is visible rather than asserted. +RECOMMENDED_THRESHOLD = 0.30 + +# Confidence floor each model class is reported at. These are not new choices -- each is +# the operating point its own write-up committed to, restated here so one table decides it +# for every row and figure instead of each caller deciding again. A model whose +# predictions carry no confidence (the chat VLMs) is unaffected either way: rescore() +# never drops an unscored prediction. +OPERATING_POINT = { + "purpose-trained": DEPLOYED_THRESHOLD, # RampNet as deployed (docs/operating_point.md) + "supervised": 0.25, # pre-registered YOLO headline (#71) + "supervised-transfer": 0.0, # Vistas mask components, at the floor #126 exported + "open-vocab": 0.0, # the 0.05 export floor, as the log reports them + "chat-vlm": 0.0, # no confidences -- no-op + "pointing": 0.0, # no confidences -- no-op + "unclassified": 0.0, # discovered leg: report it where it sits, and say so +} + +# How to say that operating point in a table cell. "0.05 floor" and "no score" are not +# the same thing and the distinction is load-bearing: a floored detector could be tuned +# to a better point and is not being, whereas a chat VLM has no confidence to tune on, so +# its single row IS the model. Collapsing both to "0.00" invites the reading that the +# chat VLMs were handed an unfairly low threshold. +OPERATING_POINT_NOTE = { + "purpose-trained": "0.55", + "supervised": "0.25", + "supervised-transfer": "export floor", + "open-vocab": "0.05 floor", + "chat-vlm": "no score", + "pointing": "no score", + "unclassified": "export floor", +} + +# Model class per roster provider. The registry knows WHO has been run; this only says +# what kind of thing each provider is, which is the one fact a results table needs and +# the roster does not carry. An unknown provider falls through to "unclassified" and is +# still scored -- silently dropping a leg someone paid for is the failure this file +# exists to prevent. +PROVIDER_CLASS = { + "rampnet": "purpose-trained", + "yolo": "supervised", + "vistas": "supervised-transfer", + "gemini": "chat-vlm", + "claude": "chat-vlm", + "qwen": "chat-vlm", + "molmo": "pointing", + "owlv2": "open-vocab", + "gdino": "open-vocab", +} + +CLASS_ORDER = ("purpose-trained", "supervised", "supervised-transfer", "chat-vlm", + "pointing", "open-vocab", "unclassified") +CLASS_LABEL = { + "purpose-trained": "purpose-trained", + "supervised": "supervised baseline", + "supervised-transfer": "supervised transfer", + "chat-vlm": "chat VLM", + "pointing": "pointing model", + "open-vocab": "open-vocab detector", + "unclassified": "unclassified", +} + +# Short, readable stand-ins for the long published ids, keyed by published name (unique +# by construction -- it is a filename stem). Anything absent falls back to the id itself, +# so a new leg is readable-but-verbose rather than missing. +DISPLAY = { + "rampnet": "RampNet", + "y11x_pano_h200": "YOLO11x (pano)", + "y11l_pano": "YOLO11l (pano)", + "y26_pano": "YOLO26 (pano)", + "gemini-3.1-pro-preview": "Gemini 3.1 Pro", + "gemini-3.7-flash": "Gemini 3.7 Flash", + "gemini-3.6-flash": "Gemini 3.6 Flash", + "Qwen/Qwen3-VL-32B-Instruct": "Qwen3-VL-32B", + "Qwen/Qwen3-VL-8B-Instruct": "Qwen3-VL-8B", + "allenai/Molmo2-8B": "Molmo2-8B", + "google/owlv2-large-patch14-ensemble": "OWLv2-large", + "IDEA-Research/grounding-dino-base": "Grounding DINO", + "claude-opus-5-effort-low": "Claude Opus 5 (low)", + "claude-opus-5-effort-high": "Claude Opus 5 (high)", + "claude-sonnet-5-effort-low": "Claude Sonnet 5 (low)", + "claude-sonnet-5-effort-high": "Claude Sonnet 5 (high)", + "mask2former-vistas-curb-cut": "Mask2Former Vistas (curb cut)", + "mask2former-vistas-curb-cut+curb": "Mask2Former Vistas (+curb)", +} + +DEFAULT_JSON = os.path.join(REPO, "analysis_out", "scoreboard.json") +DEFAULT_DOC = os.path.join(REPO, "docs", "model_scoreboard.md") +FIGURE_DIR = os.path.join(REPO, "docs", "figures") + + +def class_of(leg): + """Model class for a roster leg; an unregistered provider is named, never dropped.""" + return PROVIDER_CLASS.get(leg.provider, "unclassified") + + +def display_of(leg): + """Short readable name, falling back to the published id.""" + name = roster.published_name(leg) + return DISPLAY.get(name, name) + + +def legs(): + """Every registered leg, in roster order (which is results-table order). + + Driven by ``rampnet.roster`` rather than by scanning ``benchmark/model_detections/``: + filenames are ambiguous now that one model id can be several legs + (``claude-opus-5`` at two efforts publishes as two stems) and that a published name + can itself contain the ``__`` separator (``mask2former-vistas-curb-cut__curb``). + ``test_roster.py`` already asserts the directory and the registry agree, so the + registry is the safe side of that pair to read from. + """ + return list(roster.ROSTER) + + +def unregistered_exports(): + """Published detection files no roster entry claims — reported, never scored silently. + + ``test_roster.py`` should make this empty. It is computed anyway so that if the two + ever drift, the scoreboard says so instead of quietly omitting a paid-for run. + """ + known = {roster.published_filename(leg, split) + for leg in roster.ROSTER for split in ALL_SPLITS} + return sorted(name for name in os.listdir(PUBLISHED_DIR) + if name.endswith(".json") and name not in known) + + +@functools.lru_cache(maxsize=None) +def low_floor_panos(split): + """The low-floor cache for one split, or None. Memoized: read up to 4x per build. + + ``score`` needs it twice per split (floor + AP) and ``pooled_curve`` / + ``rampnet_marks`` need it again, and it is the same file every time. + """ + if not os.path.exists(os.path.join(CACHE_DIR, f"{split}.json")): + return None + return load_low_floor(split)[0] + + +def low_floor_floor(split): + """The lowest confidence the low-floor cache stored for a split, or None.""" + panos = low_floor_panos(split) + if panos is None: + return None + return min((c for pd in panos for *_xy, c in pd["preds"]), default=None) + + +def low_floor_report(split, radius_sq): + """RampNet's untruncated PR curve + AP for one split, or None if uncached. + + ``op_cache`` is the #54 extraction: the same panoramas, same preprocessing, same + no-TTA deployment path, every heatmap peak down to 0.05. Its >=0.55 slice is gated to + reproduce the committed records (``low_floor_sweep.py parity``), so it is the same run + seen further down rather than a different one. + """ + panos = low_floor_panos(split) + return None if panos is None else pr_curve_and_ap(panos, radius_sq) + + +def bundle_floor(preds): + """The lowest confidence present in a bundle's detections, or None if unscored.""" + return min((c for pts in preds.values() for c in + (prediction_confidence(d) for d in pts) if c is not None), default=None) + + +def uses_low_floor_cache(split, preds): + """Is RampNet's bundle for ``split`` truncated far enough above the cache to swap? + + Decided by *measuring* both floors rather than by naming splits. One function so the + AP column (``score``) and the PR curve (``pooled_curve``) can never disagree about + which source a split is read from -- a divergence that would put a curve and the AP + printed beside it on two different runs. + """ + if low_floor_panos(split) is None: + return False + bf, cf = bundle_floor(preds), low_floor_floor(split) + return bf is not None and cf is not None and bf - cf > 0.1 + + +def load_split(split): + """(records, {pano_id: GroundTruth}) for one benchmark bundle.""" + bundle = os.path.join(REPO, "benchmark", split) + records, verdicts, _ = C.load_bundle(bundle) + if verdicts is not None: + C.validate_bundle(records, verdicts) + return records, C.ground_truths_from_verdicts(records, verdicts) + gts = C.load_manual_ground_truths(bundle) + return records, {pid: gts[pid] for pid in records if pid in gts} + + +def score(leg, split, records, gts, radius_sq, op): + """One (leg, split) cell, or None when the leg was never run on that split. + + P/R/F1 are read at the operating point ``op``; AP is read from the full-range run, + because ``--op-threshold`` truncates the curve it integrates. That split is + ``compare.operating_report``'s, reused here so the scoreboard and the log cannot + disagree about what a row means. + """ + if leg.provider == "rampnet": + preds = {pid: records[pid]["detections"] for pid in gts} + else: + # publish_as, not label: a pinned leg's detections live under its own stem, and + # loading by label alone silently returns the sibling leg's file. + preds = load_detections(leg.label, split, + publish_as=roster.published_name(leg)) + if preds is None: + return None + scored = [(preds.get(pid, []), gt) for pid, gt in gts.items()] + full = aggregate([score_pano(p, g, radius_sq=radius_sq) for p, g in scored]) + rep = C.operating_report(full, scored, radius_sq, op) + + # RampNet's bundle curve is truncated at its deployed threshold; the committed + # low-floor cache carries the same run down to 0.05. See the module docstring. + # + # Substituted only where truncation actually exists, decided by measuring the two + # floors rather than by naming splits. manual_gold's bundle is ALREADY at 0.05, so + # there is nothing to fix there -- and swapping in the cache would silently trade a + # flip-TTA export for a no-TTA one, which is a different change from un-truncating a + # curve and would leave that row's AP and its P/R/F1 describing two different + # inference configs. + ap, ap_source = rep.ap, "bundle" + if leg.provider == "rampnet" and uses_low_floor_cache(split, preds): + low = low_floor_report(split, radius_sq) + if low is not None and low.ap is not None: + ap, ap_source = low.ap, "op_cache (0.05 floor)" + return { + "split": split, + "precision": rep.precision, "recall": rep.recall, "f1": rep.f1, "ap": ap, + "ap_source": ap_source, + # The AP as computed from the bundle alone -- i.e. exactly the number + # docs/model_comparison.md's per-split tables print. Carried on every row, not + # just the substituted ones, so the two documents can be diffed mechanically + # (test_every_number_matches_model_comparison) instead of by eye. + "ap_bundle": rep.ap, + "tp": rep.tp, "fp": rep.fp, "fn": rep.fn, + "n_panos": rep.n_panos, "n_gt_recall": rep.n_gt_recall, + "fp_per_pano": rep.fp / rep.n_panos if rep.n_panos else None, + } + + +def mean(values): + vals = [v for v in values if v is not None] + return sum(vals) / len(vals) if vals else None + + +def summarize(leg, cells): + """Macro-mean over the pooled US splits the leg actually ran on, + the held-out ones. + + ``coverage`` travels with every aggregate so a model averaged over one city can never + be read as one averaged over seven — and ``complete`` decides which table it lands + in, because a one-city mean has no business sitting in the same column as a + seven-city one. The held-out splits are carried individually, never folded in, for + the reasons in ``HELD_OUT``. + """ + klass = class_of(leg) + pooled = [cells[s] for s in US_SPLITS if cells.get(s)] + f1s = [c["f1"] for c in pooled] + have_ap = bool(pooled) and all(c["ap"] is not None for c in pooled) + have_bundle_ap = bool(pooled) and all(c["ap_bundle"] is not None for c in pooled) + # Micro (count-pooled) P/R alongside the macro-mean. The PR-curve figure integrates + # over concatenated panoramas, so a point drawn on those axes has to be micro too -- + # plotting the macro-mean there would put the dots and the lines in two different + # aggregations under one subtitle. The headline table stays macro. + tp = sum(c["tp"] for c in pooled) + fp = sum(c["fp"] for c in pooled) + fn = sum(c["fn"] for c in pooled) + out = { + "model": roster.published_name(leg), + "label": leg.label, + "spec": leg.spec, + "provider": leg.provider, + "standing": leg.standing, + "display": display_of(leg), + "class": klass, + "operating_point": OPERATING_POINT[klass], + "operating_point_note": OPERATING_POINT_NOTE[klass], + "coverage": f"{len(pooled)}/{len(US_SPLITS)}", + "complete": len(pooled) == len(US_SPLITS), + "pooled_splits": [s for s in US_SPLITS if cells.get(s)], + "precision": mean(c["precision"] for c in pooled), + "recall": mean(c["recall"] for c in pooled), + "f1": mean(f1s), + "ap": mean(c["ap"] for c in pooled) if have_ap else None, + # Macro-mean of the per-split BUNDLE AP -- the aggregate of exactly the numbers + # docs/model_comparison.md prints. For every model but RampNet it equals "ap"; + # for RampNet it is the 0.55-truncated figure the substitution exists to replace, + # kept so the page can show what it replaced rather than assert it. + "ap_bundle": mean(c["ap_bundle"] for c in pooled) if have_bundle_ap else None, + "ap_is_substituted": any(c["ap_source"] != "bundle" for c in pooled), + "fp_per_pano": mean(c["fp_per_pano"] for c in pooled), + "micro_precision": tp / (tp + fp) if tp + fp else None, + "micro_recall": tp / (tp + fn) if tp + fn else None, + "f1_min": min(f1s) if f1s else None, + "f1_max": max(f1s) if f1s else None, + "f1_min_split": min(pooled, key=lambda c: c["f1"])["split"] if pooled else None, + "f1_max_split": max(pooled, key=lambda c: c["f1"])["split"] if pooled else None, + "n_splits_run": len([s for s in ALL_SPLITS if cells.get(s)]), + } + for split in HELD_OUT_ORDER: + cell = cells.get(split) + for metric in ("f1", "precision", "recall", "ap"): + out[f"{split}_{metric}"] = cell[metric] if cell else None + return out + + +# Curves are pooled over the seven US splits so the figure has one line per model rather +# than seven. Unlike the headline table this pooling is MICRO (every panorama counts once, +# so a bigger split pulls harder) -- a PR curve is an integral over predictions and has no +# natural macro form. Stated on the figure. +def pooled_curve(leg, splits, radius_sq): + """(ScoreReport, marked operating points) pooled across US_SPLITS, or None. + + None when the leg carries no confidences (a chat VLM has one operating point, not a + curve) or has not run every pooled split. + """ + pano_scores, has_conf, n_seen = [], True, 0 + for split in US_SPLITS: + records, gts = splits[split] + if leg.provider == "rampnet": + bundle = {pid: records[pid]["detections"] for pid in gts} + # Same gate the AP column uses, so the curve and the AP printed in its legend + # are always read from the same source. + if uses_low_floor_cache(split, bundle): + for pd in low_floor_panos(split): + pano_scores.append(score_pano(pd["preds"], pd["gt"], + radius_sq=radius_sq)) + else: + for pid, gt in gts.items(): + pano_scores.append(score_pano(bundle[pid], gt, radius_sq=radius_sq)) + n_seen += 1 + continue + preds = load_detections(leg.label, split, publish_as=roster.published_name(leg)) + if preds is None: + return None + n_seen += 1 + for pid, gt in gts.items(): + pts = preds.get(pid, []) + if any(prediction_confidence(q) is None for q in pts): + has_conf = False + pano_scores.append(score_pano(pts, gt, radius_sq=radius_sq)) + if n_seen != len(US_SPLITS) or not has_conf: + return None + report = aggregate(pano_scores) + return report if report.pr_curve else None + + +def rampnet_marks(splits, radius_sq, thresholds=(DEPLOYED_THRESHOLD, RECOMMENDED_THRESHOLD)): + """(recall, precision) on the pooled curve at each named threshold. + + The point of drawing the curve is to choose a point on it, so the two the project has + actually argued about are marked: the deployed 0.55 and #54's recommended 0.30. + """ + panos = [] + for split in US_SPLITS: + cached = low_floor_panos(split) + if cached is None: + return {} + panos.extend(cached) + out = {} + for t in thresholds: + rep = aggregate([score_pano([q for q in pd["preds"] if q[2] >= t], pd["gt"], + radius_sq=radius_sq) for pd in panos]) + out[f"{t:.2f}"] = {"recall": rep.recall, "precision": rep.precision, "f1": rep.f1} + return out + + +def build(models=None): + """Score every (leg, split) pair from committed data. Returns the result dict. + + ``models`` filters by published name, so ``--models claude-opus-5-effort-low`` names + one leg rather than one model id. + """ + radius_sq = radius_sq_for() + wanted = set(models) if models else None + chosen = [leg for leg in legs() + if wanted is None or roster.published_name(leg) in wanted] + splits = {s: load_split(s) for s in ALL_SPLITS} + + per_model, summaries = {}, {} + for leg in chosen: + op = OPERATING_POINT[class_of(leg)] + cells = {} + for split, (records, gts) in splits.items(): + cell = score(leg, split, records, gts, radius_sq, op) + if cell is not None: + cells[split] = cell + name = roster.published_name(leg) + per_model[name] = cells + summaries[name] = summarize(leg, cells) + + # Complete rows first, then class (so the table reads as a taxonomy), then F1. + # A leg with no pooled coverage at all sorts last within its class rather than + # ranking as if it scored zero. + def key(name): + s = summaries[name] + return (not s["complete"], CLASS_ORDER.index(s["class"]), + -(s["f1"] if s["f1"] is not None else -1)) + + order = sorted(summaries, key=key) + + # Pooled PR curves for the models that carry confidences. The point arrays are + # PLOT-ONLY and are dropped before the JSON is written (scoreboard_render. + # json_payload): the two open detectors alone carry ~120k points, which is 7.7 MB of + # committed artifact for something no reader diffs and the figure rebuilds in 3 s. + # The AP, the marks and the point count survive, which is what the page cites. + curves = {} + for leg in chosen: + rep = pooled_curve(leg, splits, radius_sq) + if rep is not None: + curves[roster.published_name(leg)] = { + "recalls": list(rep.pr_curve[0]), "precisions": list(rep.pr_curve[1]), + "ap": rep.ap, + "n_points": len(rep.pr_curve[0]), + "marks": (rampnet_marks(splits, radius_sq) + if leg.provider == "rampnet" else {}), + } + return { + "pooled_splits": list(US_SPLITS), + "city_splits": list(CITY_SPLITS), + "all_splits": list(ALL_SPLITS), + "held_out": dict(HELD_OUT), + "in_distribution_split": IN_DISTRIBUTION, + "unregistered_exports": unregistered_exports(), + "curves": curves, + "splits": {s: {"n_panos": len(gts), + "n_gt": sum(len(g.gt_points) for g in gts.values()), + "pooled": s in US_SPLITS} + for s, (_, gts) in splits.items()}, + "models": [summaries[n] for n in order], + "per_split": {n: per_model[n] for n in order}, + } + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0], + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--json-out", default=DEFAULT_JSON) + ap.add_argument("--doc", default=DEFAULT_DOC) + ap.add_argument("--figure-dir", default=FIGURE_DIR) + ap.add_argument("--models", + help="Comma-separated PUBLISHED names (roster.published_name), e.g. " + "claude-opus-5-effort-low. Scores a SUBSET, so it will not " + "write the committed doc, JSON or figures unless you also pass " + "--doc/--json-out/--figure-dir explicitly. Default: every leg.") + ap.add_argument("--no-figures", action="store_true", help="Skip matplotlib entirely.") + ap.add_argument("--check", action="store_true", + help="Verify the committed doc AND analysis_out/scoreboard.json match " + "the committed data; write nothing and exit non-zero on drift.") + args = ap.parse_args() + + models = [m.strip() for m in args.models.split(",")] if args.models else None + + # Refused before scoring, not after: --check on a subset would compare a partial + # board against the full committed page and report the page as stale, which is a + # false alarm and an expensive one to sit through. + if args.check and models: + print("--check scores every leg; drop --models") + raise SystemExit(2) + + # A subset run produces a partial board. Splicing that into the committed page leaves + # a one-row headline table sitting above prose about a twelve-model field, and the + # next --check then reports the REAL page as stale -- the signal inverted. So a + # subset run is read-only against the committed artifacts unless the caller names + # different destinations. + explicit = {a.split("=")[0] for a in sys.argv[1:]} + subset_guard = bool(models) and not args.check + write_doc = not subset_guard or "--doc" in explicit + write_json_out = not subset_guard or "--json-out" in explicit + write_figs = not args.no_figures and (not subset_guard or "--figure-dir" in explicit) + + result = build(models) + + from scoreboard_render import ( # noqa: E402 + json_payload, render_tables, splice, write_json) + tables = render_tables(result) + + if args.check: + problems = [] + if not os.path.exists(args.doc): + problems.append(f"{args.doc}: missing") + else: + with open(args.doc, encoding="utf-8", newline="") as fh: + current = fh.read() + if splice(current, tables) != current: + problems.append(f"{args.doc}: generated tables are stale " + "(re-run scripts/analysis/scoreboard.py)") + # The JSON is a committed artifact too, and nothing else checks it. Compared as + # bytes, which also catches a CRLF flip that a value-level compare would miss. + if not os.path.exists(args.json_out): + problems.append(f"{args.json_out}: missing") + else: + with open(args.json_out, "rb") as fh: + on_disk = fh.read() + if on_disk != json_payload(result).encode("utf-8"): + problems.append(f"{args.json_out}: stale or reformatted " + "(re-run scripts/analysis/scoreboard.py)") + if problems: + print("\n".join(problems)) + raise SystemExit(1) + print(f"{args.doc}: current") + print(f"{args.json_out}: current") + return + + if subset_guard: + print(f"--models given ({', '.join(models)}): scoring a subset, so the committed " + "doc, JSON and figures are left alone.\nPass --doc/--json-out/--figure-dir " + "to write a partial board somewhere else.") + + if write_json_out: + write_json(args.json_out, result) + print(f"wrote {args.json_out}") + + if write_doc and os.path.exists(args.doc): + with open(args.doc, encoding="utf-8", newline="") as fh: + current = fh.read() + updated = splice(current, tables) + if updated != current: + with open(args.doc, "w", encoding="utf-8", newline="") as fh: + fh.write(updated) + print(f"updated {args.doc}") + else: + print(f"{args.doc}: already current") + elif write_doc: + print(f"{args.doc}: not found -- write the prose first, then re-run to fill " + "the generated blocks") + + if write_figs: + import scoreboard_figures + for path in scoreboard_figures.render_all(result, args.figure_dir): + print(f"wrote {path}") + + +if __name__ == "__main__": + main() diff --git a/scripts/analysis/scoreboard_figures.py b/scripts/analysis/scoreboard_figures.py new file mode 100644 index 00000000..b8d0ddf8 --- /dev/null +++ b/scripts/analysis/scoreboard_figures.py @@ -0,0 +1,587 @@ +"""The five figures for docs/model_scoreboard.md — the findings that read faster as a picture. + +Called by ``scoreboard.py``; importable on its own for iterating on a figure without +re-scoring. Matplotlib is imported inside ``render_all`` so ``--no-figures`` needs no +plotting stack at all. + +**Colour.** The four aggregate figures run on one hue: the validated categorical slot 1 +(``#2a78d6``) plus the palette's neutral inks, with the sequential blue ramp for the +heatmap. That is a deliberate reduction, not laziness. The natural design — a hue per +model class — cannot ship: three of those four are scatter/matrix forms, which are +scored on the **all-pairs** pairlist, and five categorical slots fail it outright +(``validate_palette.js``: magenta↔orange normal-vision ΔE 12.9, below the hard floor of +15). A normal-vision FAIL is the one result secondary encoding does not excuse, so the +documented remedy is to cut series or facet rather than to add a legend and hope. Model +class is therefore carried by **position** (bar order, group headers) and **marker +shape** — channels with no CVD failure mode — and colour is freed to carry emphasis. + +The PR-curve figure is the exception and gets four hues, because overlapping *lines* are +scored on the **adjacent** pairlist, which that set passes outright (worst adjacent CVD +ΔE 9.1); see ``CURVE_COLOR``. Marker shape cannot separate curves, so there is no +position-or-shape substitute to fall back on there. + +The palette's contrast WARN (slot 1 sits above 3:1, but the light neutrals do not) is +discharged the way ``plot_operating_point.py`` discharges it: every plotted number is +also in a table, in ``docs/model_scoreboard.md`` and ``analysis_out/scoreboard.json``. +""" +import os + +# Palette: validated categorical slot 1, the sequential blue ramp, and the neutral inks +# (references/palette.md). Same values plot_operating_point.py uses, so the two figures +# families sit together in one document without looking like two documents. +BLUE = "#2a78d6" +BLUE_DEEP = "#184f95" +INK = "#0b0b0b" +INK_SECONDARY = "#52514e" +INK_MUTED = "#898781" +GRID = "#e1e0d9" +BASELINE = "#c3c2b7" +SURFACE = "#fcfcfb" +MUTED_FILL = "#c9c8c2" # the un-emphasised bars: present, recessive, not competing +# Sequential blue, light -> dark (palette.md steps 100..700). Monotone in lightness, one +# hue: the documented form for a magnitude encoding. +SEQ = ["#cde2fb", "#b7d3f6", "#9ec5f4", "#86b6ef", "#6da7ec", "#5598e7", "#3987e5", + "#2a78d6", "#256abf", "#1c5cab", "#184f95", "#104281", "#0d366b"] + +# Class -> marker, the CVD-free channel that replaces a per-class hue in the scatters. +CLASS_MARKER = { + "purpose-trained": "o", + "supervised": "s", + "supervised-transfer": "P", + "chat-vlm": "^", + "pointing": "D", + "open-vocab": "v", + "unclassified": "X", +} + + +# Label placement is deterministic, not automatic: a solver would move labels whenever +# a number moved, and these figures are committed artifacts that get diffed. Each entry +# is (dx, dy) in points plus the horizontal anchor, and the default is down-and-right. +# Only points that would collide need one. +PR_NUDGE = { + "y11x_pano_h200": (11, 5, "left"), + "y11l_pano": (11, -8, "left"), + "google/owlv2-large-patch14-ensemble": (-9, 11, "right"), + "IDEA-Research/grounding-dino-base": (-9, -2, "right"), +} +GEN_NUDGE = { + # RampNet sits close to the diagonal in the top-right, so its label crosses that line + # whichever side it goes; the halo carries it rather than a nudge that cannot fit. + "rampnet": (-14, -13, "right"), + "y11l_pano": (11, 6, "left"), + "y11x_pano_h200": (11, -7, "left"), + "y26_pano": (-11, -10, "right"), + "gemini-3.7-flash": (11, 3, "left"), + # Left, not right: running right puts this label across Qwen-8B's marker 0.10 away, + # and a reader then attaches it to the wrong point. + "Qwen/Qwen3-VL-32B-Instruct": (-11, 4, "right"), + "Qwen/Qwen3-VL-8B-Instruct": (11, -8, "left"), + "google/owlv2-large-patch14-ensemble": (12, 7, "left"), + "IDEA-Research/grounding-dino-base": (12, -9, "left"), +} + + +def _halo(width=2.6): + """A SURFACE-coloured stroke behind label text. + + These are dense scatters over contours, a reference diagonal and each other, and a + label that crosses a hairline reads as damaged even though it is legible. A halo + fixes every such crossing at once, which nudging one label at a time does not — and + unlike a nudge it cannot push a label off the axes. + """ + from matplotlib import patheffects + return [patheffects.withStroke(linewidth=width, foreground=SURFACE)] + + +def _titles(ax, title, subtitle): + """Title above, subtitle below it, both above the axes and left-aligned. + + set_title() alone puts the title flush to the axes, so a subtitle placed in axes + coordinates lands ABOVE it and the two read in the wrong order. + """ + ax.set_title(title, fontsize=12.5, color=INK, loc="left", pad=30) + ax.text(0.0, 1.018, subtitle, transform=ax.transAxes, fontsize=8.6, + color=INK_SECONDARY, va="bottom") + + +def _style(ax): + """Recessive chrome: hairline solid grid, no top/right spines, muted ticks.""" + ax.set_facecolor(SURFACE) + for side in ("top", "right"): + ax.spines[side].set_visible(False) + for side in ("left", "bottom"): + ax.spines[side].set_color(BASELINE) + ax.spines[side].set_linewidth(0.8) + ax.tick_params(colors=INK_MUTED, labelsize=8.5, length=3, width=0.8) + + +def _seq_color(value, vmax): + """A step of the sequential ramp for ``value`` in [0, vmax].""" + if value is None or vmax <= 0: + return SURFACE + idx = int(round((value / vmax) * (len(SEQ) - 1))) + return SEQ[max(0, min(len(SEQ) - 1, idx))] + + +def fig_headline(result, path, plt): + """Pooled F1 per model, sorted — the board, and the size of the lead. + + Emphasis rather than a hue per class: the story here is one number (how far clear + RampNet is), which is the case the anti-pattern list says to solve with highlight-one + rather than with a full categorical palette. + """ + from scoreboard import CLASS_LABEL + + # Complete rows only: figures 1, 2 and 4 all plot a pooled mean, and a one-city mean + # drawn beside a seven-city one is the comparison the partial table exists to avoid. + # Those legs appear in the matrix (fig 3), which is per split and needs no averaging. + models = [m for m in result["models"] if m["complete"] and m["f1"] is not None] + models.sort(key=lambda m: m["f1"]) + # Class rides in the tick label rather than in a second text column: drawn + # separately it lands on top of the tick labels, because matplotlib sizes the left + # margin from the ticks alone. + labels = [f"{m['display']} · {CLASS_LABEL[m['class']]}" for m in models] + values = [m["f1"] for m in models] + is_ref = [m["model"] == "rampnet" for m in models] + + fig, ax = plt.subplots(figsize=(9.6, 6.0)) + fig.patch.set_facecolor(SURFACE) + _style(ax) + ax.grid(True, axis="x", color=GRID, lw=0.8, zorder=0) + ax.set_axisbelow(True) + + bars = ax.barh(range(len(models)), values, height=0.62, zorder=3, + color=[BLUE if r else MUTED_FILL for r in is_ref]) + for i, (bar, m) in enumerate(zip(bars, models)): + ax.text(bar.get_width() + 0.012, i, f"{m['f1']:.3f}", va="center", + fontsize=8.6, color=INK if is_ref[i] else INK_SECONDARY, + fontweight="bold" if is_ref[i] else "normal", zorder=4) + + ax.set_yticks(range(len(models))) + ax.set_yticklabels(labels, fontsize=9) + for tick, ref in zip(ax.get_yticklabels(), is_ref): + tick.set_color(INK if ref else INK_SECONDARY) + if ref: + tick.set_fontweight("bold") + + ax.set_xlim(0, 1.0) + ax.set_ylim(-0.7, len(models) - 0.3) + ax.set_xlabel("F1, macro-mean over the seven pooled US city splits", + fontsize=9.5, color=INK_SECONDARY) + + # The lead goes in the subtitle rather than into an annotated arrow: the gap between + # the top two bars is 0.38 of a row, which cannot hold a rule and a caption without + # colliding with one of them. + # + # Both lookups tolerate absence: a --models subset run may hold no RampNet row, or no + # challenger, and a figure helper is the wrong place to die on that. + runner_up = max((m for m in models if m["model"] != "rampnet"), + key=lambda m: m["f1"], default=None) + ref_f1 = next((m["f1"] for m in models if m["model"] == "rampnet"), None) + if ref_f1 is not None and runner_up is not None: + subtitle = (f"+{ref_f1 - runner_up['f1']:.3f} F1 clear of the best challenger " + f"({runner_up['display']}, {runner_up['f1']:.3f}).") + else: + subtitle = "Macro-mean over the seven pooled US city splits." + _titles(ax, "RampNet leads every off-the-shelf and supervised baseline tested", + subtitle) + # Two lines: at 7.4pt one line of this runs past the right edge and gets clipped. + n_partial = len([m for m in result["models"] if not m["complete"]]) + fig.text(0.008, 0.030, + "Operating points differ by model class: RampNet 0.55, YOLO 0.25, " + "open-vocab 0.05 floor, chat VLMs emit no score.", + fontsize=7.4, color=INK_MUTED, ha="left", va="bottom") + fig.text(0.008, 0.008, + (f"{n_partial} single-split legs are reported per split instead — " + if n_partial else "") + "see docs/model_scoreboard.md.", + fontsize=7.4, color=INK_MUTED, ha="left", va="bottom") + fig.tight_layout(rect=(0, 0.052, 1, 1)) + fig.savefig(path, dpi=170, facecolor=fig.get_facecolor()) + plt.close(fig) + return path + + +def fig_precision_recall(result, path, plt): + """Where each model sits in the P/R plane — the shape of how it fails, not just how far. + + F1 iso-contours are the reason this beats the bar chart for diagnosis: two models on + the same contour score identically and are nothing alike. + """ + from matplotlib.lines import Line2D + import numpy as np + + from scoreboard import CLASS_LABEL + + models = [m for m in result["models"] + if m["complete"] and m["precision"] is not None + and m["recall"] is not None] + + fig, ax = plt.subplots(figsize=(8.8, 7.4)) + fig.patch.set_facecolor(SURFACE) + _style(ax) + + grid = np.linspace(0.001, 1.0, 400) + rr, pp = np.meshgrid(grid, grid) + f1 = 2 * rr * pp / (rr + pp) + levels = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] + cs = ax.contour(rr, pp, f1, levels=levels, colors=GRID, linewidths=0.9, zorder=1) + # Contour labels go in the right margin, not wherever the auto-placer picks. Left to + # itself it put "F1 0.6" at recall 0.44 / precision 0.99 — on top of the two YOLO + # markers and their labels. Every iso-F1 curve crosses recall 1.0 at p = F1/(2 - F1), + # and nothing on this board reaches recall 1.0, so that column is always free. + ax.clabel(cs, fmt=lambda v: f"F1 {v:.1f}", fontsize=6.8, colors=INK_MUTED, + manual=[(1.0, lv / (2 - lv)) for lv in levels]) + + for m in models: + ref = m["model"] == "rampnet" + ax.plot(m["recall"], m["precision"], CLASS_MARKER[m["class"]], + ms=13 if ref else 9.5, color=BLUE, + alpha=1.0 if ref else 0.72, mec=SURFACE, mew=2.0, zorder=5) + dx, dy, ha = PR_NUDGE.get(m["model"], (11, -3.5, "left")) + ax.annotate(m["display"], (m["recall"], m["precision"]), + textcoords="offset points", xytext=(dx, dy), ha=ha, + fontsize=8.4, color=INK if ref else INK_SECONDARY, + fontweight="bold" if ref else "normal", zorder=6, + path_effects=_halo()) + + ax.set_xlim(0, 1.04) + ax.set_ylim(0, 1.04) + ax.set_xlabel("recall", fontsize=9.5, color=INK_SECONDARY) + ax.set_ylabel("precision", fontsize=9.5, color=INK_SECONDARY) + _titles(ax, "The same F1 hides opposite failures", + "Macro-mean over the seven pooled US city splits. Marker shape is model class.") + + seen, handles = set(), [] + for m in models: + if m["class"] in seen: + continue + seen.add(m["class"]) + handles.append(Line2D([], [], ls="", marker=CLASS_MARKER[m["class"]], ms=8, + color=BLUE, mec=SURFACE, mew=1.6, + label=CLASS_LABEL[m["class"]])) + leg = ax.legend(handles=handles, loc="lower left", frameon=False, fontsize=8.2, + labelcolor=INK_SECONDARY, handletextpad=0.5, + bbox_to_anchor=(0.005, 0.005)) + leg.set_zorder(7) + + fig.text(0.008, 0.012, + "Open-vocabulary detectors buy their recall with ~65–72 false positives per " + "panorama; Qwen-32B buys precision by barely firing.", + fontsize=7.4, color=INK_MUTED, ha="left", va="bottom") + fig.tight_layout(rect=(0, 0.028, 1, 0.96)) + fig.savefig(path, dpi=170, facecolor=fig.get_facecolor()) + plt.close(fig) + return path + + +def fig_by_split(result, path, plt): + """F1 for every (model, split) cell — consistency across cities, at a glance. + + Sequential single-hue ramp (magnitude), values printed in every cell so the figure is + also its own table, and the three held-out splits pushed to the right of a gap so + they are never read as part of the pooled headline. + """ + from scoreboard import US_SPLITS + + splits = list(result["all_splits"]) + held = [s for s in splits if s not in US_SPLITS] + ordered = list(US_SPLITS) + held + models = result["models"] + per = result["per_split"] + + # A one-column gutter between the pooled splits and the held-out ones. + gap_at = len(US_SPLITS) + ncols = len(ordered) + 1 + + # Height tracks the row count so adding a leg does not silently squash the cells. + fig, ax = plt.subplots(figsize=(11.6, 1.5 + 0.42 * len(models))) + fig.patch.set_facecolor(SURFACE) + ax.set_facecolor(SURFACE) + + vmax = 1.0 + for row, m in enumerate(models): + for i, split in enumerate(ordered): + col = i if i < gap_at else i + 1 + cell = per[m["model"]].get(split) + value = cell["f1"] if cell else None + color = _seq_color(value, vmax) if value is not None else "#f4f3f0" + ax.add_patch(plt.Rectangle((col + 0.03, row + 0.03), 0.94, 0.94, + facecolor=color, edgecolor="none", zorder=2)) + text = "—" if value is None else f"{value:.2f}" + # White ink only where the fill is dark enough to carry it. + ink = "#ffffff" if (value is not None and value >= 0.62) else INK + ax.text(col + 0.5, row + 0.5, text, ha="center", va="center", + fontsize=8.2, color=ink if value is not None else INK_MUTED, + zorder=3) + + ax.set_xlim(0, ncols) + ax.set_ylim(0, len(models)) + ax.invert_yaxis() + ax.set_xticks([(i if i < gap_at else i + 1) + 0.5 for i in range(len(ordered))]) + ax.set_xticklabels([s if s in US_SPLITS else f"{s} †" for s in ordered], + rotation=38, ha="right", fontsize=8.2, color=INK_SECONDARY) + ax.set_yticks([r + 0.5 for r in range(len(models))]) + ax.set_yticklabels([m["display"] for m in models], fontsize=8.8, + color=INK_SECONDARY) + for tick, m in zip(ax.get_yticklabels(), models): + if m["model"] == "rampnet": + tick.set_color(INK) + tick.set_fontweight("bold") + for side in ("top", "right", "left", "bottom"): + ax.spines[side].set_visible(False) + ax.tick_params(length=0) + + # No gutter caption: the gap plus the daggered column headers already carry it, and a + # rotated label there runs into the title band. + # + # The claim is bounded on purpose. RampNet does NOT have the flattest row outright -- + # OWLv2 (0.028) and Grounding DINO (0.039) are flatter, by being pinned near zero in + # every city. Consistency only means something above the floor, so the comparison is + # drawn against the models that clear it, and the threshold is stated rather than + # implied. + working = [m for m in models + if m["complete"] and m["f1"] is not None and m["f1"] >= 0.1] + spreads = [m["f1_max"] - m["f1_min"] for m in working if m["model"] != "rampnet"] + ref = next((m for m in models if m["model"] == "rampnet"), None) + if ref is not None and ref["f1_max"] is not None and spreads: + subtitle = (f"RampNet varies by {ref['f1_max'] - ref['f1_min']:.2f} across the " + f"seven pooled cities; every challenger above F1 0.1 varies by " + f"{min(spreads):.2f}–{max(spreads):.2f}.") + else: + subtitle = "F1 per (model, split). Held-out splits sit right of the gutter." + _titles(ax, "F1 by model and split", subtitle) + fig.text(0.008, 0.012, + "† held out of the pooled headline: budapest (single-rater GT at low reviewer " + "confidence), sao_paulo (non-US), manual_gold (in-distribution reference).", + fontsize=7.4, color=INK_MUTED, ha="left", va="bottom") + fig.tight_layout(rect=(0, 0.03, 1, 1)) + fig.savefig(path, dpi=170, facecolor=fig.get_facecolor()) + plt.close(fig) + return path + + +def fig_generalization(result, path, plt): + """In-distribution F1 against deployed F1 — what the architecture buys. + + The diagonal is "no domain advantage". A zero-shot model has no training distribution + to be inside, so it lands on or near the line; a model trained on the RampNet dataset + starts above the line on manual_gold and falls by however much it fails to generalize. + The vertical drop to the diagonal is that penalty, and it is the whole #51 ablation in + one distance. + """ + from matplotlib.lines import Line2D + + from scoreboard import CLASS_LABEL + + models = [m for m in result["models"] + if m["complete"] and m["f1"] is not None + and m["manual_gold_f1"] is not None] + + fig, ax = plt.subplots(figsize=(8.4, 7.4)) + fig.patch.set_facecolor(SURFACE) + _style(ax) + ax.grid(True, color=GRID, lw=0.8, zorder=0) + ax.set_axisbelow(True) + + ax.plot([0, 1], [0, 1], color=BASELINE, lw=1.2, zorder=1) + ax.text(0.965, 0.985, "no domain advantage", rotation=45, rotation_mode="anchor", + ha="right", va="bottom", fontsize=7.8, color=INK_MUTED, zorder=2) + + for m in models: + ref = m["model"] == "rampnet" + x, y = m["manual_gold_f1"], m["f1"] + ax.plot([x, x], [y, x], color=INK_MUTED, lw=0.9, ls=(0, (2, 2)), zorder=3) + ax.plot(x, y, CLASS_MARKER[m["class"]], ms=13 if ref else 9.5, color=BLUE, + alpha=1.0 if ref else 0.72, mec=SURFACE, mew=2.0, zorder=5) + # Signed: deployed minus in-distribution. A model ABOVE the line gains, and + # hardcoding a minus rendered those as "--0.05". + dx, dy, ha = GEN_NUDGE.get(m["model"], (11, -3.5, "left")) + ax.annotate(f"{m['display']} {y - x:+.2f}", (x, y), textcoords="offset points", + xytext=(dx, dy), ha=ha, fontsize=8.4, + color=INK if ref else INK_SECONDARY, + fontweight="bold" if ref else "normal", zorder=6, + path_effects=_halo()) + + ax.set_xlim(0, 1.04) + ax.set_ylim(0, 1.04) + ax.set_xlabel("in-distribution F1 (manual_gold, 1,000 GSV panoramas)", + fontsize=9.5, color=INK_SECONDARY) + ax.set_ylabel("deployed F1 (macro-mean, seven US cities)", + fontsize=9.5, color=INK_SECONDARY) + _titles(ax, "The drop from in-distribution to deployed is what generalization costs", + "Labels give deployed F1 minus in-distribution F1. Marker shape is model class.") + + seen, handles = set(), [] + for m in models: + if m["class"] in seen: + continue + seen.add(m["class"]) + handles.append(Line2D([], [], ls="", marker=CLASS_MARKER[m["class"]], ms=8, + color=BLUE, mec=SURFACE, mew=1.6, + label=CLASS_LABEL[m["class"]])) + ax.legend(handles=handles, loc="upper left", frameon=False, fontsize=8.2, + labelcolor=INK_SECONDARY, handletextpad=0.5) + + fig.text(0.008, 0.012, + "Both Gemini legs with city numbers are absent: their manual_gold detections " + "were never published (docs/model_scoreboard.md, 'What is missing').", + fontsize=7.4, color=INK_MUTED, ha="left", va="bottom") + fig.tight_layout(rect=(0, 0.028, 1, 0.96)) + fig.savefig(path, dpi=170, facecolor=fig.get_facecolor()) + plt.close(fig) + return path + + +# Four validated categorical hues for the curve families (validate_palette.js, adjacent +# pairlist -- the documented one for lines: ALL PASS, worst adjacent CVD dE 9.1). The three +# YOLO arms share one hue and separate by dash: they are one family, and spending three +# slots on them would push the set past what the all-pairs floors allow. +CURVE_COLOR = { + "rampnet": "#2a78d6", + "y11l_pano": "#eb6834", + "y11x_pano_h200": "#eb6834", + "y26_pano": "#eb6834", + "google/owlv2-large-patch14-ensemble": "#1baf7a", + "IDEA-Research/grounding-dino-base": "#eda100", +} +CURVE_DASH = { + "y11l_pano": (0, (5, 2)), + "y11x_pano_h200": (0, (1, 1.6)), + "y26_pano": (0, (4, 1.4, 1, 1.4)), +} + + +def _decimate(xs, ys, keep=1500): + """Thin a PR curve for plotting; the open detectors carry ~60k points each.""" + if len(xs) <= keep: + return xs, ys + step = len(xs) / keep + idx = sorted({int(i * step) for i in range(keep)} | {0, len(xs) - 1}) + return [xs[i] for i in idx], [ys[i] for i in idx] + + +def fig_pr_curves(result, path, plt): + """The trade-off curve, pooled over the seven US splits — how to choose a threshold. + + The headline table reports one point per model. This is the surface that point sits on, + which is what a threshold decision actually needs: a model with a calibrated score can + be moved along its curve for free, and a chat VLM cannot be moved at all. Both facts + are visible here and neither is visible in a table of F1. + + Pooling here is MICRO (concatenate every panorama, then integrate once), unlike the + macro-mean headline: a PR curve is an integral over ranked predictions and has no + natural macro form. Said on the figure so the two cannot be silently compared. + """ + from matplotlib.lines import Line2D + + curves = result.get("curves") or {} + by_name = {m["model"]: m for m in result["models"]} + + fig, ax = plt.subplots(figsize=(9.2, 7.6)) + fig.patch.set_facecolor(SURFACE) + _style(ax) + ax.grid(True, color=GRID, lw=0.8, zorder=0) + ax.set_axisbelow(True) + + # Scoreless models first, so the curves draw over them. + # + # MICRO precision/recall, not the headline table's macro-mean: these axes are the + # micro-pooled curve's, and a macro point plotted on them is a different aggregation + # under the same subtitle. The shift is small (max 0.010 in P, Qwen-32B) but "small" + # is not "labelled", and this figure's whole subject is what a point on a curve means. + for m in result["models"]: + if m["model"] in curves or not m["complete"]: + continue + pr = m.get("micro_precision"), m.get("micro_recall") + if pr[0] is None or pr[1] is None: + continue + precision, recall = pr + ax.plot(recall, precision, "o", ms=7, color=INK_MUTED, + mec=SURFACE, mew=1.6, zorder=4) + ax.annotate(m["display"], (recall, precision), + textcoords="offset points", xytext=(9, -3.5), fontsize=8, + color=INK_MUTED, zorder=5, path_effects=_halo()) + + handles = [] + for name, curve in sorted(curves.items(), + key=lambda kv: -(kv[1]["ap"] or 0)): + colour = CURVE_COLOR.get(name, INK_MUTED) + dash = CURVE_DASH.get(name, "solid") + ref = name == "rampnet" + xs, ys = _decimate(curve["recalls"], curve["precisions"]) + ax.plot(xs, ys, color=colour, lw=2.6 if ref else 1.7, ls=dash, + zorder=6 if ref else 5, solid_capstyle="round") + label = by_name.get(name, {}).get("display", name) + handles.append(Line2D([], [], color=colour, lw=2.6 if ref else 1.7, ls=dash, + label=f"{label} {curve['ap']:.3f}")) + + # The two thresholds the project has argued about, on RampNet's curve. + from scoreboard import DEPLOYED_THRESHOLD + + marks = (curves.get("rampnet") or {}).get("marks") or {} + for thr, mk in sorted(marks.items()): + deployed = abs(float(thr) - DEPLOYED_THRESHOLD) < 1e-9 + ax.plot(mk["recall"], mk["precision"], "o", ms=9, + mfc=CURVE_COLOR["rampnet"] if not deployed else SURFACE, + mec=CURVE_COLOR["rampnet"], mew=2.2, zorder=8) + ax.annotate(f"{float(thr):.2f}" + (" deployed" if deployed else " recommended (#54)"), + (mk["recall"], mk["precision"]), textcoords="offset points", + xytext=(-10, 11 if deployed else -16), ha="right", fontsize=8.2, + color=INK, zorder=9, path_effects=_halo()) + + ax.set_xlim(0, 1.02) + ax.set_ylim(0, 1.02) + ax.set_xlabel("recall", fontsize=9.5, color=INK_SECONDARY) + ax.set_ylabel("precision", fontsize=9.5, color=INK_SECONDARY) + _titles(ax, "A calibrated score is a dial; a chat VLM is a dot", + "Pooled over the seven US splits, micro — every panorama counts once, dots " + "included. Grey dots emit no confidence and cannot be moved.") + # Mid-left: the lower-left corner is where the two open-detector curves live, and a + # legend there sits on top of them. + # + # "micro" belongs in the legend TITLE, not repeated on six rows: the headline table's + # AP column is the macro-mean of the same data and lands a few thousandths away, so + # the family has to be named — but naming it six times widened the block until its + # last row ran into the Qwen-8B dot label. + leg = ax.legend(handles=handles, loc="lower left", frameon=False, fontsize=8.2, + labelcolor=INK_SECONDARY, handlelength=2.6, + title="AP, micro-pooled", alignment="left", + bbox_to_anchor=(0.01, 0.30)) + leg.get_title().set(color=INK_MUTED, fontsize=7.8) + leg.set_zorder(10) + fig.text(0.008, 0.030, + "RampNet's curve is read from analysis_out/op_cache — the #54 low-floor " + "re-extraction of the same run, no TTA.", + fontsize=7.4, color=INK_MUTED, ha="left", va="bottom") + fig.text(0.008, 0.008, + "Below 0.55 it is a LOWER bound: the ground truth was assembled from " + "detections at or above that floor, so a real ramp nobody marked scores as a " + "false positive.", + fontsize=7.4, color=INK_MUTED, ha="left", va="bottom") + fig.tight_layout(rect=(0, 0.052, 1, 0.96)) + fig.savefig(path, dpi=170, facecolor=fig.get_facecolor()) + plt.close(fig) + return path + + +FIGURES = { + "scoreboard_pr_curves.png": fig_pr_curves, + "scoreboard_f1.png": fig_headline, + "scoreboard_pr.png": fig_precision_recall, + "scoreboard_by_split.png": fig_by_split, + "scoreboard_generalization.png": fig_generalization, +} + + +def render_all(result, figure_dir): + """Write every figure; returns the paths written.""" + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + os.makedirs(figure_dir, exist_ok=True) + written = [] + for name, fn in FIGURES.items(): + written.append(fn(result, os.path.join(figure_dir, name), plt)) + return written diff --git a/scripts/analysis/scoreboard_render.py b/scripts/analysis/scoreboard_render.py new file mode 100644 index 00000000..197d4453 --- /dev/null +++ b/scripts/analysis/scoreboard_render.py @@ -0,0 +1,324 @@ +"""Markdown tables for docs/model_scoreboard.md, and the splice that keeps them current. + +Split out of ``scoreboard.py`` so the scoring is testable without a doc on disk and the +rendering is testable without re-scoring ten splits. + +The doc holds the prose; this holds the numbers. Each generated block sits between a +matched pair of HTML comments and is replaced wholesale on every run, so a number can +only ever enter the doc by coming out of the scorer. Prose written *outside* the markers +is never touched. ``scoreboard.py --check`` re-renders and compares, which is what makes a +stale summary a test failure rather than a thing someone notices six months later. +""" +import json +import os +import re + +BEGIN = "" +END = "" + +# Column header for each split, short enough that the by-split matrix stays readable. +SPLIT_HEADER = { + "richmond": "rich", + "bend": "bend", + "clovis": "clovis", + "morgantown": "morg", + "annapolis": "annap", + "paterson": "pater", + "gainesville": "gaines", + "budapest_district5": "budapest †", + "sao_paulo": "sao_paulo †", + "manual_gold": "manual_gold †", +} + + +def num(value, places=3, dash="–"): + return dash if value is None else f"{value:.{places}f}" + + +def bold(text, on=True): + return f"**{text}**" if on else text + + +def _table(header, rows, align=None): + align = align or (["---"] * len(header)) + out = ["| " + " | ".join(header) + " |", "|" + "|".join(align) + "|"] + out += ["| " + " | ".join(r) + " |" for r in rows] + return "\n".join(out) + + +def headline_table(result): + """Rows are models, columns are metrics, pooled over the seven US city splits. + + Only legs with all seven are here. A one-city mean in the same column as a + seven-city one is the exact confusion the coverage field exists to prevent, so the + partial legs get their own table instead of a footnote. + """ + from scoreboard import CLASS_LABEL, RAMPNET + + models = [m for m in result["models"] if m["complete"]] + best = {k: max((m[k] for m in models if m[k] is not None), default=None) + for k in ("precision", "recall", "f1", "ap")} + ref = next((m["f1"] for m in models if m["model"] == RAMPNET), None) + + rows = [] + for m in models: + lead = None if (ref is None or m["f1"] is None) else m["f1"] - ref + if m["model"] == RAMPNET: + delta = "—" + elif lead is None: + delta = "–" + else: + delta = f"{lead:+.3f}" + span = ("–" if m["f1_min"] is None + else f"{m['f1_min']:.2f}–{m['f1_max']:.2f}") + rows.append([ + bold(m["display"], m["model"] == RAMPNET), + CLASS_LABEL[m["class"]], + m["operating_point_note"], + bold(num(m["precision"]), m["precision"] == best["precision"]), + bold(num(m["recall"]), m["recall"] == best["recall"]), + bold(num(m["f1"]), m["f1"] == best["f1"]), + delta, + num(m["ap"]) + (" †" if m.get("ap_is_substituted") and m["ap"] else ""), + num(m["fp_per_pano"], 1), + span, + ]) + # Every metric here is the macro-mean over the seven pooled splits, AP included. The + # PR-curve figure's legend is the MICRO-pooled AP of the same data and reads a few + # thousandths different; the two are labelled wherever both appear so a reader never + # has to guess which family a number belongs to. + header = ["model", "class", "op", "P", "R", "F1", "ΔF1 vs RampNet", "AP (macro)", + "FP/pano", "F1 range"] + align = ["---", "---", "--:", "--:", "--:", "--:", "--:", "--:", "--:", ":-:"] + return _table(header, rows, align) + + +def partial_table(result): + """Legs that have not run every pooled split — reported per split, never averaged. + + Returns a one-line note instead of a table when there are none, so the block never + renders as an empty header that reads like a missing result. + """ + from scoreboard import CLASS_LABEL + + partial = [m for m in result["models"] if not m["complete"]] + if not partial: + return "*Every registered leg has run all seven pooled splits.*" + + per = result["per_split"] + rows = [] + for m in partial: + for split in result["all_splits"]: + cell = per[m["model"]].get(split) + if not cell: + continue + rows.append([ + m["display"], + CLASS_LABEL[m["class"]], + f"`{split}`", + num(cell["precision"]), + num(cell["recall"]), + num(cell["f1"]), + num(cell["ap"]), + num(cell["fp_per_pano"], 1), + f"{cell['tp']}/{cell['fp']}/{cell['fn']}", + ]) + header = ["model", "class", "split", "P", "R", "F1", "AP", "FP/pano", "tp/fp/fn"] + align = ["---", "---", "---", "--:", "--:", "--:", "--:", "--:", "--:"] + return _table(header, rows, align) + + +def by_split_table(result): + """F1 for every (model, split) pair — the matrix the per-split tables never form.""" + from scoreboard import RAMPNET, US_SPLITS + + splits = result["all_splits"] + per = result["per_split"] + # Best model per split, so each column's winner is visible without arithmetic. + best = {} + for s in splits: + vals = [cells[s]["f1"] for cells in per.values() if cells.get(s)] + best[s] = max(vals) if vals else None + + best_pooled = max((m["f1"] for m in result["models"] + if m["complete"] and m["f1"] is not None), default=None) + rows = [] + for m in result["models"]: + cells = per[m["model"]] + row = [bold(m["display"], m["model"] == RAMPNET)] + for s in splits: + if s in US_SPLITS: + cell = cells.get(s) + row.append("–" if not cell else bold(num(cell["f1"], 3), + cell["f1"] == best[s])) + if s == US_SPLITS[-1]: + # A partial row has a pooled mean, but it is a mean over a different set + # of cities, so printing it in this column would invite the comparison + # the partial table exists to prevent. + row.append("–" if not m["complete"] + else bold(num(m["f1"]), m["f1"] == best_pooled)) + for s in splits: + if s not in US_SPLITS: + cell = cells.get(s) + row.append("–" if not cell else bold(num(cell["f1"], 3), + cell["f1"] == best[s])) + rows.append(row) + + header = ["model"] + [SPLIT_HEADER[s] for s in US_SPLITS] + ["**pooled**"] + \ + [SPLIT_HEADER[s] for s in splits if s not in US_SPLITS] + align = ["---"] + ["--:"] * (len(header) - 1) + return _table(header, rows, align) + + +def threshold_table(result): + """RampNet at the two thresholds the project has argued about, pooled over the US splits. + + The headline table reports one operating point per model because most of the roster + has only one. RampNet does not, and the difference is the whole of #54, so it gets the + two rows rather than a sentence. + """ + marks = ((result.get("curves") or {}).get("rampnet") or {}).get("marks") or {} + if not marks: + return "*No low-floor cache available — run `operating_point_curve.py extract`.*" + note = {"0.55": "deployed today (`OPERATIONAL_CONFIDENCE`, auto-labeler)", + "0.30": "recommended by #54; **not yet adopted** (labeler#20 open)"} + rows = [] + for thr in sorted(marks, reverse=True): + m = marks[thr] + rows.append([f"**{thr}**", num(m["precision"]), num(m["recall"]), num(m["f1"]), + note.get(thr, "")]) + return _table(["peak threshold", "P", "R", "F1", ""], rows, + ["---", "--:", "--:", "--:", "---"]) + + +def ap_provenance_table(result): + """Where RampNet's AP on each split comes from, and what the log prints for it. + + This is the one column where the two documents disagree by design, so the mapping is + generated rather than described: ``model_comparison.md`` prints the bundle AP, this + page prints the low-floor one, and both are here side by side with the reason. The + test asserts the middle column against the log, so the correspondence is a gate. + """ + from scoreboard import RAMPNET + + cells = result["per_split"].get(RAMPNET) or {} + rows = [] + for split in result["all_splits"]: + cell = cells.get(split) + if not cell: + continue + substituted = cell["ap_source"] != "bundle" + rows.append([ + f"`{split}`", + num(cell["ap_bundle"]), + bold(num(cell["ap"]), substituted), + "`op_cache` (0.05 floor)" if substituted else "bundle — already at 0.05", + "truncated at the deployed 0.55" if substituted + else "no truncation to undo; flip-TTA export", + ]) + return _table(["split", "AP in `model_comparison.md`", "AP here", "read from", "why"], + rows, ["---", "--:", "--:", "---", "---"]) + + +def coverage_note(result): + """What each split is, how big it is, and why a held-out one is held out.""" + rows = [] + for split, info in result["splits"].items(): + why = result["held_out"].get(split) + rows.append([ + f"`{split}`", + "pooled" if info["pooled"] else "held out †", + str(info["n_panos"]), + str(info["n_gt"]), + why or "US deployment city, verdict-grade GT", + ]) + return _table(["split", "role", "panos", "GT ramps", "note"], rows, + ["---", "---", "--:", "--:", "---"]) + + +def render_tables(result): + """{block name: markdown} for every generated block in the doc.""" + return { + "headline": headline_table(result), + "thresholds": threshold_table(result), + "partial": partial_table(result), + "by-split": by_split_table(result), + "ap-provenance": ap_provenance_table(result), + "coverage": coverage_note(result), + } + + +def splice(text, tables): + """Replace each generated block in ``text``; leave everything else byte-identical. + + A block present in ``tables`` but absent from the doc is a silent no-op by design: + the doc decides which tables it wants and where, the script only decides what they + say. + """ + for name, body in tables.items(): + pattern = re.compile( + re.escape(BEGIN.format(name=name)) + r".*?" + re.escape(END.format(name=name)), + re.S) + replacement = (BEGIN.format(name=name) + "\n\n" + body + "\n\n" + + END.format(name=name)) + text = pattern.sub(lambda _m: replacement, text) + return text + + +# Decimals kept for every float in the committed JSON. Six is ~3,000x finer than +# anything the page reports (three decimals) and still coarse enough to be identical on +# every platform this runs on -- which is the whole point, see _round_floats. +JSON_PRECISION = 6 + + +def _round_floats(value, places=JSON_PRECISION): + """Round every float in a nested structure, so the artifact is byte-reproducible. + + Full-precision floats do NOT survive the trip between environments: AP comes out of + numpy, and a different numpy build reorders the last bits of an accumulation, which + changes ``repr`` and therefore the file. That made a byte-compare of this file fail + on CI's Python 3.10 while passing on 3.12 and on the author's machine -- the artifact + was not reproducible, and the check that was supposed to prove it was reproducible + was the thing that noticed. + + Rounding fixes the artifact rather than weakening the check: seventeen significant + digits of accumulation noise were never meaningful in a file whose purpose is to be + diffed by a reviewer, and at six decimals a real change is still visible thousands of + times before the page's three decimals would move. + """ + if isinstance(value, float): + return round(value, places) + if isinstance(value, dict): + return {k: _round_floats(v, places) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_round_floats(v, places) for v in value] + return value + + +def json_payload(result): + """The committed JSON, as a string — the result minus the plot-only curve arrays. + + A PR curve is one point per ranked prediction, and the two open detectors carry + ~120k between them: serialized they are 7.7 MB, 98% of the file, for something no + reader diffs and ``scoreboard.py`` rebuilds from the same committed detections in + about three seconds. What the page actually cites — the AP, RampNet's marked + thresholds, and how many points the curve had — is kept. + + Floats are rounded (``_round_floats``) so the file is identical on every platform, + and written LF-only for the same reason: this is byte-compared by ``--check``, and + Python's default newline translation on Windows would emit CRLF and make a re-run + look like a change (the imagery_manifest fix, 22dd536). + """ + slim = dict(result) + slim["curves"] = { + name: {k: v for k, v in curve.items() if k not in ("recalls", "precisions")} + for name, curve in (result.get("curves") or {}).items() + } + return json.dumps(_round_floats(slim), indent=2, sort_keys=False) + "\n" + + +def write_json(path, result): + """Write the machine-readable scoreboard (see ``json_payload``).""" + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8", newline="") as fh: + fh.write(json_payload(result)) diff --git a/scripts/model_comparison/compare.py b/scripts/model_comparison/compare.py index 02152571..1430ed11 100644 --- a/scripts/model_comparison/compare.py +++ b/scripts/model_comparison/compare.py @@ -399,7 +399,10 @@ def report_usage(detector, label, city, panos_scored, usage_log_path): } try: os.makedirs(os.path.dirname(usage_log_path) or ".", exist_ok=True) - with open(usage_log_path, "a", encoding="utf-8") as f: + # newline="" so a Windows run appends LF, not CRLF. This ledger is + # append-only and byte-compared in review; a CRLF line silently breaks that + # (the same defect imagery_manifest.py was fixed for). + with open(usage_log_path, "a", encoding="utf-8", newline="") as f: f.write(json.dumps(rec) + "\n") except OSError as e: # Print the record so the numbers survive in the run log even when the diff --git a/tests/test_model_comparison.py b/tests/test_model_comparison.py index c80796a1..795a80c5 100644 --- a/tests/test_model_comparison.py +++ b/tests/test_model_comparison.py @@ -1512,6 +1512,29 @@ class _Local: assert len(log.read_text().splitlines()) == 2 +def test_report_usage_writes_lf_on_every_platform(tmp_path): + """The spend ledger is append-only and byte-compared in review, so a CRLF line is a + real defect — and `read_text().splitlines()` above cannot see one, because it strips + \\r\\n and \\n alike. Asserted on the bytes instead. + + The writer was appending CRLF on Windows and git's autocrlf was normalising it away + on commit, so the blob looked right while every working copy was wrong. Same defect + imagery_manifest.py was fixed for in 22dd536. + """ + class _Det: + name = "gemini" + model_id = "gemini-2.5-flash" + usage = {"calls": 1, "input_tokens": 10, "output_tokens": 10, + "thoughts_tokens": 0} + + log = tmp_path / "usage_log.jsonl" + report_usage(_Det(), "gemini-2.5-flash", "richmond", 1, str(log)) + report_usage(_Det(), "gemini-2.5-flash", "bend", 1, str(log)) + raw = log.read_bytes() + assert b"\r\n" not in raw, "usage ledger written with CRLF" + assert raw.count(b"\n") == 2 and raw.endswith(b"\n") + + def test_usage_record_carries_the_rig_that_priced_it(tmp_path): # Two runs of the same model on the same bundle at different tiling cost ~6x # different input, and without the signature the log cannot tell them apart. diff --git a/tests/test_scoreboard.py b/tests/test_scoreboard.py new file mode 100644 index 00000000..c2e702cb --- /dev/null +++ b/tests/test_scoreboard.py @@ -0,0 +1,644 @@ +"""Guards on the aggregated scoreboard (docs/model_scoreboard.md). + +A summary table is a special kind of liability: it is the page people quote, and it is +the page furthest from the data that produced it. So the assertions here are about the +ways a summary goes wrong rather than about arithmetic — + +- it silently disagrees with the detailed log it summarizes (spot-checked against the + numbers committed in docs/model_comparison.md), +- it pools a split the log says must not be pooled, +- it grows a private copy of the split registry that drifts from everyone else's, +- it drops a model that was actually run, or reports a partial run as a complete one, +- it goes stale after a re-export and nothing fails. + +Pure: reads only committed bundles, committed published detections, and manual_labels. +No cache, no GPU, no credentials, no network. +""" +import json +import os +import re +import subprocess +import sys + +import pytest + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO) +sys.path.insert(0, os.path.join(REPO, "scripts", "analysis")) +sys.path.insert(0, os.path.join(REPO, "scripts", "model_comparison")) + +import low_floor_sweep as lfs # noqa: E402 +import scoreboard as sb # noqa: E402 +import scoreboard_render as sr # noqa: E402 +from rampnet import roster # noqa: E402 + + +@pytest.fixture(scope="module") +def board(): + """Score every (model, split) pair once; ~3 s, shared by the whole module.""" + return sb.build() + + +def _cell(board, model, split): + return board["per_split"][model][split] + + +def _summary(board, model): + return next(m for m in board["models"] if m["model"] == model) + + +# --------------------------------------------------------------------------- # +# the registry — one copy, not three +# --------------------------------------------------------------------------- # +def test_split_registry_is_imported_not_copied(): + """A private split list here would drift from low_floor_sweep's without failing. + + test_registries_agree_with_low_floor_sweep already ties miss_decomposition to that + registry; this extends the same contract to the scoreboard, which is the module most + likely to be tempted into 'just the nine cities'. + """ + assert sb.US_SPLITS is lfs.US_SPLITS + assert sb.ALL_SPLITS is lfs.ALL_SPLITS + assert sb.HELD_OUT is lfs.HELD_OUT + + +def test_budapest_is_never_pooled(board): + """docs/model_comparison.md: budapest's numbers 'must not be pooled with the US + splits or averaged into a headline'. The headline row must obey that literally.""" + for model in board["models"]: + assert "budapest_district5" not in model["pooled_splits"] + assert "sao_paulo" not in model["pooled_splits"] + assert "manual_gold" not in model["pooled_splits"] + assert board["pooled_splits"] == list(lfs.US_SPLITS) + + +def test_every_held_out_split_states_why(board): + """Same contract HELD_OUT carries everywhere else: an omission must be explained, + because an unexplained one is indistinguishable from a withheld result.""" + for split in set(board["all_splits"]) - set(board["pooled_splits"]): + assert board["held_out"].get(split), f"{split} held out with no documented reason" + + +# --------------------------------------------------------------------------- # +# agreement with the detailed log +# --------------------------------------------------------------------------- # +# (model, split) -> (P, R, F1) as committed in docs/model_comparison.md. +# +# This is a hand-copy, and test_every_number_matches_model_comparison below reads the log +# itself — so this dict is deliberately NOT the contract, it is a regression pin. Its job +# is to name a handful of numbers explicitly, in a form that is readable in a diff and +# that does not depend on the parser. When the #132 seam wrap moved the challengers, this +# dict went stale exactly as a hand-copy does; the full check caught the same thing across +# all 88 rows. Keep it small, and update it from the log, never from the scorer. +PUBLISHED = { + ("rampnet", "richmond"): (0.964, 0.768, 0.855), + ("rampnet", "annapolis"): (0.973, 0.738, 0.839), + ("rampnet", "budapest_district5"): (0.874, 0.510, 0.644), + ("gemini-3.1-pro-preview", "paterson"): (0.852, 0.567, 0.681), + ("Qwen/Qwen3-VL-32B-Instruct", "budapest_district5"): (0.433, 0.043, 0.079), + # These two moved with the #132 seam wrap (0.462/0.457/0.460 and 0.025/0.908/0.049 + # before it): one detection each sat across the 360 seam from its ground truth. + ("allenai/Molmo2-8B", "morgantown"): (0.466, 0.461, 0.463), + ("google/owlv2-large-patch14-ensemble", "clovis"): (0.025, 0.913, 0.049), + # The two arms that arrived with #126 and #122, at the same no-floor operating + # point their own write-ups report. + ("mask2former-vistas-curb-cut", "richmond"): (0.411, 0.697, 0.517), + ("mask2former-vistas-curb-cut+curb", "richmond"): (0.126, 0.648, 0.210), +} + + +@pytest.mark.parametrize("key,expected", sorted(PUBLISHED.items(), key=lambda kv: str(kv[0]))) +def test_matches_the_numbers_committed_in_model_comparison(board, key, expected): + model, split = key + cell = _cell(board, model, split) + got = (cell["precision"], cell["recall"], cell["f1"]) + for name, want, have in zip(("P", "R", "F1"), expected, got): + assert have == pytest.approx(want, abs=0.0006), \ + f"{model} on {split}: {name} {have:.4f} vs published {want}" + + +def test_rampnet_manual_gold_row_is_the_published_gate(board): + """manual_gold is exported at the 0.05 floor, so RampNet's headline row only lands on + the published gold-set numbers if the deployed 0.55 threshold is actually applied. + Drop the operating point and this reads 0.723/0.935 instead.""" + cell = _cell(board, "rampnet", "manual_gold") + assert cell["precision"] == pytest.approx(0.947, abs=0.0006) + assert cell["recall"] == pytest.approx(0.873, abs=0.0006) + assert cell["f1"] == pytest.approx(0.908, abs=0.0006) + + +def test_yolo_rows_are_at_the_preregistered_threshold(board): + """The #71 protocol fixes the YOLO headline at conf 0.25 before any benchmark + contact. At the 0.05 export floor these same rows read 0.660 and 0.786.""" + assert _cell(board, "y11l_pano", "richmond")["f1"] == pytest.approx(0.595, abs=0.0006) + assert _cell(board, "y11x_pano_h200", "manual_gold")["f1"] == pytest.approx(0.851, abs=0.0006) + + +def test_ap_is_read_full_range_not_truncated_at_the_operating_point(board): + """AP integrates the whole sweep, so it must come from the unthresholded run. + + RampNet's manual_gold bundle is exported at 0.05, so its AP is already untruncated + (0.917); if the operating point leaked into the AP column it would fall to the + 0.55-and-above slice. + """ + assert _cell(board, "rampnet", "manual_gold")["ap"] == pytest.approx(0.917, abs=0.0006) + + +def test_rampnet_city_ap_comes_from_the_low_floor_cache(board): + """The city bundles stop at 0.55, so an AP computed from them is a truncated curve. + + Read that way richmond is 0.763 and the pooled figure is 0.720 — which puts RampNet + BELOW the YOLO arms on AP, an artifact of the floor rather than a result. The #54 + re-extraction carries the same run down to 0.05, which is where every other scored + model is exported. + """ + cell = _cell(board, "rampnet", "richmond") + assert cell["ap_source"] == "op_cache (0.05 floor)" + assert cell["ap"] == pytest.approx(0.876, abs=0.002) + assert cell["ap_bundle"] == pytest.approx(0.763, abs=0.0006) + # ...and the pooled truncated figure is the one that inverts the ordering. + assert _summary(board, "rampnet")["ap_bundle"] == pytest.approx(0.720, abs=0.0006) + + +def test_the_substitution_is_scoped_to_actual_truncation(board): + """manual_gold's bundle is already at 0.05, so it must keep its own AP. + + Swapping the cache in there would trade a flip-TTA export for a no-TTA one (0.917 -> + 0.904) — a different change from un-truncating a curve, and it would leave one row's + AP and its P/R/F1 describing two different inference configurations. + """ + assert _cell(board, "rampnet", "manual_gold")["ap_source"] == "bundle" + + +def test_ap_ordering_is_not_an_artifact_of_the_floor(board): + """With both read at a 0.05 floor, RampNet's AP leads the supervised arms clearly.""" + rampnet = _summary(board, "rampnet")["ap"] + best_yolo = max(_summary(board, m)["ap"] + for m in ("y11l_pano", "y11x_pano_h200", "y26_pano")) + assert rampnet > best_yolo + 0.1, f"RampNet AP {rampnet:.3f} vs best YOLO {best_yolo:.3f}" + + +def test_threshold_marks_reproduce_the_published_operating_point_table(board): + """The PR figure's marked points must agree with docs/operating_point.md. + + That document's pooled row is P 0.964 / R 0.722 / F1 0.826 at the deployed 0.55 and + 0.900 raw precision at 0.30. Computed here from the same committed cache by a + different code path, so a drift in either is a real disagreement. + """ + marks = board["curves"]["rampnet"]["marks"] + assert marks["0.55"]["precision"] == pytest.approx(0.964, abs=0.0006) + assert marks["0.55"]["recall"] == pytest.approx(0.722, abs=0.0006) + assert marks["0.55"]["f1"] == pytest.approx(0.826, abs=0.0006) + assert marks["0.30"]["precision"] == pytest.approx(0.900, abs=0.0006) + + +def test_only_score_carrying_models_get_a_curve(board): + """A chat VLM has one operating point, not a curve — it must not get a fake one.""" + curves = board["curves"] + assert "rampnet" in curves and "google/owlv2-large-patch14-ensemble" in curves + for scoreless in ("gemini-3.1-pro-preview", "Qwen/Qwen3-VL-32B-Instruct", + "allenai/Molmo2-8B"): + assert scoreless not in curves + # ...and a leg that has not run every pooled split cannot be pooled into one. + assert "mask2former-vistas-curb-cut" not in curves + + +# --------------------------------------------------------------------------- # +# the whole log, not a spot-check +# --------------------------------------------------------------------------- # +# model_comparison.md's row labels -> roster published names. Both documents score the +# same committed detections, so a disagreement is a bug in one of them, never a choice. +_LOG_ROW_NAMES = { + "rampnet": "rampnet", + "rampnet @0.55": "rampnet", + "gemini-3.1-pro-preview": "gemini-3.1-pro-preview", + "gemini-3.6-flash": "gemini-3.6-flash", + "gemini-3.7-flash": "gemini-3.7-flash", + "molmo2-8B (points)": "allenai/Molmo2-8B", + "Qwen3-VL-32B-Instruct": "Qwen/Qwen3-VL-32B-Instruct", + "Qwen3-VL-8B-Instruct": "Qwen/Qwen3-VL-8B-Instruct", + "owlv2-large-patch14-ensemble": "google/owlv2-large-patch14-ensemble", + "grounding-dino-base": "IDEA-Research/grounding-dino-base", + "mask2former-vistas-curb-cut": "mask2former-vistas-curb-cut", + "mask2former-vistas-curb-cut+curb": "mask2former-vistas-curb-cut+curb", +} +# (model, split) pairs the log prints and this page deliberately does not carry, each with +# the reason. Anything else missing is a failure, not an exemption. +_LOG_ROWS_NOT_ON_THE_BOARD = { + ("gemini-3.1-pro-preview", "manual_gold"): + "manual_gold detections never published (docs/model_scoreboard.md, 'What is missing')", + ("gemini-3.6-flash", "manual_gold"): + "manual_gold detections never published (docs/model_scoreboard.md, 'What is missing')", +} +_LOG_TABLE_HEADER = "| model | P | R | F1 | AP | tp/fp/fn |" +_LOG_SPLIT_HEADING = re.compile( + r"^(?:\*\*(\w+)\*\*|#+ Result: (\w+)) \(\d[\d,]* (?:reviewed )?panos") + + +def _parse_model_comparison(): + """Every (split, model, P, R, F1, AP) row in docs/model_comparison.md's result tables. + + Reads the log rather than a transcription of it, so this test cannot pass by agreeing + with a stale copy of the numbers it is supposed to be checking. + """ + path = os.path.join(REPO, "docs", "model_comparison.md") + with open(path, encoding="utf-8") as fh: + lines = fh.read().split("\n") + + def value(cell): + cell = cell.replace("**", "").replace("*", "").strip() + return None if cell in ("–", "-", "—", "") else float(cell) + + rows, split, in_table = [], None, False + for lineno, line in enumerate(lines, 1): + heading = _LOG_SPLIT_HEADING.match(line.strip()) + if heading: + split = heading.group(1) or heading.group(2) + continue + if line.strip() == _LOG_TABLE_HEADER: + in_table = True + continue + if not in_table: + continue + cells = line.split("|")[1:-1] + if not line.startswith("|") or len(cells) != 6: + in_table = False + continue + label = cells[0].replace("**", "").replace("*", "").strip() + if set(label) <= set("-: "): + continue + assert label in _LOG_ROW_NAMES, ( + f"docs/model_comparison.md:{lineno}: unrecognized model row {label!r}. Add it " + f"to _LOG_ROW_NAMES so this row is checked rather than skipped.") + rows.append((split, _LOG_ROW_NAMES[label], value(cells[1]), value(cells[2]), + value(cells[3]), value(cells[4]), lineno)) + return rows + + +def test_the_log_parser_actually_finds_the_tables(): + """A parser that silently matches nothing would make the check below vacuous. + + docs/model_comparison.md gets restructured often; if a heading or column layout + changes, this fails loudly instead of the real test passing on an empty list. + """ + rows = _parse_model_comparison() + assert len(rows) >= 85, f"only parsed {len(rows)} rows out of model_comparison.md" + assert {s for s, *_ in rows} == set(lfs.ALL_SPLITS) + + +def test_every_number_matches_model_comparison(board): + """Every P/R/F1/AP in the log's per-split tables, against this page's scorer. + + The spot-check above pins nine hand-picked cells. This pins all of them, in both + directions: a number edited in either document without re-running fails here. + + RampNet's AP is the one deliberate difference — the log prints the bundle AP, which is + truncated at the deployed 0.55, and this page substitutes the low-floor cache. That is + checked too, against ``ap_bundle``, so the exception cannot quietly widen into + "RampNet's AP does not have to agree with anything". + """ + mismatches = [] + for split, model, P, R, F1, AP, lineno in _parse_model_comparison(): + cell = board["per_split"].get(model, {}).get(split) + if cell is None: + if (model, split) in _LOG_ROWS_NOT_ON_THE_BOARD: + continue + mismatches.append(f"model_comparison.md:{lineno} {model}/{split}: in the log, " + "absent from the scoreboard") + continue + for metric, want, have in (("P", P, cell["precision"]), ("R", R, cell["recall"]), + ("F1", F1, cell["f1"])): + if want is not None and abs(want - have) > 0.0006: + mismatches.append(f"model_comparison.md:{lineno} {model}/{split} {metric}: " + f"log {want} vs scoreboard {have:.4f}") + if AP is not None: + # The log always prints the bundle AP. So must ap_bundle -- including on the + # rows where the page then substitutes the low-floor cache. + if cell["ap_bundle"] is None or abs(AP - cell["ap_bundle"]) > 0.0011: + mismatches.append(f"model_comparison.md:{lineno} {model}/{split} AP: " + f"log {AP} vs bundle {cell['ap_bundle']}") + assert not mismatches, "\n".join(mismatches) + + +def test_only_rampnets_ap_is_allowed_to_differ_from_the_log(board): + """Scope the exception: every other model's AP must be the bundle AP unchanged. + + Without this, a future substitution applied to some other arm would sail through the + test above, because that test compares the log to ``ap_bundle`` rather than to what + the page actually prints. + """ + for model, cells in board["per_split"].items(): + for split, cell in cells.items(): + if model == "rampnet": + continue + assert cell["ap_source"] == "bundle", \ + f"{model}/{split} AP was substituted; only RampNet's may be" + assert cell["ap"] == cell["ap_bundle"] + + +def test_the_ap_provenance_table_shows_the_logs_number(board): + """The page's reconciliation table has to quote the log, not a rounded memory of it.""" + table = sr.ap_provenance_table(board) + assert "AP in `model_comparison.md`" in table + for split, log_ap in (("richmond", "0.763"), ("paterson", "0.681"), + ("manual_gold", "0.917")): + row = next(l for l in table.splitlines() if l.startswith(f"| `{split}` |")) + assert log_ap in row, f"{split} row does not carry the log's AP {log_ap}: {row}" + # manual_gold is the row that must NOT be substituted, and must say so. + gold = next(l for l in table.splitlines() if l.startswith("| `manual_gold` |")) + assert "bundle" in gold and "0.05" in gold + + +# --------------------------------------------------------------------------- # +# coverage — a run that happened must appear; one that didn't must not be invented +# --------------------------------------------------------------------------- # +def test_scores_every_registered_leg(board): + """The roster is the source of truth for who has been run, so every entry is scored. + + Hardcoding a roster here is how gemini-3.7-flash sat published-but-unscored for a + while. Reading rampnet.roster instead means a new leg reaches this page by being + registered, which is the same act that publishes its detections. + """ + scored = {m["model"] for m in board["models"]} + registered = {roster.published_name(c) for c in roster.ROSTER} + assert scored == registered, f"registered but not scored: {registered - scored}" + + +def test_no_published_file_is_left_unscored(board): + """A detections file no roster entry claims would be silently invisible here. + + test_roster.py asserts the same thing from the registry side; this is the scoreboard + refusing to present a partial view as a complete one if the two ever drift. + """ + assert board["unregistered_exports"] == [] + + +def test_a_pinned_leg_loads_its_own_detections(board): + """claude-opus-5 at two efforts is two legs sharing one label. + + They publish under different stems and must not resolve to the same file — loading by + label alone would give both rows whichever file won, and the two would read as + identical results rather than as the effort comparison they are. + """ + low = _cell(board, "claude-opus-5-effort-low", "annapolis") + high = _cell(board, "claude-opus-5-effort-high", "annapolis") + assert low["f1"] != high["f1"] + assert low["f1"] == pytest.approx(0.588, abs=0.0006) + + +def test_partial_coverage_is_reported_not_averaged_away(board): + """The two Gemini legs have city detections but no published manual_gold. + + The failure this guards is the quiet one: an aggregate over four cities printed in + the same column as one over seven. Coverage travels with the row instead. + """ + for model in ("gemini-3.1-pro-preview", "gemini-3.6-flash"): + summary = _summary(board, model) + assert summary["coverage"] == "7/7" + assert summary["complete"] is True + assert summary["manual_gold_f1"] is None + assert "manual_gold" not in board["per_split"][model] + + +def test_single_split_legs_stay_out_of_the_pooled_tables(board): + """Vistas ran richmond only; the Claude legs ran annapolis only. + + A one-city macro-mean in the pooled column would be read as a seven-city one. It is + computed (the number is real, for that one city) but must not reach the headline + table or the pooled column of the matrix. + """ + single = [m for m in board["models"] if not m["complete"]] + assert {m["model"] for m in single} == { + "mask2former-vistas-curb-cut", "mask2former-vistas-curb-cut+curb", + "claude-opus-5-effort-low", "claude-opus-5-effort-high", + "claude-sonnet-5-effort-low", "claude-sonnet-5-effort-high", + } + for m in single: + assert m["coverage"] == "1/7" + + headline = sr.headline_table(board) + matrix = sr.by_split_table(board) + for m in single: + assert m["display"] not in headline, f"{m['display']} is in the pooled headline" + assert m["display"] in matrix, f"{m['display']} vanished from the matrix" + # ...and its POOLED cell is blank rather than a one-city mean. Checked by column + # position, not by substring: a one-split leg's pooled mean equals its single + # city cell, so searching the row for that value matches the legitimate one. + row = next(l for l in matrix.splitlines() if l.startswith(f"| {m['display']} |")) + cells = [c.strip() for c in row.split("|")] + pooled = cells[2 + len(lfs.US_SPLITS)] + assert pooled == "–", f"{m['display']} pooled cell is {pooled!r}, expected a dash" + + +def test_partial_table_names_the_split_every_number_came_from(board): + table = sr.partial_table(board) + assert "`richmond`" in table and "`annapolis`" in table + assert "Claude Opus 5 (low)" in table + assert "Mask2Former Vistas (curb cut)" in table + + +def test_a_leg_from_an_unmapped_provider_is_classified_not_dropped(): + """A roster entry whose provider predates this module still has to reach the board.""" + leg = roster.Challenger(spec="newthing:x", label="x", provider="newthing", + density=None, standing=False, added="2026-01-01", note="") + assert sb.class_of(leg) == "unclassified" + assert sb.display_of(leg) == "x" + + +def test_every_roster_provider_has_a_class_and_an_operating_point(): + """A provider missing from PROVIDER_CLASS falls to 'unclassified' silently, which is + the safe failure but the wrong one to ship — catch it here instead.""" + providers = {c.provider for c in roster.ROSTER} + missing = providers - set(sb.PROVIDER_CLASS) + assert not missing, f"roster providers with no model class: {sorted(missing)}" + for klass in sb.CLASS_ORDER: + assert klass in sb.OPERATING_POINT + assert klass in sb.OPERATING_POINT_NOTE + assert klass in sb.CLASS_LABEL + for klass in sb.PROVIDER_CLASS.values(): + assert klass in sb.CLASS_ORDER + + +# --------------------------------------------------------------------------- # +# the splice — prose survives, numbers are replaced +# --------------------------------------------------------------------------- # +def test_splice_replaces_only_the_generated_block(): + doc = ("prose above\n\n" + + sr.BEGIN.format(name="headline") + "\nstale\n" + sr.END.format(name="headline") + + "\n\nprose below\n") + out = sr.splice(doc, {"headline": "| fresh |"}) + assert "prose above" in out and "prose below" in out + assert "stale" not in out and "| fresh |" in out + assert sr.splice(out, {"headline": "| fresh |"}) == out # idempotent + + +def test_splice_ignores_a_block_the_doc_does_not_want(): + doc = "just prose\n" + assert sr.splice(doc, {"headline": "| x |"}) == doc + + +def test_committed_doc_is_current(board): + """The whole point of generating the tables: staleness is a test failure. + + Re-run `python scripts/analysis/scoreboard.py` if this fails. + """ + with open(sb.DEFAULT_DOC, encoding="utf-8", newline="") as fh: + current = fh.read() + assert sr.splice(current, sr.render_tables(board)) == current, \ + "docs/model_scoreboard.md is stale — re-run scripts/analysis/scoreboard.py" + + +# --------------------------------------------------------------------------- # +# the committed JSON — the artifact nothing used to check +# --------------------------------------------------------------------------- # +def test_committed_json_is_current(board): + """analysis_out/scoreboard.json is committed, so it can go stale like the doc can. + + Compared as bytes, which also pins the LF endings: a value-level compare passes + happily on a Windows-written CRLF file, which is the trap json_artifacts keep hitting + (imagery_manifest, the usage ledger). + """ + with open(sb.DEFAULT_JSON, "rb") as fh: + on_disk = fh.read() + assert on_disk == sr.json_payload(board).encode("utf-8"), \ + "analysis_out/scoreboard.json is stale — re-run scripts/analysis/scoreboard.py" + + +def test_the_committed_json_has_no_crlf(): + """Stated separately from the byte-compare so a failure says which defect it is.""" + with open(sb.DEFAULT_JSON, "rb") as fh: + assert b"\r\n" not in fh.read() + + +def test_the_json_is_reproducible_across_environments(): + """No float in the committed JSON may carry more digits than the writer rounds to. + + A byte-compare of full-precision floats is not a portable check: AP comes out of + numpy, and a different numpy build shifts the last bits, so the file differed between + CI's 3.10 and 3.12 while every value was the same. Rounding at the writer is what + makes the artifact reproducible; this asserts the file on disk actually went through + it, because a hand-edit or an older writer would not show up in any value comparison. + """ + def deepest(value): + if isinstance(value, float): + text = repr(value) + return len(text.split(".")[1]) if "." in text and "e" not in text else 0 + if isinstance(value, dict): + return max((deepest(v) for v in value.values()), default=0) + if isinstance(value, list): + return max((deepest(v) for v in value), default=0) + return 0 + + with open(sb.DEFAULT_JSON, encoding="utf-8") as fh: + payload = json.load(fh) + assert deepest(payload) <= sr.JSON_PRECISION, \ + "a full-precision float reached the committed JSON; it will not survive a " \ + "different numpy build" + + +def test_write_json_pins_lf_on_the_writer_not_on_git(tmp_path, board): + """Round-trip through the actual writer, checked on bytes. + + Reading the file back and comparing values would pass on a CRLF write — that is how + this class of defect keeps surviving (imagery_manifest, the usage ledger). The + committed artifact has to be right regardless of the contributor's autocrlf setting. + """ + out = tmp_path / "nested" / "scoreboard.json" + sr.write_json(str(out), board) + raw = out.read_bytes() + assert b"\r\n" not in raw + assert raw.endswith(b"\n") + assert json.loads(raw)["pooled_splits"] == list(lfs.US_SPLITS) + + +def test_the_json_does_not_carry_the_curve_point_arrays(board): + """The plot-only arrays are ~120k points; serialized they were 7.7 MB of committed + artifact, 98% of the file, that no reader diffs and the figures rebuild in seconds. + + What the page cites — AP, RampNet's marked thresholds, the point count — stays. + """ + payload = json.loads(sr.json_payload(board)) + assert payload["curves"], "the curves block itself must survive" + for name, curve in payload["curves"].items(): + assert "recalls" not in curve and "precisions" not in curve, \ + f"{name}: point arrays leaked back into the committed JSON" + assert curve["ap"] is not None and curve["n_points"] > 0 + assert payload["curves"]["rampnet"]["marks"]["0.55"]["f1"] == pytest.approx(0.826, + abs=0.0006) + # In-memory, the figures still get the full curves. + assert len(board["curves"]["rampnet"]["recalls"]) == \ + payload["curves"]["rampnet"]["n_points"] + assert len(sr.json_payload(board)) < 200_000, "committed JSON is bloated again" + + +# --------------------------------------------------------------------------- # +# the CLI — --check has to cover what it claims, --models must not clobber +# --------------------------------------------------------------------------- # +def _run_scoreboard(*args): + return subprocess.run( + [sys.executable, os.path.join(REPO, "scripts", "analysis", "scoreboard.py"), *args], + capture_output=True, text=True, cwd=REPO) + + +def test_check_fails_on_a_stale_json(tmp_path): + """--check used to verify only the doc while its help said 'doc and JSON'. + + A falsified headline F1 in the committed JSON passed with exit 0, which made the one + artifact the .gitignore re-include exists for the one artifact nothing validated. + """ + payload = json.loads(open(sb.DEFAULT_JSON, encoding="utf-8").read()) + payload["models"][0]["f1"] = 0.111 + stale = tmp_path / "scoreboard.json" + stale.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8", newline="") + done = _run_scoreboard("--check", "--json-out", str(stale)) + assert done.returncode != 0, "a falsified JSON passed --check" + assert "stale" in done.stdout + + +def test_a_models_subset_does_not_touch_the_committed_page(tmp_path): + """`--models y11x_pano_h200` used to splice a ONE-ROW headline into the committed doc. + + That left YOLO11x bolded as the winner in every column, directly above prose reading + "RampNet wins by 0.221 F1" — and then --check reported the real page as stale, so the + signal inverted. It also crashed in the figures afterwards, half-written. + """ + doc_before = open(sb.DEFAULT_DOC, "rb").read() + json_before = open(sb.DEFAULT_JSON, "rb").read() + done = _run_scoreboard("--models", "y11x_pano_h200", "--no-figures") + assert done.returncode == 0, done.stdout + done.stderr + assert "left alone" in done.stdout + assert open(sb.DEFAULT_DOC, "rb").read() == doc_before + assert open(sb.DEFAULT_JSON, "rb").read() == json_before + # ...but naming a destination explicitly still writes a partial board there. + out = tmp_path / "subset.json" + done = _run_scoreboard("--models", "y11x_pano_h200", "--no-figures", + "--json-out", str(out)) + assert done.returncode == 0, done.stdout + done.stderr + assert [m["model"] for m in json.loads(out.read_text())["models"]] == ["y11x_pano_h200"] + + +def test_check_refuses_a_subset(): + """--check on a subset would compare a partial board against the full page and + report the page as stale. That is a false alarm, so it is refused instead.""" + done = _run_scoreboard("--check", "--models", "rampnet") + assert done.returncode != 0 + assert "drop --models" in done.stdout + + +def test_a_figure_helper_survives_a_board_without_rampnet(board, tmp_path): + """The subset guard protects the committed files; these two protect the process. + + Both used to raise StopIteration on any board with no RampNet row, after main() had + already rewritten the doc and the JSON. + """ + from scoreboard_figures import fig_by_split, fig_headline + plt = pytest.importorskip("matplotlib.pyplot") + import matplotlib + matplotlib.use("Agg") + + trimmed = dict(board) + trimmed["models"] = [m for m in board["models"] if m["model"] != "rampnet"] + trimmed["per_split"] = {k: v for k, v in board["per_split"].items() if k != "rampnet"} + out = str(tmp_path / "fig.png") + fig_headline(trimmed, out, plt) + fig_by_split(trimmed, out, plt)