From c5ce85860392da924208d914200aa38d50102ee0 Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Fri, 4 Sep 2026 16:41:42 +0000 Subject: [PATCH 1/2] Backfill total_s from Cam2V's model_step_wall_s in runtime metric samples Cam2V reports its per-step duration as model_step_wall_s and chunk_fps instead of the benchmark tooling's canonical total_s/model_step_s. _generated_fps_for_step and _generated_fps_summary only ever check total_s (falling back to model_step_s), and harness.py's run highlights read total_s directly, so LingBot's Cam2V scenarios never get a derived generated_fps and their step-time/throughput headline cards render empty. The runtime-metric-samples ingest path (_records_from_runtime_metric_samples) has no key-normalization step at all, unlike the stats-rows path, which already renames vendor-specific keys via _KEY_OVERRIDES. Add a small, additive alias step that backfills total_s from model_step_wall_s when a step doesn't already report total_s directly, leaving the app-reported key in place. Every downstream reader of total_s already exists (_generated_fps_for_step, _generated_fps_summary, harness.py's highlights, report.py's headline cards) and needs no change. Refs #564 Signed-off-by: Amir Fathi --- flashdreams/tests/test_benchmark_harness.py | 52 +++++++++++++++++++++ flashdreams/tools/benchmarks/metrics.py | 16 +++++++ 2 files changed, 68 insertions(+) diff --git a/flashdreams/tests/test_benchmark_harness.py b/flashdreams/tests/test_benchmark_harness.py index 6b17a14dd..eef69610c 100644 --- a/flashdreams/tests/test_benchmark_harness.py +++ b/flashdreams/tests/test_benchmark_harness.py @@ -233,6 +233,58 @@ def test_runtime_benchmark_stats_records_group_samples_by_step( assert records[1].metrics["generated_fps"] == pytest.approx(15.0) +def test_runtime_benchmark_stats_backfills_total_s_from_model_step_wall_s( + tmp_path: Path, +) -> None: + """Cam2V reports model_step_wall_s/chunk_fps, not the canonical total_s. + + The harness derives generated_fps and its run highlights from total_s + (falling back to model_step_s), so a scenario that only ever reports + model_step_wall_s drops out of every part of the pipeline keyed on those + canonical names. Backfilling total_s from the app-reported key, without + dropping that key, is what keeps both readable. + """ + stats_path = tmp_path / "stats_demo.json" + stats_path.write_text( + json.dumps( + { + "schema_version": 1, + "artifact_type": "flashdreams.runtime.demo.benchmark_stats", + "steps": [ + {"step_index": 0, "frame_count": 4}, + ], + "samples": [ + { + "name": "model_step_wall_s", + "value": 0.2, + "unit": "s", + "category": "timing", + "step_index": 0, + }, + { + "name": "chunk_fps", + "value": 20.0, + "unit": "fps", + "category": "throughput", + "step_index": 0, + }, + ], + } + ), + encoding="utf-8", + ) + + records = records_from_stats_file( + stats_path, scenario_id="cam2v-lingbot-quality-10s", source_root=tmp_path + ) + + assert len(records) == 1 + assert records[0].metrics["model_step_wall_s"] == pytest.approx(0.2) + assert records[0].metrics["chunk_fps"] == pytest.approx(20.0) + assert records[0].metrics["total_s"] == pytest.approx(0.2) + assert records[0].metrics["generated_fps"] == pytest.approx(20.0) + + def test_runtime_benchmark_stats_written_by_the_v2_sink_are_read( tmp_path: Path, ) -> None: diff --git a/flashdreams/tools/benchmarks/metrics.py b/flashdreams/tools/benchmarks/metrics.py index 60d785e76..4a35c8845 100644 --- a/flashdreams/tools/benchmarks/metrics.py +++ b/flashdreams/tools/benchmarks/metrics.py @@ -73,6 +73,11 @@ "cache_ms": "cache_seed_prune_s", "copy_ms": "gpu_to_cpu_copy_s", } +# Canonical name backfilled onto a runtime metric sample, alongside (never +# instead of) the name an integration actually reports. +_RUNTIME_METRIC_CANONICAL_ALIASES = { + "model_step_wall_s": "total_s", +} _RUNTIME_BENCHMARK_STATS_ARTIFACT_TYPE = "flashdreams.runtime.demo.benchmark_stats" _RUNTIME_METRIC_SAMPLE_PARSER = "runtime_metric_samples" @@ -359,6 +364,10 @@ def _records_from_runtime_metric_samples( metrics_by_step.setdefault(step_index, {})[name] = value sample_count_by_step[step_index] = sample_count_by_step.get(step_index, 0) + 1 + for metrics in metrics_by_step.values(): + _apply_runtime_metric_aliases(metrics) + _apply_runtime_metric_aliases(summary_metrics) + for step_index, frame_count in frame_counts_by_step.items(): metrics = metrics_by_step.setdefault(step_index, {}) metrics["generated_frame_count"] = frame_count @@ -392,6 +401,13 @@ def _records_from_runtime_metric_samples( return records +def _apply_runtime_metric_aliases(metrics: dict[str, float | int]) -> None: + """Backfill each canonical name from its alias, alongside the original key.""" + for raw_name, canonical_name in _RUNTIME_METRIC_CANONICAL_ALIASES.items(): + if canonical_name not in metrics and raw_name in metrics: + metrics[canonical_name] = metrics[raw_name] + + def _runtime_step_frame_counts(steps: object) -> dict[int, int]: if not isinstance(steps, list): return {} From 3cd9561a6bcf9a6c5ab2f5c79b25da1ad45c8d2e Mon Sep 17 00:00:00 2001 From: Amir Fathi Date: Fri, 4 Sep 2026 20:17:48 +0000 Subject: [PATCH 2/2] Assign total_s at the source in Cam2VModelLoop.step() The metrics.py alias backfilled total_s only inside the stats-file ingest path, after the runtime artifact was already written, so StepResult.metrics itself (and anything reading it directly) still carried only model_step_wall_s/chunk_fps. Setting total_s alongside model_step_wall_s in step() gives every consumer the canonical key; the parser-side alias stays for historical artifacts recorded before this change. Signed-off-by: Amir Fathi --- apps/cam2v/cam2v/session.py | 1 + apps/cam2v/tests/test_application.py | 1 + 2 files changed, 2 insertions(+) diff --git a/apps/cam2v/cam2v/session.py b/apps/cam2v/cam2v/session.py index 6d8f37758..883e7f403 100644 --- a/apps/cam2v/cam2v/session.py +++ b/apps/cam2v/cam2v/session.py @@ -355,6 +355,7 @@ def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: "postprocess_enabled": int(state.postprocess_enabled), "postprocess_comparison": int(state.config.postprocess_comparison), "postprocess_output_frames": postprocess_output_frame_count, + "total_s": model_step_wall_s, } ) if state.steady_started_at is not None: diff --git a/apps/cam2v/tests/test_application.py b/apps/cam2v/tests/test_application.py index 48349301a..733f0ef1b 100644 --- a/apps/cam2v/tests/test_application.py +++ b/apps/cam2v/tests/test_application.py @@ -220,6 +220,7 @@ def test_model_loop_maps_wasd_to_shared_camera_input_and_metrics() -> None: result.metrics["chunk_fps"] ) assert result.metrics["model_step_wall_s"] > 0 + assert result.metrics["total_s"] == result.metrics["model_step_wall_s"] ui_loop._run_message_batch() assert ui_state.status is not None assert ui_state.status.completed_blocks == 1