diff --git a/apps/cam2v/cam2v/session.py b/apps/cam2v/cam2v/session.py index 6d8f37758..883e7f403 100644 --- a/apps/cam2v/cam2v/session.py +++ b/apps/cam2v/cam2v/session.py @@ -355,6 +355,7 @@ def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: "postprocess_enabled": int(state.postprocess_enabled), "postprocess_comparison": int(state.config.postprocess_comparison), "postprocess_output_frames": postprocess_output_frame_count, + "total_s": model_step_wall_s, } ) if state.steady_started_at is not None: diff --git a/apps/cam2v/tests/test_application.py b/apps/cam2v/tests/test_application.py index 48349301a..733f0ef1b 100644 --- a/apps/cam2v/tests/test_application.py +++ b/apps/cam2v/tests/test_application.py @@ -220,6 +220,7 @@ def test_model_loop_maps_wasd_to_shared_camera_input_and_metrics() -> None: result.metrics["chunk_fps"] ) assert result.metrics["model_step_wall_s"] > 0 + assert result.metrics["total_s"] == result.metrics["model_step_wall_s"] ui_loop._run_message_batch() assert ui_state.status is not None assert ui_state.status.completed_blocks == 1 diff --git a/flashdreams/tests/test_benchmark_harness.py b/flashdreams/tests/test_benchmark_harness.py index 6b17a14dd..eef69610c 100644 --- a/flashdreams/tests/test_benchmark_harness.py +++ b/flashdreams/tests/test_benchmark_harness.py @@ -233,6 +233,58 @@ def test_runtime_benchmark_stats_records_group_samples_by_step( assert records[1].metrics["generated_fps"] == pytest.approx(15.0) +def test_runtime_benchmark_stats_backfills_total_s_from_model_step_wall_s( + tmp_path: Path, +) -> None: + """Cam2V reports model_step_wall_s/chunk_fps, not the canonical total_s. + + The harness derives generated_fps and its run highlights from total_s + (falling back to model_step_s), so a scenario that only ever reports + model_step_wall_s drops out of every part of the pipeline keyed on those + canonical names. Backfilling total_s from the app-reported key, without + dropping that key, is what keeps both readable. + """ + stats_path = tmp_path / "stats_demo.json" + stats_path.write_text( + json.dumps( + { + "schema_version": 1, + "artifact_type": "flashdreams.runtime.demo.benchmark_stats", + "steps": [ + {"step_index": 0, "frame_count": 4}, + ], + "samples": [ + { + "name": "model_step_wall_s", + "value": 0.2, + "unit": "s", + "category": "timing", + "step_index": 0, + }, + { + "name": "chunk_fps", + "value": 20.0, + "unit": "fps", + "category": "throughput", + "step_index": 0, + }, + ], + } + ), + encoding="utf-8", + ) + + records = records_from_stats_file( + stats_path, scenario_id="cam2v-lingbot-quality-10s", source_root=tmp_path + ) + + assert len(records) == 1 + assert records[0].metrics["model_step_wall_s"] == pytest.approx(0.2) + assert records[0].metrics["chunk_fps"] == pytest.approx(20.0) + assert records[0].metrics["total_s"] == pytest.approx(0.2) + assert records[0].metrics["generated_fps"] == pytest.approx(20.0) + + def test_runtime_benchmark_stats_written_by_the_v2_sink_are_read( tmp_path: Path, ) -> None: diff --git a/flashdreams/tools/benchmarks/metrics.py b/flashdreams/tools/benchmarks/metrics.py index 60d785e76..4a35c8845 100644 --- a/flashdreams/tools/benchmarks/metrics.py +++ b/flashdreams/tools/benchmarks/metrics.py @@ -73,6 +73,11 @@ "cache_ms": "cache_seed_prune_s", "copy_ms": "gpu_to_cpu_copy_s", } +# Canonical name backfilled onto a runtime metric sample, alongside (never +# instead of) the name an integration actually reports. +_RUNTIME_METRIC_CANONICAL_ALIASES = { + "model_step_wall_s": "total_s", +} _RUNTIME_BENCHMARK_STATS_ARTIFACT_TYPE = "flashdreams.runtime.demo.benchmark_stats" _RUNTIME_METRIC_SAMPLE_PARSER = "runtime_metric_samples" @@ -359,6 +364,10 @@ def _records_from_runtime_metric_samples( metrics_by_step.setdefault(step_index, {})[name] = value sample_count_by_step[step_index] = sample_count_by_step.get(step_index, 0) + 1 + for metrics in metrics_by_step.values(): + _apply_runtime_metric_aliases(metrics) + _apply_runtime_metric_aliases(summary_metrics) + for step_index, frame_count in frame_counts_by_step.items(): metrics = metrics_by_step.setdefault(step_index, {}) metrics["generated_frame_count"] = frame_count @@ -392,6 +401,13 @@ def _records_from_runtime_metric_samples( return records +def _apply_runtime_metric_aliases(metrics: dict[str, float | int]) -> None: + """Backfill each canonical name from its alias, alongside the original key.""" + for raw_name, canonical_name in _RUNTIME_METRIC_CANONICAL_ALIASES.items(): + if canonical_name not in metrics and raw_name in metrics: + metrics[canonical_name] = metrics[raw_name] + + def _runtime_step_frame_counts(steps: object) -> dict[int, int]: if not isinstance(steps, list): return {}