Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/cam2v/cam2v/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,7 @@ def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]:
"postprocess_enabled": int(state.postprocess_enabled),
"postprocess_comparison": int(state.config.postprocess_comparison),
"postprocess_output_frames": postprocess_output_frame_count,
"total_s": model_step_wall_s,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Postprocessing Time Is Excluded

When post-processing is enabled, StepResult.frame_count counts the presented post-processed frames, but total_s is set to the model-only duration and excludes the time measured by postprocess_step_wall_s. The benchmark then divides the post-processed frame count by model-only time, which overstates throughput; buffered output and final-tail frames can further distort individual steps. Use the complete loop duration for total_s, or pair the model-only duration with the model-generated frame count.

}
)
if state.steady_started_at is not None:
Expand Down
1 change: 1 addition & 0 deletions apps/cam2v/tests/test_application.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ def test_model_loop_maps_wasd_to_shared_camera_input_and_metrics() -> None:
result.metrics["chunk_fps"]
)
assert result.metrics["model_step_wall_s"] > 0
assert result.metrics["total_s"] == result.metrics["model_step_wall_s"]
ui_loop._run_message_batch()
assert ui_state.status is not None
assert ui_state.status.completed_blocks == 1
Expand Down
52 changes: 52 additions & 0 deletions flashdreams/tests/test_benchmark_harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,58 @@ def test_runtime_benchmark_stats_records_group_samples_by_step(
assert records[1].metrics["generated_fps"] == pytest.approx(15.0)


def test_runtime_benchmark_stats_backfills_total_s_from_model_step_wall_s(
tmp_path: Path,
) -> None:
"""Cam2V reports model_step_wall_s/chunk_fps, not the canonical total_s.

The harness derives generated_fps and its run highlights from total_s
(falling back to model_step_s), so a scenario that only ever reports
model_step_wall_s drops out of every part of the pipeline keyed on those
canonical names. Backfilling total_s from the app-reported key, without
dropping that key, is what keeps both readable.
"""
stats_path = tmp_path / "stats_demo.json"
stats_path.write_text(
json.dumps(
{
"schema_version": 1,
"artifact_type": "flashdreams.runtime.demo.benchmark_stats",
"steps": [
{"step_index": 0, "frame_count": 4},
],
"samples": [
{
"name": "model_step_wall_s",
"value": 0.2,
"unit": "s",
"category": "timing",
"step_index": 0,
},
{
"name": "chunk_fps",
"value": 20.0,
"unit": "fps",
"category": "throughput",
"step_index": 0,
},
],
}
),
encoding="utf-8",
)

records = records_from_stats_file(
stats_path, scenario_id="cam2v-lingbot-quality-10s", source_root=tmp_path
)

assert len(records) == 1
assert records[0].metrics["model_step_wall_s"] == pytest.approx(0.2)
assert records[0].metrics["chunk_fps"] == pytest.approx(20.0)
assert records[0].metrics["total_s"] == pytest.approx(0.2)
assert records[0].metrics["generated_fps"] == pytest.approx(20.0)


def test_runtime_benchmark_stats_written_by_the_v2_sink_are_read(
tmp_path: Path,
) -> None:
Expand Down
16 changes: 16 additions & 0 deletions flashdreams/tools/benchmarks/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,11 @@
"cache_ms": "cache_seed_prune_s",
"copy_ms": "gpu_to_cpu_copy_s",
}
# Canonical name backfilled onto a runtime metric sample, alongside (never
# instead of) the name an integration actually reports.
_RUNTIME_METRIC_CANONICAL_ALIASES = {
"model_step_wall_s": "total_s",
}
_RUNTIME_BENCHMARK_STATS_ARTIFACT_TYPE = "flashdreams.runtime.demo.benchmark_stats"
_RUNTIME_METRIC_SAMPLE_PARSER = "runtime_metric_samples"

Expand Down Expand Up @@ -359,6 +364,10 @@ def _records_from_runtime_metric_samples(
metrics_by_step.setdefault(step_index, {})[name] = value
sample_count_by_step[step_index] = sample_count_by_step.get(step_index, 0) + 1

for metrics in metrics_by_step.values():
_apply_runtime_metric_aliases(metrics)
_apply_runtime_metric_aliases(summary_metrics)

for step_index, frame_count in frame_counts_by_step.items():
metrics = metrics_by_step.setdefault(step_index, {})
metrics["generated_frame_count"] = frame_count
Expand Down Expand Up @@ -392,6 +401,13 @@ def _records_from_runtime_metric_samples(
return records


def _apply_runtime_metric_aliases(metrics: dict[str, float | int]) -> None:
"""Backfill each canonical name from its alias, alongside the original key."""
for raw_name, canonical_name in _RUNTIME_METRIC_CANONICAL_ALIASES.items():
if canonical_name not in metrics and raw_name in metrics:
metrics[canonical_name] = metrics[raw_name]


def _runtime_step_frame_counts(steps: object) -> dict[int, int]:
if not isinstance(steps, list):
return {}
Expand Down