From 9557ad6454fd17f054300d3a02e2ca3087e62818 Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Thu, 3 Sep 2026 00:41:00 -0700 Subject: [PATCH 1/2] feat: migrate Waypoint integration to v2 layout Signed-off-by: Jonathan McCaffrey --- apps/cam2v/README.md | 5 +- apps/cam2v/cam2v/__init__.py | 4 + apps/cam2v/cam2v/ui.py | 148 +++++-- apps/cam2v/tests/test_application.py | 1 + docs/source/models/waypoint.rst | 25 +- integrations/waypoint/README.md | 375 ---------------- integrations/waypoint/pyproject.toml | 32 -- integrations_v2/README.md | 4 +- .../waypoint/ADR-1-control-events.md | 26 +- integrations_v2/waypoint/README.md | 408 +++++++++++++++--- integrations_v2/waypoint/VALIDATION.md | 26 +- .../waypoint/__init__.py | 6 +- integrations_v2/waypoint/apps/__init__.py | 4 + integrations_v2/waypoint/apps/cam2v/README.md | 59 +++ .../waypoint/apps/cam2v/__init__.py | 4 + .../app.py => apps/cam2v/adapter.py} | 42 +- .../cam2v}/assets/example_controls.json | 0 .../cam2v}/control_events.py | 0 .../{waypoint_v2 => apps/cam2v}/session.py | 128 ++++-- .../waypoint/config.py | 12 +- integrations_v2/waypoint/impl/__init__.py | 4 + .../waypoint/impl}/checkpoint.py | 2 +- .../waypoint/impl}/controls.py | 2 +- .../waypoint/impl}/decoder.py | 0 .../waypoint/impl}/encoder.py | 2 +- .../waypoint/impl}/pipeline.py | 6 +- .../waypoint/impl}/scheduler.py | 0 .../waypoint/impl}/spec.py | 0 .../waypoint/impl}/transformer/__init__.py | 13 +- .../waypoint/impl}/transformer/cache.py | 2 +- .../waypoint/impl}/transformer/impl.py | 10 +- .../waypoint/impl}/transformer/network.py | 11 +- .../waypoint/impl}/transformer/norm.py | 0 .../waypoint/impl}/transformer/rope.py | 2 +- integrations_v2/waypoint/pyproject.toml | 47 +- .../test_cam2v_app.py} | 93 +++- .../waypoint/tests/test_cuda.py | 4 +- .../waypoint/tests/test_spec.py | 19 +- .../waypoint/waypoint_v2/__init__.py | 16 - pyproject.toml | 2 - uv.lock | 24 +- 41 files changed, 911 insertions(+), 657 deletions(-) delete mode 100644 integrations/waypoint/README.md delete mode 100644 integrations/waypoint/pyproject.toml rename {integrations/waypoint => integrations_v2}/waypoint/__init__.py (84%) create mode 100644 integrations_v2/waypoint/apps/__init__.py create mode 100644 integrations_v2/waypoint/apps/cam2v/README.md create mode 100644 integrations_v2/waypoint/apps/cam2v/__init__.py rename integrations_v2/waypoint/{waypoint_v2/app.py => apps/cam2v/adapter.py} (89%) rename integrations_v2/waypoint/{waypoint_v2 => apps/cam2v}/assets/example_controls.json (100%) rename integrations_v2/waypoint/{waypoint_v2 => apps/cam2v}/control_events.py (100%) rename integrations_v2/waypoint/{waypoint_v2 => apps/cam2v}/session.py (71%) rename {integrations/waypoint => integrations_v2}/waypoint/config.py (79%) create mode 100644 integrations_v2/waypoint/impl/__init__.py rename {integrations/waypoint/waypoint => integrations_v2/waypoint/impl}/checkpoint.py (99%) rename {integrations/waypoint/waypoint => integrations_v2/waypoint/impl}/controls.py (99%) rename {integrations/waypoint/waypoint => integrations_v2/waypoint/impl}/decoder.py (100%) rename {integrations/waypoint/waypoint => integrations_v2/waypoint/impl}/encoder.py (97%) rename {integrations/waypoint/waypoint => integrations_v2/waypoint/impl}/pipeline.py (96%) rename {integrations/waypoint/waypoint => integrations_v2/waypoint/impl}/scheduler.py (100%) rename {integrations/waypoint/waypoint => integrations_v2/waypoint/impl}/spec.py (100%) rename {integrations/waypoint/waypoint => integrations_v2/waypoint/impl}/transformer/__init__.py (80%) rename {integrations/waypoint/waypoint => integrations_v2/waypoint/impl}/transformer/cache.py (99%) rename {integrations/waypoint/waypoint => integrations_v2/waypoint/impl}/transformer/impl.py (96%) rename {integrations/waypoint/waypoint => integrations_v2/waypoint/impl}/transformer/network.py (98%) rename {integrations/waypoint/waypoint => integrations_v2/waypoint/impl}/transformer/norm.py (100%) rename {integrations/waypoint/waypoint => integrations_v2/waypoint/impl}/transformer/rope.py (99%) rename integrations_v2/waypoint/{waypoint_v2/tests/test_waypoint_v2.py => tests/test_cam2v_app.py} (84%) rename {integrations => integrations_v2}/waypoint/tests/test_cuda.py (95%) rename {integrations => integrations_v2}/waypoint/tests/test_spec.py (98%) delete mode 100644 integrations_v2/waypoint/waypoint_v2/__init__.py diff --git a/apps/cam2v/README.md b/apps/cam2v/README.md index bf92952ca..8e3284f35 100644 --- a/apps/cam2v/README.md +++ b/apps/cam2v/README.md @@ -13,7 +13,9 @@ application named `cam2v-`. | `Q` / `E` | Strafe left / right | | `I` / `K` | Pitch up / down | -Losing browser focus clears held keys. +Losing browser focus clears held keys. The HUD wraps each held key in brackets. +Model adapters may replace the displayed groups while keeping the same UI loop; +Waypoint shows its raw-action controls and `R` reset command this way. ## Usage @@ -21,6 +23,7 @@ Concrete launch commands live with each model adapter: - [Lingbot](../../integrations_v2/lingbot/apps/cam2v/README.md) - [HY-WorldPlay](../../integrations_v2/hy_worldplay/apps/cam2v/README.md) +- [Waypoint](../../integrations_v2/waypoint/apps/cam2v/README.md) The general command shape is: diff --git a/apps/cam2v/cam2v/__init__.py b/apps/cam2v/cam2v/__init__.py index 855c0dc40..c3aa1093c 100644 --- a/apps/cam2v/cam2v/__init__.py +++ b/apps/cam2v/cam2v/__init__.py @@ -20,6 +20,8 @@ CameraControlInput, ) from .ui import ( + Cam2VControlGroup, + Cam2VControlKey, Cam2VSlangPyUILoop, Cam2VUIState, Cam2VUIStatus, @@ -29,6 +31,8 @@ "Cam2VApplication", "Cam2VApplicationDefaults", "Cam2VConditioning", + "Cam2VControlGroup", + "Cam2VControlKey", "Cam2VGenerateStep", "Cam2VInputResolver", "Cam2VModelLoop", diff --git a/apps/cam2v/cam2v/ui.py b/apps/cam2v/cam2v/ui.py index 8088175dd..51d022922 100644 --- a/apps/cam2v/cam2v/ui.py +++ b/apps/cam2v/cam2v/ui.py @@ -21,16 +21,64 @@ ) from flashdreams.runtime_v2.user_input_events import UserInputEvents -_CAMERA_KEY_ORDER = ("w", "s", "q", "e", "a", "d", "j", "l", "i", "k") -"""Stable order used when active camera controls are displayed.""" - -_CAMERA_KEYS = frozenset(_CAMERA_KEY_ORDER) -"""Keyboard controls recognized by the shared camera pose integrator.""" - RECENT_MODEL_FPS_WINDOW_SECONDS = 2.0 """Trailing AR-step completion window displayed in the model status panel.""" +@dataclass(frozen=True, slots=True) +class Cam2VControlKey: + """One keyboard key displayed in a Cam2V control group.""" + + key: str + """Canonical runtime key used to track held state.""" + + label: str + """Short user-facing label rendered in the overlay.""" + + +@dataclass(frozen=True, slots=True) +class Cam2VControlGroup: + """One action and its associated keyboard controls.""" + + action: str + """User-facing action description.""" + + keys: tuple[Cam2VControlKey, ...] + """Keys that trigger the action.""" + + +DEFAULT_CAM2V_CONTROL_GROUPS = ( + Cam2VControlGroup( + action="Move forward / backward", + keys=(Cam2VControlKey("w", "W"), Cam2VControlKey("s", "S")), + ), + Cam2VControlGroup( + action="Strafe left / right", + keys=(Cam2VControlKey("q", "Q"), Cam2VControlKey("e", "E")), + ), + Cam2VControlGroup( + action="Yaw left / right", + keys=( + Cam2VControlKey("a", "A"), + Cam2VControlKey("d", "D"), + Cam2VControlKey("j", "J"), + Cam2VControlKey("l", "L"), + ), + ), + Cam2VControlGroup( + action="Pitch up / down", + keys=(Cam2VControlKey("i", "I"), Cam2VControlKey("k", "K")), + ), +) +"""Default keyboard groups for the shared camera pose integrator.""" + +DEFAULT_CAM2V_UI_INSTRUCTIONS = ( + "Held controls are shown in brackets.", + "Click the video before using keyboard controls.", +) +"""Default hints rendered below the Cam2V controls.""" + + @dataclass(frozen=True, slots=True) class Cam2VUIStatus: """Latest model-generation status copied to the UI loop.""" @@ -71,12 +119,19 @@ class Cam2VUIState: warmup_blocks: int """Leading blocks excluded from recent model throughput.""" + control_groups: tuple[Cam2VControlGroup, ...] = DEFAULT_CAM2V_CONTROL_GROUPS + """Action-oriented key groups displayed by the overlay.""" + + instructions: tuple[str, ...] = DEFAULT_CAM2V_UI_INSTRUCTIONS + """Short usage hints displayed below the controls.""" + + show_status: bool = True + """Whether model throughput lines are included above the controls.""" + held_keys: set[str] = field(default_factory=set) - """Camera-control keys currently held by the client.""" + """Keyboard control keys currently held by the client.""" - _keyboard_state: KeyboardState = field( - default_factory=lambda: KeyboardState(supported_keys=_CAMERA_KEYS), - ) + _keyboard_state: KeyboardState = field(init=False) """UI-thread-owned source-aware keyboard state.""" status: Cam2VUIStatus | None = None @@ -91,8 +146,14 @@ class Cam2VUIState: status_widgets: list[Any] = field(default_factory=list, init=False, repr=False) """Retained SlangPy text widgets for model status.""" + control_widgets: list[Any] = field(default_factory=list, init=False, repr=False) + """Retained SlangPy text widgets for action-oriented key groups.""" + active_keys_widget: Any | None = field(default=None, init=False, repr=False) - """Retained SlangPy text widget for active camera controls.""" + """Retained SlangPy text widget for active keyboard controls.""" + + def __post_init__(self) -> None: + self._keyboard_state = self._new_keyboard_state() def update_status(self, status: Cam2VUIStatus) -> None: """Replace the displayed model-generation status.""" @@ -101,10 +162,16 @@ def update_status(self, status: Cam2VUIStatus) -> None: def reset(self) -> None: """Clear transient controls and model status for a new generation.""" self.held_keys.clear() - self._keyboard_state = KeyboardState(supported_keys=_CAMERA_KEYS) + self._keyboard_state = self._new_keyboard_state() self.status = None self.frames_presented = 0 + def _new_keyboard_state(self) -> KeyboardState: + keys = frozenset( + control.key for group in self.control_groups for control in group.keys + ) + return KeyboardState(supported_keys=keys) + class Cam2VSlangPyUILoop(SlangPyUILoop[Cam2VUIState]): """Draw Cam2V controls and model throughput over generated video.""" @@ -142,27 +209,38 @@ def _ensure_widgets( return state.window = ui.Window( ui.screen, - "Camera controls", + "Controls", position=(16, 16), - size=(360, 280), + size=(400, 340 if state.show_status else 220), ) - state.status_widgets = [ - ui.Text(state.window, line) - for line in _status_lines(state, sampled_at=sampled_at) + if state.show_status: + state.status_widgets = [ + ui.Text(state.window, line) + for line in _status_lines(state, sampled_at=sampled_at) + ] + state.control_widgets = [ + ui.Text(state.window, _control_group_text(group, state.held_keys)) + for group in state.control_groups ] - ui.Text(state.window, "Move: W/S Strafe: Q/E") - ui.Text(state.window, "Yaw: A/D or J/L Pitch: I/K") state.active_keys_widget = ui.Text(state.window, _active_keys_text(state)) - ui.Text(state.window, "Click the video before using keyboard controls.") + for instruction in state.instructions: + ui.Text(state.window, instruction) def _refresh_widgets(state: Cam2VUIState, *, sampled_at: float) -> None: - for widget, line in zip( - state.status_widgets, - _status_lines(state, sampled_at=sampled_at), + if state.status_widgets: + for widget, line in zip( + state.status_widgets, + _status_lines(state, sampled_at=sampled_at), + strict=True, + ): + widget.text = line + for widget, group in zip( + state.control_widgets, + state.control_groups, strict=True, ): - widget.text = line + widget.text = _control_group_text(group, state.held_keys) if state.active_keys_widget is not None: state.active_keys_widget.text = _active_keys_text(state) @@ -207,15 +285,31 @@ def _status_lines( def _active_keys_text(state: Cam2VUIState) -> str: - active = [key.upper() for key in _CAMERA_KEY_ORDER if key in state.held_keys] + active = [ + control.label + for group in state.control_groups + for control in group.keys + if control.key in state.held_keys + ] return f"Active keys: {', '.join(active) if active else 'none'}" +def _control_group_text( + group: Cam2VControlGroup, + held_keys: set[str], +) -> str: + labels = [ + f"[{control.label}]" if control.key in held_keys else control.label + for control in group.keys + ] + return f"{group.action}: {' / '.join(labels)}" + + def _apply_ui_input(state: Cam2VUIState, events: UserInputEvents) -> None: for event in events.get_events(): if isinstance(event, FocusUserInputEvent) and not event.focused: state.held_keys.clear() - state._keyboard_state = KeyboardState(supported_keys=_CAMERA_KEYS) + state._keyboard_state = state._new_keyboard_state() continue if not isinstance(event, KeyboardUserInputEvent): continue @@ -229,6 +323,8 @@ def _apply_ui_input(state: Cam2VUIState, events: UserInputEvents) -> None: __all__ = [ + "Cam2VControlGroup", + "Cam2VControlKey", "Cam2VSlangPyUILoop", "Cam2VUIState", "Cam2VUIStatus", diff --git a/apps/cam2v/tests/test_application.py b/apps/cam2v/tests/test_application.py index 588cb61b8..09bdf1d33 100644 --- a/apps/cam2v/tests/test_application.py +++ b/apps/cam2v/tests/test_application.py @@ -536,6 +536,7 @@ def render( assert any(line.startswith("Recent model rate (2 s):") for line in displayed) assert state.active_keys_widget is not None assert state.active_keys_widget.text == "Active keys: W" + assert state.control_widgets[0].text == "Move forward / backward: [W] / S" ui_loop.step( 1, diff --git a/docs/source/models/waypoint.rst b/docs/source/models/waypoint.rst index eb338a78f..6a885aa43 100644 --- a/docs/source/models/waypoint.rst +++ b/docs/source/models/waypoint.rst @@ -62,7 +62,7 @@ Support summary * - Surface - FlashDreams support * - Application slug - - waypoint-1-5-1b through flashdreams-run-v2 + - cam2v-waypoint through flashdreams-run-v2 * - Input modalities - RGB/RGBA seed image; keyboard and mouse buttons; relative mouse motion; ternary scroll-wheel direction @@ -104,10 +104,10 @@ From the FlashDreams repository root: .. code-block:: bash - uv sync --package flashdreams-waypoint-v2 --inexact + uv sync --package flashdreams-waypoint --inexact -The V2 application package depends on the sibling flashdreams-waypoint model -package, so both are installed together. +The package contains the model implementation, configuration, tests, and Cam2V +application binding in the V2 integration layout. Running the model ----------------- @@ -117,7 +117,8 @@ and bundled control timeline: .. code-block:: bash - uv run --no-sync flashdreams-run-v2 waypoint-1-5-1b \ + uv run --no-sync flashdreams-run-v2 cam2v-waypoint \ + --presentation-mode on_demand \ --output-path waypoint.mp4 --stats-path waypoint.metrics.json \ -- --example-data --actions 40 --seed 464 --profile @@ -125,16 +126,18 @@ Run the same application interactively in a browser: .. code-block:: bash - uv run --no-sync flashdreams-run-v2 waypoint-1-5-1b \ + uv run --no-sync flashdreams-run-v2 cam2v-waypoint \ --mode webrtc --host 127.0.0.1 --port 8766 \ - -- --seed-image seed.png --seed 464 + -- --image-path seed.png --seed 464 -Open http://127.0.0.1:8766/. Arguments before the separator configure the V2 -runtime; arguments after it configure Waypoint. To inspect all model arguments: +Open http://127.0.0.1:8766/. The control HUD highlights held keys; press ``R`` +to reset the rollout to the starting image. Arguments before the separator +configure the V2 runtime; arguments after it configure Waypoint. To inspect all +model arguments: .. code-block:: bash - uv run --no-sync flashdreams-run-v2 waypoint-1-5-1b -- --help + uv run --no-sync flashdreams-run-v2 cam2v-waypoint -- --help Model and integration architecture ---------------------------------- @@ -170,7 +173,7 @@ reports both rather than relabeling the upstream model. The package-level design review contains component, class, use-case, and sequence diagrams: -.. button-link:: https://github.com/NVIDIA/flashdreams/blob/main/integrations/waypoint/README.md +.. button-link:: https://github.com/NVIDIA/flashdreams/blob/main/integrations_v2/waypoint/README.md :color: secondary :outline: diff --git a/integrations/waypoint/README.md b/integrations/waypoint/README.md deleted file mode 100644 index 8c4ca7651..000000000 --- a/integrations/waypoint/README.md +++ /dev/null @@ -1,375 +0,0 @@ - - -# Waypoint 1.5 architecture - -This package is the model layer for the published -[Overworld/Waypoint-1.5-1B](https://huggingface.co/Overworld/Waypoint-1.5-1B) -checkpoint. It implements checkpoint loading, controls, the autoregressive -diffusion pipeline, sparse K/V history, and the shared TAEHV codec. The -[V2 application](../../integrations_v2/waypoint/README.md) owns CLI arguments, -sessions, browser events, and presentation. - -This document is the concise design-review reference. Exact validation commands, -hardware, parity metrics, and long-rollout evidence are recorded in -[VALIDATION.md](../../integrations_v2/waypoint/VALIDATION.md). - -## Scope and decisions - -| Area | Design | -|---|---| -| Model | Dense BF16 autoregressive DiT, 24 blocks, width 2048, 32 query heads, 16 K/V heads | -| Parameter accounting | Upstream model card: 1.2B; pinned BF16 checkpoint: 1,860,823,096 serialized tensor elements | -| Inputs | Four-frame image seed plus one keyboard/mouse/wheel control per action | -| Output | One 32-channel latent and four RGB frames per action | -| Denoising | Fixed four-step rectified-flow Euler schedule: 1.0, 0.9, 0.75, 0.3, 0.0 | -| History | Dense 16-action local window; sparse 128-action global horizon in every fourth block | -| Context | 128 latent actions / 512 presented RGB frames on global-attention blocks | -| Presentation | Native TAEHV canvas, 1024x512 TCHW in [-1, 1], four frames per result | -| State | Model weights shared by the application; cache, RNG, controls, and seed isolated per session | -| Concurrency | An application lock serializes shared model/RNG work; output tensors leave the model loop detached | - -The published config has `prompt_conditioning: null`; this integration therefore -does not load a text encoder or expose a prompt. The generic upstream -`WorldEngine` has a `set_prompt` method for other configurations, but for this -checkpoint it does not instantiate a prompt encoder and `set_prompt` raises. -The pinned checkpoint also has no prompt or cross-attention tensor keys. A -prompt argument would therefore be ignored or rejected rather than condition -generation. The integration also does not currently implement the upstream -360P checkpoint, quantization, or multi-GPU execution. - -## Use cases and modalities - -```mermaid -flowchart LR - user([Interactive user]) - replay([Replay, test, or benchmark]) - seed[RGB or RGBA seed image] - live[Keyboard, mouse buttons,
relative motion, wheel] - file[Versioned JSON controls] - app((waypoint-1-5-1b)) - model[Waypoint 1.5 DiT + TAEHV] - browser[Live WebRTC video] - mp4[Deterministic MP4] - metrics[Per-action metrics JSON] - prompt[Text prompt
not supported] - - user --> seed - user --> live - replay --> seed - replay --> file - seed --> app - live --> app - file --> app - prompt -. excluded by checkpoint .-> app - app --> model - model --> browser - model --> mp4 - model --> metrics -``` - -The seed establishes visual state; it is not a continuing image-conditioning -stream. File controls select finite, reproducible MP4/metrics runs. Omitting a -control file selects an open-ended live session driven by V2 browser events. - -## Component and file view - -```mermaid -flowchart TB - subgraph core[flashdreams core and V2 runtime] - registry[application_registry.py
slug discovery] - runner[application_runner.py / session_runner.py] - api[IApplication / ISession / IModelLoop] - input[UserInputEvents / EventBuffer
transport and reset lifecycle] - sinks[WebRTCClientWindow / Mp4ClientWindow
MetricsOutputSink] - base[StreamInferencePipeline
DiffusionModel / Transformer] - taehv[recipes/taehv
Hy15TAEHVEncoder and Decoder] - end - - subgraph adapter[integrations_v2/waypoint] - app[app.py
WaypointApplication] - session[session.py
WaypointSession and ModelLoop] - events[control_events.py
WaypointControlEventAdapter] - end - - subgraph waypoint[integrations/waypoint] - config[config.py and spec.py
checkpoint contract] - pipeline[pipeline.py
WaypointInferencePipeline] - controls[controls.py / encoder.py
WaypointControl] - transformer[transformer/impl.py
WaypointTransformer] - network[transformer/network.py
WaypointDiT] - cache[transformer/cache.py
WaypointKVCache] - scheduler[scheduler.py
fixed Euler schedule] - codec[decoder.py
WaypointTAEHVDecoder] - checkpoint[checkpoint.py
strict state-dict mapping] - end - - registry --> app - runner --> app - api -. implemented by .-> app - api -. implemented by .-> session - runner --> session - runner --> input - runner --> sinks - app --> session - app --> pipeline - session --> events - session --> pipeline - input --> events - events --> controls - pipeline --> controls - pipeline --> transformer - pipeline --> scheduler - pipeline --> codec - transformer --> network - network --> cache - transformer --> checkpoint - config --> pipeline - base -. specialized by .-> pipeline - base -. specialized by .-> transformer - taehv -. reused by .-> pipeline - taehv -. specialized by .-> codec -``` - -The package boundary is intentional: `integrations/waypoint` is reusable model -inference code and imports no V2 runtime classes. `integrations_v2/waypoint` is -the thin application adapter and depends on both FlashDreams and the model -package. - -## V2 reuse and action-mapping boundary - -The V2 runtime already provides device-neutral, timestamped input events; -event fan-out to model and UI loops; reset/focus lifecycle; WebRTC transport; -and generic presentation, MP4, and metrics sinks. It deliberately delivers raw -events to each model loop. It does not currently provide an action-to-video -application, a stateful action coalescer, or a protocol from user events to a -model's control object. - -`WaypointControlEventAdapter` is therefore integration-owned. It contains two -seams that a later shared action-to-video package can separate: - -1. A model-neutral accumulator can retain held key names and mouse-button - indices, convert absolute pointer positions to relative normalized motion, - collect wheel deltas, and clear state on focus loss or reset. -2. A model-owned mapper can convert that snapshot to Waypoint's Windows - virtual-key vocabulary, apply canvas and sensitivity scaling, reduce the - wheel to a ternary value, and construct `WaypointControl`. The existing - `WaypointControlEncoder` remains responsible for control-to-tensor encoding. - -The legacy `flashdreams.runtime.mapping.InputMapping` API is not a V2 -dependency and should not be revived inside this integration. `apps/cam2v` -demonstrates the desired shared-app organization, but its keyboard-to-camera -trajectory resampling is a different semantic contract. A future -`apps/action2v`-style package should be extracted when a second interactive -world-model integration can validate the common snapshot, mapper, replay, and -control-help UI contracts. Until then, keeping this adapter explicitly named -and scoped to Waypoint avoids freezing its virtual-key assumptions into V2. - -## Class and ownership view - -```mermaid -classDiagram - direction LR - - class IApplication - class ISession - class IModelLoop - class StreamInferencePipeline - class Transformer - class StreamingEncoder - class Hy15TAEHVDecoder - - class WaypointApplication { - -pipeline - -pipeline_lock - +init(args) - +session_desc() - +create_session(desc) - } - class WaypointSession { - -seed_frames - -controls - -state - +init() - +close() - } - class WaypointModelLoop { - +step(index, events) - +reset() - +close() - } - class WaypointModelState { - +cache - +rng_state - +control_events - +controls_generated - } - class WaypointInferencePipeline { - +initialize_cache(seed_pixels) - +generate(index, cache, control) - +finalize(index, cache) - } - class WaypointControlEncoder - class WaypointTransformer - class WaypointDiT - class WaypointKVCache - class WaypointTAEHVDecoder - - IApplication <|-- WaypointApplication - ISession <|-- WaypointSession - IModelLoop <|-- WaypointModelLoop - StreamInferencePipeline <|-- WaypointInferencePipeline - StreamingEncoder <|-- WaypointControlEncoder - Transformer <|-- WaypointTransformer - Hy15TAEHVDecoder <|-- WaypointTAEHVDecoder - - WaypointApplication o-- WaypointInferencePipeline : shares weights - WaypointApplication --> WaypointSession : creates - WaypointSession *-- WaypointModelState : owns - WaypointSession --> WaypointModelLoop : registers - WaypointModelLoop --> WaypointInferencePipeline : drives - WaypointInferencePipeline *-- WaypointControlEncoder - WaypointInferencePipeline *-- WaypointTransformer - WaypointInferencePipeline *-- WaypointTAEHVDecoder - WaypointTransformer *-- WaypointDiT - WaypointDiT --> WaypointKVCache : updates -``` - -The application owns expensive immutable modules. A session owns all mutable -rollout state, including the transformer K/V cache, TAEHV state, seed tensors, -RNG state, and control adapter. `StepResult` tensors are detached before -publication so runtime queues and encoders do not retain model autograd state. - -## Initialization and action sequence - -```mermaid -sequenceDiagram - actor Client - participant Runtime as V2 runtime - participant App as WaypointApplication - participant Session as WaypointSession - participant ModelLoop as WaypointModelLoop - participant Pipeline as WaypointInferencePipeline - participant DiT as WaypointTransformer / DiT - participant Cache as K/V + TAEHV caches - participant Sink as WebRTC or MP4 sink - - Client->>Runtime: slug, runtime args, Waypoint args - Runtime->>App: init(args) - Runtime->>App: create_session(session_desc) - App->>App: lazily load one BF16 model - App-->>Runtime: new session with shared model + lock - Runtime->>Session: init() - Session->>Pipeline: initialize_cache(four seed frames) - Pipeline->>Cache: TAEHV encode and prime decoder state - Pipeline->>DiT: sigma-zero seed flow - DiT->>Cache: commit action 0 K/V - - alt step 0 - Runtime->>ModelLoop: step(0, events) - ModelLoop-->>Runtime: detached seed StepResult (4 frames) - else generated action N - Runtime->>ModelLoop: step(N, ordered input events) - ModelLoop->>Pipeline: generate(N, cache, control) - loop sigmas 1.0 to 0.3 - Pipeline->>DiT: predict provisional flow - DiT->>Cache: replace provisional action-N K/V - end - Pipeline->>Cache: decode clean latent to 4 RGB frames - ModelLoop->>Pipeline: finalize(N, cache) - Pipeline->>DiT: sigma-zero clean-state evaluation - DiT->>Cache: commit action-N K/V - ModelLoop-->>Runtime: detached TCHW StepResult - end - - Runtime->>Sink: present or encode four frames - opt browser reset - Client->>Runtime: reset event - Runtime->>ModelLoop: reset() - ModelLoop->>Pipeline: rebuild cache from retained seed - end -``` - -The four denoise evaluations may overwrite only the current provisional slot. -`finalize` performs the separate sigma-zero evaluation that commits clean K/V -state for the next action. Skipping or reordering that transition changes the -autoregressive world state. - -## Tensor and control contracts - -| Boundary | Contract | -|---|---| -| Display seed | `[4, 3, 512, 1024]` TCHW float, normalized to `[-1, 1]` | -| Codec seed | `[B, 4, 3, 512, 1024]` float in `[0, 1]` | -| One model action | `[B, 1, 32, 32, 64]` latent before 2x2 patchification | -| DiT token stream | 512 tokens per action, width 2048 | -| Public control | 256-way multi-hot buttons, `mouse_dx`, `mouse_dy`, ternary wheel | -| Model result | `[4, 3, 512, 1024]` detached TCHW float in `[-1, 1]` | - -The upstream runtime can resize a 1280x720 client image to the model's -1024x512 codec canvas and resize decoded frames back to 1280x720. FlashDreams -deliberately exposes the native 1024x512 canvas and performs no post-generation -spatial resample. - -## Cache policy - -Each action has 512 spatial tokens. Most transformer blocks retain the latest -16 actions densely. Global blocks occur at indices 3, 7, 11, 15, 19, and 23; -they span 128 actions but pin only every eighth historical action plus the -current action. On CUDA, fixed-capacity ring storage and a compiled -`FlexAttention` block mask avoid reallocating or concatenating history. CPU -tests use a compact dictionary representation as the readable reference. - -During denoising the cache is frozen: repeated evaluations replace a tail slot -without advancing history. During seed establishment and `finalize`, it is -unfrozen so the clean action is committed exactly once. - -## Control files - -Pass a JSON action timeline with `--controls-file`. Every field within an -action is optional: - -```json -{ - "schema_version": 1, - "actions": [ - {"buttons": [32], "mouse_dx": 0.1, "mouse_dy": 0.0, "scroll_wheel": 0}, - {}, - {"buttons": [1, 32]} - ] -} -``` - -`buttons` must contain IDs in `[0, 256)`; mouse values must be finite; wheel -is `-1`, `0`, or `1`. See -[ADR-1](../../integrations_v2/waypoint/ADR-1-control-events.md) for browser-event -mapping and reset/focus semantics. - -## Review boundaries and known limitations - -- One V2 application may create multiple sessions, but shared model execution - is serialized. The current WebRTC server accepts one browser client. -- Reset is deterministic for a fixed seed and control sequence and rebuilds the - cache without temporarily retaining two full GPU caches. -- The session advertises 60 FPS for playback pacing. Generation throughput is - hardware- and stack-dependent; it is not guaranteed to sustain that rate. -- The integration preserves upstream model behavior, including possible - long-rollout drift, unstable geometry, inconsistent objects, and implausible - motion. It is not a physically accurate or safety-critical simulator. -- A generic action-to-video application and shared control-help overlay remain - follow-up work. Cam2V is an organizational precedent, not a compatible - control contract for Waypoint. - -## Validation - -CPU tests cover checkpoint contracts, controls, lifecycle, reset, detached -outputs, MP4 frame accounting, and session isolation. CUDA tests cover local -and global fixed-cache attention through ring wraparound. Published-weight -validation covers official-reference parity, 15 distinct scenes at 40 actions, -one 118-action rollout, native-resolution MP4 output, and steady-state -performance on an RTX PRO 6000 Blackwell. - -See [the V2 validation record](../../integrations_v2/waypoint/VALIDATION.md) for -the exact revisions, hashes, commands, metrics, and acceptance evidence. diff --git a/integrations/waypoint/pyproject.toml b/integrations/waypoint/pyproject.toml deleted file mode 100644 index 52a87156f..000000000 --- a/integrations/waypoint/pyproject.toml +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[build-system] -requires = ["setuptools>=69", "wheel"] -build-backend = "setuptools.build_meta" - -[project] -name = "flashdreams-waypoint" -version = "0.1.0" -description = "FlashDreams integration for the Waypoint 1.5 world-model checkpoint." -readme = "README.md" -requires-python = ">=3.10" -dependencies = ["flashdreams"] - -[tool.uv.sources] -flashdreams = { workspace = true } - -[project.optional-dependencies] -dev = ["pytest>=8.0"] - -[tool.setuptools.packages.find] -include = ["waypoint*"] -exclude = ["tests"] - -[tool.pytest.ini_options] -addopts = "--import-mode=importlib -p flashdreams._pytest_plugins.marker_enforcement" -markers = [ - "ci_cpu: CPU-safe test, runs on the CPU CI runner", - "ci_gpu: requires GPU or libGL (cv2), runs on the GPU CI runner", - "manual: heavy or environment-specific test, opt-in only", -] diff --git a/integrations_v2/README.md b/integrations_v2/README.md index f01ea28cc..682a48e16 100644 --- a/integrations_v2/README.md +++ b/integrations_v2/README.md @@ -22,8 +22,8 @@ follows is already done for you. shared interactive camera-to-video application. - `hy_worldplay` — the HY-WorldPlay model and its `cam2v-hy-worldplay` binding, including live PRoPE/action camera-history adaptation. -- `waypoint` — the interactive Waypoint 1.5 image-established application - with deterministic control replay and live keyboard/mouse input. +- `waypoint` — the Waypoint 1.5 model and its `cam2v-waypoint` binding, with + deterministic control replay, live keyboard/mouse input, HUD, and reset. - `self_forcing`, `causal_forcing`, `fastvideo_causal_wan22`, `wan21`, `cosmos_predict2`, and `wan22` — model implementations with T2V adapters over the reusable `apps/t2v` package. diff --git a/integrations_v2/waypoint/ADR-1-control-events.md b/integrations_v2/waypoint/ADR-1-control-events.md index 6c772f784..3a608b5b4 100644 --- a/integrations_v2/waypoint/ADR-1-control-events.md +++ b/integrations_v2/waypoint/ADR-1-control-events.md @@ -45,10 +45,12 @@ Events are processed in V2 timestamp order. Edge state is persistent; mouse and wheel accumulators are per generated action. A reset inside a batch clears both persistent and accumulated state before later events in that batch are applied. -File-driven mode has complete precedence over live input: keyboard and mouse -events do not alter a controls-file rollout. Reset and close remain runtime -lifecycle events in either mode. Live mode samples one coalesced control for -every model-loop action. +File-driven mode has complete precedence over live movement input: keyboard and +mouse events do not alter a controls-file action. Pressing ``R`` is a model-owned +restart command in either mode; it rebuilds the cache, restores the seeded RNG, +and emits the starting image without forwarding ``R`` as a model button. Runtime +reset and close lifecycle events remain supported. Live mode samples one +coalesced control for every model-loop action. ## Reuse review @@ -60,10 +62,10 @@ semantic interpretation. The older `flashdreams.runtime.mapping.InputMapping` protocol belongs to the legacy runtime's canonical/inference-input schema and is not compatible with -the V2 loop contract. The shared `apps/cam2v` package has useful application, -session, UI, and reset organization, but its keyboard resampler produces camera -trajectories and does not handle Waypoint's button vocabulary, mouse motion, or -wheel channel. Neither is a dependency for this integration. +the V2 loop contract. The shared `apps/cam2v` model loop produces camera +trajectories and cannot represent Waypoint's button vocabulary, mouse motion, +or wheel channel. Waypoint therefore retains its action adapter while reusing +the Cam2V UI loop and parameterized control HUD. `WaypointControlEventAdapter` is intentionally named as an integration-specific adapter even though some of its mechanics can later be shared. A reusable @@ -79,9 +81,9 @@ action-to-video design should keep three boundaries: object to checkpoint-specific tensors (`WaypointControlEncoder`). A future generic action-to-video app can also own seed/replay/live-mode -selection, reset wiring, and a control-help overlay while accepting the mapper -and model pipeline as integration-provided strategies. This PR does not add -that public protocol: with only one direct consumer, doing so would make -Waypoint's Windows virtual-key and pixel-delta policies accidental framework +selection and reset wiring while accepting the mapper and model pipeline as +integration-provided strategies. This integration does not add that public +protocol: with only one direct consumer, doing so would make Waypoint's Windows +virtual-key and pixel-delta policies accidental framework contracts. The extraction should be validated jointly with the next compatible interactive world model. diff --git a/integrations_v2/waypoint/README.md b/integrations_v2/waypoint/README.md index 00703b9a5..c0a2c6df0 100644 --- a/integrations_v2/waypoint/README.md +++ b/integrations_v2/waypoint/README.md @@ -3,63 +3,375 @@ SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# Waypoint 1.5 V2 application +# Waypoint 1.5 architecture -This package adapts the independently authored `flashdreams-waypoint` model -package to FlashDreams' V2 application, session, model-loop, event, and output -APIs. Model modules are loaded once per application; the image-established -transformer/decoder cache, RNG stream, and live controls are isolated per -session. +This package is the model layer for the published +[Overworld/Waypoint-1.5-1B](https://huggingface.co/Overworld/Waypoint-1.5-1B) +checkpoint. It implements checkpoint loading, controls, the autoregressive +diffusion pipeline, sparse K/V history, the shared TAEHV codec, and the +[Cam2V application binding](apps/cam2v/README.md). Model implementation lives +under `impl/`; CLI, session, input, and presentation code lives under +`apps/cam2v/`. -## Run Waypoint +This document is the concise design-review reference. Exact validation commands, +hardware, parity metrics, and long-rollout evidence are recorded in +[VALIDATION.md](VALIDATION.md). -Write a deterministic example rollout and its step metrics: +## Scope and decisions -```bash -flashdreams-run-v2 waypoint-1-5-1b \ - --output-path waypoint.mp4 --stats-path waypoint.metrics.json \ - -- --example-data --actions 40 --seed 464 --profile +| Area | Design | +|---|---| +| Model | Dense BF16 autoregressive DiT, 24 blocks, width 2048, 32 query heads, 16 K/V heads | +| Parameter accounting | Upstream model card: 1.2B; pinned BF16 checkpoint: 1,860,823,096 serialized tensor elements | +| Inputs | Four-frame image seed plus one keyboard/mouse/wheel control per action | +| Output | One 32-channel latent and four RGB frames per action | +| Denoising | Fixed four-step rectified-flow Euler schedule: 1.0, 0.9, 0.75, 0.3, 0.0 | +| History | Dense 16-action local window; sparse 128-action global horizon in every fourth block | +| Context | 128 latent actions / 512 presented RGB frames on global-attention blocks | +| Presentation | Native TAEHV canvas, 1024x512 TCHW in [-1, 1], four frames per result | +| State | Model weights shared by the application; cache, RNG, controls, and seed isolated per session | +| Concurrency | An application lock serializes shared model/RNG work; output tensors leave the model loop detached | + +The published config has `prompt_conditioning: null`; this integration therefore +does not load a text encoder or expose a prompt. The generic upstream +`WorldEngine` has a `set_prompt` method for other configurations, but for this +checkpoint it does not instantiate a prompt encoder and `set_prompt` raises. +The pinned checkpoint also has no prompt or cross-attention tensor keys. A +prompt argument would therefore be ignored or rejected rather than condition +generation. The integration also does not currently implement the upstream +360P checkpoint, quantization, or multi-GPU execution. + +## Use cases and modalities + +```mermaid +flowchart LR + user([Interactive user]) + replay([Replay, test, or benchmark]) + seed[RGB or RGBA seed image] + live[Keyboard, mouse buttons,
relative motion, wheel] + file[Versioned JSON controls] + app((cam2v-waypoint)) + model[Waypoint 1.5 DiT + TAEHV] + browser[Live WebRTC video] + mp4[Deterministic MP4] + metrics[Per-action metrics JSON] + prompt[Text prompt
not supported] + + user --> seed + user --> live + replay --> seed + replay --> file + seed --> app + live --> app + file --> app + prompt -. excluded by checkpoint .-> app + app --> model + model --> browser + model --> mp4 + model --> metrics ``` -Use a local image and a controls JSON file in the same finite MP4 mode: +The seed establishes visual state; it is not a continuing image-conditioning +stream. File controls select finite, reproducible MP4/metrics runs. Omitting a +control file selects an open-ended live session driven by V2 browser events. + +## Component and file view + +```mermaid +flowchart TB + subgraph core[flashdreams core and V2 runtime] + registry[application_registry.py
slug discovery] + runner[application_runner.py / session_runner.py] + api[IApplication / ISession / IModelLoop] + input[UserInputEvents / EventBuffer
transport and reset lifecycle] + sinks[WebRTCClientWindow / Mp4ClientWindow
MetricsOutputSink] + base[StreamInferencePipeline
DiffusionModel / Transformer] + taehv[recipes/taehv
Hy15TAEHVEncoder and Decoder] + end -```bash -flashdreams-run-v2 waypoint-1-5-1b --output-path waypoint.mp4 \ - -- --seed-image seed.png --controls-file controls.json --seed 464 + subgraph adapter[integrations_v2/waypoint/apps/cam2v] + app[adapter.py
WaypointCam2VApplication] + session[session.py
WaypointSession and ModelLoop] + events[control_events.py
WaypointControlEventAdapter] + hud[shared Cam2V UI loop
model-specific control groups] + end + + subgraph waypoint[integrations_v2/waypoint model] + config[config.py + impl/spec.py
checkpoint contract] + pipeline[impl/pipeline.py
WaypointInferencePipeline] + controls[impl/controls.py / encoder.py
WaypointControl] + transformer[impl/transformer/impl.py
WaypointTransformer] + network[impl/transformer/network.py
WaypointDiT] + cache[impl/transformer/cache.py
WaypointKVCache] + scheduler[impl/scheduler.py
fixed Euler schedule] + codec[impl/decoder.py
WaypointTAEHVDecoder] + checkpoint[impl/checkpoint.py
strict state-dict mapping] + end + + registry --> app + runner --> app + api -. implemented by .-> app + api -. implemented by .-> session + runner --> session + runner --> input + runner --> sinks + app --> session + app --> pipeline + session --> events + session --> hud + session --> pipeline + input --> events + events --> controls + pipeline --> controls + pipeline --> transformer + pipeline --> scheduler + pipeline --> codec + transformer --> network + network --> cache + transformer --> checkpoint + config --> pipeline + base -. specialized by .-> pipeline + base -. specialized by .-> transformer + taehv -. reused by .-> pipeline + taehv -. specialized by .-> codec ``` -For live keyboard and mouse input, use the browser window and omit a controls -file: +The package boundary is internal to one V2 integration: reusable model code +stays under `impl/` and imports no V2 runtime classes, while `apps/cam2v/` owns +the application, session, input mapping, and shared HUD configuration. + +## V2 reuse and action-mapping boundary + +The V2 runtime already provides device-neutral, timestamped input events; +event fan-out to model and UI loops; reset/focus lifecycle; WebRTC transport; +and generic presentation, MP4, and metrics sinks. It deliberately delivers raw +events to each model loop. It does not currently provide an action-to-video +application, a stateful action coalescer, or a protocol from user events to a +model's control object. + +`WaypointControlEventAdapter` is therefore integration-owned. It contains two +seams that a later shared action-to-video package can separate: -```bash -flashdreams-run-v2 waypoint-1-5-1b --mode webrtc \ - -- --seed-image seed.png --seed 464 +1. A model-neutral accumulator can retain held key names and mouse-button + indices, convert absolute pointer positions to relative normalized motion, + collect wheel deltas, and clear state on focus loss or reset. +2. A model-owned mapper can convert that snapshot to Waypoint's Windows + virtual-key vocabulary, apply canvas and sensitivity scaling, reduce the + wheel to a ternary value, and construct `WaypointControl`. The existing + `WaypointControlEncoder` remains responsible for control-to-tensor encoding. + +The legacy `flashdreams.runtime.mapping.InputMapping` API is not a V2 +dependency and should not be revived inside this integration. The shared +`apps/cam2v` HUD and presentation loop are reusable, but its keyboard-to-camera +trajectory resampling is a different semantic contract. A future +`apps/action2v`-style model loop should be extracted when a second interactive +world-model integration can validate the common snapshot, mapper, and replay +contracts. Until then, keeping the action adapter explicitly named and scoped +to Waypoint avoids freezing its virtual-key assumptions into V2. + +## Class and ownership view + +```mermaid +classDiagram + direction LR + + class IApplication + class ISession + class IModelLoop + class StreamInferencePipeline + class Transformer + class StreamingEncoder + class Hy15TAEHVDecoder + + class WaypointCam2VApplication { + -pipeline + -pipeline_lock + +init(args) + +session_desc() + +create_session(desc) + } + class WaypointSession { + -seed_frames + -controls + -state + +init() + +close() + } + class WaypointModelLoop { + +step(index, events) + +reset() + +close() + } + class WaypointModelState { + +cache + +rng_state + +control_events + +controls_generated + } + class WaypointInferencePipeline { + +initialize_cache(seed_pixels) + +generate(index, cache, control) + +finalize(index, cache) + } + class WaypointControlEncoder + class WaypointTransformer + class WaypointDiT + class WaypointKVCache + class WaypointTAEHVDecoder + + IApplication <|-- WaypointCam2VApplication + ISession <|-- WaypointSession + IModelLoop <|-- WaypointModelLoop + StreamInferencePipeline <|-- WaypointInferencePipeline + StreamingEncoder <|-- WaypointControlEncoder + Transformer <|-- WaypointTransformer + Hy15TAEHVDecoder <|-- WaypointTAEHVDecoder + + WaypointCam2VApplication o-- WaypointInferencePipeline : shares weights + WaypointCam2VApplication --> WaypointSession : creates + WaypointSession *-- WaypointModelState : owns + WaypointSession --> WaypointModelLoop : registers + WaypointModelLoop --> WaypointInferencePipeline : drives + WaypointInferencePipeline *-- WaypointControlEncoder + WaypointInferencePipeline *-- WaypointTransformer + WaypointInferencePipeline *-- WaypointTAEHVDecoder + WaypointTransformer *-- WaypointDiT + WaypointDiT --> WaypointKVCache : updates ``` -Arguments after `--` belong to Waypoint. Run -`flashdreams-run-v2 waypoint-1-5-1b -- --help` for the complete list. The first -run downloads the public Waypoint 1.5 1B and TAEHV checkpoints. - -The application declares four-frame `TCHW` results on Waypoint's native -1024x512 canvas at 60 FPS playback. Seed images are resized once to that native -canvas; generated frames are presented without another spatial resample. File -controls use blocking, new-results-only presentation so MP4 output retains -every generated frame. - -Sessions created by one application intentionally start from the configured -seed, which makes the same control sequence reproducible across reset and -session recreation. Model modules are shared, and model/RNG work is serialized -by an application lock while each session restores its own RNG and cache state. -The current WebRTC runtime accepts one browser client per server process. - -See [ADR-1-control-events.md](ADR-1-control-events.md) for the live keyboard and -mouse contract. Passing `--controls-file` selects finite deterministic mode; -omitting it selects live input. `--example-data` uses the pinned public seed -and bundled 118-action timeline. See [VALIDATION.md](VALIDATION.md) for CPU, -CUDA, real-checkpoint, MP4, performance, and official-reference parity evidence. - -V2 supplies raw input transport, loop/reset lifecycle, WebRTC, and output -sinks, but it has no generic action-to-video application or event-to-action -mapping protocol today. The Waypoint-specific adapter remains local to this -package; ADR-1 records the reviewed extraction boundary for a future shared -action snapshot, mapper protocol, and control-help UI. +The application owns expensive immutable modules. A session owns all mutable +rollout state, including the transformer K/V cache, TAEHV state, seed tensors, +RNG state, and control adapter. `StepResult` tensors are detached before +publication so runtime queues and encoders do not retain model autograd state. + +## Initialization and action sequence + +```mermaid +sequenceDiagram + actor Client + participant Runtime as V2 runtime + participant App as WaypointCam2VApplication + participant Session as WaypointSession + participant ModelLoop as WaypointModelLoop + participant Pipeline as WaypointInferencePipeline + participant DiT as WaypointTransformer / DiT + participant Cache as K/V + TAEHV caches + participant Sink as WebRTC or MP4 sink + + Client->>Runtime: slug, runtime args, Waypoint args + Runtime->>App: init(args) + Runtime->>App: create_session(session_desc) + App->>App: lazily load one BF16 model + App-->>Runtime: new session with shared model + lock + Runtime->>Session: init() + Session->>Pipeline: initialize_cache(four seed frames) + Pipeline->>Cache: TAEHV encode and prime decoder state + Pipeline->>DiT: sigma-zero seed flow + DiT->>Cache: commit action 0 K/V + + alt step 0 + Runtime->>ModelLoop: step(0, events) + ModelLoop-->>Runtime: detached seed StepResult (4 frames) + else generated action N + Runtime->>ModelLoop: step(N, ordered input events) + ModelLoop->>Pipeline: generate(N, cache, control) + loop sigmas 1.0 to 0.3 + Pipeline->>DiT: predict provisional flow + DiT->>Cache: replace provisional action-N K/V + end + Pipeline->>Cache: decode clean latent to 4 RGB frames + ModelLoop->>Pipeline: finalize(N, cache) + Pipeline->>DiT: sigma-zero clean-state evaluation + DiT->>Cache: commit action-N K/V + ModelLoop-->>Runtime: detached TCHW StepResult + end + + Runtime->>Sink: present or encode four frames + opt browser reset or R key + Client->>Runtime: reset event or R key press + Runtime->>ModelLoop: reset lifecycle or step events + ModelLoop->>Pipeline: rebuild cache from retained seed + ModelLoop-->>Runtime: starting-image StepResult + end +``` + +The four denoise evaluations may overwrite only the current provisional slot. +`finalize` performs the separate sigma-zero evaluation that commits clean K/V +state for the next action. Skipping or reordering that transition changes the +autoregressive world state. + +## Tensor and control contracts + +| Boundary | Contract | +|---|---| +| Display seed | `[4, 3, 512, 1024]` TCHW float, normalized to `[-1, 1]` | +| Codec seed | `[B, 4, 3, 512, 1024]` float in `[0, 1]` | +| One model action | `[B, 1, 32, 32, 64]` latent before 2x2 patchification | +| DiT token stream | 512 tokens per action, width 2048 | +| Public control | 256-way multi-hot buttons, `mouse_dx`, `mouse_dy`, ternary wheel | +| Model result | `[4, 3, 512, 1024]` detached TCHW float in `[-1, 1]` | + +The upstream runtime can resize a 1280x720 client image to the model's +1024x512 codec canvas and resize decoded frames back to 1280x720. FlashDreams +deliberately exposes the native 1024x512 canvas and performs no post-generation +spatial resample. + +## Cache policy + +Each action has 512 spatial tokens. Most transformer blocks retain the latest +16 actions densely. Global blocks occur at indices 3, 7, 11, 15, 19, and 23; +they span 128 actions but pin only every eighth historical action plus the +current action. On CUDA, fixed-capacity ring storage and a compiled +`FlexAttention` block mask avoid reallocating or concatenating history. CPU +tests use a compact dictionary representation as the readable reference. + +During denoising the cache is frozen: repeated evaluations replace a tail slot +without advancing history. During seed establishment and `finalize`, it is +unfrozen so the clean action is committed exactly once. + +## Control files + +Pass a JSON action timeline with `--controls-file`. Every field within an +action is optional: + +```json +{ + "schema_version": 1, + "actions": [ + {"buttons": [32], "mouse_dx": 0.1, "mouse_dy": 0.0, "scroll_wheel": 0}, + {}, + {"buttons": [1, 32]} + ] +} +``` + +`buttons` must contain IDs in `[0, 256)`; mouse values must be finite; wheel +is `-1`, `0`, or `1`. See +[ADR-1](ADR-1-control-events.md) for browser-event +mapping and reset/focus semantics. + +## Review boundaries and known limitations + +- One V2 application may create multiple sessions, but shared model execution + is serialized. The current WebRTC server accepts one browser client. +- Reset is deterministic for a fixed seed and control sequence and rebuilds the + cache without temporarily retaining two full GPU caches. +- The session advertises 60 FPS for playback pacing. Generation throughput is + hardware- and stack-dependent; it is not guaranteed to sustain that rate. +- The integration preserves upstream model behavior, including possible + long-rollout drift, unstable geometry, inconsistent objects, and implausible + motion. It is not a physically accurate or safety-critical simulator. +- The shared Cam2V HUD and presentation loop are compatible with Waypoint; + Cam2V's pose-matrix model loop is not, so raw action mapping remains local. + +## Validation + +CPU tests cover checkpoint contracts, controls, lifecycle, reset, detached +outputs, MP4 frame accounting, and session isolation. CUDA tests cover local +and global fixed-cache attention through ring wraparound. Published-weight +validation covers official-reference parity, 15 distinct scenes at 40 actions, +one 118-action rollout, native-resolution MP4 output, and steady-state +performance on an RTX PRO 6000 Blackwell. + +See [the V2 validation record](VALIDATION.md) for +the exact revisions, hashes, commands, metrics, and acceptance evidence. diff --git a/integrations_v2/waypoint/VALIDATION.md b/integrations_v2/waypoint/VALIDATION.md index ab92e707c..94d4ef266 100644 --- a/integrations_v2/waypoint/VALIDATION.md +++ b/integrations_v2/waypoint/VALIDATION.md @@ -9,6 +9,24 @@ Validated on 2026-08-25 against FlashDreams main `8fd97fa3`, source PR #464 `0f178234`, and the official `world_engine` implementation at `b3f1e725dedac17ccbfaf9ee37f5e068bb44bed4`. +## V2 layout migration gates + +The 2026-09-03 migration moved all Waypoint model and application code into +`integrations_v2/waypoint`, registered `cam2v-waypoint`, and removed the legacy +`integrations/waypoint` package. The migration gates passed: + +- Ruff format and lint checks over Waypoint and the shared Cam2V HUD. +- 68 CPU tests covering model contracts, V2 sessions, HUD controls, deterministic + `R` reset, MP4 frame accounting, and the absence of the legacy directory. +- ty checks over Waypoint and Cam2V using the repository dependency environment. +- Wheel build and inspection, including the application entry point and bundled + 118-action control timeline. +- Workspace lockfile validation with a single `flashdreams-waypoint` package. + +The CUDA and published-weight measurements below are preserved from the original +model integration and V2 runtime validation; the migration changes package and +UI boundaries, not model math. + ## Artifacts and hardware - GPU: NVIDIA RTX PRO 6000 Blackwell Workstation Edition (96 GiB), driver @@ -30,13 +48,13 @@ and long-rollout outputs are under its `additional-inference/` directory. ## Automated gates - Ruff 0.12.7 check and format: passed. -- ty 0.0.53 for both integration packages: passed. +- ty 0.0.53 for the former split model and application packages: passed. - CPU model, TAEHV, V2 lifecycle, reset, input, and MP4 tests: 51 passed and 7 deselected. - CUDA fixed-cache FlexAttention equivalence, local and global layers over eight autoregressive frames: 2 passed. -- Built-wheel entry-point discovery: `waypoint-1-5-1b` resolves to - `WaypointApplication`; all 118 bundled actions are available. +- Built-wheel entry-point discovery: `cam2v-waypoint` resolves to + `WaypointCam2VApplication`; all 118 bundled actions are available. ## Real V2 inference @@ -84,7 +102,7 @@ exactly 164 frames. Final review gates also passed the complete pre-commit suite (Ruff check/fix, Ruff format, lockfile/version checks, and `ty`), 52 focused CPU tests across -TAEHV and both Waypoint packages, and the two CUDA fixed-cache tests. The broad +TAEHV and the former split Waypoint packages, and the two CUDA fixed-cache tests. The broad CPU run passed 1,584 tests with 2 skipped and 337 deselected after excluding only `integrations/omnidreams/tests/interactive_drive`; collecting that unrelated directory requires the optional `pyvirtualdisplay` and diff --git a/integrations/waypoint/waypoint/__init__.py b/integrations_v2/waypoint/__init__.py similarity index 84% rename from integrations/waypoint/waypoint/__init__.py rename to integrations_v2/waypoint/__init__.py index 590865688..d30698301 100644 --- a/integrations/waypoint/waypoint/__init__.py +++ b/integrations_v2/waypoint/__init__.py @@ -15,13 +15,13 @@ """Waypoint 1.5 integration contracts.""" -from waypoint.controls import ( +from waypoint.impl.controls import ( WaypointControl, load_controls_from_file, make_control_context, ) -from waypoint.encoder import WaypointControlEncoder, WaypointControlEncoderConfig -from waypoint.spec import WAYPOINT_1_5, WaypointModelSpec +from waypoint.impl.encoder import WaypointControlEncoder, WaypointControlEncoderConfig +from waypoint.impl.spec import WAYPOINT_1_5, WaypointModelSpec __all__ = [ "WAYPOINT_1_5", diff --git a/integrations_v2/waypoint/apps/__init__.py b/integrations_v2/waypoint/apps/__init__.py new file mode 100644 index 000000000..ad8f61817 --- /dev/null +++ b/integrations_v2/waypoint/apps/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Application bindings for Waypoint.""" diff --git a/integrations_v2/waypoint/apps/cam2v/README.md b/integrations_v2/waypoint/apps/cam2v/README.md new file mode 100644 index 000000000..4efae7822 --- /dev/null +++ b/integrations_v2/waypoint/apps/cam2v/README.md @@ -0,0 +1,59 @@ + + +# Waypoint Cam2V + +Install the Waypoint integration and launch its `cam2v-waypoint` application: + +```bash +uv sync --package flashdreams-waypoint --inexact +uv run --no-sync flashdreams-run-v2 cam2v-waypoint --mode webrtc \ + --host 0.0.0.0 --port 8089 -- --example-data +``` + +Use a local starting image with live controls: + +```bash +uv run --no-sync flashdreams-run-v2 cam2v-waypoint --mode webrtc \ + --host 0.0.0.0 --port 8089 -- \ + --image-path seed.png --seed 464 +``` + +## Controls + +| Input | Action | +| --- | --- | +| `W` / `A` / `S` / `D` | Move | +| Mouse | Look around | +| `Shift` | Sprint | +| `Space` | Jump or context action | +| `R` | Reset the rollout to the starting image | + +The on-screen HUD lists these controls and wraps held keys in brackets. Losing +window or browser focus clears held controls. Pass `--no-ui` to disable the +HUD. + +For a deterministic file-driven MP4: + +```bash +uv run --no-sync flashdreams-run-v2 cam2v-waypoint \ + --presentation-mode on_demand \ + --output-path waypoint.mp4 --stats-path waypoint.metrics.json -- \ + --example-data --actions 40 --seed 464 --profile --no-ui +``` + +The application emits four seed frames followed by four frames per action at +Waypoint's native 1024 x 512 resolution. `--seed-image` remains an alias for +`--image-path`. Run `flashdreams-run-v2 cam2v-waypoint -- --help` for all +application arguments. + +## Tests + +```bash +uv sync --package flashdreams-waypoint --group test --inexact +uv run --no-sync pytest integrations_v2/waypoint/tests -m ci_cpu +``` + +Run `-m ci_gpu` instead for the CUDA attention and cache tests. diff --git a/integrations_v2/waypoint/apps/cam2v/__init__.py b/integrations_v2/waypoint/apps/cam2v/__init__.py new file mode 100644 index 000000000..fed0ec31b --- /dev/null +++ b/integrations_v2/waypoint/apps/cam2v/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Cam2V application binding for Waypoint.""" diff --git a/integrations_v2/waypoint/waypoint_v2/app.py b/integrations_v2/waypoint/apps/cam2v/adapter.py similarity index 89% rename from integrations_v2/waypoint/waypoint_v2/app.py rename to integrations_v2/waypoint/apps/cam2v/adapter.py index 401447f56..78441aa2e 100644 --- a/integrations_v2/waypoint/waypoint_v2/app.py +++ b/integrations_v2/waypoint/apps/cam2v/adapter.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Waypoint 1.5 V2 application.""" +"""Waypoint 1.5 Cam2V application adapter.""" from __future__ import annotations @@ -17,9 +17,6 @@ import torch from loguru import logger from torch import Tensor -from waypoint import WaypointControl, load_controls_from_file -from waypoint.config import PIPELINE_WAYPOINT_1_5 -from waypoint.pipeline import WaypointInferencePipeline from flashdreams.api_v2.application import IApplication from flashdreams.core.io.disk import default_flashdreams_cache_dir @@ -31,7 +28,10 @@ SessionDesc, ) from flashdreams.runtime_v2.video_tensor import VideoTensorLayout -from waypoint_v2.session import WaypointSession +from waypoint import WaypointControl, load_controls_from_file +from waypoint.apps.cam2v.session import WaypointSession +from waypoint.config import PIPELINE_WAYPOINT_1_5 +from waypoint.impl.pipeline import WaypointInferencePipeline _OUTPUT_WIDTH = 1024 _OUTPUT_HEIGHT = 512 @@ -56,9 +56,10 @@ class _ApplicationConfig: device: torch.device profile: bool mouse_sensitivity: float + use_ui: bool -class WaypointApplication(IApplication): +class WaypointCam2VApplication(IApplication): """Load one Waypoint model and create isolated image-established sessions.""" def __init__( @@ -92,10 +93,15 @@ def init(self, commandline_args: Sequence[str]) -> None: ValueError: Inputs do not describe a valid file or live rollout. """ parser = argparse.ArgumentParser( - prog="flashdreams-run-v2 waypoint-1-5-1b --", + prog="flashdreams-run-v2 cam2v-waypoint --", description="Run Waypoint 1.5 from an image using file or live controls.", ) - parser.add_argument("--seed-image", type=Path) + parser.add_argument( + "--image-path", + "--seed-image", + dest="seed_image", + type=Path, + ) parser.add_argument("--example-data", action="store_true") parser.add_argument("--controls-file", type=Path) parser.add_argument( @@ -107,10 +113,16 @@ def init(self, commandline_args: Sequence[str]) -> None: parser.add_argument("--device", default="cuda") parser.add_argument("--profile", action="store_true") parser.add_argument("--mouse-sensitivity", type=float, default=1.0) + parser.add_argument( + "--ui", + action=argparse.BooleanOptionalAction, + default=True, + help="Show the interactive control overlay (default: enabled).", + ) args = parser.parse_args(list(commandline_args)) if args.seed_image is None and not args.example_data: - raise ValueError("pass --seed-image or --example-data") + raise ValueError("pass --image-path/--seed-image or --example-data") if args.seed is not None and args.seed < 0: raise ValueError(f"--seed must be non-negative, got {args.seed}") if not math.isfinite(args.mouse_sensitivity) or args.mouse_sensitivity < 0: @@ -135,7 +147,7 @@ def init(self, commandline_args: Sequence[str]) -> None: seed = secrets.randbits(63) if args.seed is None else args.seed if args.seed is None: - logger.info(f"[waypoint-1.5-1b] generated seed {seed}") + logger.info(f"[cam2v-waypoint] generated seed {seed}") self._config = _ApplicationConfig( seed_image=args.seed_image, controls=controls, @@ -143,6 +155,7 @@ def init(self, commandline_args: Sequence[str]) -> None: device=torch.device(args.device), profile=args.profile, mouse_sensitivity=args.mouse_sensitivity, + use_ui=args.ui, ) def session_desc(self) -> SessionDesc: @@ -150,7 +163,7 @@ def session_desc(self) -> SessionDesc: return SessionDesc( output_layout=VideoTensorLayout.tchw, backpressure_mode=BackpressureMode.BLOCK, - presentation_mode=PresentationMode.ON_DEMAND, + presentation_mode=PresentationMode.CONTINUOUS, frames_per_second_for_ui=_PLAYBACK_FPS, frames_per_second_for_step=_PLAYBACK_FPS, video_width=_OUTPUT_WIDTH, @@ -188,6 +201,7 @@ def create_session(self, session_desc: SessionDesc) -> WaypointSession: seed=config.seed, controls=config.controls, mouse_sensitivity=config.mouse_sensitivity, + use_ui=config.use_ui, ) def close(self) -> None: @@ -196,7 +210,7 @@ def close(self) -> None: def _require_config(self) -> _ApplicationConfig: if self._config is None: - raise RuntimeError("WaypointApplication.init() must run first") + raise RuntimeError("WaypointCam2VApplication.init() must run first") return self._config def _ensure_pipeline(self, config: _ApplicationConfig) -> WaypointInferencePipeline: @@ -284,7 +298,7 @@ def _download_example_image() -> Path: def create_app() -> IApplication: """Return a new lazy Waypoint 1.5 V2 application.""" - return WaypointApplication() + return WaypointCam2VApplication() -__all__ = ["WaypointApplication", "create_app", "load_seed_display_frames"] +__all__ = ["WaypointCam2VApplication", "create_app", "load_seed_display_frames"] diff --git a/integrations_v2/waypoint/waypoint_v2/assets/example_controls.json b/integrations_v2/waypoint/apps/cam2v/assets/example_controls.json similarity index 100% rename from integrations_v2/waypoint/waypoint_v2/assets/example_controls.json rename to integrations_v2/waypoint/apps/cam2v/assets/example_controls.json diff --git a/integrations_v2/waypoint/waypoint_v2/control_events.py b/integrations_v2/waypoint/apps/cam2v/control_events.py similarity index 100% rename from integrations_v2/waypoint/waypoint_v2/control_events.py rename to integrations_v2/waypoint/apps/cam2v/control_events.py diff --git a/integrations_v2/waypoint/waypoint_v2/session.py b/integrations_v2/waypoint/apps/cam2v/session.py similarity index 71% rename from integrations_v2/waypoint/waypoint_v2/session.py rename to integrations_v2/waypoint/apps/cam2v/session.py index 52db03b84..f45af99cf 100644 --- a/integrations_v2/waypoint/waypoint_v2/session.py +++ b/integrations_v2/waypoint/apps/cam2v/session.py @@ -7,20 +7,64 @@ import threading from dataclasses import dataclass +from functools import cached_property from typing import Any import torch +from cam2v import ( + Cam2VControlGroup, + Cam2VControlKey, + Cam2VSlangPyUILoop, + Cam2VUIState, +) from torch import Tensor -from waypoint import WAYPOINT_1_5, WaypointControl -from waypoint.pipeline import WaypointInferencePipeline from flashdreams.api_v2.loop import IModelLoop from flashdreams.api_v2.session import ISession +from flashdreams.runtime_v2.presentation_manager import PresentationManager from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult +from flashdreams.runtime_v2.user_input_event import ( + KeyboardInputState, + KeyboardUserInputEvent, +) from flashdreams.runtime_v2.user_input_events import UserInputEvents from flashdreams.runtime_v2.video_tensor import VideoTensorLayout -from waypoint_v2.control_events import WaypointControlEventAdapter +from waypoint import WAYPOINT_1_5, WaypointControl +from waypoint.apps.cam2v.control_events import WaypointControlEventAdapter +from waypoint.impl.pipeline import WaypointInferencePipeline + +_WAYPOINT_CONTROL_GROUPS = ( + Cam2VControlGroup( + action="Move", + keys=( + Cam2VControlKey("w", "W"), + Cam2VControlKey("a", "A"), + Cam2VControlKey("s", "S"), + Cam2VControlKey("d", "D"), + ), + ), + Cam2VControlGroup( + action="Sprint", + keys=(Cam2VControlKey("shift", "Shift"),), + ), + Cam2VControlGroup( + action="Jump / action", + keys=(Cam2VControlKey("space", "Space"),), + ), + Cam2VControlGroup( + action="Reset rollout", + keys=(Cam2VControlKey("r", "R"),), + ), +) +"""Keyboard controls displayed by the Waypoint Cam2V overlay.""" + +_WAYPOINT_UI_INSTRUCTIONS = ( + "Mouse: look around", + "Held controls are shown in brackets.", + "Click the video before using controls.", +) +"""Waypoint-specific control hints displayed below the HUD.""" @dataclass(slots=True) @@ -48,26 +92,13 @@ class WaypointModelLoop(IModelLoop[WaypointModelState]): def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: """Generate the seed result or one controlled autoregressive action.""" state = self.state - if step_index == 0: - if state.seed_emitted or state.controls_generated: - raise RuntimeError("Waypoint seed step is out of sequence") - state.seed_emitted = True - return [ - StepResult( - step_index=0, - output=state.seed_frames.detach(), - frame_count=WAYPOINT_1_5.frames_per_action, - output_layout=state.session_desc.output_layout, - metrics={"autoregressive_index": 0, "seed_frames": 4}, - ) - ] - - expected_index = state.controls_generated + 1 - if step_index != expected_index: - raise RuntimeError( - f"Waypoint action step is out of sequence: expected {expected_index}, " - f"got {step_index}" - ) + if _restart_requested(events): + self.reset() + return self._seed_result(step_index) + if not state.seed_emitted: + return self._seed_result(step_index) + + autoregressive_index = state.controls_generated + 1 control = self._control_for_step(events) cache = self._require_cache() @@ -76,14 +107,14 @@ def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: if rng is None: raise RuntimeError("Waypoint pipeline must have a deterministic seed") rng.set_state(state.rng_state) - video = state.pipeline.generate(step_index, cache, control) - stats = state.pipeline.finalize(step_index, cache) + video = state.pipeline.generate(autoregressive_index, cache, control) + stats = state.pipeline.finalize(autoregressive_index, cache) state.rng_state = rng.get_state() output = _presentation_frames(video, state.session_desc) state.controls_generated += 1 metrics: dict[str, float | int] = dict(stats or {}) - metrics.setdefault("autoregressive_index", step_index) + metrics.setdefault("autoregressive_index", autoregressive_index) metrics.setdefault("generated_frames", output.shape[0]) return [ StepResult( @@ -95,6 +126,18 @@ def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: ) ] + def _seed_result(self, step_index: int) -> list[StepResult]: + self.state.seed_emitted = True + return [ + StepResult( + step_index=step_index, + output=self.state.seed_frames.detach(), + frame_count=WAYPOINT_1_5.frames_per_action, + output_layout=self.state.session_desc.output_layout, + metrics={"autoregressive_index": 0, "seed_frames": 4}, + ) + ] + def is_finished(self) -> bool: """Finish after the seed and every file-driven control; live mode persists.""" controls = self.state.controls @@ -148,6 +191,7 @@ def __init__( seed: int, controls: tuple[WaypointControl, ...] | None, mouse_sensitivity: float, + use_ui: bool = True, ) -> None: """Create an uninitialized Waypoint session. @@ -160,6 +204,7 @@ def __init__( seed: Fixed per-session diffusion seed. controls: Finite file-driven actions, or ``None`` for live input. mouse_sensitivity: Multiplier used by the live input adapter. + use_ui: Whether to register the shared Cam2V control overlay. Raises: ValueError: The layout or seed-frame shape does not match the session. @@ -187,8 +232,14 @@ def __init__( self._seed = seed self._controls = controls self._mouse_sensitivity = mouse_sensitivity + self._use_ui = use_ui self._state: WaypointModelState | None = None + @cached_property + def _presentation_manager(self) -> PresentationManager: + """Return a frame manager initialized on the model device.""" + return PresentationManager(device=self._pipeline.device) + def init(self) -> None: """Move the seed to the model device, establish cache, and register the loop.""" dtype = self._pipeline.diffusion_model.dtype @@ -220,6 +271,22 @@ def init(self) -> None: rng_state=initial_rng_state.clone(), ) self._state = state + if self._use_ui: + self.register_ui_loop( + Cam2VSlangPyUILoop, + state=Cam2VUIState( + total_blocks=len(self._controls) + if self._controls is not None + else 1, + target_fps=self._session_desc.frames_per_second_for_step, + warmup_blocks=0, + control_groups=_WAYPOINT_CONTROL_GROUPS, + instructions=_WAYPOINT_UI_INSTRUCTIONS, + show_status=False, + ), + width=self._session_desc.video_width, + height=self._session_desc.video_height, + ) self.register_model_loop(WaypointModelLoop, state=state) @property @@ -234,6 +301,15 @@ def close(self) -> None: self._state.control_events.reset() +def _restart_requested(events: UserInputEvents) -> bool: + return any( + isinstance(event, KeyboardUserInputEvent) + and event.state is KeyboardInputState.PRESSED + and event.key.strip().lower() == "r" + for event in events.get_events() + ) + + def _seed_pixels(seed_frames: Tensor) -> Tensor: return seed_frames.add(1.0).mul(0.5).unsqueeze(0) diff --git a/integrations/waypoint/waypoint/config.py b/integrations_v2/waypoint/config.py similarity index 79% rename from integrations/waypoint/waypoint/config.py rename to integrations_v2/waypoint/config.py index a9404725b..4518221d8 100644 --- a/integrations/waypoint/waypoint/config.py +++ b/integrations_v2/waypoint/config.py @@ -7,12 +7,12 @@ from flashdreams.infra.diffusion.model import DiffusionModelConfig from flashdreams.infra.pipeline import StreamInferencePipelineConfig -from waypoint.decoder import WaypointTAEHVDecoderConfig -from waypoint.encoder import WaypointControlEncoderConfig -from waypoint.pipeline import WaypointInferencePipelineConfig -from waypoint.scheduler import WaypointEulerSchedulerConfig -from waypoint.spec import WAYPOINT_1_5 -from waypoint.transformer import WaypointTransformerConfig +from waypoint.impl.decoder import WaypointTAEHVDecoderConfig +from waypoint.impl.encoder import WaypointControlEncoderConfig +from waypoint.impl.pipeline import WaypointInferencePipelineConfig +from waypoint.impl.scheduler import WaypointEulerSchedulerConfig +from waypoint.impl.spec import WAYPOINT_1_5 +from waypoint.impl.transformer import WaypointTransformerConfig WAYPOINT_1_5_CHECKPOINT = ( "https://huggingface.co/Overworld/Waypoint-1.5-1B/resolve/main/model.safetensors" diff --git a/integrations_v2/waypoint/impl/__init__.py b/integrations_v2/waypoint/impl/__init__.py new file mode 100644 index 000000000..d912c5562 --- /dev/null +++ b/integrations_v2/waypoint/impl/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Waypoint model implementation.""" diff --git a/integrations/waypoint/waypoint/checkpoint.py b/integrations_v2/waypoint/impl/checkpoint.py similarity index 99% rename from integrations/waypoint/waypoint/checkpoint.py rename to integrations_v2/waypoint/impl/checkpoint.py index 674ebfad9..8cef57ce8 100644 --- a/integrations/waypoint/waypoint/checkpoint.py +++ b/integrations_v2/waypoint/impl/checkpoint.py @@ -22,7 +22,7 @@ import torch from torch import nn -from waypoint.spec import WAYPOINT_1_5, WaypointModelSpec +from waypoint.impl.spec import WAYPOINT_1_5, WaypointModelSpec def expected_waypoint_1_5_checkpoint_shapes( diff --git a/integrations/waypoint/waypoint/controls.py b/integrations_v2/waypoint/impl/controls.py similarity index 99% rename from integrations/waypoint/waypoint/controls.py rename to integrations_v2/waypoint/impl/controls.py index 78315b0d2..1e52891d3 100644 --- a/integrations/waypoint/waypoint/controls.py +++ b/integrations_v2/waypoint/impl/controls.py @@ -26,7 +26,7 @@ import torch from torch import Tensor -from waypoint.spec import WAYPOINT_1_5, WaypointModelSpec +from waypoint.impl.spec import WAYPOINT_1_5, WaypointModelSpec def load_controls_from_file(path: Path) -> tuple["WaypointControl", ...]: diff --git a/integrations/waypoint/waypoint/decoder.py b/integrations_v2/waypoint/impl/decoder.py similarity index 100% rename from integrations/waypoint/waypoint/decoder.py rename to integrations_v2/waypoint/impl/decoder.py diff --git a/integrations/waypoint/waypoint/encoder.py b/integrations_v2/waypoint/impl/encoder.py similarity index 97% rename from integrations/waypoint/waypoint/encoder.py rename to integrations_v2/waypoint/impl/encoder.py index 0c336fd37..3cd912044 100644 --- a/integrations/waypoint/waypoint/encoder.py +++ b/integrations_v2/waypoint/impl/encoder.py @@ -12,7 +12,7 @@ StreamingEncoder, StreamingEncoderCache, ) -from waypoint.controls import WaypointControl +from waypoint.impl.controls import WaypointControl @dataclass(kw_only=True) diff --git a/integrations/waypoint/waypoint/pipeline.py b/integrations_v2/waypoint/impl/pipeline.py similarity index 96% rename from integrations/waypoint/waypoint/pipeline.py rename to integrations_v2/waypoint/impl/pipeline.py index cfd8c553e..c1b2ea496 100644 --- a/integrations/waypoint/waypoint/pipeline.py +++ b/integrations_v2/waypoint/impl/pipeline.py @@ -17,9 +17,9 @@ StreamInferencePipelineConfig, ) from flashdreams.recipes.taehv import Hy15TAEHVEncoder, Hy15TAEHVEncoderConfig -from waypoint.controls import WaypointControl -from waypoint.decoder import WaypointTAEHVDecoder -from waypoint.transformer.impl import WaypointTransformerCache +from waypoint.impl.controls import WaypointControl +from waypoint.impl.decoder import WaypointTAEHVDecoder +from waypoint.impl.transformer.impl import WaypointTransformerCache @dataclass(kw_only=True) diff --git a/integrations/waypoint/waypoint/scheduler.py b/integrations_v2/waypoint/impl/scheduler.py similarity index 100% rename from integrations/waypoint/waypoint/scheduler.py rename to integrations_v2/waypoint/impl/scheduler.py diff --git a/integrations/waypoint/waypoint/spec.py b/integrations_v2/waypoint/impl/spec.py similarity index 100% rename from integrations/waypoint/waypoint/spec.py rename to integrations_v2/waypoint/impl/spec.py diff --git a/integrations/waypoint/waypoint/transformer/__init__.py b/integrations_v2/waypoint/impl/transformer/__init__.py similarity index 80% rename from integrations/waypoint/waypoint/transformer/__init__.py rename to integrations_v2/waypoint/impl/transformer/__init__.py index 1d6ae7900..af3774bed 100644 --- a/integrations/waypoint/waypoint/transformer/__init__.py +++ b/integrations_v2/waypoint/impl/transformer/__init__.py @@ -15,23 +15,26 @@ """Native Waypoint DiT topology and conditioning primitives.""" -from waypoint.transformer.cache import ( +from waypoint.impl.transformer.cache import ( WaypointAttentionPolicy, WaypointKVCache, WaypointKVView, ) -from waypoint.transformer.impl import ( +from waypoint.impl.transformer.impl import ( WaypointTransformer, WaypointTransformerCache, WaypointTransformerConfig, ) -from waypoint.transformer.network import ( +from waypoint.impl.transformer.network import ( WaypointDiT, WaypointDiTConfig, sinusoidal_noise_embedding, ) -from waypoint.transformer.norm import adaptive_gate, adaptive_rms_norm -from waypoint.transformer.rope import WaypointOrthoRoPEAngles, apply_waypoint_ortho_rope +from waypoint.impl.transformer.norm import adaptive_gate, adaptive_rms_norm +from waypoint.impl.transformer.rope import ( + WaypointOrthoRoPEAngles, + apply_waypoint_ortho_rope, +) __all__ = [ "WaypointDiT", diff --git a/integrations/waypoint/waypoint/transformer/cache.py b/integrations_v2/waypoint/impl/transformer/cache.py similarity index 99% rename from integrations/waypoint/waypoint/transformer/cache.py rename to integrations_v2/waypoint/impl/transformer/cache.py index 8597cf7a9..ecf39e67e 100644 --- a/integrations/waypoint/waypoint/transformer/cache.py +++ b/integrations_v2/waypoint/impl/transformer/cache.py @@ -11,7 +11,7 @@ from torch import Tensor from torch.nn.attention.flex_attention import _DEFAULT_SPARSE_BLOCK_SIZE, BlockMask -from waypoint.spec import WAYPOINT_1_5, WaypointModelSpec +from waypoint.impl.spec import WAYPOINT_1_5, WaypointModelSpec @dataclass(frozen=True, kw_only=True) diff --git a/integrations/waypoint/waypoint/transformer/impl.py b/integrations_v2/waypoint/impl/transformer/impl.py similarity index 96% rename from integrations/waypoint/waypoint/transformer/impl.py rename to integrations_v2/waypoint/impl/transformer/impl.py index a3643d5e8..95c4169db 100644 --- a/integrations/waypoint/waypoint/transformer/impl.py +++ b/integrations_v2/waypoint/impl/transformer/impl.py @@ -16,11 +16,11 @@ TransformerAutoregressiveCache, TransformerConfig, ) -from waypoint.checkpoint import load_waypoint_state_dict -from waypoint.controls import WaypointControl, make_control_context -from waypoint.spec import WaypointModelSpec -from waypoint.transformer.cache import WaypointKVCache -from waypoint.transformer.network import WaypointDiT, WaypointDiTConfig +from waypoint.impl.checkpoint import load_waypoint_state_dict +from waypoint.impl.controls import WaypointControl, make_control_context +from waypoint.impl.spec import WaypointModelSpec +from waypoint.impl.transformer.cache import WaypointKVCache +from waypoint.impl.transformer.network import WaypointDiT, WaypointDiTConfig @dataclass(kw_only=True) diff --git a/integrations/waypoint/waypoint/transformer/network.py b/integrations_v2/waypoint/impl/transformer/network.py similarity index 98% rename from integrations/waypoint/waypoint/transformer/network.py rename to integrations_v2/waypoint/impl/transformer/network.py index 3a3a207d8..340c7ccb9 100644 --- a/integrations/waypoint/waypoint/transformer/network.py +++ b/integrations_v2/waypoint/impl/transformer/network.py @@ -26,10 +26,13 @@ from torch.nn.attention.flex_attention import BlockMask from flashdreams.infra.config import InstantiateConfig -from waypoint.spec import WAYPOINT_1_5, WaypointModelSpec -from waypoint.transformer.cache import WaypointKVCache -from waypoint.transformer.norm import adaptive_gate, adaptive_rms_norm -from waypoint.transformer.rope import WaypointOrthoRoPEAngles, apply_waypoint_ortho_rope +from waypoint.impl.spec import WAYPOINT_1_5, WaypointModelSpec +from waypoint.impl.transformer.cache import WaypointKVCache +from waypoint.impl.transformer.norm import adaptive_gate, adaptive_rms_norm +from waypoint.impl.transformer.rope import ( + WaypointOrthoRoPEAngles, + apply_waypoint_ortho_rope, +) # Compile the pure fixed-attention operation so FlexAttention receives the diff --git a/integrations/waypoint/waypoint/transformer/norm.py b/integrations_v2/waypoint/impl/transformer/norm.py similarity index 100% rename from integrations/waypoint/waypoint/transformer/norm.py rename to integrations_v2/waypoint/impl/transformer/norm.py diff --git a/integrations/waypoint/waypoint/transformer/rope.py b/integrations_v2/waypoint/impl/transformer/rope.py similarity index 99% rename from integrations/waypoint/waypoint/transformer/rope.py rename to integrations_v2/waypoint/impl/transformer/rope.py index 4377a26e3..85d55170d 100644 --- a/integrations/waypoint/waypoint/transformer/rope.py +++ b/integrations_v2/waypoint/impl/transformer/rope.py @@ -20,7 +20,7 @@ import torch from torch import Tensor, nn -from waypoint.spec import WAYPOINT_1_5, WaypointModelSpec +from waypoint.impl.spec import WAYPOINT_1_5, WaypointModelSpec def apply_waypoint_ortho_rope(tokens: Tensor, cosine: Tensor, sine: Tensor) -> Tensor: diff --git a/integrations_v2/waypoint/pyproject.toml b/integrations_v2/waypoint/pyproject.toml index 3ab36dc1f..dc87b23e8 100644 --- a/integrations_v2/waypoint/pyproject.toml +++ b/integrations_v2/waypoint/pyproject.toml @@ -6,35 +6,44 @@ requires = ["setuptools>=69", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "flashdreams-waypoint-v2" +name = "flashdreams-waypoint" version = "0.1.0" -description = "FlashDreams V2 application for the Waypoint 1.5 world model." +description = "Waypoint 1.5 Cam2V integration for FlashDreams v2." readme = "README.md" requires-python = ">=3.10" dependencies = [ "flashdreams", - "flashdreams-waypoint", + "flashdreams-cam2v", "pillow>=10", ] -[project.entry-points."flashdreams.applications_v2"] -waypoint-1-5-1b = "waypoint_v2.app:create_app" - [tool.uv.sources] flashdreams = { workspace = true } -flashdreams-waypoint = { workspace = true } +flashdreams-cam2v = { workspace = true } -[tool.setuptools.packages.find] -include = ["waypoint_v2*"] -exclude = ["tests"] +[project.optional-dependencies] +dev = [ + "pytest>=8.0", +] -[tool.setuptools.package-data] -waypoint_v2 = ["assets/*.json"] - -[tool.pytest.ini_options] -addopts = "--import-mode=importlib -p flashdreams._pytest_plugins.marker_enforcement" -markers = [ - "ci_cpu: CPU-safe test, runs on the CPU CI runner", - "ci_gpu: requires GPU or libGL (cv2), runs on the GPU CI runner", - "manual: heavy or environment-specific test, opt-in only", +[project.entry-points."flashdreams.applications_v2"] +"cam2v-waypoint" = "waypoint.apps.cam2v.adapter:create_app" + +[tool.setuptools] +packages = [ + "waypoint", + "waypoint.apps", + "waypoint.apps.cam2v", + "waypoint.apps.cam2v.assets", + "waypoint.impl", + "waypoint.impl.transformer", ] + +[tool.setuptools.package-dir] +waypoint = "." + +[tool.setuptools.package-data] +"waypoint.apps.cam2v" = ["assets/*.json"] + +[tool.uv] +managed = true diff --git a/integrations_v2/waypoint/waypoint_v2/tests/test_waypoint_v2.py b/integrations_v2/waypoint/tests/test_cam2v_app.py similarity index 84% rename from integrations_v2/waypoint/waypoint_v2/tests/test_waypoint_v2.py rename to integrations_v2/waypoint/tests/test_cam2v_app.py index a6d2cbce2..3f87dace3 100644 --- a/integrations_v2/waypoint/waypoint_v2/tests/test_waypoint_v2.py +++ b/integrations_v2/waypoint/tests/test_cam2v_app.py @@ -13,14 +13,19 @@ from typing import Any, cast import pytest +import tomli as tomllib import torch +from cam2v import Cam2VSlangPyUILoop from numpy import uint64 from torch import Tensor from waypoint import WaypointControl -from waypoint.pipeline import WaypointInferencePipeline -from waypoint_v2.app import WaypointApplication, load_seed_display_frames -from waypoint_v2.control_events import WaypointControlEventAdapter -from waypoint_v2.session import WaypointModelLoop, WaypointSession +from waypoint.apps.cam2v.adapter import ( + WaypointCam2VApplication, + load_seed_display_frames, +) +from waypoint.apps.cam2v.control_events import WaypointControlEventAdapter +from waypoint.apps.cam2v.session import WaypointModelLoop, WaypointSession +from waypoint.impl.pipeline import WaypointInferencePipeline from flashdreams.api_v2.user_input_event import UserInputEvent from flashdreams.runtime_v2.mp4_client_window import Mp4ClientWindow @@ -42,6 +47,8 @@ pytestmark = pytest.mark.ci_cpu +_PACKAGE_ROOT = Path(__file__).resolve().parents[1] + class _FakeDiffusionModel: dtype = torch.float32 @@ -133,6 +140,7 @@ def _session( *, controls: tuple[WaypointControl, ...] | None, seed: int = 7, + use_ui: bool = False, ) -> WaypointSession: session_desc = _desc() return WaypointSession( @@ -143,6 +151,7 @@ def _session( seed=seed, controls=controls, mouse_sensitivity=1.0, + use_ui=use_ui, ) @@ -156,21 +165,37 @@ def _empty_events() -> UserInputEvents: return UserInputEvents([]) +def test_package_uses_the_complete_v2_layout() -> None: + """Keep model code, Cam2V binding, and entry point in one V2 package.""" + manifest = tomllib.loads((_PACKAGE_ROOT / "pyproject.toml").read_text()) + + assert manifest["project"]["name"] == "flashdreams-waypoint" + assert "flashdreams-cam2v" in manifest["project"]["dependencies"] + assert manifest["project"]["entry-points"]["flashdreams.applications_v2"] == { + "cam2v-waypoint": "waypoint.apps.cam2v.adapter:create_app" + } + assert (_PACKAGE_ROOT / "config.py").is_file() + assert (_PACKAGE_ROOT / "impl").is_dir() + assert (_PACKAGE_ROOT / "apps" / "cam2v" / "adapter.py").is_file() + assert not (_PACKAGE_ROOT / "impl" / "runner.py").exists() + assert not (_PACKAGE_ROOT.parents[1] / "integrations" / "waypoint").exists() + + def test_application_description_is_cheap_and_mp4_complete() -> None: """Session metadata is available before args, downloads, or model loading.""" - app = WaypointApplication() + app = WaypointCam2VApplication() session_desc = app.session_desc() assert session_desc.output_layout is VideoTensorLayout.tchw assert session_desc.video_width == 1024 assert session_desc.video_height == 512 assert session_desc.frames_per_second_for_step == 60 assert session_desc.backpressure_mode.value == "block" - assert session_desc.presentation_mode is PresentationMode.ON_DEMAND + assert session_desc.presentation_mode is PresentationMode.CONTINUOUS def test_application_requires_a_seed_source_and_actions_require_a_file() -> None: """Invalid argument combinations fail without constructing model state.""" - app = WaypointApplication() + app = WaypointCam2VApplication() with pytest.raises(ValueError, match="seed-image"): app.init([]) with pytest.raises(ValueError, match="actions requires"): @@ -180,7 +205,7 @@ def test_application_requires_a_seed_source_and_actions_require_a_file() -> None def test_invalid_session_contract_precedes_image_or_model_work() -> None: """Layout and size rejection happen before image decode or checkpoint setup.""" calls: list[str] = [] - app = WaypointApplication( + app = WaypointCam2VApplication( seed_loader=lambda path: calls.append(f"seed:{path}") or torch.empty(0), pipeline_factory=lambda seed, device, profile: calls.append("pipeline") or _pipeline(seed), @@ -216,7 +241,7 @@ def pipeline_factory( factory_calls.append((seed, device, profile)) return fake_pipeline - app = WaypointApplication( + app = WaypointCam2VApplication( pipeline_factory=pipeline_factory, seed_loader=lambda path: seed_frames, ) @@ -344,6 +369,54 @@ def test_reset_rebuilds_cache_and_replays_first_action_deterministically() -> No assert loop.state.cache is None +def test_r_key_restarts_the_rollout_and_internal_action_index() -> None: + """Pressing R emits the seed and makes the next action AR index one.""" + fake = _FakePipeline(seed=37) + session = _session(cast(WaypointInferencePipeline, fake), controls=None, seed=37) + session.init() + loop = cast(WaypointModelLoop, session.model_loop) + loop.step(0, _empty_events()) + first = loop.step(1, _empty_events())[0].read_output().clone() + first_cache = fake.initialized_caches[-1] + + restart = loop.step( + 2, + _events( + KeyboardUserInputEvent( + timestamp=uint64(0), key="r", state=KeyboardInputState.PRESSED + ) + ), + )[0] + replay = loop.step(3, _empty_events())[0].read_output() + + assert restart.step_index == 2 + assert restart.metrics == {"autoregressive_index": 0, "seed_frames": 4} + assert torch.equal(restart.read_output(), loop.state.seed_frames) + assert torch.equal(first, replay) + assert [call[0] for call in fake.generate_calls] == [1, 1] + assert fake.initialized_caches[-1] is not first_cache + + +def test_session_registers_waypoint_controls_with_the_shared_cam2v_hud() -> None: + """Expose Waypoint actions and reset through the shared Cam2V overlay.""" + session = _session(_pipeline(), controls=None, use_ui=True) + + session.init() + + ui_loop = cast(Cam2VSlangPyUILoop, session.ui_loop) + controls = { + group.action: tuple(control.label for control in group.keys) + for group in ui_loop.state.control_groups + } + assert controls == { + "Move": ("W", "A", "S", "D"), + "Sprint": ("Shift",), + "Jump / action": ("Space",), + "Reset rollout": ("R",), + } + assert not ui_loop.state.show_status + + def test_two_sessions_share_modules_but_keep_cache_and_rng_state_isolated() -> None: """Interleaved sessions replay the same seeded sequence independently.""" fake = _FakePipeline(seed=43) @@ -358,6 +431,7 @@ def test_two_sessions_share_modules_but_keep_cache_and_rng_state_isolated() -> N seed=43, controls=controls, mouse_sensitivity=1.0, + use_ui=False, ) second = WaypointSession( pipeline=pipeline, @@ -367,6 +441,7 @@ def test_two_sessions_share_modules_but_keep_cache_and_rng_state_isolated() -> N seed=43, controls=controls, mouse_sensitivity=1.0, + use_ui=False, ) first.init() second.init() diff --git a/integrations/waypoint/tests/test_cuda.py b/integrations_v2/waypoint/tests/test_cuda.py similarity index 95% rename from integrations/waypoint/tests/test_cuda.py rename to integrations_v2/waypoint/tests/test_cuda.py index 6fcf1fbbe..e8f47dffd 100644 --- a/integrations/waypoint/tests/test_cuda.py +++ b/integrations_v2/waypoint/tests/test_cuda.py @@ -9,8 +9,8 @@ import torch import torch.nn.functional as F from waypoint import WAYPOINT_1_5 -from waypoint.transformer import WaypointAttentionPolicy, WaypointKVCache -from waypoint.transformer.network import _compiled_fixed_attention +from waypoint.impl.transformer import WaypointAttentionPolicy, WaypointKVCache +from waypoint.impl.transformer.network import _compiled_fixed_attention pytestmark = pytest.mark.ci_gpu diff --git a/integrations/waypoint/tests/test_spec.py b/integrations_v2/waypoint/tests/test_spec.py similarity index 98% rename from integrations/waypoint/tests/test_spec.py rename to integrations_v2/waypoint/tests/test_spec.py index fcf6913d8..081bb5d05 100644 --- a/integrations/waypoint/tests/test_spec.py +++ b/integrations_v2/waypoint/tests/test_spec.py @@ -28,18 +28,21 @@ load_controls_from_file, make_control_context, ) -from waypoint.checkpoint import ( +from waypoint.config import PIPELINE_WAYPOINT_1_5 +from waypoint.impl.checkpoint import ( expected_waypoint_1_5_checkpoint_keys, expected_waypoint_1_5_checkpoint_shapes, load_waypoint_state_dict, validate_waypoint_1_5_checkpoint_keys, validate_waypoint_1_5_checkpoint_shapes, ) -from waypoint.config import PIPELINE_WAYPOINT_1_5 -from waypoint.decoder import WaypointTAEHVDecoder -from waypoint.encoder import WaypointControlEncoderConfig -from waypoint.pipeline import WaypointInferencePipeline, WaypointInferencePipelineConfig -from waypoint.transformer import ( +from waypoint.impl.decoder import WaypointTAEHVDecoder +from waypoint.impl.encoder import WaypointControlEncoderConfig +from waypoint.impl.pipeline import ( + WaypointInferencePipeline, + WaypointInferencePipelineConfig, +) +from waypoint.impl.transformer import ( WaypointAttentionPolicy, WaypointDiTConfig, WaypointKVCache, @@ -49,8 +52,8 @@ adaptive_rms_norm, apply_waypoint_ortho_rope, ) -from waypoint.transformer.impl import WaypointTransformerCache -from waypoint.transformer.network import ( +from waypoint.impl.transformer.impl import WaypointTransformerCache +from waypoint.impl.transformer.network import ( _ConditionHead, _ControlFusion, _WaypointAttention, diff --git a/integrations_v2/waypoint/waypoint_v2/__init__.py b/integrations_v2/waypoint/waypoint_v2/__init__.py deleted file mode 100644 index d05c2db0e..000000000 --- a/integrations_v2/waypoint/waypoint_v2/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""FlashDreams V2 application for Waypoint 1.5.""" - -from waypoint_v2.app import WaypointApplication, create_app -from waypoint_v2.control_events import WaypointControlEventAdapter -from waypoint_v2.session import WaypointModelLoop, WaypointSession - -__all__ = [ - "WaypointApplication", - "WaypointControlEventAdapter", - "WaypointModelLoop", - "WaypointSession", - "create_app", -] diff --git a/pyproject.toml b/pyproject.toml index ce9428f4c..3c82aced1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,6 @@ extraPaths = [ "integrations_v2/self_forcing", "integrations_v2/wan21", "integrations_v2/wan22", - "integrations/waypoint", "integrations_v2/color_fade", "integrations_v2/null_model", "integrations_v2/red_screen", @@ -108,7 +107,6 @@ extra-paths = [ "integrations_v2/self_forcing", "integrations_v2/wan21", "integrations_v2/wan22", - "integrations/waypoint", "integrations_v2/color_fade", "integrations_v2/null_model", "integrations_v2/red_screen", diff --git a/uv.lock b/uv.lock index b35fa6bfa..e5084d1a4 100644 --- a/uv.lock +++ b/uv.lock @@ -41,7 +41,6 @@ members = [ "flashdreams-wan21", "flashdreams-wan22", "flashdreams-waypoint", - "flashdreams-waypoint-v2", "ludus-renderer", "omnidreams-game-engine", ] @@ -1629,9 +1628,11 @@ provides-extras = ["dev"] [[package]] name = "flashdreams-waypoint" version = "0.1.0" -source = { editable = "integrations/waypoint" } +source = { editable = "integrations_v2/waypoint" } dependencies = [ { name = "flashdreams" }, + { name = "flashdreams-cam2v" }, + { name = "pillow" }, ] [package.optional-dependencies] @@ -1642,27 +1643,12 @@ dev = [ [package.metadata] requires-dist = [ { name = "flashdreams", editable = "flashdreams" }, + { name = "flashdreams-cam2v", editable = "apps/cam2v" }, + { name = "pillow", specifier = ">=10" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, ] provides-extras = ["dev"] -[[package]] -name = "flashdreams-waypoint-v2" -version = "0.1.0" -source = { editable = "integrations_v2/waypoint" } -dependencies = [ - { name = "flashdreams" }, - { name = "flashdreams-waypoint" }, - { name = "pillow" }, -] - -[package.metadata] -requires-dist = [ - { name = "flashdreams", editable = "flashdreams" }, - { name = "flashdreams-waypoint", editable = "integrations/waypoint" }, - { name = "pillow", specifier = ">=10" }, -] - [[package]] name = "flip-evaluator" version = "1.7" From 2bc6ad86ad954d53a85e7c9ce11895126e29e9a4 Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Thu, 3 Sep 2026 09:35:37 -0700 Subject: [PATCH 2/2] fix: preserve Waypoint finite replay semantics Signed-off-by: Jonathan McCaffrey --- docs/source/models/waypoint.rst | 11 +++++++---- integrations_v2/waypoint/apps/cam2v/README.md | 9 ++++++--- integrations_v2/waypoint/apps/cam2v/adapter.py | 4 ++-- integrations_v2/waypoint/tests/test_cam2v_app.py | 6 +++--- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/docs/source/models/waypoint.rst b/docs/source/models/waypoint.rst index 6a885aa43..0491e497e 100644 --- a/docs/source/models/waypoint.rst +++ b/docs/source/models/waypoint.rst @@ -127,13 +127,16 @@ Run the same application interactively in a browser: .. code-block:: bash uv run --no-sync flashdreams-run-v2 cam2v-waypoint \ - --mode webrtc --host 127.0.0.1 --port 8766 \ + --mode webrtc --presentation-mode continuous \ + --host 127.0.0.1 --port 8766 \ -- --image-path seed.png --seed 464 Open http://127.0.0.1:8766/. The control HUD highlights held keys; press ``R`` -to reset the rollout to the starting image. Arguments before the separator -configure the V2 runtime; arguments after it configure Waypoint. To inspect all -model arguments: +to reset the rollout to the starting image. Continuous presentation keeps the +interactive HUD responsive; the application default remains on-demand so +finite replays contain each generated frame exactly once. Arguments before the +separator configure the V2 runtime; arguments after it configure Waypoint. To +inspect all model arguments: .. code-block:: bash diff --git a/integrations_v2/waypoint/apps/cam2v/README.md b/integrations_v2/waypoint/apps/cam2v/README.md index 4efae7822..31a5c4d18 100644 --- a/integrations_v2/waypoint/apps/cam2v/README.md +++ b/integrations_v2/waypoint/apps/cam2v/README.md @@ -10,14 +10,15 @@ Install the Waypoint integration and launch its `cam2v-waypoint` application: ```bash uv sync --package flashdreams-waypoint --inexact uv run --no-sync flashdreams-run-v2 cam2v-waypoint --mode webrtc \ - --host 0.0.0.0 --port 8089 -- --example-data + --presentation-mode continuous --host 0.0.0.0 --port 8089 -- \ + --example-data ``` Use a local starting image with live controls: ```bash uv run --no-sync flashdreams-run-v2 cam2v-waypoint --mode webrtc \ - --host 0.0.0.0 --port 8089 -- \ + --presentation-mode continuous --host 0.0.0.0 --port 8089 -- \ --image-path seed.png --seed 464 ``` @@ -33,7 +34,9 @@ uv run --no-sync flashdreams-run-v2 cam2v-waypoint --mode webrtc \ The on-screen HUD lists these controls and wraps held keys in brackets. Losing window or browser focus clears held controls. Pass `--no-ui` to disable the -HUD. +HUD. Interactive examples opt into continuous presentation for prompt HUD +updates. Waypoint otherwise defaults to on-demand presentation so finite +replays contain each generated frame exactly once. For a deterministic file-driven MP4: diff --git a/integrations_v2/waypoint/apps/cam2v/adapter.py b/integrations_v2/waypoint/apps/cam2v/adapter.py index 78441aa2e..a5bbb7b5b 100644 --- a/integrations_v2/waypoint/apps/cam2v/adapter.py +++ b/integrations_v2/waypoint/apps/cam2v/adapter.py @@ -159,11 +159,11 @@ def init(self, commandline_args: Sequence[str]) -> None: ) def session_desc(self) -> SessionDesc: - """Return Waypoint's native 1024x512 TCHW presentation contract.""" + """Return Waypoint's finite, native 1024x512 TCHW replay contract.""" return SessionDesc( output_layout=VideoTensorLayout.tchw, backpressure_mode=BackpressureMode.BLOCK, - presentation_mode=PresentationMode.CONTINUOUS, + presentation_mode=PresentationMode.ON_DEMAND, frames_per_second_for_ui=_PLAYBACK_FPS, frames_per_second_for_step=_PLAYBACK_FPS, video_width=_OUTPUT_WIDTH, diff --git a/integrations_v2/waypoint/tests/test_cam2v_app.py b/integrations_v2/waypoint/tests/test_cam2v_app.py index 3f87dace3..7e876726a 100644 --- a/integrations_v2/waypoint/tests/test_cam2v_app.py +++ b/integrations_v2/waypoint/tests/test_cam2v_app.py @@ -181,8 +181,8 @@ def test_package_uses_the_complete_v2_layout() -> None: assert not (_PACKAGE_ROOT.parents[1] / "integrations" / "waypoint").exists() -def test_application_description_is_cheap_and_mp4_complete() -> None: - """Session metadata is available before args, downloads, or model loading.""" +def test_application_description_is_cheap_and_preserves_finite_replay() -> None: + """Default metadata preserves exact replay without loading model state.""" app = WaypointCam2VApplication() session_desc = app.session_desc() assert session_desc.output_layout is VideoTensorLayout.tchw @@ -190,7 +190,7 @@ def test_application_description_is_cheap_and_mp4_complete() -> None: assert session_desc.video_height == 512 assert session_desc.frames_per_second_for_step == 60 assert session_desc.backpressure_mode.value == "block" - assert session_desc.presentation_mode is PresentationMode.CONTINUOUS + assert session_desc.presentation_mode is PresentationMode.ON_DEMAND def test_application_requires_a_seed_source_and_actions_require_a_file() -> None: