diff --git a/docs/source/api/serving.rst b/docs/source/api/serving.rst index 9d988cc02..78e4dc31f 100644 --- a/docs/source/api/serving.rst +++ b/docs/source/api/serving.rst @@ -41,6 +41,37 @@ The reusable applications provide concrete examples of the shared serving stack: - OmniDreams model wiring under ``integrations_v2/omnidreams/`` and serving under ``apps/interactive_drive/``. +Application-served browser UI +----------------------------- + +The v2 WebRTC runtime serves one minimal viewer at ``/`` for every application. +An application that implements ``IWebUiProvider`` +(``flashdreams.api_v2.web_ui``) also has its own page and session endpoints +served: + +.. list-table:: + :header-rows: 1 + :widths: 34 66 + + * - Route + - Serves + * - ``GET /request_session`` + - ``index.html`` from the application's web root. + * - ``GET /`` + - Any other file in that web root, resolved inside it only. + * - ``GET /api/session/initial_scene`` + - The application's scene, verbatim as JSON. + * - ``GET /api/session/first_frame`` + - The session's first frame, or ``404`` before one exists. + * - ``POST /api/session/input`` + - One page submission, answered with the resulting scene. + +These routes answer ``404`` for an application that does not implement the +protocol, so adding it changes nothing for the others. The serving layer never +inspects a scene: prompt, event, and upload semantics belong to the +application. ``integrations_v2/lingbot/`` is the reference implementation; see +:doc:`/models/lingbot_world`. + Launch patterns --------------- diff --git a/docs/source/models/lingbot_world.rst b/docs/source/models/lingbot_world.rst index ac4c9d589..9673116e4 100644 --- a/docs/source/models/lingbot_world.rst +++ b/docs/source/models/lingbot_world.rst @@ -76,6 +76,35 @@ Application arguments follow ``--``. Run ``flashdreams-run-v2 cam2v-lingbot -- --help`` for custom first-frame, intrinsics, prompt, and motion-normalizer inputs. +Browser UI +---------- + +The run above serves two clients: + +- ``http://:8089/`` — the runtime's minimal viewer, offered by every v2 + application. +- ``http://:8089/request_session`` — the Lingbot scene UI, with a preset + picker, first-frame upload, prompt box, and buttons for the scene's text + events. + +A scene is a prompt, a first frame, and a catalog of text events. Triggering an +event replaces the rollout's cross-attention text context in place, so the +scene changes without restarting the session; clearing it restores the base +prompt. Changing the prompt — including by picking another preset — swaps that +context too, so the scene changes while the rollout keeps running. The first +frame is the exception: a rollout cannot replace the frame it was initialized +from, so pick a preset before connecting to start from its image. + +Append ``?manual`` to skip auto-connect — only one session runs per process, so +a stray browser tab can otherwise claim it — or ``?preset=`` to open a +specific scene, such as ``?preset=water-blaster``. + +The page and its presets ship in ``integrations_v2/lingbot/apps/cam2v/web/``. +It is served because ``LingbotCam2VApplication`` implements +``flashdreams.api_v2.web_ui.IWebUiProvider``, which also adds the +``/api/session/initial_scene``, ``/api/session/first_frame``, and +``/api/session/input`` endpoints the page calls. + Sample data is downloaded from the `LingBot-World v2 repository `_. Valid ``--example-idx`` values are ``0, 1, 2, 5``. Note the single GPU command might run diff --git a/docs/source/quickstart/index.rst b/docs/source/quickstart/index.rst index e31d575d4..462d6f505 100644 --- a/docs/source/quickstart/index.rst +++ b/docs/source/quickstart/index.rst @@ -62,9 +62,10 @@ and streams the generated camera view to a browser over WebRTC: interactive-drive-omnidreams --mode webrtc \ --host 0.0.0.0 --port 8089 -Then open ``http://:8089/request_session`` in a browser on the same network -(use ``localhost`` on the same machine). The first launch spends several -minutes loading checkpoints and compiling kernels; later launches reuse +Then open ``http://:8089/`` in a browser on the same network (use +``localhost`` on the same machine). An application serving its own page, such +as Lingbot, also offers it at ``/request_session``. The first launch spends +several minutes loading checkpoints and compiling kernels; later launches reuse the cached assets. Inspect the application arguments without loading checkpoints: diff --git a/flashdreams/flashdreams/api_v2/loop.py b/flashdreams/flashdreams/api_v2/loop.py index 12d23e528..cf941c915 100644 --- a/flashdreams/flashdreams/api_v2/loop.py +++ b/flashdreams/flashdreams/api_v2/loop.py @@ -5,6 +5,7 @@ from __future__ import annotations +import logging import queue import threading import time @@ -14,6 +15,7 @@ from enum import Enum from typing import TYPE_CHECKING, Any, Generic, TypeVar, final +from loguru import logger from torch import Tensor from flashdreams.runtime_v2.event_buffer import EventBuffer @@ -24,6 +26,8 @@ ) from flashdreams.runtime_v2.user_input_events import UserInputEvents +_LOGGER = logging.getLogger(__name__) + if TYPE_CHECKING: from flashdreams.runtime_v2.presentation_manager import PresentationManager from flashdreams.runtime_v2.session_desc import SessionDesc @@ -471,6 +475,9 @@ def _parse_lifecycle_events( ) -> _LoopRunResult | None: """Return the terminal lifecycle request, prioritizing close.""" if any(isinstance(event, CloseUserInputEvent) for event in events): + # A client can end the whole run this way, which otherwise looks from + # the outside like the server exiting on its own. + logger.info("A client sent a close event; stopping the session.") return _LoopRunResult(stop_requested=True) return None diff --git a/flashdreams/flashdreams/api_v2/web_ui.py b/flashdreams/flashdreams/api_v2/web_ui.py new file mode 100644 index 000000000..666bb7aec --- /dev/null +++ b/flashdreams/flashdreams/api_v2/web_ui.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Optional application hook for serving an application-owned browser UI. + +The v2 WebRTC server ships one minimal viewer for every application. An +application that wants its own richer page -- a scene picker, an event panel, +a heads-up display -- implements :class:`IWebUiProvider` and gets its files +served alongside that viewer, plus three endpoints the page can call. + +The serving layer stays generic: it copies :meth:`IWebUiProvider.initial_scene` +into a JSON response and hands :meth:`IWebUiProvider.apply_session_input` the +decoded request body without inspecting either. What a "scene" is, and which +inputs a page may change, belong entirely to the application. + +Applications that do not implement this protocol are unaffected: the server +registers none of these routes and serves its built-in viewer as before. +""" + +from collections.abc import Mapping +from pathlib import Path +from typing import Any, Protocol, runtime_checkable + + +@runtime_checkable +class IWebUiProvider(Protocol): + """An application that serves its own browser UI.""" + + def web_root(self) -> Path: + """Return the directory holding the application's web assets. + + Its ``index.html`` is served at ``/request_session``; the remaining + files are served by name. Paths are resolved inside this directory + only, so a request cannot escape it. + """ + ... + + def initial_scene(self) -> Mapping[str, Any]: + """Return what the page needs to render before any frame arrives. + + Returned verbatim as JSON from ``GET /api/session/initial_scene`` and + again from ``POST /api/session/input``, so a page can render the + result of its own change without a second request. + """ + ... + + def first_frame(self) -> tuple[bytes, str] | None: + """Return the session's first frame as ``(data, content_type)``. + + ``None`` when the session has no first frame to show yet, which the + server reports as ``404`` rather than an error. + """ + ... + + def apply_session_input(self, payload: Mapping[str, Any]) -> Mapping[str, Any]: + """Apply one page-submitted change and return the resulting scene. + + Args: + payload: Decoded request body. Multipart uploads arrive with file + parts as ``bytes`` and every other part as ``str``. + + Returns: + The scene as :meth:`initial_scene` would report it afterwards. + + Raises: + ValueError: The payload is not a change this application accepts. + The server reports it as ``400`` with the message. + """ + ... + + +__all__ = ["IWebUiProvider"] diff --git a/flashdreams/flashdreams/runtime_v2/README.md b/flashdreams/flashdreams/runtime_v2/README.md index 4c239a3eb..7bbcd76e9 100644 --- a/flashdreams/flashdreams/runtime_v2/README.md +++ b/flashdreams/flashdreams/runtime_v2/README.md @@ -260,6 +260,40 @@ replacement must keep the negotiated layout, frame rates, width, and height. What a sink expects of the pixel values it is handed is part of the result contract, in [`api_v2`](../api_v2/README.md#what-a-step-returns). +## Serving an application's own browser UI + +The WebRTC mode ships one minimal viewer at `/`, which every application gets. +An application wanting a richer page — a scene picker, an event panel, a +heads-up display — implements `IWebUiProvider` from `flashdreams.api_v2.web_ui` +and returns four things: the directory holding its web assets, the scene the +page renders before any frame arrives, the session's first frame, and what to +do with one page submission. + +Doing so adds five routes to that run: + +| Route | Serves | +| --- | --- | +| `GET /request_session` | `index.html` from the application's web root | +| `GET /` | any other file in that web root, resolved inside it only | +| `GET /api/session/initial_scene` | `initial_scene()`, verbatim as JSON | +| `GET /api/session/first_frame` | `first_frame()`, or 404 before one exists | +| `POST /api/session/input` | `apply_session_input()`, then the resulting scene | + +The serving layer stays generic: it copies the scene into a JSON response and +hands the decoded request body back without inspecting either, so what a scene +is belongs to the application. JSON, form, and multipart bodies all arrive as +one flat mapping, with uploaded files as `bytes` under their field name. An +application raising `ValueError` becomes a 400 carrying its message. + +The wiring runs through `ClientWindowMode.attach_application`, called once +after the window is created. Only `_WebRTCMode` overrides it, and only for an +application implementing the protocol — every other application keeps exactly +the routes it had before, since the handlers answer 404 without a provider. + +`ClientWindowMode.create` builds the window before the application attaches, +and aiohttp freezes its router at startup, so these routes are registered up +front and `WebRTCServer.serve_web_ui()` fills in the application behind them. + ## Adding a mode A mode is one way to watch a run. Subclass `ClientWindowMode` in diff --git a/flashdreams/flashdreams/runtime_v2/application_runner.py b/flashdreams/flashdreams/runtime_v2/application_runner.py index 4e518d62d..a71b4b94d 100644 --- a/flashdreams/flashdreams/runtime_v2/application_runner.py +++ b/flashdreams/flashdreams/runtime_v2/application_runner.py @@ -9,6 +9,8 @@ import time from collections.abc import Sequence +from loguru import logger + from flashdreams.api_v2.application import IApplication from flashdreams.api_v2.client_window import IClientWindow from flashdreams.api_v2.output_sink import OutputSink @@ -82,10 +84,17 @@ def run( try: self._application.init(commandline_args) next_session_desc: SessionDesc | None = session_desc + session_count = 0 while next_session_desc is not None: if deadline is not None and time.monotonic() >= deadline: + logger.info( + "Application run ending: the {}s timeout expired.", + timeout_seconds, + ) break session = self._application.create_session(next_session_desc) + session_count += 1 + logger.info("Session {} starting.", session_count) session_run_started = True remaining_seconds = ( None if deadline is None else max(0.0, deadline - time.monotonic()) @@ -100,8 +109,27 @@ def run( except BaseException: # ``run_session`` closes the window on every failure. window_needs_close = False + logger.info( + "Session {} failed; the run ends with it.", session_count + ) raise window_needs_close = next_session_desc is not None + # A run that stops without saying so is indistinguishable from + # a crash to whoever is watching the server, so every way out + # of this loop says which one it was. + if next_session_desc is None: + logger.info( + "Session {} ended and asked for no replacement, so the " + "application run is complete. A session ends when the " + "client closes the window, the model finishes its " + "requested steps, or the UI stops.", + session_count, + ) + else: + logger.info( + "Session {} ended and requested a replacement.", + session_count, + ) finally: if window_needs_close: _close_client_window(self._client_window) diff --git a/flashdreams/flashdreams/runtime_v2/cli.py b/flashdreams/flashdreams/runtime_v2/cli.py index e1e98366a..509a8d904 100644 --- a/flashdreams/flashdreams/runtime_v2/cli.py +++ b/flashdreams/flashdreams/runtime_v2/cli.py @@ -79,6 +79,7 @@ def entrypoint(argv: Sequence[str] | None = None) -> None: return session_desc = _session_desc(application, parsed) window = mode.create(parsed) + mode.attach_application(window, application) _report(mode.starting(window)) # The session's UI and client input decide when the run ends. metrics_output_sink = ( diff --git a/flashdreams/flashdreams/runtime_v2/client_window_factory.py b/flashdreams/flashdreams/runtime_v2/client_window_factory.py index b5d018b2b..2ab4f4817 100644 --- a/flashdreams/flashdreams/runtime_v2/client_window_factory.py +++ b/flashdreams/flashdreams/runtime_v2/client_window_factory.py @@ -14,7 +14,9 @@ from pathlib import Path from typing import TYPE_CHECKING, cast +from flashdreams.api_v2.application import IApplication from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.web_ui import IWebUiProvider from flashdreams.runtime_v2.mp4_client_window import Mp4ClientWindow if TYPE_CHECKING: @@ -49,6 +51,16 @@ def create(self, parsed_args: argparse.Namespace) -> IClientWindow: ValueError: Whatever :meth:`check_arguments` reports. """ + def attach_application( + self, client_window: IClientWindow, application: IApplication + ) -> None: + """Give the window whatever the application offers this mode. + + Called once, after :meth:`create` and before the run. Most modes want + nothing: a file has no place to put an application's browser UI. + """ + del client_window, application + def starting(self, client_window: IClientWindow) -> str | None: """Return what to tell the user before the run, such as where to watch.""" del client_window @@ -100,6 +112,13 @@ def create(self, parsed_args: argparse.Namespace) -> IClientWindow: return WebRTCClientWindow(host=parsed_args.host, port=parsed_args.port) + def attach_application( + self, client_window: IClientWindow, application: IApplication + ) -> None: + """Serve the application's own browser UI when it has one.""" + if isinstance(application, IWebUiProvider): + cast("WebRTCClientWindow", client_window).serve_web_ui(application) + def starting(self, client_window: IClientWindow) -> str | None: """Return where to connect, which nobody can guess when the port is free.""" server = cast("WebRTCClientWindow", client_window).server diff --git a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py index 59535f190..e7dba1a18 100644 --- a/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py +++ b/flashdreams/flashdreams/runtime_v2/serving/webrtc_server.py @@ -16,16 +16,19 @@ from dataclasses import dataclass from fractions import Fraction from importlib.resources import files +from pathlib import Path from typing import Any, Literal, cast import numpy as np import torch from aiohttp import web +from aiohttp.multipart import BodyPartReader from aiortc import MediaStreamTrack, RTCPeerConnection, RTCSessionDescription from aiortc.mediastreams import MediaStreamError from av import VideoFrame from loguru import logger +from flashdreams.api_v2.web_ui import IWebUiProvider from flashdreams.runtime_v2.cuda_utils import resolve_cuda_device from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult @@ -48,6 +51,9 @@ _BROWSER_PAGE = _WEB_RESOURCES.joinpath("index.html").read_text(encoding="utf-8") _BROWSER_SCRIPT = _WEB_RESOURCES.joinpath("app.js").read_text(encoding="utf-8") +_MAX_SESSION_INPUT_BYTES = 16 * 1024 * 1024 +"""Ceiling on one session-input request, sized for a first-frame upload.""" + _INTERACTIVE_FRAME_QUEUE_SIZE = 2 """Send-ready frames retained while the media sender is temporarily busy.""" @@ -295,6 +301,72 @@ async def _next_queued_frame(self) -> _QueuedRGBFrame: await self._frame_available.wait() +async def _read_session_input_payload(request: web.Request) -> dict[str, Any]: + """Decode one session-input request into a flat payload. + + JSON bodies are returned as-is. Form and multipart bodies become a flat + mapping too, so an application handles one shape either way: file parts + arrive as ``bytes`` under their field name, with the uploaded filename + and content type under ``_filename`` and ``_content_type``. + + Raises: + web.HTTPBadRequest: The body is malformed or not an object. + web.HTTPRequestEntityTooLarge: The body exceeds the size ceiling. + """ + content_type = request.content_type or "" + if content_type == "application/json": + try: + json_body = await request.json() + except json.JSONDecodeError as error: + raise web.HTTPBadRequest(reason="Body must be valid JSON.") from error + if not isinstance(json_body, dict): + raise web.HTTPBadRequest(reason="Body must be a JSON object.") + return json_body + + payload: dict[str, Any] = {} + total_bytes = 0 + if content_type.startswith("multipart/"): + try: + reader = await request.multipart() + except (AssertionError, ValueError) as error: + raise web.HTTPBadRequest(reason="Body must be valid multipart.") from error + while True: + part = await reader.next() + if part is None: + break + if not isinstance(part, BodyPartReader) or part.name is None: + continue + if part.filename: + data = bytearray() + while chunk := await part.read_chunk(): + data.extend(chunk) + total_bytes += len(chunk) + if total_bytes > _MAX_SESSION_INPUT_BYTES: + raise web.HTTPRequestEntityTooLarge( + max_size=_MAX_SESSION_INPUT_BYTES, + actual_size=total_bytes, + ) + payload[part.name] = bytes(data) + payload[f"{part.name}_filename"] = part.filename + payload[f"{part.name}_content_type"] = part.headers.get( + "Content-Type", "application/octet-stream" + ) + continue + text = await part.text() + total_bytes += len(text) + if total_bytes > _MAX_SESSION_INPUT_BYTES: + raise web.HTTPRequestEntityTooLarge( + max_size=_MAX_SESSION_INPUT_BYTES, + actual_size=total_bytes, + ) + payload[part.name] = text + return payload + + for key, value in (await request.post()).items(): + payload[key] = value if isinstance(value, str) else str(value) + return payload + + class WebRTCServer: """Own the HTTP, signaling, input buffering, and media transport.""" @@ -304,12 +376,16 @@ def __init__( host: str = "127.0.0.1", port: int = 0, startup_timeout_seconds: float = 10.0, + web_ui: IWebUiProvider | None = None, ) -> None: """ Args: host: Interface on which the HTTP server listens. port: Listening port. Zero asks the operating system to choose one. startup_timeout_seconds: Maximum time to wait for server startup. + web_ui: Application serving its own browser UI. ``None`` serves + only the built-in viewer, which is what every application + that does not implement the protocol gets. Raises: RuntimeError: The server cannot start. @@ -325,6 +401,7 @@ def __init__( self._host = host self._port = port self._startup_timeout_seconds = startup_timeout_seconds + self._web_ui = web_ui self._input_callback: Callable[[UserInputEvent], None] | None = None self._started = threading.Event() self._startup_error: BaseException | None = None @@ -668,6 +745,17 @@ async def _start_server(self) -> None: app.router.add_get("/app.js", self._serve_browser_script) app.router.add_get("/healthz", self._health) app.router.add_post("/api/webrtc/offer", self._offer) + # Registered unconditionally because aiohttp freezes the router once + # the runner starts, and the application attaches after this server + # is constructed. Without a web-UI application every one of these + # answers 404, exactly as an unrouted path did before they existed. + app.router.add_get("/request_session", self._serve_web_ui_page) + app.router.add_get("/api/session/initial_scene", self._initial_scene) + app.router.add_get("/api/session/first_frame", self._first_frame) + app.router.add_post("/api/session/input", self._session_input) + # Last, so every named route above wins over a file of the same name + # in the application's web root. + app.router.add_get("/{web_asset:.*}", self._serve_web_ui_asset) runner = web.AppRunner(app) await runner.setup() address_family = socket.AF_INET6 if ":" in self._host else socket.AF_INET @@ -694,6 +782,81 @@ async def _serve_browser_script(self, _: web.Request) -> web.Response: """Return the browser client's JavaScript.""" return web.Response(text=_BROWSER_SCRIPT, content_type="text/javascript") + def serve_web_ui(self, web_ui: IWebUiProvider) -> None: + """Serve an application's own page and session routes. + + Callable after startup: the routes are already registered and answer + 404 until this supplies the application behind them. + + Args: + web_ui: Application whose web root and session data to serve. + + Raises: + RuntimeError: The server is closed. + """ + if self._closed: + raise RuntimeError("Cannot serve a web UI from a closed WebRTC server.") + self._web_ui = web_ui + + def _require_web_ui(self) -> IWebUiProvider: + """Return the web-UI application, which the routes guarantee exists.""" + web_ui = self._web_ui + if web_ui is None: + raise web.HTTPNotFound() + return web_ui + + def _resolve_web_asset(self, relative_path: str) -> Path: + """Return an existing file inside the application's web root. + + Raises: + web.HTTPNotFound: The path escapes the web root or names nothing. + """ + root = Path(self._require_web_ui().web_root()).resolve() + try: + resolved = (root / relative_path).resolve() + except OSError as error: + raise web.HTTPNotFound() from error + # Containment is checked after resolving, so neither "..", a symlink, + # nor an absolute path can reach a file outside the web root. + if not resolved.is_file() or not resolved.is_relative_to(root): + raise web.HTTPNotFound() + return resolved + + async def _serve_web_ui_page(self, _: web.Request) -> web.StreamResponse: + """Return the application's own page.""" + return web.FileResponse(self._resolve_web_asset("index.html")) + + async def _serve_web_ui_asset(self, request: web.Request) -> web.StreamResponse: + """Return one file from the application's web root.""" + return web.FileResponse( + self._resolve_web_asset(request.match_info["web_asset"]) + ) + + async def _initial_scene(self, _: web.Request) -> web.Response: + """Return what the application says its session currently shows.""" + web_ui = self._require_web_ui() + scene = await asyncio.to_thread(web_ui.initial_scene) + return web.json_response(dict(scene)) + + async def _first_frame(self, _: web.Request) -> web.Response: + """Return the session's first frame, or 404 before one exists.""" + web_ui = self._require_web_ui() + frame = await asyncio.to_thread(web_ui.first_frame) + if frame is None: + raise web.HTTPNotFound() + data, content_type = frame + return web.Response(body=data, content_type=content_type) + + async def _session_input(self, request: web.Request) -> web.Response: + """Hand one page-submitted change to the application.""" + web_ui = self._require_web_ui() + payload = await _read_session_input_payload(request) + try: + scene = await asyncio.to_thread(web_ui.apply_session_input, payload) + except ValueError as error: + raise web.HTTPBadRequest(reason=str(error)) from error + return web.json_response(dict(scene)) + async def _health(self, _: web.Request) -> web.Response: """Report whether the server has an open session and client.""" return web.json_response( diff --git a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py index 218671520..83e5ceb72 100644 --- a/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py +++ b/flashdreams/flashdreams/runtime_v2/webrtc_client_window.py @@ -10,6 +10,7 @@ from numpy import uint64 from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.api_v2.web_ui import IWebUiProvider from flashdreams.runtime_v2.serving.webrtc_server import WebRTCServer from flashdreams.runtime_v2.session_desc import SessionDesc from flashdreams.runtime_v2.step_result import StepResult @@ -72,6 +73,17 @@ def handle_input(event: UserInputEvent) -> None: self.server.register_input_callback(handle_input) + def serve_web_ui(self, web_ui: IWebUiProvider) -> None: + """Serve an application's own browser UI from this window's server. + + Construction is specific to this implementation; it is not part of the + ``IClientWindow`` protocol. + + Args: + web_ui: Application whose page and session routes to serve. + """ + self.server.serve_web_ui(web_ui) + def request_hide_cursor(self, hide_cursor: bool) -> None: """Show or hide the cursor in the browser window.""" self.server.configure_cursor( diff --git a/flashdreams/test_v2/test_webrtc_web_ui.py b/flashdreams/test_v2/test_webrtc_web_ui.py new file mode 100644 index 000000000..b4b126ef1 --- /dev/null +++ b/flashdreams/test_v2/test_webrtc_web_ui.py @@ -0,0 +1,286 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU tests for the application-served browser UI on the v2 WebRTC server.""" + +# ruff: noqa: E402 - optional WebRTC imports must follow importorskip. + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import pytest +import pytest_asyncio + +pytestmark = pytest.mark.ci_cpu + +pytest.importorskip("aiohttp") +pytest.importorskip("aiortc") + +from aiohttp import web +from aiohttp.test_utils import TestClient, TestServer + +from flashdreams.api_v2.web_ui import IWebUiProvider +from flashdreams.runtime_v2.serving.webrtc_server import WebRTCServer + + +class StubWebUi: + """Smallest application that satisfies :class:`IWebUiProvider`.""" + + def __init__(self, root: Path) -> None: + self._root = root + self.applied: list[dict[str, Any]] = [] + self.frame: tuple[bytes, str] | None = (b"\xff\xd8jpeg", "image/jpeg") + + def web_root(self) -> Path: + return self._root + + def initial_scene(self) -> Mapping[str, Any]: + return {"prompt": "a scene", "event_catalog": [{"event_id": "storm"}]} + + def first_frame(self) -> tuple[bytes, str] | None: + return self.frame + + def apply_session_input(self, payload: Mapping[str, Any]) -> Mapping[str, Any]: + if payload.get("prompt") == "": + raise ValueError("Prompt must not be empty.") + self.applied.append(dict(payload)) + return {"prompt": payload.get("prompt", "unchanged")} + + +@pytest.fixture +def web_root(tmp_path: Path) -> Path: + """Return a web root holding one of each thing a page loads.""" + root = tmp_path / "web" + (root / "assets").mkdir(parents=True) + (root / "index.html").write_text("page", encoding="utf-8") + (root / "adapter.js").write_text("// adapter", encoding="utf-8") + (root / "assets" / "logo.svg").write_text("", encoding="utf-8") + (tmp_path / "outside.txt").write_text("SECRET", encoding="utf-8") + return root + + +@pytest.fixture +def server() -> WebRTCServer: + """Return a server with only the state its HTTP handlers read. + + ``__init__`` starts a thread and binds a socket, neither of which these + tests need; the handlers under test are the ones the running server + registers. + """ + instance = WebRTCServer.__new__(WebRTCServer) + instance._web_ui = None + instance._closed = False + instance._client_connected = False + instance._hide_cursor = False + instance._lock_cursor_to_window = False + instance._session_desc = None + return instance + + +@pytest_asyncio.fixture +async def client(server: WebRTCServer) -> TestClient: + """Return a client for the same routes ``_start_server`` registers.""" + app = web.Application() + app.router.add_get("/", server._serve_browser) + app.router.add_get("/app.js", server._serve_browser_script) + app.router.add_get("/healthz", server._health) + app.router.add_get("/request_session", server._serve_web_ui_page) + app.router.add_get("/api/session/initial_scene", server._initial_scene) + app.router.add_get("/api/session/first_frame", server._first_frame) + app.router.add_post("/api/session/input", server._session_input) + app.router.add_get("/{web_asset:.*}", server._serve_web_ui_asset) + async with TestClient(TestServer(app)) as test_client: + yield test_client + + +def test_stub_satisfies_the_protocol(web_root: Path) -> None: + assert isinstance(StubWebUi(web_root), IWebUiProvider) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", + ["/request_session", "/adapter.js", "/api/session/initial_scene", "/api/session/first_frame"], +) +async def test_routes_are_absent_without_an_application( + client: TestClient, path: str +) -> None: + """An application without a UI must see no behaviour change.""" + assert (await client.get(path)).status == 404 + + +@pytest.mark.asyncio +async def test_builtin_routes_survive_a_web_ui( + client: TestClient, server: WebRTCServer, web_root: Path +) -> None: + """The catch-all must not shadow the runtime's own viewer.""" + server.serve_web_ui(StubWebUi(web_root)) + + assert (await client.get("/")).status == 200 + assert (await client.get("/app.js")).status == 200 + assert (await client.get("/healthz")).status == 200 + + +@pytest.mark.asyncio +async def test_page_and_assets_are_served( + client: TestClient, server: WebRTCServer, web_root: Path +) -> None: + server.serve_web_ui(StubWebUi(web_root)) + + page = await client.get("/request_session") + assert page.status == 200 + assert await page.text() == "page" + assert (await (await client.get("/adapter.js")).text()) == "// adapter" + assert (await client.get("/assets/logo.svg")).status == 200 + assert (await client.get("/missing.js")).status == 404 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "attack", + ["/../outside.txt", "/..%2Foutside.txt", "/assets/../../outside.txt"], +) +async def test_paths_cannot_escape_the_web_root( + client: TestClient, server: WebRTCServer, web_root: Path, attack: str +) -> None: + server.serve_web_ui(StubWebUi(web_root)) + + assert (await client.get(attack)).status == 404 + + +@pytest.mark.asyncio +async def test_initial_scene_is_returned_verbatim( + client: TestClient, server: WebRTCServer, web_root: Path +) -> None: + """Serving must not interpret a scene: what the application says, ships.""" + web_ui = StubWebUi(web_root) + server.serve_web_ui(web_ui) + + response = await client.get("/api/session/initial_scene") + + assert response.status == 200 + assert await response.json() == dict(web_ui.initial_scene()) + + +@pytest.mark.asyncio +async def test_first_frame_is_served_with_its_content_type( + client: TestClient, server: WebRTCServer, web_root: Path +) -> None: + server.serve_web_ui(StubWebUi(web_root)) + + response = await client.get("/api/session/first_frame") + + assert response.status == 200 + assert response.headers["Content-Type"] == "image/jpeg" + assert await response.read() == b"\xff\xd8jpeg" + + +@pytest.mark.asyncio +async def test_absent_first_frame_is_not_an_error( + client: TestClient, server: WebRTCServer, web_root: Path +) -> None: + web_ui = StubWebUi(web_root) + web_ui.frame = None + server.serve_web_ui(web_ui) + + assert (await client.get("/api/session/first_frame")).status == 404 + + +@pytest.mark.asyncio +async def test_json_input_reaches_the_application( + client: TestClient, server: WebRTCServer, web_root: Path +) -> None: + web_ui = StubWebUi(web_root) + server.serve_web_ui(web_ui) + + response = await client.post("/api/session/input", json={"prompt": "new scene"}) + + assert response.status == 200 + assert (await response.json())["prompt"] == "new scene" + assert web_ui.applied[-1] == {"prompt": "new scene"} + + +@pytest.mark.asyncio +async def test_form_input_reaches_the_application( + client: TestClient, server: WebRTCServer, web_root: Path +) -> None: + web_ui = StubWebUi(web_root) + server.serve_web_ui(web_ui) + + response = await client.post("/api/session/input", data={"prompt": "formy"}) + + assert response.status == 200 + assert web_ui.applied[-1] == {"prompt": "formy"} + + +@pytest.mark.asyncio +async def test_multipart_upload_arrives_as_bytes( + client: TestClient, server: WebRTCServer, web_root: Path +) -> None: + """A page posts a first frame beside text fields; both must survive.""" + web_ui = StubWebUi(web_root) + server.serve_web_ui(web_ui) + + response = await client.post( + "/api/session/input", + data={ + "prompt": "with image", + "text_events": json.dumps([{"event_id": "storm", "prompt": "wind"}]), + "image": b"\x89PNGfake", + }, + ) + + assert response.status == 200 + applied = web_ui.applied[-1] + assert applied["image"] == b"\x89PNGfake" + assert isinstance(applied["text_events"], str) + assert applied["image_content_type"].startswith("application/") + + +@pytest.mark.asyncio +async def test_application_rejection_is_a_bad_request( + client: TestClient, server: WebRTCServer, web_root: Path +) -> None: + """A ValueError is the application's way of saying 400, not 500.""" + server.serve_web_ui(StubWebUi(web_root)) + + assert (await client.post("/api/session/input", json={"prompt": ""})).status == 400 + + +@pytest.mark.asyncio +async def test_malformed_json_is_a_bad_request( + client: TestClient, server: WebRTCServer, web_root: Path +) -> None: + server.serve_web_ui(StubWebUi(web_root)) + + response = await client.post( + "/api/session/input", + data="{not json", + headers={"Content-Type": "application/json"}, + ) + + assert response.status == 400 + + +@pytest.mark.asyncio +async def test_non_object_json_is_a_bad_request( + client: TestClient, server: WebRTCServer, web_root: Path +) -> None: + server.serve_web_ui(StubWebUi(web_root)) + + response = await client.post("/api/session/input", json=["not", "an", "object"]) + + assert response.status == 400 + + +def test_a_closed_server_refuses_a_web_ui( + server: WebRTCServer, web_root: Path +) -> None: + server._closed = True + + with pytest.raises(RuntimeError, match="closed"): + server.serve_web_ui(StubWebUi(web_root)) diff --git a/integrations_v2/lingbot/README.md b/integrations_v2/lingbot/README.md index 182b992fc..5a1b449c7 100644 --- a/integrations_v2/lingbot/README.md +++ b/integrations_v2/lingbot/README.md @@ -36,6 +36,93 @@ uv run --no-sync flashdreams-run-v2 cam2v-lingbot \ Shared [`apps/cam2v`](../../apps/cam2v/README.md) documentation lists controls, application arguments, and development commands. +## Browser UI + +This application serves its own page, so the run above offers two clients: + +- `http://:8089/` — the runtime's minimal viewer, the same one every v2 + application gets. +- `http://:8089/request_session` — the Lingbot scene UI: a preset + picker, a first-frame upload, a prompt box, and buttons for the scene's text + events. + +The page lives in [`apps/cam2v/web/`](apps/cam2v/web) and is served because +`LingbotCam2VApplication` implements `IWebUiProvider` +(`flashdreams.api_v2.web_ui`). That protocol is also what adds +`/api/session/initial_scene`, `/api/session/first_frame`, and +`POST /api/session/input`; an application without it gets none of them. + +Useful URL parameters: + +| Parameter | Effect | +| --- | --- | +| `?manual` | Do not auto-connect. Only one session runs per process, so a stray tab can otherwise claim it before the tab you meant to use. | +| `?preset=` | Open straight to a preset, slug being its lowercased name with spaces as hyphens, e.g. `?preset=water-blaster`. | + +Digit keys `1`-`9` jump to the matching preset while no text field has focus. +`C` clears the active event; `R` restarts the scene, discarding the rollout's +accumulated state so it re-seeds from its first frame — which is how to +recover after prompt swaps have left the world looking like a blend of +scenes. The current prompt is re-applied to the fresh rollout. + +### Scenes and text events + +A scene is a prompt, a first frame, and a catalog of text events. Triggering an +event swaps the rollout's cross-attention text context in place, so the scene +changes without restarting the session; clearing it restores the scene's base +prompt. The reserved `user_prompt` event id carries free-form text typed into +the page instead of a catalog entry. + +Catalog rules live in [`apps/cam2v/scene.py`](apps/cam2v/scene.py): at most 20 +events, 64-character labels, 1000-character event prompts, and ids drawn from +letters, numbers, `_`, `.`, `:` and `-`. The built-in presets are in +[`apps/cam2v/web/scene_presets.json`](apps/cam2v/web/scene_presets.json). + +Changing the prompt — including by picking another preset — swaps the text +context the same way, so the scene changes while the rollout keeps running. The +first frame is the exception: a rollout cannot replace the frame it was +initialized from, so picking a preset in the browser changes its events and +wording while the world still looks like whatever the session started on. To +start *in* a preset, name it when launching: `PRESET=noir-alley-combat bash +run.sh`. + +## Scripts + +For boxes without uv, [`setup.sh`](setup.sh) builds a plain `venv` with pip and +installs `flashdreams`, `apps/cam2v`, and this package as editable: + +```bash +bash setup.sh # TORCH_INDEX=cu130 to change the CUDA wheel index +bash run.sh # then open http://:8089/request_session +``` + +The camera-controls overlay is off by default here, because it is composited +into the generated frames rather than drawn by the browser, and so cannot be +dismissed from the page. `--ui` turns it on for its timing readout. A UI loop +runs either way: it is the only thing that can ask the runtime for a +replacement session, which is how the page switches scenes. + +`run.sh` uses `./.venv`, or an already-active `VIRTUAL_ENV`, or whatever +`flashdreams-run-v2` is on `PATH`. Arguments are forwarded to the +application. Settings are environment variables and go *before* the command +— `LIGHT=1 bash run.sh`, not `bash run.sh LIGHT=1`: + +| Variable | Default | Effect | +| --- | --- | --- | +| `APP` | `cam2v-lingbot` | Application slug; any entry point in `pyproject.toml`. | +| `HOST` / `PORT` | `0.0.0.0` / `8089` | Where to serve. | +| `LIGHT` | `0` | `LIGHT=1` generates 512x288 at 12 fps for GPUs that cannot keep up at the native 832x464. | +| `WIDTH` / `HEIGHT` / `FPS` | unset | Override the size directly; wins over `LIGHT`. | +| `PRESET` | unset | Start the rollout on a built-in preset's own image and prompt, by slug (`noir-alley-combat`). Unset runs the bundled example scene. | +| `BLOCKS` | `100000` | Blocks to generate before the run ends — set high so a run lasts until you stop it. `BLOCKS=20` for a quick smoke test. The application's own default is 20, roughly fifteen seconds. | +| `VENV` | `./.venv` | Environment to activate. | + +`LIGHT` reduces what the model generates, not just what is encoded: fewer +pixels cost less end to end but look worse. The page scales whatever is +generated up to the window, so a smaller stream is softer, not smaller. There +is no encoder to choose — the v2 server hands raw frames to aiortc's software +encoder. + ## Programmatic pipeline access ```python diff --git a/integrations_v2/lingbot/SCENE_PRESETS.md b/integrations_v2/lingbot/SCENE_PRESETS.md new file mode 100644 index 000000000..f88b2fb41 --- /dev/null +++ b/integrations_v2/lingbot/SCENE_PRESETS.md @@ -0,0 +1,253 @@ +# LingBot Scene Presets + +## Overview +Scene presets allow users to quickly load predefined scene configurations (prompt + events) into the LingBot initial scene panel. + +## Features +- **Built-in presets**: 5 default scene presets (Dragon, Jet Ski Cruise, Noir Alley Combat, Water Blaster, Circuit Racer) +- **Default on load**: "Dragon" auto-applies when the page opens (no manual selection needed) +- **Save custom presets**: Save current scene configuration as a new preset +- **Persistent storage**: Presets saved in browser localStorage +- **Quick access**: Dropdown selector in "Quick Start" section +- **Images**: every preset's start image is committed under `apps/cam2v/web/assets/` and referenced by relative path, so the picker renders with no network access + +## Usage + +### Loading a Preset +1. Open LingBot WebRTC interface (the "Dragon" preset is pre-selected by default) +2. Find "Quick Start" dropdown in Initial Scene panel +3. Select a different preset from the dropdown +4. Preset prompt + events + start image auto-populate the form; the dropdown stays on the selected preset name + +### In-Game Event Hotkeys +Once connected, each preset's events are playable via keyboard — +**Player Controls events use digit-key shortcuts (1-9)**, **Director +Controls events use letter-key shortcuts** from a fixed pool (`b, f, g, +h, m, n, o, p, r, t, u, v, x, y, z`) that deliberately excludes the +movement keys (w/a/s/d/q/e/i/j/k/l) so hotkeys never collide with driving +input. **Jump** always gets **Space** and **Crouch** always gets **Ctrl** +instead of a digit, matching common game convention — both also render as +their own row right next to the movement key grid instead of the general +event button list, since they're movement actions, not narrative +triggers. Every other event button shows its assigned digit/letter hotkey +(e.g. "Portal (1)", "Storm Rolls In (B)"). Press **c** to Clear the active +event — present and wired the same way on every game, since Clear isn't +part of any preset's catalog. Ignored while typing in a text field. Player +events beyond the digit pool or director events beyond the 15-letter pool +still work by click, just without a shown hotkey. + +### Director Controls +Each preset's events are split into two categories, matching the original +REACTOR case files' `actor: "character"` (player-triggered) vs. +`actor: "environment"` (narrative/pacing) events. Both live in the *same* +panel, switched with a single **toggle switch** at the top — not two +separate panels or tab buttons. The panel's own heading also swaps between +"Player Controls" / "Director Controls" to match: +- **Off (Player)** — the regular action buttons + a "Custom Prompt" box + + the movement key grid (w/a/s/d/q/e/i/j/k/l). +- **On (Director)** — the environment/pacing events (weather, hazards, + wildlife, etc.) + its own separate "Director Prompt" box, so a director + can send free-form direction text independently of the player's prompt. + The movement key grid hides — a director doesn't drive movement. + +The toggle only appears once director mode is on, reached either by adding +`?director` to the page URL (e.g. `?manual&director`) — which starts the +toggle already on — or by clicking **"Enable Director Mode"** in Player +Controls for any preset that has director events (no URL edit needed). +Player events get digit hotkeys, director events get letter hotkeys, so +the two never collide even though only one side's buttons are visible at +once. The health bar stays visible in both Player and Director Controls. + +Both tabs' events are always uploaded to the server together regardless +of `?director` — the shared WebRTC protocol has no player/director +distinction, so this split is purely a client-side UI/visibility choice. + +Hand-added events (via "Add" in the Initial Scene panel's Text Events +list, not from a built-in preset) default to Player and can be flipped to +Director with the **Director** checkbox on that event's own row — the live +Player/Director Controls buttons update immediately when toggled. + +### Sharing a Preset via URL +Add `?preset=` to the page URL to land directly on a specific built-in +preset instead of the "Dragon" default — the slug is the preset name, +lowercased with spaces replaced by hyphens (e.g. `Water Blaster` → +`water-blaster`). Example: `http://:8089/request_session?preset=circuit-racer`. +Unknown/missing slugs fall back to the default preset. This only shares +*which game loads*, not a live/running session — WebRTC still allows only +one active session per server process. + +### Saving a Preset +1. Edit prompt and text events in the Initial Scene panel +2. Click "Save" button next to Quick Start dropdown +3. Enter a name for the preset (e.g., "My Custom Scene") +4. Preset is saved to browser localStorage +5. New preset appears in dropdown for future use + +## Data Format + +Presets are stored as JSON array in `localStorage["lingbot-presets"]`: + +```json +[ + { + "name": "Preset Name", + "prompt": "Scene prompt text", + "events": [ + { + "event_id": "unique-id", + "label": "Event Label", + "prompt": "Event prompt text", + "health": -10 + } + ], + "directorEvents": [], + "hud": { "maxHealth": 100 } + } +] +``` + +`health` is optional (omit for no health effect); `directorEvents` uses the +same shape as `events`. + +## Adding a New Built-in Preset (Game) + +Built-in presets live in +`integrations_v2/lingbot/apps/cam2v/web/scene_presets.json` — a plain JSON +array, fetched at page load by `loadScenePresets()` in `adapter.js` (not +inlined in the JS, so it can be hand-edited without touching code). To add +one: + +1. **Write the base prompt** (1-3 sentences): subject + environment + style, + third person (or first person for cockpit/POV scenes like Circuit + Racer/Water Blaster). No camera-motion or input language — this app + drives movement via the `w/a/s/d/q/e/i/j/k/l` keys and text events, not + prose. +2. **Write events**: each a short one-sentence imperative/descriptive + clause (`{ event_id, label, prompt, health? }`). `event_id` is a short + lowercase token, unique within that preset's own `events`/`directorEvents` + arrays combined (ids can repeat *across* presets — each preset's list is + independent). Mix character-triggered actions (tricks, attacks) — put + these in `events`, **at least 5 of them** — with environment beats + (weather, other characters/vehicles appearing) — put these in + `directorEvents` (see "Director Controls" above). Add an optional numeric + `health` field where it makes narrative sense — negative for a hit/risk + (e.g. a hazard or a combat move), positive for a recovery/reward beat, + omit it entirely for a purely cosmetic trick with no stakes. **`events.length + + directorEvents.length` must not exceed 20** — that's a hard server-side + cap (`MAX_TEXT_EVENTS` in `apps/cam2v/scene.py`) applied to the *combined* total, + not separately per category, since the server has no player/director + concept at all (only a generic `category` string tag) — both arrays + upload together as one flat list; going over it makes every connect + attempt fail with "At most 20 text events are supported." `applyPreset()` + also logs a client-side warning at selection time if a preset is over + budget, so this should be caught before it reaches a failed connect. + - If you have access to `REACTOR_js-sdk`'s `lib/lingbot-cases/*.json` + (a richer, layered `base`/`camera`/`movement`/`events` scene format + used by a different app), you can mine its `scene.base.default` and + `scene.events[].detail` text for content and compress it down to this + flatter prompt+events shape — drop the camera/movement layers and any + `EXACTLY ONE ...` frame-count guard clauses, since this app doesn't use + that layering. Reference copies of a few are kept in + `apps/cam2v/web/assets/sources/` for exactly this purpose. +3. **Pick a start image**: commit it under `apps/cam2v/web/assets/` and + reference it by relative path (`assets/.jpg`), the way the built-in + presets do — it ships with the package and the page needs no network + access. A public `https://` URL still works if you would rather not commit + a binary. Note the v1 server's remote-URL fetch, with its SSRF guard + against private hosts, has no v2 equivalent yet: a URL is handed to the + browser to load, not fetched server-side. +4. **Add the object** to the array in `scene_presets.json` (strict JSON — + double-quoted keys, no trailing commas, no comments): + ```json + { + "name": "My Game", + "image": "assets/my-game.jpg", + "prompt": "...", + "events": [ + { "event_id": "thing1", "label": "Thing One", "prompt": "..." } + ], + "directorEvents": [], + "hud": { "maxHealth": 100 } + } + ``` +5. **(Optional) Make it the default**: the preset at index `0` is + auto-applied on page load via `presetSelect.value = "0"; applyPreset(0)` + in `mount()` — reorder the array (or edit those two lines) to change + which preset that is. +6. **Reload the page** — `adapter.js` is served straight off disk on every + request (the v2 server serves the application's `web_root()`), so a + browser refresh picks up the change with no server restart needed, as + long as you're running an editable (`pip install -e`) install. +7. Update the **Built-in Presets** list below and the preset count in + **Features** to keep this doc in sync. + +## Storage Location +- **Browser**: localStorage under key `lingbot-presets` +- **Persistence**: Survives page refresh, tab close +- **Scope**: Browser/domain specific +- **Cleared when**: Browser cache is cleared, incognito mode + +## Built-in Presets + +Event counts below are `player + director = total` (against the 20-event +combined cap — see step 2 under "Adding a New Built-in Preset"). Every +built-in preset has at least 5 player events. + +### 1. Dragon (default) +- **Prompt**: "A soaring journey through a fantasy jungle on the back of a flying creature. The wind whips past the rider's blue hands gripping the reins, causing the leather straps to vibrate, as the aerial voyage carries them toward an ancient gothic castle, its stonework growing clearer as it nears. Floating landmasses and cascading waterfalls fill the fantastical landscape below." +- **Player events (4)**: Jump, Portal, Storm, Fireworks. +- **Director events (6)**: Rival Dragon Appears, Wind Gust Rocks Mount, Castle Guards Fire Arrows, Meteor Shower, Aurora Lights the Sky, Griffin Gives Chase. +- 4 + 6 = 10 total. Portal/Storm/Fireworks are `DEFAULT_TEXT_EVENTS` from `apps/cam2v/scene.py` (also on `main`), not invented for this preset — Jump and all director events are additions specific to this preset (no source JSON exists for Dragon; the app's own example-00 default). Auto-selects on load. + +### 2. Jet Ski Cruise +- **Prompt**: "Turquoise water near a sandy beach lined with palm trees. A man in a red life vest riding a white and red jet ski, keeping it on top of the water at all times." +- **Player events (4)**: Jump, One-Hand Wave, Jump Off, Jump Back On. +- **Director events (9)**: Dolphins Leap, Storm Rolls In, Rogue Wave, Shark Lunges, Waterspout Forms, Fuel Runs Low, Thrown from the Jet Ski, Fuel Cache Spotted, Calm Water Break. +- 4 + 9 = 13 total. Uses "Fuel" as its HUD label (`hud.healthLabel`) instead of "Health". Jump Off/Jump Back On are a matched risk/recovery pair (-5/+8); One-Hand Wave awards a small `+3` trick bonus; Fuel Cache Spotted (+10) and Calm Water Break (+5) are dedicated recovery beats so a full playthrough always has a way to regain fuel, not just drain it. + +### 3. Noir Alley Combat +- **Prompt**: "A narrow urban alley at night, dark brick walls and heavy rain, shiny puddles on wet asphalt, yellow police tape, blue and red ambient light. A lone uniformed police officer in dark blue tactical gear holding a flashlight." +- **Player events (7)**: Jump, Draw Pistol (fires exactly one shot, health -8), Crouch, Punch Combo, Roundhouse Kick, Baton Strike, Dodge Roll. +- **Director events (5)**: Player Falls to Ground, Enemy Appears, Enemy Attacks, Chicken Walks In, Drone Appears. +- 7 + 5 = 12 total. Enemy Appears/Attacks were changed from plural to singular ("a single figure... no second figure, no group") to keep the model from generating a crowd when only one enemy is intended. + +### 4. Water Blaster +- **Prompt**: "First-person point of view aiming out across a colourful floating inflatable aqua park on a calm green quarry lake under bright summer sun. A bare hand grips a blue and red toy water blaster at the lower right of the frame." +- **Player events (6)**: Jump, Crouch, Splash Blast, Raise Float Shield, Green Slime Blast, Dive. +- **Director events (4)**: Rival Blaster Ambush, Bathers Get Super Soakers, Crocodile Lunges, Giant Balloon Drops. +- 6 + 5 = 11 total. + +### 5. Circuit Racer +- **Prompt**: "First-person cockpit view from inside a Formula 1 race car, gloved hands on the wheel and the glowing dash ahead, speeding down a sunlit asphalt racing circuit lined with red-and-white kerbs." +- **Player events (5)**: Jump, Kick Up Sparks, Lock-Up Smoke, Drift, DRS Boost. +- **Director events (7)**: Rain Sweeps In, Sun Glare, Tunnel Section, Road Fire, Checkered Flag, Puddle on the Track, Oil Slick Ahead. +- 5 + 7 = 12 total. + +Noir Alley Combat, Water Blaster, Jet Ski Cruise, and Circuit Racer prompts/events are adapted from the richer layered scene definitions in `REACTOR_js-sdk`'s `lib/lingbot-cases/*.json` (kept as reference copies under `apps/cam2v/web/assets/sources/` in this repo) down to this app's flatter prompt+events format. Their start images are committed under `apps/cam2v/web/assets/` and referenced by relative path. + +## Browser Console Access + +View all saved presets: +```javascript +JSON.parse(localStorage.getItem("lingbot-presets")) +``` + +Clear all presets: +```javascript +localStorage.removeItem("lingbot-presets") +``` + +## Implementation Files +- **UI**: `integrations_v2/lingbot/apps/cam2v/web/adapter.js` + - Top of file: Preset data definitions (`scenePresets`) + - `makeSceneCard()`: Scene card HTML with dropdown + Save button + - `saveCurrentPreset()` / `updatePresetDropdown()` / `loadSavedPresets()` / `applyPreset()`: Load/Save preset functions + - `mount()`: selects + applies preset index 0 ("Dragon") on load, before `loadInitialScene()` runs + - `presetSelect.addEventListener` / `savePresetButton.addEventListener`: Event listeners + +## Future Enhancements +1. **Server-side storage**: Save presets to backend database +2. **Export/Import**: Download presets as JSON file +3. **Sharing**: Share presets via link or code +4. **Categories**: Organize presets by category +5. **Versioning**: Track preset changes over time diff --git a/integrations_v2/lingbot/apps/cam2v/adapter.py b/integrations_v2/lingbot/apps/cam2v/adapter.py index 2d4794f22..7023b24c0 100644 --- a/integrations_v2/lingbot/apps/cam2v/adapter.py +++ b/integrations_v2/lingbot/apps/cam2v/adapter.py @@ -6,13 +6,28 @@ from __future__ import annotations import dataclasses +import mimetypes +import shutil +import tempfile +import threading +from collections.abc import Mapping +from pathlib import Path from typing import Any import torch -from cam2v import Cam2VApplication, Cam2VApplicationDefaults +from cam2v import ( + Cam2VApplication, + Cam2VApplicationDefaults, + Cam2VSlangPyUILoop, + generate_camera_step, +) +from loguru import logger from flashdreams.api_v2.application import IApplication +from flashdreams.api_v2.session import ISession from flashdreams.infra.config import derive_config +from flashdreams.runtime_v2.session_desc import SessionDesc +from lingbot.apps.cam2v.scene import SceneState from lingbot.config import ( PIPELINE_LINGBOT_WORLD_FAST, PIPELINE_LINGBOT_WORLD_FAST_TAEHV_WINDOW15_SINK3, @@ -40,8 +55,32 @@ """Lingbot defaults for the reusable Cam2V application.""" +class _SilentCam2VUILoop(Cam2VSlangPyUILoop): + """Cam2V UI loop that presents frames without drawing the overlay. + + The camera-controls panel is drawn into the generated frames before they + are encoded, so it is part of the video and cannot be dismissed from the + browser. ``--no-ui`` removes it by registering no UI loop at all, but a UI + loop is the only thing able to ask the runtime for a replacement session, + which is how the page switches scenes. This keeps the loop and drops only + the drawing: no widgets are built, so nothing is composited over the + frame, and the state the overlay would have displayed goes unmaintained. + """ + + def step_ui(self, ui: Any, step_index: int, events: Any) -> Any: + """Return the current model frame, building no widgets.""" + del ui, step_index, events + return self.presented_model_frame() + + class LingbotCam2VApplication(Cam2VApplication): - """Lingbot World specialization of the shared Cam2V application.""" + """Lingbot World specialization of the shared Cam2V application. + + Also serves the browser UI under ``web/``: a scene picker whose presets + carry a prompt, a first frame, and a catalog of text events. Implementing + :class:`IWebUiProvider` is what makes the v2 WebRTC server route + ``/request_session`` and the ``/api/session`` endpoints here. + """ def __init__(self, pipeline_config: Any | None = None) -> None: """Select the pipeline config used by this Cam2V application. @@ -58,7 +97,318 @@ def __init__(self, pipeline_config: Any | None = None) -> None: pipeline_config, enable_sync_and_profile=False ), ) + # Wrap the generation step so a requested prompt swap is applied on + # the model thread between chunks, where the rollout cache is idle. + defaults = dataclasses.replace(defaults, generate_step=self._generate_step) super().__init__(defaults=defaults) + self._scene: SceneState | None = None + self._pending_prompt: str | None = None + self._active_prompt: str | None = None + self._prompt_lock = threading.Lock() + self._prompt_embeddings: dict[str, Any] = {} + self._active_cache: Any | None = None + self._ui_loop: Any | None = None + self._draw_overlay: bool | None = None + self._scene_dir: Path | None = None + + def _configure_argument_parser(self, parser: Any) -> None: + """Default the overlay off, since it is drawn into the video itself. + + The shared application defaults ``--ui`` on, which composites the + camera-controls panel into the generated frames -- part of the video, + not something the browser can dismiss. ``--ui`` still turns it on for + the timing readout. + """ + super()._configure_argument_parser(parser) + parser.set_defaults(ui=False) + + def create_session(self, session_desc: SessionDesc) -> ISession: + """Create the rollout and describe it for the browser UI.""" + # --ui selects which UI loop runs, not whether one runs at all. A UI + # loop is the only thing that can ask the runtime for a replacement + # session, so registering none -- what --no-ui does -- would silently + # disable switching scenes from the page. The flag keeps the meaning + # its name implies: draw the overlay, or do not. + # + # Recorded once: forcing _use_ui on below would otherwise be read back + # as "the overlay was asked for" by the next session, so a scene + # switch brought it back. + if self._draw_overlay is None: + self._draw_overlay = self._use_ui + draw_overlay = self._draw_overlay + self._use_ui = True + session = super().create_session(session_desc) + # The shared session registers its loops without keeping one, so the + # bound method is wrapped to capture what it registers. + register_ui_loop = session.register_ui_loop + + def capture_ui_loop(loop_type: Any, **kwargs: Any) -> Any: + if loop_type is Cam2VSlangPyUILoop and not draw_overlay: + loop_type = _SilentCam2VUILoop + loop = register_ui_loop(loop_type, **kwargs) + self._ui_loop = loop + return loop + + session.register_ui_loop = capture_ui_loop # type: ignore[method-assign] + self._ui_loop = None + scene = self._scene + if scene is None: + self._scene = self._build_scene(session_desc) + else: + # A page may have built the scene already, before the runtime got + # this far; take the resolution the session actually resolved to. + scene.video_width = session_desc.video_width + scene.video_height = session_desc.video_height + # A new rollout re-encodes from its own prompt, so nothing carries + # over from the previous one. + with self._prompt_lock: + self._pending_prompt = None + self._active_prompt = None + return session + + def _generate_step( + self, + pipeline: Any, + autoregressive_index: int, + cache: Any, + camera_input: Any, + ) -> torch.Tensor: + """Apply any pending text swap, then generate the chunk as usual.""" + if cache is not self._active_cache: + # A reset discards the cache, and the next step re-seeds it from + # the session's own conditioning -- which restores the prompt the + # rollout started with. Re-apply whatever the page has chosen + # since, so a reset clears the drifted image without also + # reverting the scene. + self._active_cache = cache + self._active_prompt = None + scene = self._scene + if scene is not None and scene.active_prompt() != "": + with self._prompt_lock: + self._pending_prompt = scene.active_prompt() + with self._prompt_lock: + prompt = self._pending_prompt + self._pending_prompt = None + if prompt is not None and prompt != self._active_prompt: + self._swap_text_context(pipeline, cache, prompt) + self._active_prompt = prompt + return generate_camera_step( + pipeline, autoregressive_index, cache, camera_input + ) + + def _swap_text_context(self, pipeline: Any, cache: Any, prompt: str) -> None: + """Replace the rollout's cross-attention text context with ``prompt``. + + Only the static text context changes: the self-attention KV cache + stays intact, so the rollout keeps its horizon and the swap shows up + from this chunk on. + + Raises: + RuntimeError: This pipeline's transformer cannot swap text. + """ + transformer = pipeline.diffusion_model.transformer + replace_text_embeddings = getattr(transformer, "replace_text_embeddings", None) + if not callable(replace_text_embeddings): + raise RuntimeError( + "Lingbot text events need a pipeline whose transformer supports " + "replace_text_embeddings; this pipeline does not." + ) + embeddings = self._prompt_embeddings.get(prompt) + if embeddings is None: + # initialize_cache releases the one-shot encoders once the rollout + # starts, so the text encoder is reloaded before encoding here. + pipeline._ensure_oneshot_encoders_loaded() + embeddings = pipeline.text_encoder([prompt]).to( + device=torch.device(self._device) + ) + self._prompt_embeddings[prompt] = embeddings + replace_text_embeddings(cache.transformer_cache, embeddings) + logger.info("Lingbot text context updated: {}", prompt) + + def _build_scene(self, session_desc: SessionDesc | None = None) -> SceneState: + """Describe the session this application runs, or is about to. + + Resolves the conditioning the same way :meth:`create_session` does, so + the page shows the prompt and first frame the rollout actually starts + from rather than only what was passed on the command line. Example + data, in particular, supplies both and neither appears in the raw + arguments. + """ + input_values = dict(self._input_values or {}) + desc = session_desc or self.session_desc() + prompt = str(input_values.get("prompt", "")) + image_path = str(input_values.get("image_path") or "") + try: + conditioning = self.defaults.input_resolver( + { + **input_values, + "pixel_height": desc.video_height, + "pixel_width": desc.video_width, + "fps": desc.frames_per_second_for_step, + } + ) + except Exception as error: # noqa: BLE001 - the page still needs a scene + # Resolving can download example data, so it is allowed to fail + # without taking the browser UI with it. + logger.debug("Lingbot scene falling back to raw inputs: {}", error) + else: + prompt = str(getattr(conditioning, "prompt", "") or prompt) + image_path = str(getattr(conditioning, "first_frame_path", "") or image_path) + return SceneState( + prompt=prompt, + model=getattr(self._pipeline_config, "name", type(self).__name__), + video_width=desc.video_width, + video_height=desc.video_height, + first_frame_path=image_path, + ) + + def _require_scene(self) -> SceneState: + """Return the scene, building it if no session has started yet. + + The HTTP server is serving before the runtime creates a session, so a + browser that connects while the model is still loading would otherwise + be told the session has not started -- which is exactly when its page + needs the scene. + """ + scene = self._scene + if scene is None: + scene = self._build_scene() + self._scene = scene + return scene + + def close(self) -> None: + """Release the pipeline, and the scratch directory for uploads.""" + scene_dir = self._scene_dir + self._scene_dir = None + if scene_dir is not None: + shutil.rmtree(scene_dir, ignore_errors=True) + super().close() + + # --- IWebUiProvider --------------------------------------------------- + def web_root(self) -> Path: + """Return the directory holding this application's page and assets.""" + return Path(__file__).parent / "web" + + def initial_scene(self) -> Mapping[str, Any]: + """Return the current scene for the page to render.""" + return self._require_scene().as_dict() + + def first_frame(self) -> tuple[bytes, str] | None: + """Return the frame the session starts from, uploaded or resolved. + + A resolved first frame is a path on the server, so it is read here + rather than handed to the page as a URL it could not load. + """ + scene = self._require_scene() + if scene.image_bytes is not None: + return scene.image_bytes, scene.image_content_type + if not scene.first_frame_path: + return None + path = Path(scene.first_frame_path) + try: + data = path.read_bytes() + except OSError as error: + logger.debug("Lingbot first frame unreadable: {}", error) + return None + content_type, _ = mimetypes.guess_type(path.name) + return data, content_type or "application/octet-stream" + + def apply_session_input(self, payload: Mapping[str, Any]) -> Mapping[str, Any]: + """Apply one page submission, steering the rollout when it can. + + A triggered text event swaps the rollout's text conditioning in place + so it takes effect without restarting. Prompt, first-frame, and + catalog changes are recorded for the next session, since the frame a + rollout was initialized from cannot be replaced mid-flight. + + Raises: + ValueError: The submission is malformed or names an unknown event. + """ + scene = self._require_scene() + before = (scene.image_bytes, scene.image_url) + prompt = scene.apply(payload) + if (scene.image_bytes, scene.image_url) != before: + # A different first frame is a different scene, and a rollout + # cannot swap the frame it was initialized from -- so ask for a + # new session rather than steering the current one, which would + # only blend the new prompt into the old world. + self._restart_on(scene) + elif prompt is not None: + # Handed to the model thread rather than applied here: this runs + # on the HTTP thread, where the rollout cache is in use. + with self._prompt_lock: + self._pending_prompt = prompt + return scene.as_dict() + + def _restart_on(self, scene: SceneState) -> None: + """Ask the runtime for a session that starts from ``scene``. + + Does nothing when the scene names no image, or when no UI loop has + registered yet -- the only thing able to make the request. The scene + is recorded either way, so the next session starts from it. + + Raises: + ValueError: The scene names an image that cannot seed a rollout. + """ + if not scene.image_url and scene.image_bytes is None: + return + first_frame = self._materialize_first_frame(scene) + if first_frame is None: + # The scene asked for an image and none could be produced. Failing + # here surfaces it as a 400 on the page rather than leaving the + # rollout quietly generating from the frame it already had. + raise ValueError( + f"No usable start image for this scene: {scene.image_url!r} " + "could not be resolved. Preset images are paths under the " + "application's web/ directory; a remote URL is loaded by the " + "browser and cannot seed a rollout." + ) + if str(first_frame) == scene.first_frame_path: + # The page posts its scene on every connect, so without this a + # reconnect rebuilt the session -- and paid the warmup again -- + # for a frame the rollout had already started from. + return + scene.first_frame_path = str(first_frame) + # Only the frame and prompt are overridden. The resolver takes an + # explicit image_path over the example's own ("image_path or + # example_dir / image.jpg"), while example data still supplies the + # intrinsics and poses it requires and nothing else provides -- + # turning it off here made every replacement session fail with + # "Lingbot Cam2V requires intrinsic_path". + self._input_values = { + **(self._input_values or {}), + "prompt": scene.prompt, + "image_path": str(first_frame), + } + ui_loop = self._ui_loop + if ui_loop is None: + logger.info("Lingbot scene recorded; it starts on the next session.") + return + ui_loop.request_new_session(self.session_desc()) + logger.info("Lingbot restarting on a new scene: {}", first_frame.name) + + def _materialize_first_frame(self, scene: SceneState) -> Path | None: + """Return a readable path for the scene's first frame. + + Uploads are written out, and a page-supplied URL is resolved inside + the web root, which is where the built-in presets keep their images. + A URL pointing anywhere else is not fetched -- the server does not + make outbound requests on a page's behalf. + """ + if scene.image_bytes is not None: + if self._scene_dir is None: + self._scene_dir = Path(tempfile.mkdtemp(prefix="lingbot-scene-")) + suffix = mimetypes.guess_extension(scene.image_content_type) or ".jpg" + path = self._scene_dir / f"first_frame{suffix}" + path.write_bytes(scene.image_bytes) + return path + if scene.image_url.startswith(("http://", "https://")): + return None + root = self.web_root().resolve() + candidate = (root / scene.image_url).resolve() + if not candidate.is_file() or not candidate.is_relative_to(root): + return None + return candidate def create_app() -> IApplication: diff --git a/integrations_v2/lingbot/apps/cam2v/scene.py b/integrations_v2/lingbot/apps/cam2v/scene.py new file mode 100644 index 000000000..b39034108 --- /dev/null +++ b/integrations_v2/lingbot/apps/cam2v/scene.py @@ -0,0 +1,453 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Scene state behind the Lingbot browser UI: prompt, first frame, events. + +The browser page picks a scene -- a prompt, a first frame, and a catalog of +text events it can trigger during the rollout -- and this module holds what it +picked. The serving layer stays generic: it copies :meth:`SceneState.as_dict` +into a JSON response and hands :meth:`SceneState.apply` the decoded request +body, without knowing what any of it means. + +Text events are the model-facing half. Each carries a prompt that replaces the +rollout's text conditioning while the event is active, so triggering "Storm" +mid-rollout steers generation without restarting the session. The reserved +``user_prompt`` id carries free-form text supplied at request time instead of a +catalog entry. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any + +MAX_TEXT_EVENTS = 20 +"""Ceiling on a client-supplied catalog, matching the page's own limit.""" + +MAX_TEXT_EVENT_LABEL_CHARS = 64 +"""Ceiling on one event's button label.""" + +MAX_TEXT_EVENT_PROMPT_CHARS = 1_000 +"""Ceiling on one event's prompt, well above any written for the presets.""" + +MAX_PROMPT_CHARS = 2_000 +"""Ceiling on the scene prompt itself.""" + +MAX_IMAGE_BYTES = 15 * 1024 * 1024 +"""Ceiling on an uploaded first frame.""" + +USER_PROMPT_EVENT_ID = "user_prompt" +"""Reserved id for free-form text supplied at request time. + +Not a catalog entry, so it deliberately bypasses the membership check in +:meth:`SceneState.resolve_event_prompt`. Must match the literal in +``web/adapter.js`` and ``lingbot/impl/input_mapping.py``. +""" + +_TEXT_EVENT_ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,64}$") + +_CLEAR_STATES = frozenset({"clear", "release", "off", "none"}) +_TRIGGER_STATES = frozenset({"trigger", "hold", "on"}) + + +def normalize_prompt_text(prompt: str) -> str: + """Collapse prompt whitespace into a single line.""" + return " ".join(prompt.split()) + + +def _normalize_field(value: object) -> str: + return normalize_prompt_text(str(value)) if value is not None else "" + + +def _slugify_event_id(label: str, index: int) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", label.lower()).strip("-") + return (slug or f"event-{index + 1}")[:64] + + +@dataclass(frozen=True, slots=True) +class TextEventSpec: + """One text event the page can trigger, addressed by stable id.""" + + event_id: str + """Stable identifier the page sends back to trigger this event.""" + + label: str + """Short client-facing button label.""" + + prompt: str + """Text conditioning activated while this event is active.""" + + category: str = "environment" + """Client-facing group used to organize the event controls.""" + + def as_public_dict(self) -> dict[str, str]: + """Return the client-facing event payload.""" + return { + "event_id": self.event_id, + "label": self.label, + "prompt": self.prompt, + "category": self.category, + } + + +DEFAULT_TEXT_EVENTS: tuple[TextEventSpec, ...] = ( + TextEventSpec( + event_id="portal", + label="Portal", + prompt=( + "A luminous magical portal opens in the scene, casting colored light " + "and swirling particles into the environment." + ), + ), + TextEventSpec( + event_id="storm", + label="Storm", + prompt=( + "A dramatic storm rolls in with dark clouds, wind, rain, and flashes " + "of lightning reshaping the atmosphere." + ), + ), + TextEventSpec( + event_id="fireworks", + label="Fireworks", + prompt=( + "Bright fireworks burst overhead, filling the sky with colorful sparks " + "and reflections across the scene." + ), + ), +) +"""Events advertised when the page supplies no catalog of its own.""" + + +def normalize_text_events(raw_events: object) -> tuple[TextEventSpec, ...]: + """Validate and normalize a client-supplied text-event catalog. + + Args: + raw_events: Decoded ``text_events`` value from the page. + + Returns: + The catalog, with ids filled in from labels where the page omitted + them. + + Raises: + ValueError: The catalog is malformed, over a size limit, or contains + a duplicate id. + """ + if not isinstance(raw_events, (list, tuple)): + raise ValueError("Text events must be a list.") + + text_events: list[TextEventSpec] = [] + seen_ids: set[str] = set() + for index, raw_event in enumerate(raw_events): + if isinstance(raw_event, TextEventSpec): + event_id = raw_event.event_id.strip() + label = normalize_prompt_text(raw_event.label) + prompt = normalize_prompt_text(raw_event.prompt) + category = normalize_prompt_text(raw_event.category) or "custom" + elif isinstance(raw_event, Mapping): + label = _normalize_field(raw_event.get("label")) + prompt = _normalize_field(raw_event.get("prompt")) + # Left unslugified until after the blank-row check below, so a + # row with nothing in it does not acquire an id and become an + # event with a missing prompt. + event_id = _normalize_field( + raw_event.get("event_id", raw_event.get("id")) + ) + category = _normalize_field(raw_event.get("category")) + else: + raise ValueError("Each text event must be an object.") + + # A wholly empty entry is a trailing blank row in the page's editor, + # not an error. + if not event_id and not label and not prompt: + continue + if not event_id: + event_id = _slugify_event_id(label, index) + if not prompt: + raise ValueError("Text event prompt is required.") + if not label: + label = event_id + if len(label) > MAX_TEXT_EVENT_LABEL_CHARS: + raise ValueError( + f"Text event labels must be <= {MAX_TEXT_EVENT_LABEL_CHARS} characters." + ) + if len(prompt) > MAX_TEXT_EVENT_PROMPT_CHARS: + raise ValueError( + "Text event prompts must be " + f"<= {MAX_TEXT_EVENT_PROMPT_CHARS} characters." + ) + if not _TEXT_EVENT_ID_RE.fullmatch(event_id): + raise ValueError( + "Text event ids must be 1-64 characters using only letters, " + "numbers, '_', '.', ':', or '-'." + ) + if event_id in seen_ids: + raise ValueError(f"Duplicate text event id={event_id!r}.") + seen_ids.add(event_id) + text_events.append( + TextEventSpec( + event_id=event_id, + label=label, + prompt=prompt, + category=category or "custom", + ) + ) + + if len(text_events) > MAX_TEXT_EVENTS: + raise ValueError(f"At most {MAX_TEXT_EVENTS} text events are supported.") + return tuple(text_events) + + +def normalize_event_state(state: object) -> str: + """Return a trigger or clear state, defaulting anything else to trigger. + + Raises: + ValueError: The state is a word this does not recognize. + """ + normalized = str(state or "trigger").strip().lower() or "trigger" + if normalized in _CLEAR_STATES or normalized in _TRIGGER_STATES: + return normalized + raise ValueError( + "Event state must be one of trigger, hold, on, clear, release, off, none." + ) + + +def is_clear_state(state: str) -> bool: + """Return whether ``state`` asks to restore the scene's base prompt.""" + return state in _CLEAR_STATES + + +@dataclass +class SceneState: + """What the page has chosen for the session, and what it may change. + + Holds no model objects: the application owns those and reads this for the + prompt and events to apply. Kept separate so the catalog rules are + testable without a pipeline. + """ + + prompt: str + """Base prompt, restored whenever an active event is cleared.""" + + model: str + """Model variant name, shown by the page.""" + + video_width: int + """Generated frame width, shown by the page.""" + + video_height: int + """Generated frame height, shown by the page.""" + + default_image_url: str = "" + """First frame the application starts with, absent a page upload.""" + + image_url: str = "" + """First frame the page selected, by URL.""" + + image_bytes: bytes | None = None + """First frame the page uploaded, which wins over :attr:`image_url`.""" + + image_content_type: str = "image/jpeg" + """Content type reported for :attr:`image_bytes`.""" + + first_frame_path: str = "" + """File the session starts from, served through the first-frame endpoint. + + A resolved first frame is a path on the server, which a browser cannot + load, so it is read and served rather than handed over as a URL. + """ + + text_events: tuple[TextEventSpec, ...] = DEFAULT_TEXT_EVENTS + """Catalog the page may trigger from.""" + + active_event_id: str | None = None + """Event currently steering generation, or ``None`` for the base prompt.""" + + input_source: str = "default" + """``"default"`` until the page submits a change, then ``"uploaded"``.""" + + _event_prompts: dict[str, str] = field(default_factory=dict, repr=False) + """Prompt by event id, rebuilt whenever the catalog changes.""" + + def __post_init__(self) -> None: + self.prompt = normalize_prompt_text(self.prompt) + self._reindex_events() + + def _reindex_events(self) -> None: + self._event_prompts = { + event.event_id: event.prompt for event in self.text_events + } + # A catalog change can retire the active event; fall back to the base + # prompt rather than leaving a dangling id the page cannot clear. + if self.active_event_id is not None: + if self.active_event_id not in self._event_prompts: + if self.active_event_id != USER_PROMPT_EVENT_ID: + self.active_event_id = None + + def as_dict(self) -> dict[str, Any]: + """Return the scene payload the page renders itself from.""" + return { + "first_frame_url": "/api/session/first_frame", + "image_url": self.image_url or self.default_image_url, + "default_image_url": self.default_image_url, + # True only when /api/session/first_frame has something to + # answer with: uploaded bytes, or a file the session starts from. + # A URL is not fetched server-side -- the page loads it directly + # from ``image_url`` -- so reporting one here would send the page + # to an endpoint that can only 404, once per action. + "has_first_frame": bool(self.image_bytes or self.first_frame_path), + "prompt": self.prompt, + "input_source": self.input_source, + "model": self.model, + "capabilities": {"text_events": bool(self.text_events)}, + "event_catalog": [event.as_public_dict() for event in self.text_events], + "active_event_id": self.active_event_id, + "resolution": {"width": self.video_width, "height": self.video_height}, + } + + def active_prompt(self) -> str: + """Return the text the rollout should currently be conditioned on. + + The active event's prompt while one is active, otherwise the scene's + base prompt. Used to restore conditioning after a reset re-seeds the + rollout from the session's own prompt. + """ + if self.active_event_id is None: + return self.prompt + return self._event_prompts.get(self.active_event_id, self.prompt) + + def resolve_event_prompt( + self, *, event_id: str, state: str, prompt: str = "" + ) -> str | None: + """Return the prompt an event asks for, and record it as active. + + Returns: + The text to condition on, or ``None`` when nothing changes. + + Raises: + ValueError: The event is not in the catalog, or a free-form + request carried no text. + """ + state = normalize_event_state(state) + event_id = event_id.strip() + if is_clear_state(state) or not event_id: + self.active_event_id = None + return self.prompt + + if event_id == USER_PROMPT_EVENT_ID: + free_form = normalize_prompt_text(prompt) + if not free_form: + raise ValueError("A free-form prompt requires prompt text.") + if len(free_form) > MAX_PROMPT_CHARS: + raise ValueError(f"Prompt must be <= {MAX_PROMPT_CHARS} characters.") + self.active_event_id = event_id + return free_form + + event_prompt = self._event_prompts.get(event_id) + if event_prompt is None: + supported = ", ".join(sorted(self._event_prompts)) or "none" + raise ValueError(f"Unknown event_id={event_id!r}. Supported: {supported}") + self.active_event_id = event_id + return event_prompt + + def apply(self, payload: Mapping[str, Any]) -> str | None: + """Apply one page submission and return any prompt to condition on. + + Args: + payload: Decoded request body. ``prompt``, ``image``/``image_url``, + and ``text_events`` set the scene; ``event_id``/``state`` + trigger one event from the catalog. + + Returns: + Text to condition the rollout on now, or ``None`` when the change + only takes effect for the next session. + + Raises: + ValueError: The submission is malformed or names an unknown event. + """ + changed = False + + raw_events = payload.get("text_events", payload.get("events")) + if isinstance(raw_events, str) and raw_events.strip(): + try: + raw_events = json.loads(raw_events) + except json.JSONDecodeError as error: + raise ValueError("Text events must be valid JSON.") from error + if raw_events is not None and not isinstance(raw_events, str): + self.text_events = normalize_text_events(raw_events) + self._reindex_events() + changed = True + + prompt_changed = False + raw_prompt = payload.get("prompt") + if isinstance(raw_prompt, str) and raw_prompt.strip(): + prompt = normalize_prompt_text(raw_prompt) + if len(prompt) > MAX_PROMPT_CHARS: + raise ValueError(f"Prompt must be <= {MAX_PROMPT_CHARS} characters.") + prompt_changed = prompt != self.prompt + self.prompt = prompt + changed = True + + image = payload.get("image") + if isinstance(image, bytes) and image: + if len(image) > MAX_IMAGE_BYTES: + raise ValueError( + f"First-frame image must be <= {MAX_IMAGE_BYTES} bytes." + ) + content_type = str(payload.get("image_content_type", "image/jpeg")) + if not content_type.startswith("image/"): + raise ValueError("Uploaded first frame must be an image.") + self.image_bytes = image + self.image_content_type = content_type + # An upload supersedes any previously chosen URL. + self.image_url = "" + changed = True + else: + raw_url = payload.get("image_url") + if isinstance(raw_url, str) and raw_url.strip(): + self.image_url = raw_url.strip() + self.image_bytes = None + changed = True + + event_id = payload.get("event_id") + if isinstance(event_id, str) and event_id.strip(): + return self.resolve_event_prompt( + event_id=event_id, + state=str(payload.get("state", "trigger")), + prompt=str(payload.get("prompt", "")), + ) + + if not changed: + raise ValueError( + "Submit a prompt, an image, an image URL, text events, or an event." + ) + self.input_source = "uploaded" + if prompt_changed: + # Steer the running rollout rather than waiting for a session that + # may never come: the runtime builds one session per process, so a + # prompt recorded for "next time" would never be seen. Switching + # scenes mid-rollout is exactly what the text-context swap is for. + # Any active event is superseded by the new base prompt. + self.active_event_id = None + return self.prompt + return None + + +__all__ = [ + "DEFAULT_TEXT_EVENTS", + "MAX_IMAGE_BYTES", + "MAX_PROMPT_CHARS", + "MAX_TEXT_EVENTS", + "MAX_TEXT_EVENT_LABEL_CHARS", + "MAX_TEXT_EVENT_PROMPT_CHARS", + "USER_PROMPT_EVENT_ID", + "SceneState", + "TextEventSpec", + "is_clear_state", + "normalize_event_state", + "normalize_prompt_text", + "normalize_text_events", +] diff --git a/integrations_v2/lingbot/apps/cam2v/web/adapter.css b/integrations_v2/lingbot/apps/cam2v/web/adapter.css new file mode 100644 index 000000000..5ab147aa9 --- /dev/null +++ b/integrations_v2/lingbot/apps/cam2v/web/adapter.css @@ -0,0 +1,459 @@ +/* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ +/* SPDX-License-Identifier: Apache-2.0 */ + +.firstFramePreview { + position: absolute; + inset: 0; + opacity: 0; + transition: opacity 220ms ease; +} + +.stageVideo, +.firstFramePreview { + /* One rule for both, so the preview standing in for the video cannot + change size or framing on connect. Shown centred at the stream's own + pixel size -- taken from the session resolution reported by + /api/session/initial_scene (see applyVideoSizing) -- never upscaled, so + what reaches the browser is what the model generated. It only shrinks + when the window is smaller than the stream. */ + /* Anchored near the top rather than centred: the Player Controls panel + occupies the lower left, and a centred frame put the model's output + behind it. Clears the header, then stays put. */ + inset: clamp(84px, 11vh, 148px) auto auto 50%; + width: min( + 100vw, + var(--lingbot-video-width, 832px), + var(--lingbot-video-width-from-vh, 179.31vh) + ); + height: auto; + max-height: min(100vh, var(--lingbot-video-height, 464px)); + aspect-ratio: var(--lingbot-video-aspect, 832 / 464); + transform: translateX(-50%); + object-fit: cover; + object-position: center; +} + +body.is-ready-preview .firstFramePreview { + opacity: 1; +} + +.gameOverOverlay { + position: absolute; + inset: 0; + display: flex; + align-items: center; + justify-content: center; + font-size: clamp(32px, 6vw, 72px); + font-weight: 900; + letter-spacing: 0.12em; + color: #ff4d4d; + text-shadow: 0 0 24px rgba(255, 77, 77, 0.85), 0 2px 6px rgba(0, 0, 0, 0.9); + background: rgba(0, 0, 0, 0.35); + pointer-events: none; + z-index: 5; +} + +.gameOverOverlay[hidden] { + display: none; +} + +.healthBar { + display: grid; + gap: 4px; + margin-bottom: 12px; +} + +.healthBarLabel { + display: flex; + justify-content: space-between; + font-size: 12px; + opacity: 0.8; +} + +.healthBarTrack { + height: 10px; + border-radius: 5px; + background: rgba(255, 255, 255, 0.14); + overflow: hidden; +} + +.healthBarFill { + height: 100%; + width: 100%; + background: #4caf50; + transition: width 200ms ease, background-color 200ms ease; +} + +.healthBarFill.is-low { + background: #e53935; +} + +.sceneCard { + position: absolute; + top: clamp(120px, 15vh, 168px); + left: clamp(18px, 3vw, 52px); + display: grid; + gap: 12px; + width: min(560px, calc(100vw - 36px)); + max-height: min(680px, calc(100vh - 224px)); + padding: 16px 18px 18px; + overflow: auto; +} + +.sceneCard[hidden], +.eventControls[hidden], +.eventButtons[hidden], +.promptControlGroup[hidden], +.controlsModeToggle[hidden], +.healthBar[hidden] { + display: none; +} + +.controlsModeToggle { + display: flex; + align-items: center; + gap: 8px; + grid-column: 1 / -1; + margin-bottom: 4px; + background: none; + border: none; + padding: 0; + cursor: pointer; +} + +.controlsModeToggleTrack { + position: relative; + width: 34px; + height: 18px; + border-radius: 9px; + background: rgba(255, 255, 255, 0.2); + transition: background-color 150ms ease; + flex-shrink: 0; +} + +.controlsModeToggleKnob { + position: absolute; + top: 2px; + left: 2px; + width: 14px; + height: 14px; + border-radius: 50%; + background: #fff; + transition: transform 150ms ease; +} + +.controlsModeToggle.is-on .controlsModeToggleTrack { + background: #4caf50; +} + +.controlsModeToggle.is-on .controlsModeToggleKnob { + transform: translateX(16px); +} + +.firstFrameSourceRow { + display: grid; + grid-template-columns: 86px minmax(0, 1fr) 86px; + gap: 8px; + min-height: 68px; +} + +.firstFrameSourceRow[data-mode="upload"] { + grid-template-columns: minmax(0, 1fr) 86px 86px; +} + +.sourcePane { + min-width: 0; + overflow: hidden; + border: 1px solid rgba(255, 255, 255, 0.13); + border-radius: 8px; + background: rgba(255, 255, 255, 0.045); +} + +.firstFrameSourceRow[data-mode="upload"] .sourcePaneUpload, +.firstFrameSourceRow[data-mode="url"] .sourcePaneUrl { + border-color: rgba(142, 240, 28, 0.38); + background: rgba(255, 255, 255, 0.075); +} + +.sourceModeButton, +.uploadControl { + width: 100%; + min-height: 68px; + border: 0; + background: transparent; + color: var(--text); + cursor: pointer; + font-size: 0.78rem; + font-weight: 800; +} + +.uploadControl { + display: none; + align-items: center; + justify-content: center; + padding: 0 14px; +} + +.uploadControl input { + position: absolute; + width: 1px; + height: 1px; + opacity: 0; +} + +.firstFrameSourceRow[data-mode="upload"] .sourcePaneUpload .uploadControl { + display: flex; +} + +.firstFrameSourceRow[data-mode="upload"] .sourcePaneUpload .sourceModeButton, +.firstFrameSourceRow[data-mode="url"] .sourcePaneUrl .sourceModeButton { + display: none; +} + +.promptControl, +.textEventEditor, +.urlControl { + display: grid; + gap: 6px; + color: var(--muted); + font-size: 0.76rem; + font-weight: 700; +} + +.urlControl { + display: none; + padding: 8px; +} + +.firstFrameSourceRow[data-mode="url"] .sourcePaneUrl .urlControl { + display: grid; +} + +.promptControl textarea, +.promptControl input, +.textEventPrompt, +.textEventLabel, +.urlControl input { + width: 100%; + padding: 10px 12px; + border: 1px solid rgba(255, 255, 255, 0.18); + border-radius: 6px; + outline: 0; + background: rgba(4, 6, 7, 0.62); + color: var(--text); + font: 0.84rem/1.3 inherit; +} + +.promptControl textarea { + min-height: 94px; + resize: vertical; +} + +.promptControl textarea:focus, +.promptControl input:focus, +.textEventPrompt:focus, +.textEventLabel:focus, +.urlControl input:focus { + outline: 2px solid rgba(142, 240, 28, 0.38); + outline-offset: 2px; +} + +.urlUpdateButton, +.textEventAddButton, +.textEventRemoveButton { + border: 1px solid rgba(142, 240, 28, 0.42); + border-radius: 7px; + background: rgba(142, 240, 28, 0.14); + color: var(--text); + cursor: pointer; + font-size: 0.76rem; + font-weight: 800; +} + +.urlUpdateButton { + min-height: 68px; +} + +.firstFrameUpdateRow { + min-height: 0; +} + +.fieldStatus { + color: var(--muted); + font-size: 0.76rem; + font-weight: 700; +} + +.fieldStatus[data-state="error"] { + color: var(--danger); +} + +.fieldStatus[data-state="success"] { + color: var(--accent); +} + +.fieldStatus[data-state="pending"] { + color: var(--warning); +} + +.textEventHeader { + display: flex; + align-items: center; + justify-content: space-between; + min-height: 32px; +} + +.textEventAddButton, +.textEventRemoveButton { + min-width: 34px; + min-height: 30px; +} + +.textEventRemoveButton { + border-color: rgba(255, 255, 255, 0.18); + background: rgba(255, 255, 255, 0.06); + color: var(--muted); +} + +.textEventList, +.textEventFields { + display: grid; + gap: 8px; +} + +.textEventRow { + display: grid; + grid-template-columns: minmax(0, 1fr) 34px; + gap: 8px; + align-items: start; +} + +.textEventPrompt { + min-height: 62px; + resize: vertical; +} + +.textEventDirectorToggle { + display: flex; + align-items: center; + gap: 6px; + font-size: 12px; + color: var(--muted); + cursor: pointer; +} + +.eventControls { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 8px; + margin-top: 16px; + padding-top: 14px; + border-top: 1px solid rgba(255, 255, 255, 0.14); +} + +.eventButtons { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.eventControls .promptControlGroup { + grid-column: 1 / -1; + margin-top: 8px; +} + +.eventButton { + min-height: 34px; + padding: 0 12px; + border: 1px solid rgba(255, 255, 255, 0.20); + border-radius: 6px; + background: rgba(12, 14, 15, 0.62); + color: var(--text); + cursor: pointer; + font-size: 0.78rem; + font-weight: 800; +} + +.eventButton.is-active { + border-color: rgba(99, 216, 255, 0.72); + background: rgba(99, 216, 255, 0.16); + color: #dff7ff; +} + +.eventButtonHealth { + margin-left: 6px; + font-weight: 800; +} + +.eventButtonHealth.is-negative { + color: #ff8a8a; +} + +.eventButtonHealth.is-positive { + color: #8ef01c; +} + +@media (min-width: 901px) { + .sceneCard, + .controlCard { + width: min(560px, calc(100vw - 36px)); + } + + .sceneCard { + max-height: clamp(280px, calc(100vh - 500px), 430px); + } + + .controlRows { + grid-template-columns: repeat(2, minmax(0, 1fr)); + column-gap: 24px; + } + + .controlRow { + grid-template-columns: minmax(96px, 176px) 1fr; + gap: 12px; + } +} + +@media (max-width: 900px) { + .stage { + min-height: max(100svh, 1660px); + } + + .sceneCard { + top: 270px; + right: 18px; + left: 18px; + width: auto; + max-height: 570px; + } + + .controlCard { + top: 860px; + bottom: auto; + } + + .logCard { + top: 1216px; + bottom: auto; + } +} + +@media (max-width: 560px) { + .stage { + min-height: max(100svh, 1900px); + } + + .logCard { + top: 1395px; + } + + .firstFrameSourceRow, + .firstFrameSourceRow[data-mode="upload"] { + grid-template-columns: 74px minmax(0, 1fr); + } + + .urlUpdateButton { + grid-column: 1 / -1; + min-height: 44px; + } +} diff --git a/integrations_v2/lingbot/apps/cam2v/web/adapter.js b/integrations_v2/lingbot/apps/cam2v/web/adapter.js new file mode 100644 index 000000000..53e8b372b --- /dev/null +++ b/integrations_v2/lingbot/apps/cam2v/web/adapter.js @@ -0,0 +1,1145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const mockMode = new URLSearchParams(window.location.search).has("mock") +// ?director shows the separate Director Controls panel (environment/pacing +// events, e.g. weather and hazards) alongside Player Controls -- hidden by +// default since this is a second operator's role, not the regular player's. +// Also toggleable at runtime via the "Director Mode" button (no URL edit +// needed); mutable for that reason. +let directorMode = new URLSearchParams(window.location.search).has("director") + +// Letter hotkeys for event buttons, in assignment order. Excludes the +// reserved movement keys (w/a/s/d/q/e/i/j/k/l) and "c" (Clear's own +// hotkey) and "r" (Restart's) so they never collide with driving input +// or each other. Letters +// run out before director-heavy presets do -- events past the pool size +// simply render without a hotkey, same as the old 9-key digit cap. +const EVENT_HOTKEY_LETTERS = ["b", "f", "g", "h", "m", "n", "o", "p", "t", "u", "v", "x", "y", "z"] + +// Jump/Crouch are player movement actions, not narrative/pacing triggers -- +// rendered in their own row next to the movement key grid instead of the +// general event button list. +const MOVEMENT_ACTION_EVENT_IDS = new Set(["jump", "crouch"]) + +function presetSlug(name) { + return name.toLowerCase().trim().replace(/\s+/g, "-") +} + +function findPresetIndexBySlug(slug) { + if (!slug) return -1 + const normalized = slug.toLowerCase().trim() + return scenePresets.findIndex((preset) => presetSlug(preset.name) === normalized) +} + +// Loaded from scene_presets.json (a sibling file, fetched relative to this +// module) rather than inlined here, so presets/events can be hand-edited +// without touching JS. Populated by loadScenePresets(), awaited in mount() +// before anything that reads it (dropdown build, default-preset apply). +let scenePresets = [] + +async function loadScenePresets() { + const url = new URL("./scene_presets.json", import.meta.url).href + const response = await fetch(url) + if (!response.ok) { + throw new Error(`scene_presets.json fetch failed (${response.status})`) + } + const data = await response.json() + if (!Array.isArray(data)) { + throw new Error("scene_presets.json must contain a JSON array.") + } + scenePresets = data +} + +const controls = [ + { + label: "Drive / Turn", + keys: [ + { key: "w", label: "Forward" }, + { key: "a", label: "Turn left" }, + { key: "s", label: "Backward" }, + { key: "d", label: "Turn right" }, + ], + }, + { + label: "Strafe", + keys: [ + { key: "q", label: "Strafe left" }, + { key: "e", label: "Strafe right" }, + ], + }, + { + label: "Pitch", + keys: [ + { key: "i", label: "Pitch up" }, + { key: "k", label: "Pitch down" }, + ], + }, + { + label: "Look", + keys: [ + { key: "j", label: "Look left" }, + { key: "l", label: "Look right" }, + ], + }, +] + +let context = null +let initialScene = null +let initialSceneLocked = false +let promptEdited = false +let textEventsEdited = false +// The upload that carries the current catalog to the server, if one is +// still in flight. An event triggered before it lands would be rejected +// with "Unknown event_id", since the server only knows what it was told. +let pendingCatalogUpload = null +let firstFrameUrlEdited = false +let firstFrameInputMode = "url" +let selectedFirstFrameFile = null +let selectedFirstFrameUrl = null +let firstFrameSelectionCommitted = false +let activeEventId = null +let textEventDrafts = [] +let textEventSequence = 0 + +let preview = null +let gameOverOverlay = null +let sceneCard = null +let presetSelect = null +let savePresetButton = null +let firstFrameSourceRow = null +let uploadModeButton = null +let urlModeButton = null +let firstFrameInput = null +let firstFrameUrlInput = null +let firstFrameUrlUpdateButton = null +let firstFrameUrlStatus = null +let firstFrameName = null +let promptInput = null +let textEventList = null +let addTextEventButton = null +let eventControls = null +let eventButtons = null +let actionButtons = null +let clearEventButton = null +let restartButton = null +let livePromptInput = null +let livePromptSubmitButton = null +let directorButtons = null +let controlsModeToggle = null +let enableDirectorModeButton = null +let playerPromptGroup = null +// The shared movement key grid (w/a/s/d/q/e/i/j/k/l) lives outside our own +// panel in the shared page, addressable by its fixed id -- hidden while +// the Director tab is active since a director doesn't drive movement. +const movementControlRows = document.getElementById("controlRows") +// The shared panel's own heading text, also addressable by a fixed id -- +// swapped between "Player Controls" / "Director Controls" to match +// whichever mode is active. +const controlsPanelTitleText = document.getElementById("controlsPanelTitleText") +let healthBar = null +let healthBarFill = null +let healthBarValue = null +let healthBarLabelText = null +// Purely a client-side cosmetic HUD -- the server/runtime has no concept of +// health, this just tracks event `health` deltas locally (matching the +// original REACTOR case files' HUD) so the presets feel game-like. +let currentHealth = 100 +let maxHealth = 100 +let directorPromptGroup = null +let directorPromptInput = null +let directorPromptSubmitButton = null +// Player Controls is always the default view, even when ?director is set +// (which only reveals the toggle switch) -- Director Controls requires an +// explicit toggle click. +let showingDirectorControls = false +let currentPreset = null +// The image the selected preset asks for. Held separately because the +// firstFrameUrlEdited guard is cleared once an upload completes, after +// which a scene arriving from the server would overwrite the field with +// whatever the session happens to be running -- showing dragon.jpg while +// every other control said Jet Ski Cruise. +let presetImageUrl = null + +function makeSceneCard() { + const panel = document.createElement("section") + panel.className = "sceneCard overlayPanel" + panel.setAttribute("aria-label", "Initial Scene") + panel.innerHTML = ` + Initial Scene +
+ +
+ + +
+
+
+
+ + +
+
+ +
+ + +
+
+ +
+
+ +
+
+ + +
+
+
+ Text Events + +
+
+
+ ` + return panel +} + +function makeEventControls() { + const root = document.createElement("div") + root.className = "eventControls" + root.hidden = true + root.innerHTML = ` + +
+
+ Health + 100/100 +
+
+
+ +
+ + + + +
+ + +
+ + ` + return root +} + +function bindElements() { + presetSelect = sceneCard.querySelector("#scenePresetsSelect") + savePresetButton = sceneCard.querySelector(".savePresetButton") + firstFrameSourceRow = sceneCard.querySelector(".firstFrameSourceRow") + uploadModeButton = sceneCard.querySelector(".uploadModeButton") + urlModeButton = sceneCard.querySelector(".urlModeButton") + firstFrameInput = sceneCard.querySelector(".firstFrameInput") + firstFrameUrlInput = sceneCard.querySelector(".firstFrameUrlInput") + firstFrameUrlUpdateButton = sceneCard.querySelector(".urlUpdateButton") + firstFrameUrlStatus = sceneCard.querySelector(".fieldStatus") + firstFrameName = sceneCard.querySelector(".firstFrameName") + promptInput = sceneCard.querySelector(".promptControl textarea") + textEventList = sceneCard.querySelector(".textEventList") + addTextEventButton = sceneCard.querySelector(".textEventAddButton") + eventButtons = eventControls.querySelector(".eventButtons:not(.actionButtons):not(.directorButtons)") + actionButtons = eventControls.querySelector(".actionButtons") + // Jump/Crouch render as their own row right after the shared movement + // key grid, so they read as part of "movement" rather than the general + // event/trigger list below. + if (movementControlRows) movementControlRows.after(actionButtons) + clearEventButton = eventControls.querySelector(".eventButtonClear") + restartButton = eventControls.querySelector(".eventButtonRestart") + directorButtons = eventControls.querySelector(".directorButtons") + controlsModeToggle = eventControls.querySelector(".controlsModeToggle") + // Physically move ahead of the shared movement key grid (a separate, + // earlier DOM sibling this panel doesn't otherwise control) so it's + // genuinely the first control in the panel, not just first within our + // own content. + if (movementControlRows) movementControlRows.before(controlsModeToggle) + enableDirectorModeButton = eventControls.querySelector(".enableDirectorModeButton") + playerPromptGroup = eventControls.querySelector(".playerPromptGroup") + healthBar = eventControls.querySelector(".healthBar") + // Query healthBar's own descendants BEFORE relocating it below -- once + // moved out of eventControls, eventControls.querySelector(...) can no + // longer find them (they're no longer inside its subtree), which left + // these all null and silently broke every health bar update. + healthBarFill = healthBar.querySelector(".healthBarFill") + healthBarValue = healthBar.querySelector(".healthBarValue") + healthBarLabelText = healthBar.querySelector(".healthBarLabelText") + // Also moved ahead of the movement grid, right after the toggle (so + // order is: toggle, health bar, movement grid, then the rest of this + // panel's own content). + if (movementControlRows) movementControlRows.before(healthBar) + livePromptInput = eventControls.querySelector(".playerPromptGroup .promptControl input") + livePromptSubmitButton = eventControls.querySelector(".playerPromptGroup .promptSubmitButton") + directorPromptGroup = eventControls.querySelector(".directorPromptGroup") + directorPromptInput = eventControls.querySelector(".directorPromptGroup .promptControl input") + directorPromptSubmitButton = eventControls.querySelector(".directorPromptGroup .promptSubmitButton") +} + +function makeTextEventId(label = "") { + const slug = String(label) + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 48) + textEventSequence += 1 + return `${slug || "event"}-${textEventSequence}` +} + +function makeTextEventDraft(item = {}) { + const label = String(item.label || "").trim() + return { + event_id: String(item.event_id || item.id || "").trim() || makeTextEventId(label), + label, + prompt: String(item.prompt || "").trim(), + // Player by default -- only true for a preset's own directorEvents + // (tagged explicitly in applyPreset) or a custom event flipped via the + // Director checkbox in the Text Events editor. + isDirector: Boolean(item.isDirector), + } +} + +function setFirstFrameInputMode(mode) { + if (mode !== "upload" && mode !== "url") { + return + } + firstFrameInputMode = mode + firstFrameSourceRow.dataset.mode = mode + uploadModeButton.setAttribute("aria-pressed", mode === "upload" ? "true" : "false") + urlModeButton.setAttribute("aria-pressed", mode === "url" ? "true" : "false") +} + +function setFirstFrameStatus(message = "", state = "idle") { + firstFrameUrlStatus.textContent = message + firstFrameUrlStatus.hidden = !message + firstFrameUrlStatus.dataset.state = state +} + +function defaultFirstFrameName() { + return initialScene?.has_first_frame ? "Example Image" : "Choose Image" +} + +function forgetPresetImage() { + // A hand-typed URL or an uploaded file replaces the preset's own image, so + // it should stop overriding what the panel shows. + presetImageUrl = null +} + +function clearSelectedFile() { + selectedFirstFrameFile = null + firstFrameSelectionCommitted = false + firstFrameInput.value = "" + if (selectedFirstFrameUrl) { + URL.revokeObjectURL(selectedFirstFrameUrl) + selectedFirstFrameUrl = null + } +} + +function updatePreview() { + const selected = selectedFirstFrameUrl && firstFrameSelectionCommitted + // A picked preset (or manually typed URL) that hasn't been pushed to the + // server yet via "Update" -- must take priority over the server's own + // default first-frame endpoint below, otherwise applyInitialScene()'s + // unconditional updatePreview() call stomps the just-applied preset + // image back to the server default on every load/re-fetch. + const pendingUrl = firstFrameUrlEdited && firstFrameUrlInput.value.trim() + const initial = initialScene?.has_first_frame && initialScene?.first_frame_url + if (selected) { + preview.src = selectedFirstFrameUrl + } else if (pendingUrl) { + preview.src = pendingUrl + } else if (initial) { + const separator = initialScene.first_frame_url.includes("?") ? "&" : "?" + preview.src = `${initialScene.first_frame_url}${separator}t=${Date.now()}` + } + document.body.classList.toggle( + "is-ready-preview", + !context.isVideoVisible() && Boolean(selected || pendingUrl || initial), + ) +} + +function setSessionLocked(locked) { + initialSceneLocked = locked + sceneCard.hidden = locked + for (const input of sceneCard.querySelectorAll("input, textarea, button")) { + input.disabled = locked + } +} + +function renderTextEventEditor() { + textEventList.replaceChildren() + for (const [index, draft] of textEventDrafts.entries()) { + const row = document.createElement("div") + row.className = "textEventRow" + const fields = document.createElement("div") + fields.className = "textEventFields" + const label = document.createElement("input") + label.className = "textEventLabel" + label.maxLength = 64 + label.placeholder = "Label" + label.value = draft.label + const prompt = document.createElement("textarea") + prompt.className = "textEventPrompt" + prompt.rows = 2 + prompt.maxLength = 1000 + prompt.placeholder = "Event prompt" + prompt.value = draft.prompt + const directorToggle = document.createElement("label") + directorToggle.className = "textEventDirectorToggle" + const directorCheckbox = document.createElement("input") + directorCheckbox.type = "checkbox" + directorCheckbox.checked = Boolean(draft.isDirector) + const directorToggleText = document.createElement("span") + directorToggleText.textContent = "Director" + directorToggle.append(directorCheckbox, directorToggleText) + const remove = document.createElement("button") + remove.className = "textEventRemoveButton" + remove.type = "button" + remove.textContent = "X" + remove.setAttribute("aria-label", `Remove text event ${index + 1}`) + for (const input of [label, prompt, directorCheckbox]) { + input.disabled = initialSceneLocked + input.addEventListener("focus", context.releaseControls) + } + label.addEventListener("input", () => { + draft.label = label.value + textEventsEdited = true + }) + prompt.addEventListener("input", () => { + draft.prompt = prompt.value + textEventsEdited = true + }) + directorCheckbox.addEventListener("change", () => { + draft.isDirector = directorCheckbox.checked + textEventsEdited = true + // Live Player/Director Controls buttons need to reflect which panel + // this event now belongs to immediately, same as label/prompt edits + // becoming visible on the next render. + renderEventControls() + }) + remove.disabled = initialSceneLocked + remove.addEventListener("click", () => { + textEventDrafts.splice(index, 1) + textEventsEdited = true + renderTextEventEditor() + renderEventControls() + }) + fields.append(label, prompt, directorToggle) + row.append(fields, remove) + textEventList.append(row) + } +} + +function collectTextEvents() { + const events = [] + const usedIds = new Set() + for (const draft of textEventDrafts) { + const label = draft.label.trim() + const prompt = draft.prompt.trim() + if (!label && !prompt) { + continue + } + if (!prompt) { + throw new Error("Each text event needs a prompt.") + } + let eventId = String(draft.event_id || "").trim() || makeTextEventId(label) + while (usedIds.has(eventId)) { + eventId = makeTextEventId(label) + } + draft.event_id = eventId + usedIds.add(eventId) + events.push({ event_id: eventId, label: label || eventId, prompt, category: "custom" }) + } + return events +} + +let eventHotkeyMap = new Map() + +function getEventHealthDelta(eventId) { + // Looked up from currentPreset's own definitions (not the transient + // render catalog) so it still works once connected, after the server's + // echoed event_catalog -- which has no health field -- takes over as the + // render source. + const fromPlayer = currentPreset?.events?.find((item) => item.event_id === eventId) + const fromDirector = currentPreset?.directorEvents?.find((item) => item.event_id === eventId) + const health = (fromPlayer ?? fromDirector)?.health + return Number.isFinite(health) ? health : 0 +} + +function resetHealth(preset) { + maxHealth = Number(preset?.hud?.maxHealth) || 100 + currentHealth = maxHealth + if (healthBarLabelText) healthBarLabelText.textContent = preset?.hud?.healthLabel || "Health" + renderHealthBar() +} + +function renderHealthBar() { + if (gameOverOverlay) gameOverOverlay.hidden = currentHealth > 0 + if (!healthBarFill) return + const pct = maxHealth > 0 ? Math.max(0, Math.min(100, (currentHealth / maxHealth) * 100)) : 0 + healthBarFill.style.width = `${pct}%` + healthBarFill.classList.toggle("is-low", pct <= 25) + healthBarValue.textContent = `${Math.round(currentHealth)}/${Math.round(maxHealth)}` +} + +function applyHealthDelta(delta, label = "") { + if (!Number.isFinite(delta) || delta === 0) return + currentHealth = Math.max(0, Math.min(maxHealth, currentHealth + delta)) + renderHealthBar() + const sign = delta > 0 ? "+" : "" + const healthLabel = healthBarLabelText?.textContent || "health" + context?.logEvent( + `${label ? `${label}: ` : ""}${healthLabel.toLowerCase()} ${sign}${delta} (now ${Math.round(currentHealth)}/${Math.round(maxHealth)})`, + { source: "client" }, + ) +} + +function isDirectorEventId(eventId) { + return Boolean(currentPreset?.directorEvents?.some((item) => item.event_id === eventId)) +} + +// textEventDrafts items carry their own isDirector flag (set in +// applyPreset()/the Text Events editor's Director checkbox); anything else +// (e.g. the server-echoed event_catalog, before any draft carries the +// field) falls back to the preset-membership check above. +function eventIsDirector(item) { + return typeof item.isDirector === "boolean" ? item.isDirector : isDirectorEventId(item.event_id) +} + +function makeEventButton(item, hotkeyLetter, healthDelta = 0) { + const eventId = String(item.event_id || "").trim() + if (!eventId) return null + const label = String(item.label || eventId) + const button = document.createElement("button") + button.className = "eventButton" + button.type = "button" + const hotkeyText = + hotkeyLetter === "space" ? "Space" : hotkeyLetter === "control" ? "Ctrl" : hotkeyLetter ? hotkeyLetter.toUpperCase() : null + button.append(document.createTextNode(hotkeyText ? `${label} (${hotkeyText})` : label)) + if (Number.isFinite(healthDelta) && healthDelta !== 0) { + const healthTag = document.createElement("span") + healthTag.className = `eventButtonHealth ${healthDelta > 0 ? "is-positive" : "is-negative"}` + healthTag.textContent = healthDelta > 0 ? `+${healthDelta}` : String(healthDelta) + button.append(healthTag) + } + button.classList.toggle("is-active", activeEventId === eventId) + button.addEventListener("click", () => sendTextEvent(eventId, "trigger")) + return button +} + +function renderEventControls() { + // A picked preset's events (textEventDrafts, not yet pushed to the + // server via connect/Send) must take priority over the server's last- + // known event_catalog -- otherwise applyInitialScene()'s unconditional + // call here stomps the just-applied preset's events back to whatever + // the server currently has (its default catalog on first load). + const catalog = textEventsEdited + ? textEventDrafts + : Array.isArray(initialScene?.event_catalog) ? initialScene.event_catalog : [] + const playerItems = catalog.filter((item) => !eventIsDirector(item)) + const directorItems = catalog.filter((item) => eventIsDirector(item)) + + // Player events get digit hotkeys (1-9), director events get letter + // hotkeys from EVENT_HOTKEY_LETTERS -- the two keyspaces never collide, + // so both live in the same eventHotkeyMap. Jump always gets Space and + // Crouch always gets Ctrl instead of a digit, matching common game + // convention. Events past either pool's size still render, just without + // a hotkey. + eventHotkeyMap = new Map() + let digitIndex = 0 + const nextDigit = () => (digitIndex < 9 ? String(++digitIndex) : null) + let letterIndex = 0 + const nextLetter = () => EVENT_HOTKEY_LETTERS[letterIndex++] ?? null + + eventButtons.replaceChildren() + actionButtons.replaceChildren() + for (const item of playerItems) { + const eventId = String(item.event_id || "").trim() + const hotkey = eventId === "jump" ? "space" : eventId === "crouch" ? "control" : nextDigit() + const button = makeEventButton(item, hotkey, getEventHealthDelta(eventId)) + if (!button) continue + if (hotkey) eventHotkeyMap.set(hotkey, eventId) + const container = MOVEMENT_ACTION_EVENT_IDS.has(eventId) ? actionButtons : eventButtons + container.append(button) + } + clearEventButton.classList.toggle("is-active", activeEventId === null) + + directorButtons.replaceChildren() + for (const item of directorItems) { + const letter = nextLetter() + const eventId = String(item.event_id || "").trim() + const button = makeEventButton(item, letter, getEventHealthDelta(eventId)) + if (!button) continue + if (letter) eventHotkeyMap.set(letter, eventId) + directorButtons.append(button) + } + + // In director mode, Player and Director share one panel with a single + // toggle switch on top swapping which button grid (and which + // custom-prompt box) is visible -- otherwise (the common case) it's just + // Player Controls with no toggle at all. When director mode isn't on yet + // but this preset actually has director events, offer "Enable Director + // Mode" instead of requiring a URL edit. + const hasDirectorContent = directorMode && directorItems.length > 0 + enableDirectorModeButton.hidden = directorMode || directorItems.length === 0 + controlsModeToggle.hidden = !hasDirectorContent + if (!hasDirectorContent) showingDirectorControls = false + const showDirector = hasDirectorContent && showingDirectorControls + controlsModeToggle.classList.toggle("is-on", showDirector) + controlsModeToggle.setAttribute("aria-checked", String(showDirector)) + controlsPanelTitleText.textContent = showDirector ? "Director Controls" : "Player Controls" + eventButtons.hidden = showDirector + actionButtons.hidden = showDirector || actionButtons.children.length === 0 + directorButtons.hidden = !showDirector + directorPromptGroup.hidden = !showDirector + playerPromptGroup.hidden = showDirector + if (movementControlRows) movementControlRows.hidden = showDirector + eventControls.hidden = playerItems.length === 0 && directorItems.length === 0 +} + +function enableDirectorMode() { + directorMode = true + const url = new URL(window.location.href) + url.searchParams.set("director", "") + window.history.replaceState(null, "", url) + renderEventControls() +} + +function setDirectorView(showDirector) { + showingDirectorControls = showDirector + renderEventControls() +} + +function saveCurrentPreset() { + const name = prompt("Preset name:", "My Scene").trim() + if (!name) return + const preset = { + name, + prompt: promptInput.value.trim(), + events: textEventDrafts.map(d => ({ event_id: d.event_id, label: d.label, prompt: d.prompt })) + } + scenePresets.push(preset) + localStorage.setItem("lingbot-presets", JSON.stringify(scenePresets)) + updatePresetDropdown() + alert(`Preset "${name}" saved!`) +} + +function updatePresetDropdown() { + presetSelect.innerHTML = ` + + ${scenePresets.map((p, i) => ``).join("")} + ` +} + +function loadSavedPresets() { + try { + const saved = localStorage.getItem("lingbot-presets") + if (saved) { + const customPresets = JSON.parse(saved) + scenePresets.push(...customPresets) + } + } catch (err) { + console.error("Failed to load saved presets:", err) + } +} + +function applyPreset(presetIndex) { + const preset = scenePresets[Number(presetIndex)] + if (!preset) return + currentPreset = preset + resetHealth(preset) + context.logEvent(`preset selected: ${preset.name}`, { source: "client" }) + // Server hard cap (session.py: _MAX_TEXT_EVENTS). Catch an over-budget + // preset here, at selection time, instead of only discovering it via a + // failed connect attempt later. + const totalEventCount = preset.events.length + (preset.directorEvents?.length ?? 0) + if (totalEventCount > 20) { + context.logEvent( + `preset "${preset.name}" has ${totalEventCount} events (player + director combined), ` + + "over the server's 20-event limit -- connecting will fail until it's trimmed.", + { source: "client", level: "error" }, + ) + } + const url = new URL(window.location.href) + url.searchParams.set("preset", presetSlug(preset.name)) + window.history.replaceState(null, "", url) + promptInput.value = preset.prompt + promptEdited = true + // The full catalog (player + director) always uploads to the server -- + // "director" is a client-side UI distinction only (which panel a button + // renders in, and whether that panel is visible at all), the shared + // WebRTC protocol has no such concept, so both must be known server-side + // for either panel's buttons to actually do anything once clicked. + textEventDrafts = [ + ...preset.events.map((item) => makeTextEventDraft({ ...item, isDirector: false })), + ...(preset.directorEvents ?? []).map((item) => makeTextEventDraft({ ...item, isDirector: true })), + ] + textEventsEdited = true + renderTextEventEditor() + // Push the preset to the server now rather than waiting for a connect. + // The event buttons are live before connecting, and the server rejects an + // id it has not been told about -- "Unknown event_id='jump'" while the page + // showed Jump, because the catalog only travelled in beforeConnect. + pendingCatalogUpload = uploadSessionInput({ includeFirstFrame: true }) + .catch((error) => { + context.logEvent(`preset not applied: ${error.message}`, { + source: "client", + level: "error", + }) + }) + .finally(() => { + pendingCatalogUpload = null + }) + // Also refresh the live Player Controls buttons immediately, not just + // the editable Text Events list -- otherwise switching games mid-session + // only updates on the next connect/upload, not on selection itself. + renderEventControls() + if (preset.image) { + clearSelectedFile() + setFirstFrameInputMode("url") + firstFrameUrlInput.value = preset.image + firstFrameUrlEdited = true + presetImageUrl = preset.image + firstFrameName.textContent = "Upload Image" + setFirstFrameStatus("URL not updated", "pending") + // Show the preset's image immediately, ahead of the "Update" commit + // step -- picking a preset should visibly change the panel, not just + // silently populate the URL field. + preview.src = preset.image + document.body.classList.toggle("is-ready-preview", !context.isVideoVisible()) + } + context.releaseControls() +} + +function applyInitialScene(scene) { + initialScene = scene + if (!promptEdited && typeof scene.prompt === "string") { + promptInput.value = scene.prompt + } + const sceneImageUrl = typeof scene.image_url === "string" + ? scene.image_url + : (typeof scene.default_image_url === "string" ? scene.default_image_url : "") + // A chosen preset wins over the session's own image: the page should show + // the scene the user picked, not the one the rollout has not switched to + // yet. + const imageUrl = presetImageUrl || sceneImageUrl + if (!selectedFirstFrameFile && !firstFrameUrlEdited && imageUrl) { + firstFrameUrlInput.value = imageUrl + setFirstFrameInputMode("url") + } + firstFrameName.textContent = firstFrameUrlInput.value.trim() ? "Upload Image" : defaultFirstFrameName() + activeEventId = scene.active_event_id || null + if (!textEventsEdited) { + textEventDrafts = Array.isArray(scene.event_catalog) + ? scene.event_catalog.map((item) => makeTextEventDraft(item)) + : [] + renderTextEventEditor() + } + renderEventControls() + context.setModelName(scene.model || "Lingbot") + applyVideoSizing(scene.resolution) + context.setResolution(scene.resolution?.width, scene.resolution?.height) + updatePreview() +} + +function applyVideoSizing(resolution) { + const width = Number(resolution?.width) + const height = Number(resolution?.height) + if ( + !Number.isFinite(width) || + !Number.isFinite(height) || + width <= 0 || + height <= 0 + ) { + return + } + const style = document.documentElement.style + style.setProperty("--lingbot-video-width", `${width}px`) + style.setProperty("--lingbot-video-height", `${height}px`) + style.setProperty("--lingbot-video-width-from-vh", `${(width / height) * 100}vh`) + style.setProperty("--lingbot-video-aspect", `${width} / ${height}`) +} + +function mockInitialScene() { + return { + prompt: "Drive through a cinematic city street at sunset.", + has_first_frame: false, + model: "Lingbot", + resolution: { width: 832, height: 464 }, + event_catalog: [ + { event_id: "portal", label: "Portal", prompt: "A luminous portal opens." }, + { event_id: "storm", label: "Storm", prompt: "A dramatic storm rolls in." }, + ], + } +} + +async function loadInitialScene() { + if (mockMode) { + applyInitialScene(mockInitialScene()) + return + } + const response = await fetch("/api/session/initial_scene") + if (!response.ok) { + throw new Error(`initial scene failed (${response.status})`) + } + applyInitialScene(await response.json()) +} + +function validateImageUrl(value) { + const imageUrl = value.trim() + let parsed = null + try { + // Resolved against the page, so a packaged relative path -- which is how + // the built-in presets ship their images ("assets/circuit.jpg") -- is as + // valid as an absolute URL. The entered value is returned unchanged, so a + // relative preset URL stays relative. + parsed = new URL(imageUrl, window.location.href) + } catch { + throw new Error("Enter an http(s) image URL or a path like assets/name.jpg.") + } + if (!["http:", "https:"].includes(parsed.protocol)) { + throw new Error("Enter an http(s) image URL or a path like assets/name.jpg.") + } + return imageUrl +} + +async function uploadSessionInput({ includeFirstFrame = false } = {}) { + const prompt = promptInput.value.trim() + const hasPrompt = promptEdited && Boolean(prompt) + const hasFile = includeFirstFrame && firstFrameInputMode === "upload" && selectedFirstFrameFile + let imageUrl = firstFrameUrlInput.value.trim() + const hasUrl = includeFirstFrame && firstFrameInputMode === "url" && Boolean(imageUrl) + const textEvents = textEventsEdited ? collectTextEvents() : null + if (!hasPrompt && !hasFile && !hasUrl && textEvents === null) { + return + } + if (hasUrl) { + imageUrl = validateImageUrl(imageUrl) + } + if (mockMode) { + applyInitialScene({ + ...mockInitialScene(), + prompt: hasPrompt ? prompt : initialScene.prompt, + event_catalog: textEvents ?? initialScene.event_catalog, + active_event_id: activeEventId, + }) + } else { + const form = new FormData() + if (hasPrompt) form.append("prompt", prompt) + if (hasFile) form.append("image", selectedFirstFrameFile, selectedFirstFrameFile.name) + if (hasUrl) form.append("image_url", imageUrl) + if (textEvents !== null) form.append("text_events", JSON.stringify(textEvents)) + const response = await fetch("/api/session/input", { method: "POST", body: form }) + if (!response.ok) { + const text = (await response.text()).trim().replace(/^\d+:\s*/, "") + throw new Error(text || `input upload failed (${response.status})`) + } + applyInitialScene(await response.json()) + } + promptEdited = false + textEventsEdited = false + firstFrameUrlEdited = false +} + +async function updateFirstFrame() { + if (initialSceneLocked) return + try { + if (firstFrameInputMode === "upload" && !selectedFirstFrameFile) { + throw new Error("Choose an image file.") + } + if (firstFrameInputMode === "url") { + firstFrameUrlInput.value = validateImageUrl(firstFrameUrlInput.value) + clearSelectedFile() + } + setFirstFrameStatus("Updating...", "pending") + firstFrameUrlUpdateButton.disabled = true + await uploadSessionInput({ includeFirstFrame: true }) + firstFrameSelectionCommitted = true + setFirstFrameStatus("Updated", "success") + updatePreview() + } catch (error) { + setFirstFrameStatus(error.message, "error") + context.logEvent(`first frame update failed: ${error.message}`, { source: "client", level: "error" }) + } finally { + firstFrameUrlUpdateButton.disabled = initialSceneLocked + } +} + +async function sendTextEvent(eventId, state, promptValue = null) { + const label = state === "clear" ? "clear event" : `event:${eventId}` + const payload = { type: "event", event_id: eventId, state } + if (promptValue !== null) { + payload.prompt = promptValue + } + // Wait for the catalog this event belongs to, so clicking a button the + // moment a preset loads cannot beat its own events to the server. + if (pendingCatalogUpload) { + await pendingCatalogUpload + } + if (!context.sendCommand(payload, label)) { + return + } + if (state === "trigger") { + const eventLabel = + currentPreset?.events?.find((item) => item.event_id === eventId)?.label + ?? currentPreset?.directorEvents?.find((item) => item.event_id === eventId)?.label + ?? eventId + applyHealthDelta(getEventHealthDelta(eventId), eventLabel) + } + setSessionLocked(true) +} + +function attachListeners() { + presetSelect.addEventListener("change", (e) => { + if (e.target.value) applyPreset(e.target.value) + }) + // Digit keys (1-9) trigger player events, letter keys trigger director + // events (see the "(X)" hotkey suffix rendered in renderEventControls() + // / eventHotkeyMap), "c" triggers Clear (present on every game, not tied + // to any preset's catalog) -- this only fires once controls are + // actually live. + window.addEventListener("keydown", (event) => { + // Control itself is a valid hotkey (Crouch) -- only block it as a + // held modifier for some other key (e.g. Ctrl+C), same as Meta/Alt. + if (event.metaKey || event.altKey || event.repeat) return + if (event.ctrlKey && event.key !== "Control") return + if (eventControls.hidden) return + const activeTag = document.activeElement?.tagName + if (activeTag === "INPUT" || activeTag === "TEXTAREA") return + const key = event.key === " " ? "space" : event.key === "Control" ? "control" : event.key.toLowerCase() + if (key === "c") { + sendTextEvent(activeEventId || "clear", "clear") + return + } + if (key === "r") { + restartButton?.click() + return + } + const eventId = eventHotkeyMap.get(key) + if (eventId) { + if (key === "space") event.preventDefault() + sendTextEvent(eventId, "trigger") + } + }) + controlsModeToggle.addEventListener("click", () => setDirectorView(!showingDirectorControls)) + enableDirectorModeButton.addEventListener("click", enableDirectorMode) + savePresetButton.addEventListener("click", saveCurrentPreset) + uploadModeButton.addEventListener("click", () => { + setFirstFrameInputMode("upload") + context.releaseControls() + }) + urlModeButton.addEventListener("click", () => { + setFirstFrameInputMode("url") + context.releaseControls() + }) + firstFrameInput.addEventListener("change", () => { + setFirstFrameInputMode("upload") + forgetPresetImage() + const [file] = firstFrameInput.files + selectedFirstFrameFile = file || null + firstFrameSelectionCommitted = false + if (selectedFirstFrameUrl) URL.revokeObjectURL(selectedFirstFrameUrl) + selectedFirstFrameUrl = selectedFirstFrameFile ? URL.createObjectURL(selectedFirstFrameFile) : null + firstFrameName.textContent = selectedFirstFrameFile?.name || defaultFirstFrameName() + firstFrameUrlInput.value = "" + firstFrameUrlEdited = false + setFirstFrameStatus(selectedFirstFrameFile ? "Image not updated" : "", "pending") + }) + firstFrameUrlInput.addEventListener("input", () => { + setFirstFrameInputMode("url") + if (selectedFirstFrameFile) clearSelectedFile() + firstFrameUrlEdited = true + forgetPresetImage() + firstFrameName.textContent = firstFrameUrlInput.value.trim() ? "Upload Image" : defaultFirstFrameName() + setFirstFrameStatus(firstFrameUrlInput.value.trim() ? "URL not updated" : "", "pending") + }) + firstFrameUrlUpdateButton.addEventListener("click", () => void updateFirstFrame()) + promptInput.addEventListener("input", () => { promptEdited = true }) + const promptSubmitButton = sceneCard.querySelector(".promptSubmitButton") + promptSubmitButton.addEventListener("click", () => { + const promptText = promptInput.value.trim() + if (promptText) { + sendTextEvent("user_prompt", "trigger", promptText) + } + }) + addTextEventButton.addEventListener("click", () => { + textEventDrafts.push(makeTextEventDraft()) + textEventsEdited = true + renderTextEventEditor() + context.releaseControls() + }) + clearEventButton.addEventListener("click", () => sendTextEvent(activeEventId || "clear", "clear")) + // A reset discards the rollout cache, so the next chunk re-seeds from + // the session's first frame. Prompt swaps leave the world looking like + // where it started -- a jet ski in a castle jungle -- and this is what + // clears that without restarting the server. + restartButton.addEventListener("click", () => { + if (context.sendCommand({ type: "reset" }, "restart scene")) { + resetHealth(currentPreset) + context.logEvent("scene restarted", { source: "client" }) + } + }) + const submitLivePrompt = () => { + const promptText = livePromptInput.value.trim() + if (promptText) { + sendTextEvent("user_prompt", "trigger", promptText) + } + } + livePromptSubmitButton.addEventListener("click", submitLivePrompt) + livePromptInput.addEventListener("keydown", (event) => { + if (event.key === "Enter") { + event.preventDefault() + submitLivePrompt() + } + }) + const submitDirectorPrompt = () => { + const promptText = directorPromptInput.value.trim() + if (promptText) { + sendTextEvent("user_prompt", "trigger", promptText) + } + } + directorPromptSubmitButton.addEventListener("click", submitDirectorPrompt) + directorPromptInput.addEventListener("keydown", (event) => { + if (event.key === "Enter") { + event.preventDefault() + submitDirectorPrompt() + } + }) + for (const input of [firstFrameUrlInput, promptInput, addTextEventButton, livePromptInput, directorPromptInput]) { + input.addEventListener("focus", context.releaseControls) + } +} + +export default { + modelName: "Lingbot", + stylesheet: new URL("./adapter.css?v=lingbot-video-size-v13", import.meta.url).href, + controls, + + async mount(sharedContext) { + context = sharedContext + try { + await loadScenePresets() + } catch (error) { + context.logEvent(`scene_presets.json unavailable: ${error.message}`, { source: "client", level: "error" }) + } + loadSavedPresets() + preview = document.createElement("img") + preview.className = "firstFramePreview" + preview.alt = "" + preview.setAttribute("aria-hidden", "true") + gameOverOverlay = document.createElement("div") + gameOverOverlay.className = "gameOverOverlay" + gameOverOverlay.textContent = "GAME OVER" + gameOverOverlay.hidden = true + sceneCard = makeSceneCard() + eventControls = makeEventControls() + context.slots.stage.append(preview, gameOverOverlay) + context.slots.panel.append(sceneCard) + context.slots.controls.append(eventControls) + bindElements() + updatePresetDropdown() + setFirstFrameInputMode("url") + attachListeners() + // ?preset= (e.g. "water-blaster") shares a direct link to a + // specific game; falls back to index 0 ("Dragon") otherwise. + const requestedPreset = new URLSearchParams(window.location.search).get("preset") + const presetIndex = findPresetIndexBySlug(requestedPreset) + const defaultPresetIndex = presetIndex >= 0 ? presetIndex : 0 + presetSelect.value = String(defaultPresetIndex) + applyPreset(defaultPresetIndex) + try { + await loadInitialScene() + } catch (error) { + context.logEvent(`initial scene unavailable: ${error.message}`, { source: "client", level: "error" }) + } + }, + + async beforeConnect() { + // resetHealth() otherwise only runs on preset selection -- reconnecting + // without re-picking a preset left currentHealth carried over from + // whatever it was at disconnect (e.g. 0, from a prior playthrough), + // instead of a fresh session actually starting at full health/fuel. + resetHealth(currentPreset) + await uploadSessionInput({ includeFirstFrame: true }) + }, + + onActionSent() { + // No updatePreview() here: it refetched the first frame on every key + // press, cache-busted, and the preview is hidden behind the video by the + // time any action is sent. + setSessionLocked(true) + }, + + onControlMessage(payload) { + if (payload.type === "chunk_done" && Object.prototype.hasOwnProperty.call(payload, "active_event_id")) { + activeEventId = payload.active_event_id || null + renderEventControls() + // Previously the Initial Scene panel only hid once onActionSent() + // fired (the first movement key or event trigger) -- generation + // already starts right after connect, so it sat visible/editable + // over a live session until the player happened to press something. + setSessionLocked(true) + return false + } + if (payload.type === "event_ack") { + activeEventId = payload.active_event_id || null + renderEventControls() + context.logEvent(`event ${payload.event_id} ${payload.state}`) + return true + } + return false + }, + + onInitialScene(scene) { + // The v1 server acknowledged a text event over the control channel; the + // v2 endpoint answers the POST with the resulting scene instead, so the + // active-event highlight is taken from here. Only that is read: the + // panel's own fields may hold edits the player is still making. + activeEventId = scene?.active_event_id || null + renderEventControls() + }, + + onVideoVisibilityChanged(visible) { + // Video becoming visible is what "a session is live" looks like on v2. + // The v1 server announced it with a "chunk_done" control message, which + // v2 never sends -- so the Initial Scene panel used to sit visible and + // editable over a running session until the player happened to press a + // key, which is what finally triggered onActionSent(). + setSessionLocked(Boolean(visible)) + updatePreview() + }, + + onDisconnect() { + setSessionLocked(false) + updatePreview() + }, +} diff --git a/integrations_v2/lingbot/apps/cam2v/web/assets/circuit.jpg b/integrations_v2/lingbot/apps/cam2v/web/assets/circuit.jpg new file mode 100644 index 000000000..d71b88f3e Binary files /dev/null and b/integrations_v2/lingbot/apps/cam2v/web/assets/circuit.jpg differ diff --git a/integrations_v2/lingbot/apps/cam2v/web/assets/dragon.jpg b/integrations_v2/lingbot/apps/cam2v/web/assets/dragon.jpg new file mode 100644 index 000000000..d8f09ed54 Binary files /dev/null and b/integrations_v2/lingbot/apps/cam2v/web/assets/dragon.jpg differ diff --git a/integrations_v2/lingbot/apps/cam2v/web/assets/horizontal-dark.svg b/integrations_v2/lingbot/apps/cam2v/web/assets/horizontal-dark.svg new file mode 100644 index 000000000..89b68f9d1 --- /dev/null +++ b/integrations_v2/lingbot/apps/cam2v/web/assets/horizontal-dark.svg @@ -0,0 +1,192 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/integrations_v2/lingbot/apps/cam2v/web/assets/horizontal-light.svg b/integrations_v2/lingbot/apps/cam2v/web/assets/horizontal-light.svg new file mode 100644 index 000000000..a491910db --- /dev/null +++ b/integrations_v2/lingbot/apps/cam2v/web/assets/horizontal-light.svg @@ -0,0 +1,182 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/integrations_v2/lingbot/apps/cam2v/web/assets/jet-ski-cruise.jpg b/integrations_v2/lingbot/apps/cam2v/web/assets/jet-ski-cruise.jpg new file mode 100644 index 000000000..8fe363c8b Binary files /dev/null and b/integrations_v2/lingbot/apps/cam2v/web/assets/jet-ski-cruise.jpg differ diff --git a/integrations_v2/lingbot/apps/cam2v/web/assets/noir-alley-combat.jpg b/integrations_v2/lingbot/apps/cam2v/web/assets/noir-alley-combat.jpg new file mode 100644 index 000000000..77b2b6046 Binary files /dev/null and b/integrations_v2/lingbot/apps/cam2v/web/assets/noir-alley-combat.jpg differ diff --git a/integrations_v2/lingbot/apps/cam2v/web/assets/sources/f1-race.json b/integrations_v2/lingbot/apps/cam2v/web/assets/sources/f1-race.json new file mode 100644 index 000000000..94e6155ae --- /dev/null +++ b/integrations_v2/lingbot/apps/cam2v/web/assets/sources/f1-race.json @@ -0,0 +1,182 @@ +{ + "id": "car_racing_08", + "name": "Circuit Racer", + "description": "First-person cockpit car racing — key 1 = kick up sparks, key 2 = drift, key 3 = crash, key 4 = chicane flick, key 5 = lock-up smoke, key 6 = DRS boost, key 7 = ride the kerb", + "image": { + "label": "Circuit Racer", + "src": "/lingbot-cases/circuit.jpg" + }, + "scene": { + "base": { + "default": "First-person onboard cockpit view from inside a Formula 1 race car, looking out over the nose and the halo through the windscreen down a wide smooth asphalt racing circuit ahead. The world contains EXACTLY ONE start-finish gantry spanning the track ahead at a fixed position AND EXACTLY ONE grandstand on the left at a fixed position. Crisp painted red-and-white kerbs edge the track, gravel run-off beyond, banner-lined fences and distant hills under a clear blue sky. Bright sunny racing-game look, motion-blurred asphalt rushing beneath the nose, photorealistic." + }, + "player": { + "default": "The driver's gloved hands gripping the steering wheel with its lit digital dash display at the centre foreground. EXACTLY ONE steering wheel with its glowing dash display held in the driver's gloved hands at the centre of the frame at a fixed position." + }, + "camera": { + "default": { + "static": "First-person cockpit view, the steering wheel and glowing dash held steady at the centre of the frame in the driver's gloved hands, the track visible ahead through the halo and windscreen. Neither the viewpoint nor the car moves on its own; arrow-key look-input is the only source of camera motion, panning the view around the fixed cockpit while held.", + "dynamic": "Strict first-person cockpit view, the steering wheel and dash holding steady at the centre of the frame as the viewpoint advances forward down the track through the windscreen and halo; the car does not turn on its own — look-input becomes the car changing heading, the circuit sweeping past the cockpit." + } + }, + "movement": { + "default": { + "static": "The car idles on the track, the steering wheel steady in the driver's gloved hands and the dash lights glowing, the view over the nose barely trembling with the idling engine; the track ahead holds still. The viewpoint does not surge forward or accelerate on its own unless an action is triggered.", + "dynamic": "The viewpoint rushes forward down the circuit, the asphalt and painted kerbs streaking past beneath the nose and the barriers and grandstands sweeping by on either side, the wheel shifting in the gloved hands as the track blurs ahead through the halo and windscreen." + } + }, + "events": [ + { + "name": "Kick Up Sparks", + "actor": "character", + "detail": "The car crests a rise and the floor grounds out on the asphalt — a bright shower of orange sparks streams out from under the nose and sprays up past the halo and windscreen, trailing away behind, before the car settles back level on its suspension. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone." + }, + { + "name": "Drift", + "actor": "character", + "detail": "The gloved hands throw the wheel hard over and the whole cockpit view slews sideways, the kerbs and barriers swinging across the windscreen as the rear steps out in a smoking slide, then the hands catch it and the view snaps back straight down the track. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone." + }, + { + "name": "Crash", + "actor": "character", + "detail": "The view lurches violently as the car slams into the barrier ahead — the windscreen fills with flying carbon-fibre debris and spinning sky and track, the wheel wrenching in the gloved hands and the dash flashing red warnings, before it grinds to a juddering, shuddering halt. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone." + }, + { + "name": "Chicane Flick", + "actor": "character", + "detail": "The gloved hands snap the wheel left then hard right and the cockpit whips through a tight chicane, the red-and-white kerbs and barriers flicking across the windscreen in a quick left-right S as the car darts through the esses, then straightens out down the track. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone." + }, + { + "name": "Lock-Up Smoke", + "actor": "character", + "detail": "The driver stamps the brakes hard into the corner and the front tyres lock — thick white tyre smoke boils up off the wheels and plumes across the windscreen, the nose diving and the view juddering as the car slides toward the apex, before it hooks in and settles back level. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone.", + "health": 5 + }, + { + "name": "DRS Boost", + "actor": "character", + "detail": "The gloved thumb flicks the DRS on the steering wheel and the dash lights up, and the view surges away down the straight as drag drops off — the track and barriers streaking faster past the halo and windscreen, then it eases as the wing closes. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone.", + "health": 0 + }, + { + "name": "Ride the Kerb", + "actor": "character", + "detail": "The view drops onto the red-and-white kerb at the corner and the whole cockpit judders and rattles violently, the wheel shaking in the gloved hands and the horizon bouncing through the windscreen, before the car settles back onto the smooth asphalt. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone.", + "health": -5 + }, + { + "name": "Rain Sweeps In", + "actor": "environment", + "requires": { + "notFired": [ + "Sun Glare", + "Clear Dry Track" + ] + }, + "detail": "Dark storm clouds roll over and rain sweeps across the windscreen — droplets streaking and beading over the glass and halo, the track ahead darkening and glistening wet, spray kicking up off the nose; the rain stays, slicking the view ahead. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone.", + "count": 1 + }, + { + "name": "Rabbit on the Track", + "actor": "environment", + "chance": 0.08, + "detail": "A rabbit darts out onto the track ahead in the windscreen and stops in the racing line directly in the car's path, ears twitching, staying put on the tarmac as the view rushes toward it. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone." + }, + { + "name": "Road Fire", + "actor": "environment", + "chance": 0.05, + "detail": "EXACTLY ONE single wall of fire erupts across the track ahead in the windscreen — one lone wall of orange flames and black smoke rising off the tarmac and spanning the full width of the road directly in the car's path, no second fire and no other flames anywhere else; the view bears down on it and blasts through, and NO further fire appears after it, only the one, heat shimmering over the halo.", + "count": 1 + }, + { + "actor": "environment", + "name": "Puddle on the Track", + "requires": { + "fired": [ + "Rain Sweeps In" + ] + }, + "detail": "A wide sheet of standing water lies across the track ahead in the windscreen; the view rushes toward it and blasts through, a wall of spray exploding up over the halo and glass, the cockpit hydroplaning for a moment before the track grips again. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone.", + "health": -5 + }, + { + "actor": "environment", + "name": "Oil Slick Ahead", + "requires": { + "fired": [ + "Rain Sweeps In" + ] + }, + "detail": "A dark oil slick spreads across the track ahead in the windscreen; the view hits it and slews sideways, the barriers swinging across the glass as the rear slides, before it snaps back straight down the track. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone.", + "health": -8 + }, + { + "actor": "environment", + "name": "Tunnel Section", + "detail": "The track ahead dives into a dark tunnel — the bright sky vanishes from the windscreen and the walls close in, strings of overhead lights strobing past over the halo, before the view bursts back out into open daylight. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone.", + "health": 0 + }, + { + "actor": "environment", + "name": "Sun Glare", + "requires": { + "notFired": [ + "Rain Sweeps In", + "Clear Dry Track" + ] + }, + "detail": "The low sun drops to the end of the straight and blazes straight into the windscreen — a blinding wash of golden glare flooding the cockpit and washing out the track ahead, lens flare streaking across the halo and glass, before the view crests into shade. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone.", + "health": -5, + "count": 1 + }, + { + "name": "Clear Dry Track", + "actor": "environment", + "requires": { + "notFired": [ + "Rain Sweeps In", + "Sun Glare" + ] + }, + "detail": "The clouds break and the circuit runs clear and dry ahead in the windscreen — bright even daylight over dry grey tarmac, the racing line sharp and grippy, no rain on the glass and no water on the surface, just clean fast conditions as the view races on down the track. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone.", + "count": 1 + }, + { + "name": "Checkered Flag", + "actor": "environment", + "baseVersion": "empty", + "cameraVersion": "empty", + "movementVersion": "empty", + "requires": { + "minChunks": 160, + "minHealth": 1 + }, + "win": true, + "detail": "The car sweeps across the start-finish line to take the chequered flag — a trackside marshal waving the black-and-white chequered flag high over the gantry as the car flashes past the line and eases off into a victory slowdown down the pit straight, the packed grandstands rising to their feet. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone.", + "health": 0 + }, + { + "name": "Crashed Out", + "actor": "environment", + "baseVersion": "empty", + "cameraVersion": "empty", + "movementVersion": "empty", + "requires": { + "maxHealth": 0 + }, + "detail": "The car snaps out of control and spears off the track into the barriers — the nose crumpling and carbon debris flying as it slams into the tyre wall and grinds to a dead stop, steam rising from the wreck and the race over for the driver as marshals wave yellow flags across the track. EXACTLY ONE of each thing shown — a single car on a single track and only one of any object described, no second car, no duplicate and no clone.", + "health": 0 + } + ], + "hud": { + "show": true, + "maxHealth": 100, + "health": 100, + "inventory": [] + }, + "jumpPrompt": "The track kinks up into a ramp and the viewpoint launches off the crest — the nose and halo tilting up into open sky and the horizon dropping away, before the car slams back down onto the asphalt with a hard jolt through the wheel and suspension.", + "crouchPrompt": "The view sinks lower into the cockpit toward the wheel and the nose, dropping to a low, fast, close-to-the-track vantage that emphasises the sense of speed.", + "standPrompt": "The view lifts smoothly back up to its normal cockpit height behind the wheel, returning to the standard forward-looking vantage over the nose." + } +} diff --git a/integrations_v2/lingbot/apps/cam2v/web/assets/sources/gta-car.json b/integrations_v2/lingbot/apps/cam2v/web/assets/sources/gta-car.json new file mode 100644 index 000000000..58fc84ad9 --- /dev/null +++ b/integrations_v2/lingbot/apps/cam2v/web/assets/sources/gta-car.json @@ -0,0 +1,104 @@ +{ + "id": "gta_lowrider_13", + "name": "GTA Lowrider", + "hidden": true, + "description": "Third-person GTA lowrider street — key 1 = enter car & drive, key 2 = exit car, key 3 = honk horn, key 4 = headlights, key 5 = spray tag, key 6 = hydraulics bounce", + "image": { + "label": "GTA Lowrider", + "src": "/lingbot-cases/gta-car.jpg" + }, + "scene": { + "base": { + "default": "A sun-baked city street with a parked teal-green 1964 lowrider convertible with gleaming chrome wire wheels and graffiti tags sprayed across its body. The world contains EXACTLY ONE row of tall palm trees on the left at a fixed position AND EXACTLY ONE graffiti-covered brick wall on the right at a fixed position AND EXACTLY ONE glowing storefront sign straight ahead at a fixed position AND EXACTLY ONE teal lowrider car in the centre at a fixed position. Low pastel storefronts and tangled overhead power lines line a wide empty asphalt street, cracked pavement, warm golden-hour sunlight and long shadows under a hazy orange sunset sky. Cel-shaded GTA comic-book art style, bold black outlines, saturated colours, high contrast.", + "driving": "A teal-green 1964 lowrider convertible seen from directly behind as it cruises down a sun-baked city street, the young man now sitting in the driver's seat with one arm resting on the wheel, his flat-brim black cap and black t-shirt visible from behind through the open cabin, chrome wire wheels spinning and the graffiti-tagged bodywork gleaming. The world contains EXACTLY ONE row of tall palm trees on the left at a fixed position AND EXACTLY ONE graffiti-covered brick wall on the right at a fixed position AND EXACTLY ONE glowing storefront sign straight ahead at a fixed position. Low pastel storefronts and tangled overhead power lines line a wide asphalt street, warm golden-hour sunlight and long shadows under a hazy orange sunset sky. Cel-shaded GTA comic-book art style, bold black outlines, saturated colours, high contrast." + }, + "player": { + "default": "A young man seen from behind — wearing a flat-brim black baseball cap, a loose black t-shirt, grey jeans and white sneakers, tattoos down his arms, holding a green spray-paint can in his right hand. EXACTLY ONE young man — a single lone person, no duplicate and no clone.", + "driving": "" + }, + "camera": { + "default": { + "static": "Third-person view from behind the young man, his back centred in frame at constant size beside the parked lowrider, the green spray can visible in his hand — never a first-person view. Neither the man nor the camera moves; look-input is the only camera motion, arcing around the stationary centred man while held.", + "dynamic": "Strict third-person rear view from close behind and slightly above the young man, his back and shoulders filling frame centre beside the car — never first-person. It holds a fixed rear position and does not rotate around him; look-input becomes the man changing heading." + }, + "driving": { + "static": "Third-person chase view from directly behind the lowrider, the car centred in frame at constant size with the driver visible from behind — never a first-person or in-cabin view. Neither the car nor the camera moves; look-input arcs the camera around the stationary centred car while held.", + "dynamic": "Strict third-person chase-cam from close behind and slightly above the lowrider, its rear end and spinning wire wheels filling frame centre — never first-person. It holds a fixed position behind the car and tracks it forward down the street; look-input becomes the car changing heading." + } + }, + "movement": { + "default": { + "static": "The young man stands still beside the lowrider, weight settled, shaking the spray can with a faint rattle as heat shimmers off the asphalt and the palm fronds sway in a light breeze.", + "dynamic": "The young man strides forward down the street past the parked car, sneakers scuffing the warm asphalt, the spray can swinging at his side as storefronts and palms slide past." + }, + "driving": { + "static": "The lowrider idles low on its suspension, engine rumbling, the driver's arm resting on the wheel and exhaust haze curling from the tailpipe as heat ripples off the street ahead.", + "dynamic": "The lowrider rolls forward down the street at a cruising pace, wire wheels spinning and chrome catching the sunset light, storefronts, palms and power lines sweeping past on either side." + } + }, + "events": [ + { + "name": "Enter Car", + "actor": "character", + "baseVersion": "driving", + "cameraVersion": "driving", + "movementVersion": "driving", + "detail": "The young man steps around to the driver's door of the teal lowrider, pulls it open, drops into the seat and pulls the door shut; the viewpoint settles into a chase position behind the car as the engine turns over and rumbles to life, and he stays behind the wheel. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + }, + { + "name": "Exit Car", + "actor": "character", + "baseVersion": "default", + "cameraVersion": "default", + "movementVersion": "default", + "detail": "The lowrider rolls to a stop and the engine cuts; the young man pushes the driver's door open and climbs back out onto the street beside the car, standing once more on foot with the green spray can in hand. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + }, + { + "name": "Honk Horn", + "actor": "character", + "detail": "The driver leans on the horn and a loud blaring honk blasts out across the street, echoing off the storefronts and scattering a few pedestrians, before easing off. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + }, + { + "name": "Headlights", + "actor": "character", + "detail": "The lowrider's headlights snap on, throwing two bright cones of light down the street ahead and lighting up the asphalt, and stay lit. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + }, + { + "name": "Spray Tag", + "actor": "character", + "detail": "The young man shakes the can and sprays a burst of bright green paint onto the wall ahead, leaving a fresh dripping graffiti tag that stays sprayed on the brick, then lowers the can. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + }, + { + "name": "Hydraulics Bounce", + "actor": "character", + "detail": "The lowrider's hydraulic suspension kicks hard, hopping the front end up off the asphalt and slamming it back down, the whole car bouncing on its wire wheels before settling low again. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + }, + { + "name": "Police Arrive", + "actor": "environment", + "detail": "The young man stays where he is, unchanged in pose and position, while police cruisers race in with sirens wailing and red-and-blue lights flashing, screeching to a stop at angles across the street as officers pile out and take cover; the parked cruisers and officers stay, blocking the road. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + }, + { + "name": "Nightfall", + "actor": "environment", + "detail": "The warm orange sunset drains from the sky as night falls over the street — the sky deepens to dark blue and black, the storefront signs and streetlights flicker on casting neon pools across the wet-looking asphalt, and from then on the scene stays lit only by artificial light. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + }, + { + "name": "Rival Lowrider Rolls Up", + "actor": "environment", + "detail": "A rival crew's crimson-red lowrider rolls up and stops across the street ahead, engine growling, and stays parked there with its crew leaning against it, sizing up the young man and his car. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + } + ], + "hud": { + "show": true, + "maxHealth": 100, + "health": 100, + "inventory": [ + "a green spray can" + ] + }, + "jumpPrompt": "The young man springs upward off both feet, leaping high off the warm asphalt, his sneakers lifting clear of the ground before he drops back down and lands in a low crouch.", + "crouchPrompt": "The camera lowers toward the ground as the young man crouches down low, bending his knees and ducking into a compact, hunched stance beside the car; the viewpoint sinks smoothly to a low, near-ground vantage and settles there, close to the ground.", + "standPrompt": "The young man straightens back up out of the crouch, rising to full standing height as the camera lifts smoothly back to its normal eye-level vantage." + } +} diff --git a/integrations_v2/lingbot/apps/cam2v/web/assets/sources/jet-ski-cruise.json b/integrations_v2/lingbot/apps/cam2v/web/assets/sources/jet-ski-cruise.json new file mode 100644 index 000000000..9b412263e --- /dev/null +++ b/integrations_v2/lingbot/apps/cam2v/web/assets/sources/jet-ski-cruise.json @@ -0,0 +1,224 @@ +{ + "id": "case2_1012", + "name": "Reactor: Jet Ski Cruise", + "description": "Third-person jet ski freestyle — tricks: spin 360, nose pop, slalom weave, superman, one-hand wave, donut spray; jump off / jump back on; director: sharks, dolphins, storms, waterspout, whale, island, sea turtle, volcano, rogue wave, low fuel, sunset, etc.", + "objective": { + "summary": "Stay on the jet ski and preserve your fuel — keep riding on the surface near the safe shallow water and don't run dry.", + "director": "Work against the rider: try to throw them off the jet ski and lure them out into deeper, more dangerous open water away from the safe shallow beach. Favor events that unseat, endanger, or draw the rider seaward — Rogue Wave, Waterspout Forms, Storm Rolls In, Shark Lunges, and ultimately Thrown from the Jet Ski — and push the scene toward the deep open sea on the horizon, wasting the rider's fuel and stranding them. Escalate the danger over time and build on what has already happened; do not just repeat the same event back-to-back." + }, + "image": { + "label": "Reactor: Jet Ski Cruise", + "src": "/lingbot-cases/jet-ski-cruise.jpg" + }, + "scene": { + "base": { + "default": "Turquoise water near a sandy beach lined with palm trees, a distant rocky outcrop on the horizon. Sunlit coastal atmosphere with light glinting off the calm sea. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges, or goes underwater, always riding on top of the water." + }, + "player": { + "default": "A man in a red life vest riding a single white and red jet ski, seated at the handlebars and in control. EXACTLY ONE rider on EXACTLY ONE jet ski — a single lone rider, no second jet ski, no duplicate and no clone." + }, + "camera": { + "default": { + "static": "Third-person view, the man on the jet ski locked at the exact centre of the frame at constant size and distance. Neither the man nor the camera moves on its own; arrow-key look-input is the only source of camera motion, arcing the camera around the stationary, centred rider only while held.", + "dynamic": "Strict third-person rear view, the man on the jet ski locked at the exact centre of the frame as the camera holds a fixed position behind him and tracks him forward. The camera does not rotate around the rider; look-input becomes the jet ski changing heading." + }, + "overboard": { + "static": "Third-person view of the man bobbing in the water in his red life vest, centred in frame; the camera holds steady near the water surface, and only arrow-key look-input arcs it slowly around him while held.", + "dynamic": "Third-person view holding on the man treading water in the swell, the camera near the surface; it does not move on its own — look-input pans slowly around him." + } + }, + "movement": { + "default": { + "static": "The man sits upright on the jet ski, hands resting on the handlebars, the craft bobbing gently on the swell as small wavelets lap against the hull and the calm water settles around it. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "dynamic": "The jet ski surges forward across the water, its rear thruster churning the turquoise sea into a wake of white foam, spray fanning out to either side as the man leans into the ride and the hull skips over the swell. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water." + }, + "overboard": { + "static": "The man floats and treads water in his red life vest, bobbing on the swell and wiping water from his face, the empty jet ski drifting nearby; he stays in the water and does not climb back aboard on his own.", + "dynamic": "The man swims through the water in his red life vest toward the drifting jet ski, arms pulling through the swell with spray around him." + } + }, + "events": [ + { + "name": "Spin 360", + "actor": "character", + "detail": "The rider whips the jet ski into a tight spinning 360, the hull skating around and sweeping a full ring of spray across the water, then settles back facing forward. EXACTLY ONE jet ski and ONE rider in frame — a single lone rider, no second jet ski, no duplicate and no clone. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": 0 + }, + { + "name": "Nose Pop", + "actor": "character", + "detail": "The rider yanks the handlebars back and pops the nose of the jet ski high into the air, riding up on its tail with the front lifted skyward and spray fanning off the back, then drops the nose back down and levels out. EXACTLY ONE jet ski and ONE rider in frame — a single lone rider, no second jet ski, no duplicate and no clone. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": 0 + }, + { + "name": "Slalom Weave", + "actor": "character", + "detail": "The rider snaps the jet ski into a fast slalom, weaving sharply left and right through the swell in quick zig-zags and throwing alternating fans of spray off each side, then straightens back onto a level heading. EXACTLY ONE jet ski and ONE rider in frame — a single lone rider, no second jet ski, no duplicate and no clone. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": 0 + }, + { + "name": "Superman", + "actor": "character", + "detail": "The rider stretches out flat off the back of the jet ski in a superman trick, gripping the seat with both hands as his legs stream out behind over the water, holding the pose as the craft skims forward, then pulls back up into the seat. EXACTLY ONE jet ski and ONE rider in frame — a single lone rider, no second jet ski, no duplicate and no clone. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": 0 + }, + { + "name": "One-Hand Wave", + "actor": "character", + "detail": "The rider lifts one hand clean off the handlebars and waves it high overhead, the other hand still gripping the bars as the jet ski drives steadily forward across the turquoise water, then brings the hand back down and re-grips the handlebars. EXACTLY ONE jet ski and ONE rider in frame — a single lone rider, no second jet ski, no duplicate and no clone. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": 0 + }, + { + "name": "Donut Spray", + "actor": "character", + "detail": "The rider leans the jet ski into a tight continuous circle, carving one full donut after another and throwing a sweeping ring of spray across the turquoise water, then straightens back out and rides forward. EXACTLY ONE jet ski and ONE rider in frame — a single lone rider, no second jet ski, no duplicate and no clone. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": 0 + }, + { + "name": "Jump Off", + "actor": "character", + "cameraVersion": "overboard", + "movementVersion": "overboard", + "detail": "The rider stands up and leaps clean off the jet ski, splashing down INTO the turquoise water beside it and surfacing in his red life vest — now IN the water, the rider swims through the waves with steady overarm strokes, kicking and paddling through the water as the now-riderless jet ski coasts to a stop nearby; the rider is in the water swimming, not on the jet ski. EXACTLY ONE jet ski and ONE rider, a single lone rider, no second jet ski, no duplicate and no clone. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": 0 + }, + { + "name": "Jump Back On", + "actor": "character", + "requires": { "firedAny": ["Jump Off", "Thrown from the Jet Ski"] }, + "detail": "The rider swims the last stroke to the drifting jet ski, grabs the handlebars and hauls himself up out of the water back onto the seat, settling into his riding position and gunning the throttle to surge forward across the water again. EXACTLY ONE jet ski and ONE rider, a single lone rider, no second jet ski, no duplicate and no clone. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": 0 + }, + { + "name": "Calm Open Water", + "actor": "environment", + "requires": { "notFired": ["Shark Appears", "Dolphins Leap Alongside", "Whale Breaches"] }, + "detail": "The open sea stretches out ahead calm and clear, the turquoise water glassy and undisturbed — no fins, no creatures, no wildlife of any kind breaking the surface, just the rider and the bright empty water as the jet ski cruises on across the peaceful swell. EXACTLY ONE rider on EXACTLY ONE jet ski — a single lone rider, no second jet ski, no duplicate and no clone. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": 0, + "count": 1 + }, + { + "name": "Shark Appears", + "actor": "environment", + "requires": { "notFired": ["Dolphins Leap Alongside", "Whale Breaches", "Calm Open Water"] }, + "detail": "A single tall, upright, rigid, sharply pointed SHARK's dorsal fin — a dark grey stiff triangular shark fin standing straight up out of the water, unmistakably a shark and NOT a dolphin, never a curved or hooked dolphin fin, never rolling or arcing like a dolphin — rises and knifes in a straight steady line through the turquoise water ahead, slicing across the surface and crossing the jet ski's path — ONLY the top fin shows above the water, the shark's body staying hidden below the surface; the fin circles on the surface nearby for a short while, then turns away and dives back down beneath the water, slipping into the deep and vanishing from sight — the shark is gone and does not come back. EXACTLY ONE shark in the water — a single lone shark, no second shark, no duplicate and no clone. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "count": 1 + }, + { + "name": "Shark Lunges", + "actor": "environment", + "detail": "The tall, upright, rigid, sharply pointed SHARK's dorsal fin — a dark grey stiff triangular shark fin standing straight up out of the water, unmistakably a shark and NOT a dolphin's curved or hooked fin, never rolling or arcing like a dolphin — surges fast across the surface straight toward the jet ski, cutting a sharp wake right up alongside it in a rush of spray — ONLY the top fin breaks the surface, the shark's body staying below the water — then it veers off and keeps pace through the wake. EXACTLY ONE shark in the water — a single lone shark, no second shark, no duplicate and no clone. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "requires": { + "fired": [ + "Shark Appears" + ] + }, + "health": -15 + }, + { + "name": "Dolphins Leap Alongside", + "actor": "environment", + "requires": { "notFired": ["Shark Appears", "Whale Breaches", "Calm Open Water"] }, + "detail": "A pod of sleek dolphins surfaces and leaps in formation alongside the jet ski, arcing out of the turquoise water and diving back in one after another, spray glittering in the sunlight as they keep pace beside the rider, then peel away back into the sea. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": 5, + "count": 1 + }, + { + "name": "Storm Rolls In", + "actor": "environment", + "detail": "The sky darkens as heavy storm clouds roll across the sun, the wind whips up, and the calm turquoise sea churns into grey chop crested with whitecaps while cold rain sweeps across the water, streaking the surface and drumming on the jet ski. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": -10 + }, + { + "name": "Waterspout Forms", + "actor": "environment", + "detail": "A towering waterspout twists up from the sea on the horizon ahead — a swirling column of water and mist spinning between the dark clouds and the churning surface, dragging a ring of spray around its base as it looms out across the water ahead. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": -15, + "count": 1 + }, + { + "name": "Whale Breaches", + "actor": "environment", + "requires": { "notFired": ["Shark Appears", "Dolphins Leap Alongside", "Calm Open Water"] }, + "detail": "The rider on the jet ski remains the main subject, unchanged, while off to one side EXACTLY ONE massive whale breaches out of the deep — a single huge dark body rising clear of the sea and crashing back down in an enormous explosion of white spray and rolling waves, before it slips back beneath the surface. There is only ONE whale, a single lone whale, no second whale, no pod, no duplicate and no clone of it. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": 0, + "count": 1 + }, + { + "name": "Island Appears", + "actor": "environment", + "chance": 0.12, + "detail": "A rocky island rises into view on the horizon ahead — a single dark craggy outcrop of stone and green scrub sitting out on the open water, growing steadily larger as the jet ski cruises toward it across the turquoise sea. EXACTLY ONE island on the horizon — a single lone island, no second island, no duplicate and no clone. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "count": 1 + }, + { + "name": "Sea Turtle", + "actor": "environment", + "chance": 0.1, + "detail": "A single large green sea turtle glides gently through the clear turquoise water just ahead of the jet ski, its patterned shell and flippers slowly paddling as it surfaces for a breath and then drifts calmly along near the rider. EXACTLY ONE sea turtle in the water — a single lone turtle, no second turtle, no duplicate and no clone. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "count": 1 + }, + { + "name": "Volcanic Island Erupts", + "actor": "environment", + "requires": { "fired": ["Island Appears"] }, + "detail": "The distant rocky outcrop on the horizon erupts into a volcano — glowing orange lava streaming down its slopes and a towering plume of black ash and smoke rising into the sky, embers raining into the sea around its base and lighting the water with a deep red glow across the swell. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": -10, + "count": 1 + }, + { + "name": "Rogue Wave", + "actor": "environment", + "detail": "A towering rogue wave rears up ahead — a steep wall of dark water rising far above the jet ski, its crest curling and foaming as it sweeps toward the rider, then it barrels down in a churning collapse of white spray and heaving surf before the sea levels back out. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": -15 + }, + { + "name": "Fuel Runs Low", + "actor": "environment", + "requires": { "minChunks": 80 }, + "detail": "The jet ski's engine begins to strain and sputter as the fuel runs low — the motor coughing and losing power far out on the open water, well away from the safe shallows, the craft labouring on across the swell as the tank drops toward empty. EXACTLY ONE rider on EXACTLY ONE jet ski — a single lone rider, no second jet ski, no duplicate and no clone. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": -40 + }, + { + "name": "Thrown from the Jet Ski", + "actor": "environment", + "requires": { "firedAny": ["Shark Lunges", "Rogue Wave"] }, + "cameraVersion": "overboard", + "movementVersion": "overboard", + "detail": "The jet ski hits a wave and bucks hard and the rider is thrown clean off, flung sideways off the craft and crashing straight down INTO the turquoise water in a burst of white spray, plunging under and surfacing in his red life vest. From then on he is IN THE WATER — floating, swimming and treading water in the sea in his life vest, the now-riderless jet ski drifting to a stop nearby; he is fully in the water and NOT on the jet ski, NOT flung up into the air, and does not climb back aboard on his own. EXACTLY ONE man in the water, a single lone rider, no second man, no duplicate and no clone.", + "health": -30 + }, + { + "name": "Rides into the Sunset", + "actor": "environment", + "win": true, + "baseVersion": "empty", + "cameraVersion": "empty", + "movementVersion": "empty", + "requires": { "minChunks": 320, "minHealth": 1, "notFired": ["Thrown from the Jet Ski"] }, + "detail": "The sun sinks low toward the horizon and the sky blazes into deep orange, gold and pink as the rider cruises on across the calm, glittering water, the jet ski gliding smoothly over the gentle golden swell straight into the sunset, its wake shining in the warm light as the day's ride winds down and the sea spreads out ahead all molten copper and rose. EXACTLY ONE rider on EXACTLY ONE jet ski — a single lone rider, no second jet ski, no duplicate and no clone. The jet ski ALWAYS stays up on top of the water, above the surface of the waves at all times — it never sinks, dips below, submerges or goes underwater, always riding on top of the water.", + "health": 0 + }, + { + "name": "Runs Out of Fuel", + "actor": "environment", + "baseVersion": "empty", + "cameraVersion": "empty", + "movementVersion": "empty", + "requires": { "minChunks": 320, "maxHealth": 0 }, + "detail": "The jet ski's engine sputters, coughs and dies as the last of the fuel runs dry, and the craft coasts to a slow stop far out on the open water — drifting silently on the swell with no power, the rider stranded alone on the still, empty sea as the wake fades away to nothing around the dead hull. EXACTLY ONE rider on EXACTLY ONE jet ski — a single lone rider, no second jet ski, no duplicate and no clone.", + "health": 0 + } + ], + "jumpPrompt": "The man and his jet ski leap up off the water, the hull lifting clear of the sea surface and rising into the air for a moment before dropping back down with a splash.", + "crouchPrompt": "The camera lowers toward the ground as the character crouches down low, bending the knees and ducking the head into a compact, hunched stance; the viewpoint sinks smoothly to a low, near-ground vantage and settles there, close to the floor.", + "standPrompt": "The character straightens back up out of the crouch, rising to full standing height as the camera lifts smoothly back to its normal eye-level vantage.", + "hud": { + "show": true, + "maxHealth": 100, + "health": 100, + "healthLabel": "Fuel", + "inventory": [] + } + } +} diff --git a/integrations_v2/lingbot/apps/cam2v/web/assets/sources/noir-alley-combat.json b/integrations_v2/lingbot/apps/cam2v/web/assets/sources/noir-alley-combat.json new file mode 100644 index 000000000..6b266815b --- /dev/null +++ b/integrations_v2/lingbot/apps/cam2v/web/assets/sources/noir-alley-combat.json @@ -0,0 +1,95 @@ +{ + "id": "case1_0036_combat", + "name": "Noir Alley Combat", + "hidden": true, + "description": "Third-person noir alley melee — key 1 = punch combo, key 2 = roundhouse kick, key 3 = baton strike, key 4 = grapple takedown, key 5 = dodge roll", + "image": { + "label": "Noir Alley Combat", + "src": "/lingbot-cases/noir-alley-combat.jpg" + }, + "objective": { + "summary": "Clear the alley — win the fight without dropping below zero health.", + "director": "Escalate the threat as the fight goes on: start with rain and gloom, then roll in fog, spawn a second attacker ahead, and ramp the pressure the longer the player survives.", + "success": [ + "the alley is clear and the officer is still standing" + ], + "failure": [ + "the officer's health reaches zero" + ], + "durationChunks": 20 + }, + "scene": { + "base": { + "default": "A narrow urban alley at night. The world contains EXACTLY ONE tall street lamp on the right at a fixed position AND EXACTLY ONE glowing neon shop sign on the left at a fixed position AND EXACTLY ONE shop door straight ahead at a fixed position AND EXACTLY ONE green dumpster on the right at a fixed position. Dark brick walls, heavy rain falling, shiny puddles on the wet asphalt, yellow police tape, blue and red ambient light. Cinematic noir night, reflective wet surfaces." + }, + "player": { + "default": "A lone uniformed police officer in dark blue tactical gear, holding a flashlight in one hand whose bright white beam cuts through the darkness down the alley ahead. EXACTLY ONE officer — a single lone officer, no duplicate and no clone." + }, + "camera": { + "default": { + "static": "Third-person view from behind the officer, his back and body centred in frame at constant size, his hands and any weapon visible ahead — never a first-person view. Neither officer nor camera moves; look-input is the only camera motion, arcing around the stationary centred officer while held.", + "dynamic": "Strict third-person rear view from close behind and slightly above the officer, his back and body filling frame centre, his hands and any weapon visible ahead — never first-person. It holds a fixed rear position and does not rotate around him; look-input becomes the officer changing heading." + } + }, + "movement": { + "default": { + "static": "The officer stands still on the wet asphalt in a loose ready stance, weight settled, only his shoulders rising and falling with slow breaths and his gloved hands flexing as rain streams off his tactical gear and drips from his knuckles into the spreading puddles.", + "dynamic": "The officer advances at a steady prowling pace down the alley, boots splashing through the puddles and kicking up fine spray, his guard up and gear shifting with each stride as the rain streaks past and the wet asphalt slides beneath him." + } + }, + "events": [ + { + "name": "Punch Combo", + "actor": "character", + "health": -5, + "detail": "The officer snaps forward with a fast three-punch combination — jab, cross, then a heavy hook — his gloved fists driving straight ahead through the falling rain, water spraying off his knuckles with each strike, before he pulls his guard back up and settles. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + }, + { + "name": "Roundhouse Kick", + "actor": "character", + "health": -5, + "detail": "The officer plants his lead foot and whips his rear leg around in a fast roundhouse kick, his boot sweeping in a wide arc ahead of him and flinging a sheet of spray off the wet asphalt, then he brings the leg back down and re-settles into his stance. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + }, + { + "name": "Baton Strike", + "actor": "character", + "health": -8, + "detail": "The officer draws a collapsible steel baton, flicks it open with a sharp snap, and swings it down and across in a swift overhead strike ahead of him; the metal glints in the blue and red light and hisses through the rain before he lowers it back to his side. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + }, + { + "name": "Grapple Takedown", + "actor": "character", + "health": -12, + "detail": "The officer lunges forward, seizes an unseen opponent ahead of him, pivots his hips and hurls them down onto the wet asphalt with a heavy splash of spray, then straightens back up over the spot, breathing hard. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + }, + { + "name": "Dodge Roll", + "actor": "character", + "health": 10, + "detail": "The officer drops into a low crouch and executes a fast forward roll across the wet asphalt, tucking his shoulder and rising smoothly back to his feet in a ready stance. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + }, + { + "name": "Enemies Appear", + "actor": "environment", + "detail": "A group of dark, armed figures appears far down the road at the distant far end of the alley, small and silhouetted against the neon glow through the falling rain, then starts advancing up the wet street toward the officer, steadily growing larger as they close the distance; they keep coming. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it.", + "count": 3 + }, + { + "name": "Enemies Attack", + "actor": "environment", + "detail": "The figures that came up the road close the last of the distance and attack the officer all at once — fists, knives, and clubs swinging in from straight ahead as they drive him back against the wet brick wall under the flickering neon. The main subject stays EXACTLY ONE in frame — a single main character, no second copy of the main subject, no duplicate and no clone of it." + } + ], + "hud": { + "show": true, + "maxHealth": 100, + "health": 100, + "inventory": [ + "a steel baton" + ] + }, + "jumpPrompt": "The officer springs upward off both feet, leaping high off the wet asphalt, his boots lifting clear of the ground before he drops back down and lands in a low crouch.", + "crouchPrompt": "The camera lowers toward the ground as the character crouches down low, bending the knees and ducking the head into a compact, hunched stance; the viewpoint sinks smoothly to a low, near-ground vantage and settles there, close to the floor.", + "standPrompt": "The character straightens back up out of the crouch, rising to full standing height as the camera lifts smoothly back to its normal eye-level vantage." + } +} diff --git a/integrations_v2/lingbot/apps/cam2v/web/assets/sources/watergun.json b/integrations_v2/lingbot/apps/cam2v/web/assets/sources/watergun.json new file mode 100644 index 000000000..7e854cb21 --- /dev/null +++ b/integrations_v2/lingbot/apps/cam2v/web/assets/sources/watergun.json @@ -0,0 +1,157 @@ +{ + "id": "watergun_aquapark", + "name": "Water Blaster", + "description": "First-person water-blaster FPS at a floating aqua park — keys 1-9: rapid fire, pump/reload, splash blast, melee bash, raise float shield, duck behind cover, spray screen, dive dodge, sprint; + green slime blast (fires the green quarry water); Space = jump gap, C = crouch behind cover", + "image": { + "label": "Water Blaster", + "src": "/lingbot-cases/watergun.jpg" + }, + "scene": { + "base": { + "default": "First-person point of view aiming out across a colourful floating inflatable aqua park on a calm green quarry lake under bright summer sun. The world contains EXACTLY ONE large ring-shaped blue inflatable obstacle platform straight ahead at a fixed position AND EXACTLY ONE person in swimwear standing on the floats ahead at a fixed position AND EXACTLY ONE rocky tree-lined bank across the water in the background at a fixed position. Interconnected inflatable platforms in blue, green and yellow float on the dark green water, swimmers in life vests and swimsuits scattered across them, sunlight glinting off the ripples. Photorealistic first-person action-cam look, saturated colours, bright midday light.", + "downed": "First-person point of view plunged down into the green quarry water after being knocked off the floats — the view is half-submerged at the waterline, water sloshing across the lens, bubbles and ripples all around, the colourful inflatable platforms bobbing above at the surface. The player is in the water, no longer up on the floats. Photorealistic first-person action-cam look." + }, + "player": { + "default": "A bare hand at the lower right of the frame gripping a blue and red toy water blaster. EXACTLY ONE water blaster held at the lower right at a fixed position.", + "downed": "" + }, + "camera": { + "default": { + "static": "First-person view with the blue and red water blaster and the hand gripping it held at the lower right of the frame at constant size — never a third-person body, no character seen from behind. Neither the player nor the camera moves on its own; arrow-key look-input is the only source of motion, swinging the aim/view across the aqua park while held.", + "dynamic": "Strict first-person view, the water blaster held ready at the lower right of the frame as the camera advances forward across the floating platforms — never a third-person body. Look-input aims and turns the view left and right; the blaster stays fixed at the lower right." + } + }, + "movement": { + "default": { + "static": "The player stands still on a bobbing inflatable platform, the water blaster raised and steady at the lower right, the floats rocking gently on the ripples and water lapping at their edges as swimmers move in the distance.", + "dynamic": "The player wades and clambers forward across the bobbing inflatable platforms, the water blaster held up at the lower right, the floats pitching underfoot and water splashing up as the aqua park rushes past on either side." + } + }, + "events": [ + { + "name": "Rapid Fire", + "actor": "character", + "detail": "The hand pumps and unloads a rapid burst from the water blaster, a stuttering volley of water jets spraying out one after another across the floats in a scatter of spray and mist, then eases off.", + "removeItem": "water" + }, + { + "name": "Pump Reload", + "actor": "character", + "detail": "The other hand grabs the pump handle and racks the water blaster back and forth to pressurise it, water sloshing in the clear reservoir, then brings it back up to the ready position at the lower right.", + "addItem": "water" + }, + { + "name": "Splash Blast", + "actor": "character", + "detail": "The player unleashes a wide fan of water from the blaster, a broad sweeping spray that douses everything on the platform ahead in a sheet of glittering droplets, then settles.", + "removeItem": "water" + }, + { + "name": "Melee Bash", + "actor": "character", + "detail": "The player swings the water blaster forward like a bat, cracking it against a close-up target with a splash, then pulls it back to the ready position at the lower right.", + "health": 0 + }, + { + "name": "Raise Float Shield", + "actor": "character", + "detail": "The free hand hauls up a clear inflatable board and holds it flat across the front of the view like a shield, incoming jets of water hammering into it and sheeting off to the sides in bursts of spray, then lowers it back down as the water blaster returns to the ready position at the lower right.", + "health": 0 + }, + { + "name": "Duck Behind Cover", + "actor": "character", + "detail": "The player drops down behind the raised edge of a big inflatable platform, the view dipping low behind the bobbing float as water jets streak overhead and splatter across the cover, then rises back up with the water blaster raised at the lower right.", + "health": 0 + }, + { + "name": "Sprint Charge", + "actor": "character", + "detail": "The player breaks into a hard sprint straight ahead, pounding across the pitching inflatable platforms with the water blaster held up, the floats and swimmers rushing past faster and water splashing up with every step, then eases back to a steady ready jog.", + "health": 0 + }, + { + "name": "Green Slime Blast", + "actor": "character", + "detail": "The player sucks the dark green quarry water up into the blaster and unloads it as a thick, glowing green slime stream, a heavy rope of bright green lake water arcing across the aqua park and slamming into the target — leaving BOLD, vivid neon-green splat marks that burst outward and STAIN wherever they land. Every inflatable platform and any opponent it hits is left coated in bright, saturated green splotches, thick dripping streaks and glistening slime that cling to the surface as strong, high-contrast green stains and stay clearly marked long after the stream cuts off.", + "removeItem": "water" + }, + { + "name": "Swim", + "actor": "character", + "detail": "The view drops to water level as the player jumps off the inflatable float into the green quarry lake and swims forward, arms and hands stroking through the water at the lower frame and spray fanning up ahead as the colourful aqua-park platforms bob past on either side. First-person view, never a third-person body." + }, + { + "name": "Dive", + "actor": "character", + "detail": "The view plunges underwater as the player dives down beneath the surface of the green quarry lake, the water closing over the frame into murky green light shot through with rising bubbles and shafts of sunlight, the undersides of the inflatable floats and swimmers' legs drifting overhead through the rippling surface. First-person view, never a third-person body." + }, + { + "name": "Rival Blaster Ambush", + "actor": "environment", + "detail": "A rival player in a life vest pops up from behind an inflatable platform ahead and opens fire with their own water blaster, jets of water arcing straight toward the camera and splattering across the lens, the rival ducking and weaving as they keep firing.", + "health": -12 + }, + { + "name": "Rival Shoots Back", + "actor": "environment", + "health": -10, + "detail": "A rival player in a life vest on the floats ahead returns fire, sending a hard jet of water straight back at the camera that splatters and streams across the lens, the view flinching and blurring under the soaking before the water runs off. EXACTLY ONE rival in frame — a single opponent, no duplicate and no clone." + }, + { + "name": "Rival Slimes You", + "actor": "environment", + "health": -12, + "detail": "A rival player ahead unloads a thick rope of glowing PURPLE slime straight at the camera — a heavy jet of vivid neon-purple goo, a completely DIFFERENT colour from the player's own green slime — that splatters across the lens and clings, oozing and dripping down the view in bold, saturated purple streaks and splotches that stay clearly marked before they slowly run off. EXACTLY ONE rival in frame — a single opponent, no duplicate and no clone." + }, + { + "name": "Bathers Get Super Soakers", + "actor": "environment", + "health": -12, + "detail": "The director hands out huge super soakers to every bather in the aqua park, and the whole park opens fire at once — swimmers on every inflatable platform ahead and to both sides raising big bright pump-action water blasters and unleashing arcing jets of water toward the camera from all directions, crossing streams and spray filling the air as the bathers duck, weave and keep pumping. From then on the other bathers STAY armed with super soakers, firing back across the floats." + }, + { + "name": "Player Falls In", + "actor": "environment", + "detail": "A swimmer standing on the floats ahead loses their footing, arms wheeling, and topples off the inflatable into the green water with a big splash — they vanish under the surface for a moment, then bob back up in a life vest where the empty float now sits." + }, + { + "name": "Crocodile Lunges", + "actor": "environment", + "health": -20, + "detail": "A big crocodile surges up out of the green quarry water beside the inflatable floats, jaws gaping wide as it lunges straight at the camera in a burst of spray and thrashing water, snapping close before it twists and slides back below the surface. From then on the crocodile lurks in the water, cruising the aqua park with just its eyes and snout above the surface. EXACTLY ONE crocodile in the water — a single lone crocodile, no duplicate and no clone." + }, + { + "name": "Wave Surge", + "actor": "environment", + "detail": "The calm lake churns up into rolling swells that sweep across the aqua park, the inflatable platforms pitching and heaving hard on the waves, spray flying off their edges as everything bobs violently." + }, + { + "name": "Storm Rolls In", + "actor": "environment", + "detail": "Dark grey clouds sweep across the sky and blot out the summer sun, the light dropping to a flat overcast gloom, rain beginning to pockmark the lake surface and the water turning choppy and slate-green across the whole aqua park." + }, + { + "name": "Giant Balloon Drops", + "actor": "environment", + "detail": "A huge water balloon plummets down and bursts on the platform ahead in an enormous explosion of water, drenching the whole float in a wall of spray — then it is GONE, nothing left but the soaked, dripping platform and settling ripples where it hit." + }, + { + "name": "Float Deflates", + "actor": "environment", + "detail": "One of the large inflatable platforms ahead splits and rapidly deflates, folding and sagging down flat into the water and sinking below the surface — it is GONE, leaving a wide gap of open green water where the float used to be, the gap staying open." + } + ], + "hud": { + "show": true, + "maxHealth": 100, + "health": 100, + "inventory": [ + "water" + ] + }, + "jumpPrompt": "The player leaps up and forward off the edge of the inflatable platform, sailing over a gap of open water with the blaster held out, then lands splashing onto the next float and steadies.", + "crouchPrompt": "The view drops low as the player crouches down behind the raised edge of an inflatable platform for cover, only the top of the water blaster peeking over, and holds low.", + "standPrompt": "The player rises back up from behind the inflatable to full standing height, the water blaster lifting back up to the ready position at the lower right, the view returning to its normal vantage." + } +} diff --git a/integrations_v2/lingbot/apps/cam2v/web/assets/watergun.jpg b/integrations_v2/lingbot/apps/cam2v/web/assets/watergun.jpg new file mode 100644 index 000000000..2dddbe61c Binary files /dev/null and b/integrations_v2/lingbot/apps/cam2v/web/assets/watergun.jpg differ diff --git a/integrations_v2/lingbot/apps/cam2v/web/index.html b/integrations_v2/lingbot/apps/cam2v/web/index.html new file mode 100644 index 000000000..c550212b6 --- /dev/null +++ b/integrations_v2/lingbot/apps/cam2v/web/index.html @@ -0,0 +1,92 @@ + + + + + + + + FlashDreams WebRTC Drive + + + +
+
+

FlashDreams WebRTC Drive

+ + + +
+ + +
+ +
+ +
+ Status +
+ + Idle +
+ + +
+
+ Flow + waiting +
+
+ +
+ +
+

Player Controls

+
+
+
+ +
+

Client Logs

+
+
+ + Waiting +
+
+ +
+
+ FPS + -- +
+
+ Latency + -- +
+
+ Resolution + -- +
+
+ Step + -- +
+
+ World Model + World Model +
+
+
+
+ + + + diff --git a/integrations_v2/lingbot/apps/cam2v/web/request_session.css b/integrations_v2/lingbot/apps/cam2v/web/request_session.css new file mode 100644 index 000000000..f38783e62 --- /dev/null +++ b/integrations_v2/lingbot/apps/cam2v/web/request_session.css @@ -0,0 +1,605 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/ + +:root { + color-scheme: dark; + font-family: Inter, ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, sans-serif; + --panel-bg: rgba(16, 17, 18, 0.70); + --panel-border: rgba(255, 255, 255, 0.15); + --panel-shadow: 0 20px 60px rgba(0, 0, 0, 0.35); + --text: #f3f7f0; + --muted: #b9c0bf; + --accent: #8ef01c; + --accent-strong: #a9ff2f; + --danger: #ff685f; + --warning: #ffbf4f; + --cyan: #63d8ff; +} + +* { + box-sizing: border-box; +} + +html, +body { + min-height: 100%; +} + +body { + margin: 0; + overflow: hidden; + background: #050607; + color: var(--text); +} + +button { + font: inherit; +} + +select { + font: inherit; +} + +.appShell { + min-height: 100vh; +} + +.stage { + position: relative; + min-height: 100vh; + overflow: hidden; + background: #050607; + isolation: isolate; +} + +.stageBackdrop, +.idleCanvas, +.stageVideo, +.stageVignette { + position: absolute; + inset: 0; + width: 100%; + height: 100%; +} + +.stageBackdrop { + z-index: 0; + background: + linear-gradient(180deg, #14181c 0%, #0b1013 48%, #050607 100%), + linear-gradient(115deg, rgba(99, 216, 255, 0.18), rgba(142, 240, 28, 0.10) 42%, rgba(255, 191, 79, 0.12)); +} + +.idleCanvas { + z-index: 1; + display: block; + background: transparent; + opacity: 1; + transition: opacity 220ms ease; +} + +.stageVideo { + z-index: 2; + display: block; + object-fit: cover; + opacity: 0; + background: #050607; + transition: opacity 220ms ease; +} + +.modelStageSlot { + position: absolute; + z-index: 2; + inset: 0; + pointer-events: none; +} + +.modelPanelSlot { + display: contents; +} + +.modelStatusSlot:empty, +.modelControlSlot:empty { + display: none; +} + +body.has-video .stageVideo { + opacity: 1; +} + +body.has-video .idleCanvas, +body.has-video .stageBackdrop { + opacity: 0; +} + +.stageVignette { + z-index: 3; + pointer-events: none; + background: none; +} + +.srOnly { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + +.overlayPanel { + z-index: 4; + border: 1px solid var(--panel-border); + border-radius: 8px; + background: var(--panel-bg); + box-shadow: var(--panel-shadow); + backdrop-filter: blur(18px) saturate(1.15); +} + +.brandOverlay { + position: absolute; + z-index: 4; + top: clamp(18px, 3vw, 42px); + left: clamp(18px, 3vw, 52px); + width: clamp(260px, 25vw, 390px); + filter: drop-shadow(0 4px 18px rgba(0, 0, 0, 0.55)); +} + +.brandLogo { + display: block; + width: 100%; + height: auto; +} + +.statusLine strong, +.metric strong, +.logEntry time { + color: var(--accent-strong); +} + +.statusCard { + position: absolute; + top: clamp(18px, 3vw, 42px); + right: clamp(18px, 3vw, 42px); + width: min(240px, calc(100vw - 36px)); + padding: 16px 18px; + display: grid; + gap: 10px; +} + +.panelLabel { + color: var(--muted); + font-size: 0.72rem; + font-weight: 700; + letter-spacing: 0.12em; + text-transform: uppercase; +} + +.statusLine { + display: flex; + align-items: center; + gap: 10px; + min-height: 30px; + font-size: clamp(1.1rem, 2vw, 1.35rem); +} + +.statusDot, +.liveDot { + display: inline-block; + width: 10px; + height: 10px; + flex: 0 0 auto; + border-radius: 999px; + background: var(--muted); + box-shadow: 0 0 0 rgba(255, 255, 255, 0); +} + +body[data-status="connected"] .statusDot, +body[data-status="waiting"] .statusDot, +body[data-status="generating"] .statusDot, +.liveDot { + background: #47d65a; + box-shadow: 0 0 16px rgba(71, 214, 90, 0.62); +} + +body[data-status="connecting"] .statusDot { + background: var(--warning); + box-shadow: 0 0 16px rgba(255, 191, 79, 0.54); +} + +body[data-status="error"] .statusDot { + background: var(--danger); + box-shadow: 0 0 16px rgba(255, 104, 95, 0.54); +} + +.connectButton { + width: 100%; + min-height: 36px; + border: 1px solid rgba(142, 240, 28, 0.45); + border-radius: 6px; + background: rgba(142, 240, 28, 0.12); + color: var(--text); + cursor: pointer; + font-weight: 700; +} + +.connectButton:hover { + background: rgba(142, 240, 28, 0.20); +} + +.connectButton:disabled { + cursor: not-allowed; + opacity: 0.52; +} + +.postprocessField { + display: grid; + gap: 6px; + color: var(--muted); + font-size: 0.72rem; + font-weight: 700; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.postprocessField[hidden] { + display: none; +} + +.postprocessField select { + width: 100%; + min-height: 34px; + border: 1px solid rgba(255, 255, 255, 0.22); + border-radius: 6px; + background: rgba(12, 14, 15, 0.86); + color: var(--text); + padding: 0 9px; + text-transform: none; +} + +.postprocessField select:disabled { + cursor: not-allowed; + opacity: 0.52; +} + +.flowLine { + display: grid; + grid-template-columns: auto 1fr; + gap: 8px; + align-items: center; + min-width: 0; + color: var(--muted); + font-size: 0.78rem; +} + +.flowLine strong { + min-width: 0; + overflow: hidden; + color: var(--text); + text-overflow: ellipsis; + white-space: nowrap; +} + +.controlCard { + position: absolute; + left: clamp(18px, 3vw, 48px); + bottom: clamp(96px, 12vh, 132px); + width: min(380px, calc(100vw - 36px)); + padding: 18px 20px 20px; +} + +.controlCard h2, +.logCard h2 { + display: flex; + align-items: center; + gap: 10px; + margin: 0 0 18px; + font-size: 1.08rem; + font-weight: 740; + letter-spacing: 0; + white-space: nowrap; +} + +.controlCard h2 .panelAccentBar, +.logCard h2 span { + width: 3px; + height: 22px; + border-radius: 999px; + background: var(--accent); + box-shadow: 0 0 14px rgba(142, 240, 28, 0.42); +} + +.controlRows { + display: grid; + gap: 12px; +} + +.controlRows[hidden] { + display: none; +} + +.controlRow { + display: grid; + grid-template-columns: 176px 1fr; + gap: 18px; + align-items: center; + min-height: 38px; + color: var(--text); + font-size: 0.95rem; +} + +.keyCluster { + display: grid; + grid-auto-flow: column; + grid-auto-columns: 38px; + gap: 8px; + justify-content: start; +} + +.keyClusterWide { + grid-auto-columns: 38px; +} + +.controlKey { + width: 38px; + height: 38px; + border: 1px solid rgba(255, 255, 255, 0.22); + border-bottom-color: rgba(255, 255, 255, 0.34); + border-radius: 6px; + background: rgba(12, 14, 15, 0.62); + color: #f9fbff; + cursor: pointer; + font-weight: 750; + line-height: 1; + box-shadow: inset 0 -2px 0 rgba(0, 0, 0, 0.30); + touch-action: none; + user-select: none; +} + +.controlKey:hover { + border-color: rgba(255, 255, 255, 0.42); + background: rgba(255, 255, 255, 0.09); +} + +.controlKey.is-active { + border-color: rgba(142, 240, 28, 0.78); + background: rgba(142, 240, 28, 0.26); + color: var(--accent-strong); + box-shadow: + 0 0 18px rgba(142, 240, 28, 0.32), + inset 0 0 0 1px rgba(142, 240, 28, 0.18); + transform: translateY(1px); +} + +.logCard { + position: absolute; + right: clamp(18px, 3vw, 42px); + bottom: clamp(78px, 9vh, 98px); + width: min(400px, calc(100vw - 36px)); + padding: 18px 20px 16px; +} + +.logList { + display: grid; + align-content: start; + gap: 6px; + min-height: 150px; + max-height: 230px; + overflow: hidden; + padding-bottom: 12px; + border-bottom: 1px solid rgba(255, 255, 255, 0.16); + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.82rem; + line-height: 1.35; +} + +.logEntry { + display: grid; + grid-template-columns: 76px 1fr; + gap: 8px; + min-width: 0; + color: #f3f6f7; +} + +.logEntry span { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.logEntry.is-error span { + color: #ffb0aa; +} + +.logEntry.is-client time { + color: var(--cyan); +} + +.logFooter { + display: flex; + align-items: center; + gap: 8px; + padding-top: 12px; + color: var(--muted); + font-size: 0.86rem; +} + +.metricsBar { + position: absolute; + z-index: 4; + left: 50%; + bottom: clamp(20px, 4vh, 40px); + display: grid; + grid-template-columns: + minmax(max-content, 0.7fr) + minmax(max-content, 1fr) + minmax(max-content, 1.15fr) + minmax(max-content, 0.7fr) + minmax(max-content, 1.45fr); + width: min(960px, calc(100vw - 36px)); + overflow: hidden; + border: 1px solid rgba(255, 255, 255, 0.14); + border-radius: 8px; + background: rgba(18, 20, 22, 0.74); + box-shadow: 0 12px 38px rgba(0, 0, 0, 0.32); + backdrop-filter: blur(16px) saturate(1.12); + transform: translateX(-50%); +} + +.metric { + display: grid; + grid-template-columns: max-content max-content; + align-items: center; + justify-content: center; + gap: 10px; + min-width: max-content; + min-height: 42px; + padding: 0 16px; + border-right: 1px solid rgba(255, 255, 255, 0.11); + font-size: 0.88rem; + text-align: center; +} + +.metric:last-child { + border-right: 0; +} + +.metric span { + color: var(--muted); + white-space: nowrap; +} + +.metric strong { + text-align: left; + white-space: nowrap; + font-weight: 760; +} + +body[data-status="generating"] .statusLine strong { + animation: statusPulse 1.2s ease-in-out infinite; +} + +@keyframes statusPulse { + 0%, + 100% { + opacity: 1; + } + 50% { + opacity: 0.62; + } +} + +@media (max-width: 900px) { + body { + overflow: auto; + } + + .stage { + min-height: max(100svh, 980px); + } + + .brandOverlay { + top: 18px; + left: 18px; + width: min(310px, calc(100vw - 36px)); + } + + .statusCard { + top: 88px; + left: 18px; + right: auto; + } + + .controlCard, + .logCard { + left: 18px; + right: 18px; + width: auto; + } + + .controlCard { + bottom: 440px; + } + + .logCard { + bottom: 164px; + } + + .metricsBar { + left: 18px; + right: 18px; + bottom: 18px; + grid-template-columns: repeat(2, minmax(0, 1fr)); + width: auto; + transform: none; + } + + .metric { + padding: 0 14px; + } + + .metric:nth-child(2n) { + border-right: 0; + } + + .metric:last-child { + grid-column: 1 / -1; + } +} + +@media (max-width: 520px) { + .controlRow { + grid-template-columns: 1fr; + gap: 8px; + } + + .keyCluster { + grid-auto-columns: minmax(34px, 1fr); + } + + .controlKey { + width: 100%; + } + + .logList { + min-height: 112px; + max-height: 150px; + } + + .metricsBar { + grid-template-columns: 1fr; + } + + .metric { + min-height: 36px; + border-right: 0; + border-bottom: 1px solid rgba(255, 255, 255, 0.11); + } + + .metric:last-child { + grid-column: auto; + border-bottom: 0; + } +} + +.promptGenerationPanel { + position: absolute; + z-index: 4; + left: clamp(18px, 3vw, 52px); + bottom: clamp(92px, 13vw, 150px); + width: min(430px, calc(100vw - 36px)); + padding: 16px 18px; +} +.promptGenerationField { display: grid; gap: 7px; color: var(--muted); font-size: .72rem; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; } +.promptGenerationField textarea { width: 100%; resize: vertical; min-height: 92px; border: 1px solid rgba(255,255,255,.22); border-radius: 6px; background: rgba(12,14,15,.86); color: var(--text); padding: 9px; font: inherit; text-transform: none; letter-spacing: normal; } +.promptGenerationActions { display: flex; flex-wrap: wrap; gap: 8px; margin-top: 12px; } +.promptGenerationActions button { min-height: 34px; border: 1px solid rgba(142,240,28,.45); border-radius: 6px; background: rgba(142,240,28,.12); color: var(--text); cursor: pointer; font-weight: 700; padding: 0 10px; } +.promptGenerationActions button:disabled { cursor: not-allowed; opacity: .5; } +.promptGenerationHint { margin: 10px 0 0; color: var(--muted); font-size: .78rem; line-height: 1.35; } +.controlCard[hidden] { display: none; } +.promptDurationInput { width: 8rem; min-height: 34px; margin-top: 12px; border: 1px solid rgba(255,255,255,.22); border-radius: 6px; background: rgba(12,14,15,.86); color: var(--text); padding: 0 9px; } diff --git a/integrations_v2/lingbot/apps/cam2v/web/request_session.js b/integrations_v2/lingbot/apps/cam2v/web/request_session.js new file mode 100644 index 000000000..2a43aae11 --- /dev/null +++ b/integrations_v2/lingbot/apps/cam2v/web/request_session.js @@ -0,0 +1,1430 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const mockMode = new URLSearchParams(window.location.search).has("mock") +// ?manual skips the automatic connect-on-load attempt (still available via +// the Connect button) -- avoids a stray/restored tab silently racing into +// the single active WebRTC session and locking out the tab you meant to use. +const manualConnectMode = new URLSearchParams(window.location.search).has("manual") + +/** + * @typedef {Object} WebRTCModelAdapter + * @property {string=} modelName + * @property {string=} stylesheet + * @property {Array<{label: string, keys: Array}>=} controls + * @property {{postprocess?: boolean}=} capabilities + * @property {{endpoint: string, label?: string, placeholder?: string, generateLabel?: string}=} promptGeneration + * @property {(context: Object) => (void|Promise)=} mount + * @property {(context: Object) => (void|Promise)=} beforeConnect + * @property {(action: Object, context: Object) => void=} onActionSent + * @property {(payload: Object, context: Object) => boolean=} onControlMessage + * @property {(visible: boolean, context: Object) => void=} onVideoVisibilityChanged + * @property {(context: Object) => void=} onDisconnect + */ + +const connectButton = document.getElementById("connectButton") +const statusText = document.getElementById("statusText") +const flowText = document.getElementById("flowText") +const eventLog = document.getElementById("eventLog") +const logState = document.getElementById("logState") +const remoteVideo = document.getElementById("remoteVideo") +const idleCanvas = document.getElementById("idleCanvas") +const fpsValue = document.getElementById("fpsValue") +const latencyValue = document.getElementById("latencyValue") +const resolutionValue = document.getElementById("resolutionValue") +const stepValue = document.getElementById("stepValue") +const modelValue = document.getElementById("modelValue") +const postprocessField = document.getElementById("postprocessField") +const postprocessSelect = document.getElementById("postprocessSelect") +const modelStageSlot = document.getElementById("modelStageSlot") +const modelStatusSlot = document.getElementById("modelStatusSlot") +const modelPanelSlot = document.getElementById("modelPanelSlot") +const modelControlSlot = document.getElementById("modelControlSlot") +const controlRows = document.getElementById("controlRows") + +const keyAliases = new Map([ + ["arrowup", "w"], + ["arrowleft", "a"], + ["arrowdown", "s"], + ["arrowright", "d"], +]) +const keySources = new Map() +const heldKeyOrder = new Map() +const activeKeys = new Set() +const frameTimes = [] +const pendingActions = [] +const maxPendingActions = 32 +const heartbeatIntervalMs = 2000 +const autoConnectMaxAttempts = 3 +const autoConnectRetryDelayMs = 750 + +let allowedKeys = new Set() +let controlButtons = [] +/** @type {WebRTCModelAdapter|null} */ +let modelAdapter = null + +let peerConnection = null +let controlChannel = null +let statsTimer = null +let videoMetricsTimer = null +let heartbeatTimer = null +let inferenceInFlight = false +let connected = false +let disconnecting = false +let heldKeySequence = 0 +let postprocessAvailable = false +let liveVideoStream = null + +const metrics = { + fps: null, + targetFps: null, + latencyMs: null, + rttMs: null, + resolution: null, + step: null, + model: "World Model", +} + +function normalizeKey(rawKey) { + const key = String(rawKey || "").toLowerCase() + return keyAliases.get(key) || key +} + +function isEditableControlTarget(target) { + if (!target || typeof target !== "object") { + return false + } + if (target.isContentEditable === true) { + return true + } + const tagName = typeof target.tagName === "string" ? target.tagName.toLowerCase() : "" + if (["input", "textarea", "select"].includes(tagName)) { + return true + } + return typeof target.closest === "function" + && target.closest("input, textarea, select, [contenteditable]") !== null +} + +function formatTime() { + return new Date().toLocaleTimeString([], { hour12: false }) +} + +function firstFinite(...values) { + for (const value of values) { + const number = Number(value) + if (Number.isFinite(number)) { + return number + } + } + return null +} + +function formatMs(value) { + if (!Number.isFinite(value)) { + return "--" + } + if (value >= 1000) { + return `${(value / 1000).toFixed(1)} s` + } + return `${Math.round(value)} ms` +} + +function logEvent(message, { source = "server", level = "info" } = {}) { + const consoleMessage = `[FlashDreams WebRTC][${source}] ${message}` + if (level === "error") { + console.error(consoleMessage) + } else { + console.info(consoleMessage) + } + + const entry = document.createElement("div") + entry.className = `logEntry is-${source}` + if (level === "error") { + entry.classList.add("is-error") + } + + const time = document.createElement("time") + time.textContent = `[${formatTime()}]` + const body = document.createElement("span") + body.textContent = message + entry.append(time, body) + eventLog.prepend(entry) + + while (eventLog.children.length > 36) { + eventLog.lastElementChild.remove() + } +} + +function setStatus(message, state = message.toLowerCase()) { + statusText.textContent = message + document.body.dataset.status = state + logState.textContent = state === "idle" ? "Waiting" : message +} + +function setFlow(message) { + flowText.textContent = message +} + +function setVideoVisible(visible) { + document.body.classList.toggle("has-video", visible) + modelAdapter?.onVideoVisibilityChanged?.(visible, modelContext) +} + +function renderControls(groups) { + controlRows.replaceChildren() + allowedKeys = new Set() + for (const group of groups) { + if (!group || !Array.isArray(group.keys) || group.keys.length === 0) { + continue + } + const row = document.createElement("div") + row.className = "controlRow" + const cluster = document.createElement("div") + cluster.className = group.keys.length > 2 ? "keyCluster keyClusterWide" : "keyCluster" + for (const item of group.keys) { + const key = normalizeKey(typeof item === "string" ? item : item?.key) + if (!key) { + continue + } + allowedKeys.add(key) + const button = document.createElement("button") + button.className = "controlKey" + button.type = "button" + button.dataset.controlKey = key + button.textContent = key.toUpperCase() + button.setAttribute("aria-label", typeof item === "string" ? key : (item.label || key)) + cluster.append(button) + } + const label = document.createElement("span") + label.textContent = String(group.label || "Controls") + row.append(cluster, label) + controlRows.append(row) + } + controlButtons = Array.from(controlRows.querySelectorAll("[data-control-key]")) +} + +function setPostprocessDisabled(disabled) { + postprocessSelect.disabled = disabled || !postprocessAvailable +} + +async function loadPostprocessOptions() { + const payload = mockMode + ? { default_preset: "rtx-super-resolution", presets: ["rtx-super-resolution"] } + : await fetch("/api/postprocess/options").then(async (response) => { + if (!response.ok) { + throw new Error(`post-process options failed (${response.status})`) + } + return response.json() + }) + const presets = Array.isArray(payload.presets) ? payload.presets : [] + const defaultPreset = typeof payload.default_preset === "string" + ? payload.default_preset + : "" + postprocessAvailable = Boolean(defaultPreset && presets.includes(defaultPreset)) + postprocessField.hidden = !postprocessAvailable + postprocessSelect.replaceChildren(new Option("Off", "")) + for (const preset of presets) { + if (typeof preset === "string" && preset) { + postprocessSelect.append(new Option(preset, preset)) + } + } + postprocessSelect.value = postprocessAvailable ? defaultPreset : "" + setPostprocessDisabled(false) + if (postprocessAvailable) { + logEvent(`post-process=${postprocessSelect.value}`, { source: "client" }) + } +} + +async function configurePostprocessSession() { + if (!postprocessAvailable || mockMode) { + return + } + const postprocessPreset = postprocessSelect.value + const response = await fetch("/api/session/input", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ postprocess_preset: postprocessPreset }), + }) + if (!response.ok) { + const text = await response.text() + throw new Error(`session configuration failed (${response.status}): ${text}`) + } + logEvent(`post-process=${postprocessPreset || "off"}`, { source: "client" }) +} + +// This page was written for the v1 server's control protocol, which carried +// key presses, text events, heartbeats, and disconnects over the data channel +// as one message family. The v2 runtime's channel takes device events only +// (keyboard, mouse, focus, touch, ...) and rejects anything else with +// "Unsupported browser event type", so messages are translated here rather +// than in the shared server, where a page's protocol does not belong. +function toRuntimeEvent(payload) { + const type = payload?.type + if (type === "action") { + const action = payload.action || {} + if (action.event === "keydown" || action.event === "keyup") { + return { type: "keyboard", key: action.key, pressed: action.event === "keydown" } + } + // "step" asked the v1 server to generate one chunk; the v2 runtime + // generates continuously, so there is nothing to ask for. + return null + } + if (type === "disconnect") { + // Deliberately dropped rather than translated to the runtime's "close". + // On v1 this released the peer connection; on v2 a close event ends the + // whole application run, so refreshing the tab killed the server. The + // server keeps the session for a reconnect on its own. + return null + } + // The v2 runtime holds the peer connection open without a keepalive. + if (type === "heartbeat") { + return null + } + return payload +} + +// Text events are session state rather than device input, so they travel over +// the session endpoint the application serves, not the data channel. +function postSessionEvent(payload) { + const body = { event_id: payload.event_id, state: payload.state } + if (typeof payload.prompt === "string") { + body.prompt = payload.prompt + } + fetch("/api/session/input", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }) + .then(async (response) => { + if (!response.ok) { + const detail = await response.text() + throw new Error(`${response.status}: ${detail.slice(0, 200)}`) + } + return response.json() + }) + .then((scene) => { + modelAdapter?.onInitialScene?.(scene, modelContext) + }) + .catch((error) => { + logEvent(`event failed: ${error.message}`, { source: "client", level: "error" }) + }) + return true +} + +function sendModelMessage(payload) { + if (payload?.type === "event") { + return postSessionEvent(payload) + } + if (!connected || !controlChannel || controlChannel.readyState !== "open") { + return false + } + const event = toRuntimeEvent(payload) + if (event === null) { + return true + } + controlChannel.send(JSON.stringify(event)) + return true +} + +function sendModelCommand(payload, label = "model command") { + if (!sendModelMessage(payload)) { + setFlow("connect session first") + return false + } + inferenceInFlight = true + if (promptGenerationControls) promptGenerationControls.generate.disabled = true + setStatus("Generating", "generating") + setFlow(`sent ${label}`) + logEvent(label, { source: "client" }) + return true +} + +const modelContext = { + slots: { + stage: modelStageSlot, + status: modelStatusSlot, + panel: modelPanelSlot, + controls: modelControlSlot, + }, + isVideoVisible: () => document.body.classList.contains("has-video"), + logEvent, + releaseControls: releaseAllKeys, + sendCommand: sendModelCommand, + setModelName(name) { + if (typeof name === "string" && name) { + metrics.model = name + renderMetrics() + } + }, + setResolution(width, height) { + if (Number.isFinite(Number(width)) && Number.isFinite(Number(height))) { + metrics.resolution = `${Number(width)}x${Number(height)}` + renderMetrics() + } + }, +} + +async function loadModelAdapter() { + let adapter = {} + let serverAcceptedKeys = [] + const stylesheetHrefs = new Set() + // The v2 runtime serves this page out of the application's own web root, so + // the adapter sits beside it and needs no discovery endpoint. /api/ui/config + // still wins where a server provides one, keeping the v1 behaviour intact. + let adapterModule = "/adapter.js?v=lingbot-v2-protocol-v4" + stylesheetHrefs.add("/adapter.css") + try { + const response = await fetch("/api/ui/config") + if (response.ok) { + const config = await response.json() + if (typeof config.model_stylesheet === "string" && config.model_stylesheet) { + stylesheetHrefs.add(config.model_stylesheet) + } + if (Array.isArray(config.accepted_keys)) { + serverAcceptedKeys = config.accepted_keys + } + if (typeof config.adapter_module === "string" && config.adapter_module) { + adapterModule = config.adapter_module + } + } + } catch (error) { + logEvent(`model UI config unavailable, using bundled adapter: ${error.message}`, { source: "client", level: "info" }) + } + try { + const module = await import(adapterModule) + if (module.default && typeof module.default === "object") { + adapter = module.default + } + } catch (error) { + logEvent(`model UI unavailable: ${error.message}`, { source: "client", level: "error" }) + } + + modelAdapter = adapter + if (typeof adapter.stylesheet === "string" && adapter.stylesheet) { + stylesheetHrefs.add(adapter.stylesheet) + } + for (const href of stylesheetHrefs) { + const stylesheet = document.createElement("link") + stylesheet.rel = "stylesheet" + stylesheet.href = href + document.head.append(stylesheet) + } + const genericControls = serverAcceptedKeys.length > 0 + ? [{ label: "Controls", keys: serverAcceptedKeys }] + : [] + const modelControls = Array.isArray(adapter.controls) + ? adapter.controls + : genericControls + renderControls(modelControls) + if (typeof adapter.modelName === "string") { + modelContext.setModelName(adapter.modelName) + } + if (adapter.capabilities?.postprocess === true) { + try { + await loadPostprocessOptions() + } catch (error) { + postprocessAvailable = false + postprocessField.hidden = true + setPostprocessDisabled(false) + logEvent(`post-process unavailable: ${error.message}`, { + source: "client", + level: "error", + }) + } + } + configurePromptGeneration(adapter.promptGeneration) + document.querySelector(".controlCard")?.toggleAttribute("hidden", adapter.promptGeneration?.hideControls === true) + await adapter.mount?.(modelContext) +} + +function renderMetrics() { + const fps = firstFinite(metrics.fps, metrics.targetFps) + const latency = firstFinite(metrics.latencyMs, metrics.rttMs) + fpsValue.textContent = Number.isFinite(fps) ? String(Math.round(fps)) : "--" + latencyValue.textContent = formatMs(latency) + resolutionValue.textContent = metrics.resolution || "--" + stepValue.textContent = metrics.step === null ? "--" : String(metrics.step) + modelValue.textContent = metrics.model || "World Model" +} + +function recordActionSent(action) { + pendingActions.push({ + sentAt: performance.now(), + label: actionLabel(action), + }) + while (pendingActions.length > maxPendingActions) { + pendingActions.shift() + } +} + +function takeObservedActionLatency(now = performance.now()) { + if (pendingActions.length === 0) { + return null + } + const oldest = pendingActions[0] + pendingActions.length = 0 + return Math.max(0, now - oldest.sentAt) +} + +function updateMetricsFromChunk(payload) { + const observedLatencyMs = takeObservedActionLatency() + metrics.targetFps = firstFinite(payload.fps, payload.target_fps, metrics.targetFps) + metrics.latencyMs = firstFinite( + payload.latency_ms, + payload.control_latency_ms, + observedLatencyMs, + payload.lag_ms, + payload.gen_ms, + metrics.latencyMs + ) + metrics.step = Number.isFinite(Number(payload.chunk_index)) + ? Number(payload.chunk_index) + : metrics.step + metrics.model = typeof payload.model === "string" && payload.model ? payload.model : metrics.model + + if (typeof payload.resolution === "string") { + metrics.resolution = payload.resolution + } else if (payload.resolution && typeof payload.resolution === "object") { + const width = Number(payload.resolution.width) + const height = Number(payload.resolution.height) + if (Number.isFinite(width) && Number.isFinite(height)) { + metrics.resolution = `${width}x${height}` + } + } + renderMetrics() +} + +function updateMetricsFromVideo() { + if (remoteVideo.videoWidth > 0 && remoteVideo.videoHeight > 0) { + metrics.resolution = `${remoteVideo.videoWidth}x${remoteVideo.videoHeight}` + renderMetrics() + } +} + +function resizeIdleCanvas(ctx) { + const rect = idleCanvas.getBoundingClientRect() + const dpr = Math.min(window.devicePixelRatio || 1, 2) + const width = Math.max(1, Math.floor(rect.width * dpr)) + const height = Math.max(1, Math.floor(rect.height * dpr)) + if (idleCanvas.width !== width || idleCanvas.height !== height) { + idleCanvas.width = width + idleCanvas.height = height + } + ctx.setTransform(dpr, 0, 0, dpr, 0, 0) + return { width: rect.width, height: rect.height } +} + +function drawRouteRibbon(ctx, width, height, t) { + const xBase = width * 0.74 + const yBase = height * 0.28 + ctx.save() + ctx.globalAlpha = 0.62 + ctx.lineWidth = 2 + ctx.strokeStyle = "rgba(99, 216, 255, 0.72)" + ctx.setLineDash([10, 14]) + ctx.lineDashOffset = -t * 24 + ctx.beginPath() + ctx.moveTo(xBase - 92, yBase + 132) + ctx.bezierCurveTo(xBase - 36, yBase + 36, xBase + 42, yBase + 76, xBase + 86, yBase - 16) + ctx.bezierCurveTo(xBase + 116, yBase - 76, xBase + 8, yBase - 92, xBase - 20, yBase - 34) + ctx.stroke() + + ctx.setLineDash([]) + for (let i = 0; i < 8; i += 1) { + const phase = (i / 8 + t * 0.08) % 1 + const angle = phase * Math.PI * 2 + const x = xBase + Math.cos(angle) * 84 + const y = yBase + Math.sin(angle * 1.7) * 72 + ctx.fillStyle = i % 2 === 0 ? "rgba(142, 240, 28, 0.72)" : "rgba(99, 216, 255, 0.62)" + ctx.beginPath() + ctx.arc(x, y, 3.5, 0, Math.PI * 2) + ctx.fill() + } + ctx.restore() +} + +function drawIdleScene(now) { + const ctx = idleCanvas.getContext("2d") + if (!ctx) { + return + } + + const { width, height } = resizeIdleCanvas(ctx) + const t = now * 0.001 + const horizon = height * 0.46 + + const sky = ctx.createLinearGradient(0, 0, 0, height) + sky.addColorStop(0, "#314553") + sky.addColorStop(0.42, "#76919c") + sky.addColorStop(0.66, "#152024") + sky.addColorStop(1, "#060707") + ctx.fillStyle = sky + ctx.fillRect(0, 0, width, height) + + const sunGlow = ctx.createRadialGradient(width * 0.22, height * 0.22, 8, width * 0.22, height * 0.22, width * 0.42) + sunGlow.addColorStop(0, "rgba(255, 204, 112, 0.62)") + sunGlow.addColorStop(0.36, "rgba(255, 204, 112, 0.20)") + sunGlow.addColorStop(1, "rgba(255, 204, 112, 0)") + ctx.fillStyle = sunGlow + ctx.fillRect(0, 0, width, height) + + ctx.fillStyle = "rgba(24, 39, 42, 0.82)" + for (let i = 0; i < 12; i += 1) { + const x = width * (0.02 + i * 0.075) + const buildingWidth = width * (0.035 + (i % 3) * 0.012) + const buildingHeight = height * (0.11 + ((i * 7) % 5) * 0.018) + ctx.fillRect(x, horizon - buildingHeight, buildingWidth, buildingHeight) + } + + const ground = ctx.createLinearGradient(0, horizon, 0, height) + ground.addColorStop(0, "#273331") + ground.addColorStop(1, "#0a0c0c") + ctx.fillStyle = ground + ctx.fillRect(0, horizon, width, height - horizon) + + const road = ctx.createLinearGradient(width * 0.5, horizon, width * 0.5, height) + road.addColorStop(0, "#424c4f") + road.addColorStop(1, "#121516") + ctx.fillStyle = road + ctx.beginPath() + ctx.moveTo(width * 0.42, horizon + 8) + ctx.lineTo(width * 0.58, horizon + 8) + ctx.lineTo(width * 0.80, height) + ctx.lineTo(width * 0.20, height) + ctx.closePath() + ctx.fill() + + ctx.strokeStyle = "rgba(255, 255, 255, 0.42)" + ctx.lineWidth = 2 + ctx.beginPath() + ctx.moveTo(width * 0.42, horizon + 8) + ctx.lineTo(width * 0.20, height) + ctx.moveTo(width * 0.58, horizon + 8) + ctx.lineTo(width * 0.80, height) + ctx.stroke() + + const dashOffset = (t * 92) % 58 + for (let i = -2; i < 14; i += 1) { + const y = horizon + 20 + i * 58 + dashOffset + const scale = Math.max(0, Math.min(1, (y - horizon) / (height - horizon))) + const dashHeight = 18 + scale * 38 + const wobble = Math.sin(t * 0.8 + scale * 3.2) * width * 0.012 + ctx.strokeStyle = "rgba(255, 222, 114, 0.74)" + ctx.lineWidth = 2 + scale * 3 + ctx.beginPath() + ctx.moveTo(width * 0.50 + wobble, y) + ctx.lineTo(width * 0.50 + wobble * 1.3, y + dashHeight) + ctx.stroke() + } + + ctx.save() + ctx.translate(width * 0.5, height * 0.78 + Math.sin(t * 2.1) * 4) + ctx.fillStyle = "rgba(142, 240, 28, 0.74)" + ctx.beginPath() + ctx.moveTo(0, -34) + ctx.lineTo(22, 24) + ctx.lineTo(0, 12) + ctx.lineTo(-22, 24) + ctx.closePath() + ctx.fill() + ctx.strokeStyle = "rgba(255, 255, 255, 0.52)" + ctx.lineWidth = 2 + ctx.stroke() + ctx.restore() + + drawRouteRibbon(ctx, width, height, t) + + ctx.fillStyle = `rgba(255, 255, 255, ${0.06 + Math.sin(t * 1.4) * 0.018})` + ctx.fillRect(0, 0, width, height) + + if (!document.body.classList.contains("has-video")) { + recordFrame(now) + } + window.requestAnimationFrame(drawIdleScene) +} + +function recordFrame(timestamp) { + const now = Number.isFinite(timestamp) ? timestamp : performance.now() + frameTimes.push(now) + while (frameTimes.length > 0 && now - frameTimes[0] > 1200) { + frameTimes.shift() + } + if (frameTimes.length >= 2) { + const elapsed = frameTimes[frameTimes.length - 1] - frameTimes[0] + metrics.fps = elapsed > 0 ? ((frameTimes.length - 1) * 1000) / elapsed : metrics.fps + renderMetrics() + } +} + +function updateControlHighlights() { + activeKeys.clear() + for (const [key, sources] of keySources.entries()) { + if (sources.size > 0) { + activeKeys.add(key) + } + } + for (const button of controlButtons) { + const key = button.dataset.controlKey + button.classList.toggle("is-active", activeKeys.has(key)) + button.setAttribute("aria-pressed", activeKeys.has(key) ? "true" : "false") + } +} + +function actionLabel(action) { + return `${action.event}${action.key ? `:${action.key}` : ""}` +} + +function sendControlAction(action) { + if (!connected || !controlChannel || controlChannel.readyState !== "open") { + return false + } + + const event = toRuntimeEvent({ type: "action", action }) + if (event === null) { + return false + } + inferenceInFlight = true + controlChannel.send(JSON.stringify(event)) + modelAdapter?.onActionSent?.(action, modelContext) + recordActionSent(action) + setStatus("Generating", "generating") + setFlow(`sent ${actionLabel(action)}, waiting=${inferenceInFlight}`) + logEvent(`control ${actionLabel(action)}`, { source: "client" }) + return true +} + +function enqueueAction(action) { + const sent = sendControlAction(action) + if (!sent) { + setFlow(connected ? `not_sent ${actionLabel(action)}` : "connect session first") + } +} + +function setKeyHeld(key, source, held) { + const normalized = normalizeKey(key) + if (!allowedKeys.has(normalized)) { + return + } + + let sources = keySources.get(normalized) + if (!sources) { + sources = new Set() + keySources.set(normalized, sources) + } + + const wasActive = sources.size > 0 + if (held) { + sources.add(source) + } else { + sources.delete(source) + } + const isActive = sources.size > 0 + updateControlHighlights() + + if (held && !wasActive && isActive) { + heldKeySequence += 1 + heldKeyOrder.set(normalized, heldKeySequence) + enqueueAction({ event: "keydown", key: normalized }) + } + if (!held && wasActive && !isActive) { + heldKeyOrder.delete(normalized) + enqueueAction({ event: "keyup", key: normalized }) + } +} + +function releaseAllKeys() { + for (const key of Array.from(keySources.keys())) { + const sources = keySources.get(key) + if (sources && sources.size > 0) { + sources.clear() + heldKeyOrder.delete(key) + updateControlHighlights() + enqueueAction({ event: "keyup", key }) + } + } +} + +function handleControlMessage(rawMessage) { + let payload + try { + payload = JSON.parse(rawMessage) + } catch { + logEvent(`invalid control payload: ${rawMessage}`, { level: "error" }) + return + } + + if (payload.type === "chunk_done") { + inferenceInFlight = false + updateMetricsFromChunk(payload) + const genMs = firstFinite(payload.gen_ms) + const lagMs = firstFinite(payload.lag_ms) + const queueDepth = firstFinite(payload.queue_depth) + const parts = [ + `chunk_done index=${payload.chunk_index}`, + `frames=${payload.num_frames}`, + ] + if (Number.isFinite(Number(payload.enqueued_frames))) { + parts.push(`enqueued=${payload.enqueued_frames}`) + } + if (genMs !== null) { + parts.push(`gen=${Math.round(genMs)}ms`) + } + if (lagMs !== null) { + parts.push(`lag=${Math.round(lagMs)}ms`) + } + if (metrics.latencyMs !== null) { + parts.push(`latency=${Math.round(metrics.latencyMs)}ms`) + } + if (queueDepth !== null) { + parts.push(`queue=${queueDepth}`) + } + logEvent(parts.join(", ")) + const isGenerating = activeKeys.size > 0 + setStatus( + isGenerating ? "Generating" : "Connected", + isGenerating ? "generating" : "connected" + ) + setFlow(`chunk ${payload.chunk_index} complete`) + modelAdapter?.onControlMessage?.(payload, modelContext) + return + } + + if (modelAdapter?.onControlMessage?.(payload, modelContext)) { + return + } + + if (payload.type === "generation_complete") { + inferenceInFlight = false + if (promptGenerationControls) { + promptGenerationControls.generate.disabled = false + promptGenerationControls.download.disabled = false + } + setStatus("Connected", "connected") + setFlow("generation complete; ready for another prompt") + logEvent("generation complete", { source: "server" }) + stopPromptRecording() + const playbackEndpoint = promptGenerationControls?.config.playbackEndpoint + if (playbackEndpoint) { + remoteVideo.pause() + remoteVideo.srcObject = null + remoteVideo.src = `${playbackEndpoint}?generation=${Date.now()}` + remoteVideo.loop = true + remoteVideo.playbackRate = 1 + void remoteVideo.play() + } + return + } + + if (payload.type === "server_log") { + logEvent(payload.message || "server log") + return + } + + if (payload.type === "busy") { + logEvent(`server busy: ${payload.message}`, { level: "error" }) + setStatus("Connected", "connected") + return + } + + if (payload.type === "error") { + inferenceInFlight = false + if (promptGenerationControls) promptGenerationControls.generate.disabled = false + logEvent(`server error: ${payload.message}`, { level: "error" }) + setStatus("Error", "error") + setFlow("server error") + return + } + + logEvent(`server message: ${rawMessage}`) +} + +async function waitForIceGatheringComplete(pc) { + if (pc.iceGatheringState === "complete") { + return + } + await new Promise((resolve) => { + const onStateChange = () => { + if (pc.iceGatheringState === "complete") { + pc.removeEventListener("icegatheringstatechange", onStateChange) + resolve() + } + } + pc.addEventListener("icegatheringstatechange", onStateChange) + }) +} + +async function pollWebRtcStats() { + if (!peerConnection) { + return + } + try { + const stats = await peerConnection.getStats() + for (const report of stats.values()) { + if ( + report.type === "candidate-pair" && + report.state === "succeeded" && + Number.isFinite(report.currentRoundTripTime) + ) { + metrics.rttMs = report.currentRoundTripTime * 1000 + } + if ( + report.type === "inbound-rtp" && + (report.kind === "video" || report.mediaType === "video") && + Number.isFinite(report.framesPerSecond) + ) { + metrics.fps = report.framesPerSecond + } + } + renderMetrics() + } catch (error) { + logEvent(`stats unavailable: ${error.message}`, { source: "client" }) + } +} + +function startStatsPolling() { + if (statsTimer !== null) { + return + } + statsTimer = window.setInterval(() => { + void pollWebRtcStats() + }, 1000) +} + +function stopStatsPolling() { + if (statsTimer !== null) { + window.clearInterval(statsTimer) + statsTimer = null + } +} + +function resetPeerHandles(pc = peerConnection, channel = controlChannel) { + if (peerConnection === pc) { + peerConnection = null + } + if (controlChannel === channel) { + controlChannel = null + } +} + +async function dumpPeerStats(reason) { + if (!peerConnection) { + return + } + try { + const stats = await peerConnection.getStats() + const reports = new Map() + for (const report of stats.values()) { + reports.set(report.id, report) + } + console.group(`[FlashDreams WebRTC] peer stats: ${reason}`) + for (const report of stats.values()) { + if (report.type !== "candidate-pair") { + continue + } + const local = reports.get(report.localCandidateId) + const remote = reports.get(report.remoteCandidateId) + console.info({ + id: report.id, + state: report.state, + nominated: report.nominated, + writable: report.writable, + local: local + ? `${local.candidateType} ${local.protocol} ${local.address || local.ip}:${local.port}` + : report.localCandidateId, + remote: remote + ? `${remote.candidateType} ${remote.protocol} ${remote.address || remote.ip}:${remote.port}` + : report.remoteCandidateId, + }) + } + console.groupEnd() + } catch (error) { + console.warn("[FlashDreams WebRTC] getStats failed", error) + } +} + +function sendHeartbeat() { + if (!controlChannel || controlChannel.readyState !== "open") { + return + } + // The v2 runtime keeps the peer connection open on its own and rejects a + // heartbeat as an unknown event type, so nothing is sent. + +} + +function startHeartbeat() { + if (heartbeatTimer !== null) { + return + } + sendHeartbeat() + heartbeatTimer = window.setInterval(sendHeartbeat, heartbeatIntervalMs) +} + +function stopHeartbeat() { + if (heartbeatTimer !== null) { + window.clearInterval(heartbeatTimer) + heartbeatTimer = null + } +} + +function wait(delayMs) { + return new Promise((resolve) => window.setTimeout(resolve, delayMs)) +} + +function isTransientFetchError(error) { + return ( + error instanceof TypeError + && /fetch|network|load failed/i.test(String(error.message || "")) + ) +} + +function markSessionConnected(pc, channel) { + if ( + connected || + peerConnection !== pc || + controlChannel !== channel || + pc.connectionState !== "connected" || + channel.readyState !== "open" + ) { + return + } + connected = true + connectButton.disabled = false + connectButton.textContent = "Disconnect" + setStatus("Connected", "connected") + setFlow("waiting for input") + startHeartbeat() + startStatsPolling() + if (pendingPromptGeneration) { + pendingPromptGeneration = false + triggerPromptGeneration() + } +} + +function disconnectSession({ notify = true } = {}) { + if (disconnecting) { + return + } + disconnecting = true + releaseAllKeys() + stopHeartbeat() + stopStatsPolling() + connected = false + connectButton.disabled = false + connectButton.textContent = "Connect Session" + setPostprocessDisabled(false) + stopPromptRecording() + modelAdapter?.onDisconnect?.(modelContext) + if (notify && controlChannel && controlChannel.readyState === "open") { + try { + controlChannel.send(JSON.stringify({ type: "disconnect" })) + } catch { + // The browser may already be tearing the page down. + } + } + if (controlChannel && controlChannel.readyState !== "closed") { + controlChannel.close() + } + if (peerConnection) { + peerConnection.close() + } + resetPeerHandles() + // pc.close() stops incoming frames but doesn't reset the