From f46086de89043671fc869fae58fe50bb81f35022 Mon Sep 17 00:00:00 2001 From: DrHepa Date: Mon, 21 Sep 2026 19:50:54 +0200 Subject: [PATCH 1/2] feat(workflows): support scene model artifacts --- README.md | 12 + api/README.md | 1 + api/routers/generation.py | 107 +++++++-- api/runner.py | 50 ++++- api/schemas/generation.py | 13 +- api/services/artifact_input.py | 29 +++ api/services/extension_process.py | 55 ++++- api/services/generator_registry.py | 53 ++++- api/services/generators/base.py | 21 +- api/services/scene_input.py | 138 ++++++++++++ api/tests/test_extension_process.py | 28 +++ api/tests/test_generation_router.py | 3 + api/tests/test_generator_registry.py | 47 ++++ api/tests/test_runner.py | 25 +++ api/tests/test_scene_generation.py | 115 ++++++++++ api/tests/test_scene_input.py | 67 ++++++ .../main/artifact-registry-service.test.ts | 12 + electron/main/artifact-registry-service.ts | 3 +- .../main/extension-install-utils.test.mjs | 44 ++++ electron/main/extension-install-utils.ts | 34 ++- electron/main/ipc-handlers.ts | 15 +- src/areas/workflows/WorkflowsPage.tsx | 12 +- src/areas/workflows/mockExtensions.ts | 6 +- src/areas/workflows/nodes/ExtensionNode.tsx | 2 + src/areas/workflows/nodes/LoadSceneNode.tsx | 134 ++++++++++++ src/areas/workflows/preflight.test.mjs | 35 +++ src/areas/workflows/preflight.ts | 26 ++- src/areas/workflows/workflowRunStore.ts | 66 ++++-- src/areas/workflows/workflowSceneRun.test.mjs | 60 +++++ .../workflows/workflowSceneSource.test.mjs | 29 +++ src/areas/workflows/workflowSceneSource.ts | 205 ++++++++++++++++++ src/shared/stores/workflowsStore.ts | 2 +- src/shared/types/artifacts.ts | 14 ++ src/shared/types/electron.d.ts | 6 +- 34 files changed, 1392 insertions(+), 77 deletions(-) create mode 100644 api/services/artifact_input.py create mode 100644 api/services/scene_input.py create mode 100644 api/tests/test_scene_generation.py create mode 100644 api/tests/test_scene_input.py create mode 100644 src/areas/workflows/nodes/LoadSceneNode.tsx create mode 100644 src/areas/workflows/workflowSceneRun.test.mjs create mode 100644 src/areas/workflows/workflowSceneSource.test.mjs create mode 100644 src/areas/workflows/workflowSceneSource.ts diff --git a/README.md b/README.md index b162cf23..95a0b6ff 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,18 @@ original behavior. ## Workflows Start with a basic workflow first. For example, on the "Workflows" tab, try: Image -> Generate Mesh -> Add to Scene. Make sure there is a connection between each of the steps. Go to the "Generate" tab, make sure the workflow is selected, then click on "Generate 3D Model". Click on "Settings/Logs/Errors" to see any issues. +Model extensions may also declare `scene` as a node input or output. A scene is +a workspace directory containing `scene-manifest.json` with schema +`modly.scene-manifest.v1`; it is not an arbitrary JSON file. Use the **Load +Scene** workflow node to select and validate an existing scene directory. +Scene-capable generators implement `generate_artifact(input_kind, +artifact_path, ...)`; legacy image generators and `POST /generate/from-image` +remain unchanged. The generic `POST /generate/from-artifact` boundary currently +accepts only `scene`, leaving future artifact kinds to separate reviewed changes. +For this first contract, `scene` is model-only and must be declared as the single +`input` value (not inside `inputs`); process and mixed-input scene nodes are rejected. +Model nodes may still accept multiple images and produce a scene. + ## Modly CLI diff --git a/api/README.md b/api/README.md index ee45bc3f..cdcd4d6b 100644 --- a/api/README.md +++ b/api/README.md @@ -29,6 +29,7 @@ uvicorn main:app --host 127.0.0.1 --port 8765 --reload | GET | `/model/status` | Model download / load status | | GET | `/model/download` | SSE stream of download progress | | POST | `/generate/from-image` | Start image-to-3D job | +| POST | `/generate/from-artifact` | Start a typed-artifact model job (`scene` only) | | GET | `/generate/status/{job_id}` | Poll job status | ## Model diff --git a/api/routers/generation.py b/api/routers/generation.py index 8481deb4..7c355014 100644 --- a/api/routers/generation.py +++ b/api/routers/generation.py @@ -4,7 +4,8 @@ import time import traceback import uuid -from typing import Dict +from pathlib import Path +from typing import Dict, Optional, Union from fastapi import APIRouter, File, Form, UploadFile, HTTPException, BackgroundTasks from services.generators.base import smooth_progress, GenerationCancelled @@ -14,7 +15,8 @@ # binding captured at import would keep writing output to the old directory. import services.generator_registry as registry from services.generator_registry import generator_registry -from schemas.generation import JobStatus +from schemas.generation import GenerateFromArtifactRequest, JobStatus +from services.artifact_input import TypedArtifactInput, validate_artifact_input router = APIRouter(tags=["generation"]) @@ -105,6 +107,7 @@ async def generate_from_image( # Verify the requested model exists in the registry try: generator_registry.get_generator(model_id) + output_kind = generator_registry.get_manifest(model_id).get("output", "mesh") except ValueError as e: raise HTTPException(400, str(e)) @@ -131,11 +134,49 @@ async def generate_from_image( _jobs[job_id] = job _cancel_events[job_id] = threading.Event() - background_tasks.add_task(_run_generation, job_id, image_bytes, full_params, collection) + background_tasks.add_task( + _run_generation, job_id, image_bytes, full_params, collection, output_kind, model_id + ) return {"job_id": job_id} +_RESERVED_ARTIFACT_PARAMS = { + "artifact_path", "input_kind", "input_path", "scene_path", "scene_manifest_path", +} + + +@router.post("/from-artifact") +async def generate_from_artifact( + request: GenerateFromArtifactRequest, + background_tasks: BackgroundTasks, +): + """Queue a validated typed artifact without serializing it as image bytes.""" + try: + manifest = generator_registry.get_manifest(request.model_id) + except (KeyError, ValueError) as exc: + raise HTTPException(400, str(exc)) from exc + declared = manifest.get("inputs") or [manifest.get("input", "image")] + if request.input_kind not in declared: + raise HTTPException(400, f"Model {request.model_id} does not accept {request.input_kind} input") + try: + artifact = validate_artifact_input(registry.WORKSPACE_DIR, request.input_kind, request.input_path) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + + params = {k: v for k, v in request.params.items() if k not in _RESERVED_ARTIFACT_PARAMS} + params["scene_manifest_path"] = str(artifact.path) + collection = sanitize_collection(request.collection) + job_id = str(uuid.uuid4()) + _purge_old_jobs() + _jobs[job_id] = JobStatus(job_id=job_id, status="pending", progress=0) + _cancel_events[job_id] = threading.Event() + background_tasks.add_task( + _run_generation, job_id, artifact, params, collection, manifest.get("output", "mesh"), request.model_id + ) + return {"job_id": job_id} + + @router.get("/status/{job_id}") async def job_status(job_id: str): @@ -170,7 +211,14 @@ async def cancel_job(job_id: str): return {"cancelled": True} -async def _run_generation(job_id: str, image_bytes: bytes, params: dict, collection: str = "Default") -> None: +async def _run_generation( + job_id: str, + model_input: Union[bytes, TypedArtifactInput], + params: dict, + collection: str = "Default", + output_kind: str = "mesh", + model_id: Optional[str] = None, +) -> None: job = _jobs[job_id] job.status = "running" @@ -189,8 +237,14 @@ def progress_cb(pct: int, step: str = "") -> None: # Check if the model needs to be loaded BEFORE calling get_active(), # because get_active() loads the model in a blocking manner. # active_status() is an instantaneous operation (simple dict lookup). - if not generator_registry.active_status()["loaded"]: - active = generator_registry.active_status() + get_generator = (lambda: generator_registry.get_ready_generator(model_id)) \ + if model_id is not None else generator_registry.get_active + status_reader = getattr(generator_registry, "model_status", None) + status = (status_reader(model_id) if model_id is not None and status_reader + else generator_registry.active_status() if model_id is None + else {"name": model_id, "downloaded": True, "loaded": False}) + if not status["loaded"]: + active = status model_name = active['name'] init_label = f"Downloading {model_name}…" if not active['downloaded'] else f"Loading {model_name}…" progress_cb(0, init_label) @@ -202,11 +256,11 @@ def progress_cb(pct: int, step: str = "") -> None: ) load_thread.start() try: - gen = await loop.run_in_executor(None, generator_registry.get_active) + gen = await loop.run_in_executor(None, get_generator) finally: stop_load_evt.set() else: - gen = await loop.run_in_executor(None, generator_registry.get_active) + gen = await loop.run_in_executor(None, get_generator) if job_id in _cancelled: return @@ -217,18 +271,39 @@ def progress_cb(pct: int, step: str = "") -> None: gen.outputs_dir = coll_dir cancel_event = _cancel_events.get(job_id) - import inspect - supports_cancel = "cancel_event" in inspect.signature(gen.generate).parameters - output_path = await loop.run_in_executor( - None, - lambda: gen.generate(image_bytes, params, progress_cb, cancel_event) - if supports_cancel - else gen.generate(image_bytes, params, progress_cb), - ) + if isinstance(model_input, TypedArtifactInput): + # Revalidate just before crossing the inference boundary. The + # subprocess runner repeats this check inside the worker. + from services.artifact_input import revalidate_artifact_input + model_input = revalidate_artifact_input(registry.WORKSPACE_DIR, model_input) + import inspect + supports_cancel = "cancel_event" in inspect.signature(gen.generate_artifact).parameters + output_path = await loop.run_in_executor( + None, + lambda: gen.generate_artifact(model_input.kind, model_input.path, params, progress_cb, cancel_event) + if supports_cancel else gen.generate_artifact(model_input.kind, model_input.path, params, progress_cb), + ) + else: + import inspect + supports_cancel = "cancel_event" in inspect.signature(gen.generate).parameters + output_path = await loop.run_in_executor( + None, + lambda: gen.generate(model_input, params, progress_cb, cancel_event) + if supports_cancel else gen.generate(model_input, params, progress_cb), + ) if job_id in _cancelled: return + output_path = Path(output_path).resolve(strict=True) + if output_kind == "scene": + from services.scene_input import validate_scene_input + try: + output_relative = output_path.relative_to(registry.WORKSPACE_DIR.resolve()) + output_path = validate_scene_input(registry.WORKSPACE_DIR, output_relative.as_posix()) + except (OSError, ValueError) as exc: + raise ValueError("Generated scene output is not a valid workspace scene") from exc + job.status = "done" job.progress = 100 _completed_at[job_id] = time.monotonic() diff --git a/api/runner.py b/api/runner.py index 0dd21392..0dd17ddb 100644 --- a/api/runner.py +++ b/api/runner.py @@ -151,6 +151,32 @@ def _apply_manifest_metadata(gen, manifest: dict, node: dict) -> None: gen._params_schema = node.get("params_schema") or manifest.get("params_schema", []) +def decode_model_input(msg: dict): + """Decode legacy image bytes or revalidate a typed artifact in the worker.""" + if "input" not in msg: + return base64.b64decode(msg["image_b64"]) + value = msg["input"] + if not isinstance(value, dict) or set(value) != {"kind", "path"}: + raise ValueError("Typed artifact input must contain exactly kind and path") + from services.artifact_input import TypedArtifactInput, revalidate_artifact_input + typed = TypedArtifactInput(kind=value.get("kind"), path=Path(value.get("path", ""))) + return revalidate_artifact_input(WORKSPACE_DIR, typed) + + +def validate_requested_model(msg: dict, manifest: dict, node: dict) -> None: + """Reject requests routed to a worker for a different manifest node.""" + requested = msg.get("model_id") + if requested is None: # Backward compatibility with already-running legacy hosts. + return + expected = manifest["id"] + if node.get("id"): + expected = f"{expected}/{node['id']}" + if requested != expected: + raise ValueError( + f"Generation request model '{requested}' does not match worker model '{expected}'" + ) + + # ------------------------------------------------------------------ # # Main loop # ------------------------------------------------------------------ # @@ -201,10 +227,17 @@ def main() -> None: # ---- generate -------------------------------------------- elif action == "generate": + validate_requested_model(msg, manifest, node) cancel_evt = threading.Event() _cancel[rid] = cancel_evt - image_bytes = base64.b64decode(msg["image_b64"]) + model_input = decode_model_input(msg) params = msg.get("params", {}) + if hasattr(model_input, "kind"): + if not isinstance(params, dict): + raise ValueError("Model params must be an object") + reserved = {"artifact_path", "input_kind", "input_path", "scene_path", "scene_manifest_path"} + params = {key: value for key, value in params.items() if key not in reserved} + params["scene_manifest_path"] = str(model_input.path) if msg.get("outputs_dir"): gen.outputs_dir = Path(msg["outputs_dir"]) gen.outputs_dir.mkdir(parents=True, exist_ok=True) @@ -217,7 +250,20 @@ def progress_cb(pct: int, step: str = "") -> None: send({"type": "log", "level": "warning", "message": ("Model was not loaded (earlier setup failure?); " "reloaded before generating.")}) - output_path = gen.generate(image_bytes, params, progress_cb, cancel_evt) + if hasattr(model_input, "kind"): + output_path = gen.generate_artifact( + model_input.kind, model_input.path, params, progress_cb, cancel_evt + ) + else: + output_path = gen.generate(model_input, params, progress_cb, cancel_evt) + if node.get("output") == "scene": + from services.scene_input import validate_scene_input + resolved_output = Path(output_path).resolve(strict=True) + try: + relative_output = resolved_output.relative_to(WORKSPACE_DIR.resolve(strict=True)) + except (OSError, ValueError) as exc: + raise ValueError("Generated scene output is outside the workspace") from exc + output_path = validate_scene_input(WORKSPACE_DIR, relative_output.as_posix()) send({"type": "done", "id": rid, "output_path": str(output_path)}) except Exception as exc: # Detect GenerationCancelled by name to avoid import issues diff --git a/api/schemas/generation.py b/api/schemas/generation.py index 7ed6ca62..04c18c85 100644 --- a/api/schemas/generation.py +++ b/api/schemas/generation.py @@ -1,5 +1,5 @@ -from typing import Literal, Optional -from pydantic import BaseModel +from typing import Any, Literal, Optional +from pydantic import BaseModel, Field class JobStatus(BaseModel): @@ -9,3 +9,12 @@ class JobStatus(BaseModel): step: Optional[str] = None # Human-readable current step output_url: Optional[str] = None error: Optional[str] = None + + +class GenerateFromArtifactRequest(BaseModel): + """Generic typed-artifact request. Only scene is public in this release.""" + input_kind: Literal["scene"] + input_path: str + model_id: str + collection: str = "Workflows" + params: dict[str, Any] = Field(default_factory=dict) diff --git a/api/services/artifact_input.py b/api/services/artifact_input.py new file mode 100644 index 00000000..165e29ec --- /dev/null +++ b/api/services/artifact_input.py @@ -0,0 +1,29 @@ +"""Typed model artifact inputs shared by the API, worker bridge, and runner.""" +from dataclasses import dataclass +from pathlib import Path + +from services.scene_input import validate_scene_input + +SUPPORTED_ARTIFACT_INPUTS = frozenset({"scene"}) + + +@dataclass(frozen=True) +class TypedArtifactInput: + kind: str + path: Path + + +def validate_artifact_input(workspace: Path, kind: str, input_path: str) -> TypedArtifactInput: + if kind not in SUPPORTED_ARTIFACT_INPUTS: + raise ValueError(f"Unsupported artifact input kind: {kind}") + if kind == "scene": + return TypedArtifactInput(kind="scene", path=validate_scene_input(workspace, input_path)) + raise ValueError(f"Unsupported artifact input kind: {kind}") + + +def revalidate_artifact_input(workspace: Path, value: TypedArtifactInput) -> TypedArtifactInput: + try: + relative = value.path.resolve(strict=True).relative_to(workspace.resolve(strict=True)) + except (OSError, ValueError) as exc: + raise ValueError("Artifact input is outside the workspace") from exc + return validate_artifact_input(workspace, value.kind, relative.as_posix()) diff --git a/api/services/extension_process.py b/api/services/extension_process.py index 67565d36..3a39bfc5 100644 --- a/api/services/extension_process.py +++ b/api/services/extension_process.py @@ -291,16 +291,18 @@ def generate( progress_cb: Optional[Callable[[int, str], None]] = None, cancel_event: Optional[threading.Event] = None, ) -> Path: - from services.generators.base import GenerationCancelled + return self._generate_request( + {"image_b64": base64.b64encode(image_bytes).decode()}, + params, progress_cb, cancel_event, + ) - req_id = str(uuid.uuid4()) - self._send({ - "action": "generate", - "id": req_id, - "image_b64": base64.b64encode(image_bytes).decode(), - "params": params, - "outputs_dir": str(self.outputs_dir) if self.outputs_dir else None, - }) + def _receive_generation( + self, + req_id: str, + progress_cb: Optional[Callable[[int, str], None]], + cancel_event: Optional[threading.Event], + ) -> Path: + from services.generators.base import GenerationCancelled # Grace period after sending a cooperative cancel before hard-killing # the subprocess. Long enough to let generators that check cancel_event @@ -377,6 +379,41 @@ def generate( elif t == "log": print(f"[{self.MODEL_ID}] {msg.get('message', '')}", file=sys.stderr) + def generate_artifact( + self, + input_kind: str, + artifact_path: Path, + params: dict, + progress_cb: Optional[Callable[[int, str], None]] = None, + cancel_event: Optional[threading.Event] = None, + ) -> Path: + """Send a typed artifact envelope to the isolated runner.""" + from services.artifact_input import TypedArtifactInput, revalidate_artifact_input + from services.generator_registry import WORKSPACE_DIR + + validated = revalidate_artifact_input( + WORKSPACE_DIR, TypedArtifactInput(kind=input_kind, path=artifact_path) + ) + return self._generate_request( + {"input": {"kind": validated.kind, "path": str(validated.path)}}, + params, progress_cb, cancel_event, + ) + + def _generate_request( + self, + input_payload: dict, + params: dict, + progress_cb: Optional[Callable[[int, str], None]], + cancel_event: Optional[threading.Event], + ) -> Path: + req_id = str(uuid.uuid4()) + self._send({ + "action": "generate", "id": req_id, "model_id": self.MODEL_ID, + **input_payload, "params": params, + "outputs_dir": str(self.outputs_dir) if self.outputs_dir else None, + }) + return self._receive_generation(req_id, progress_cb, cancel_event) + def params_schema(self) -> list: return self._params_schema diff --git a/api/services/generator_registry.py b/api/services/generator_registry.py index 348a42cb..de1750ca 100644 --- a/api/services/generator_registry.py +++ b/api/services/generator_registry.py @@ -452,6 +452,25 @@ def _discover_extensions( node for node in raw_nodes if isinstance(node, dict) and node.get("id") ] + allowed_io = {"image", "text", "mesh", "audio", "scene"} + for node in nodes: + declared_inputs = node.get("inputs") or [node.get("input", "image")] + if (not isinstance(declared_inputs, list) + or any(value not in allowed_io for value in declared_inputs)): + raise ValueError( + f'model node "{node.get("id", "unknown")}" has an unsupported input type' + ) + if node.get("output", "mesh") not in allowed_io: + raise ValueError( + f'model node "{node.get("id", "unknown")}" has an unsupported output type' + ) + if "scene" in declared_inputs and ( + "inputs" in node or node.get("input", "image") != "scene" + ): + raise ValueError( + f'model node "{node.get("id", "unknown")}" must declare scene ' + 'as its single input field' + ) # Markers left while setup or runtime registration is unfinished: # the folder is not ready to be loaded. The readable manifest lets @@ -540,6 +559,7 @@ def _discover_extensions( "hf_include_prefixes": node.get("hf_include_prefixes", []), "params_schema": node.get("params_schema", manifest.get("params_schema", [])), "input": node.get("input", "image"), + "inputs": node.get("inputs"), "output": node.get("output", "mesh"), } if model_sources is not None: @@ -612,6 +632,9 @@ def initialize( ) # Subprocess mode: wrap in ExtensionProcess gen = ExtensionProcess(ext_dir, manifest) + # Pin the subprocess envelope to the exact registry key; + # multi-node workers must never fall back to an extension ID. + gen.MODEL_ID = model_id gen.model_dir = MODELS_DIR / model_id gen.outputs_dir = WORKSPACE_DIR else: @@ -704,10 +727,13 @@ def _assert_not_quarantined(model_id: str) -> None: def get_active(self) -> BaseGenerator: """Returns the active generator. Downloads and loads if necessary.""" - self._assert_not_quarantined(self._active_id) - gen = self._generators[self._active_id] - downloaded = self._is_downloaded(self._active_id, gen) - if "model_sources" in self._manifests[self._active_id] and not downloaded: + return self.get_ready_generator(self._active_id) + + def get_ready_generator(self, model_id: str) -> BaseGenerator: + """Load and return exactly ``model_id`` without consulting active state.""" + gen = self.get_generator(model_id) + downloaded = self._is_downloaded(model_id, gen) + if "model_sources" in self._manifests[model_id] and not downloaded: raise RuntimeError( "Model sources are incomplete. Download this node's weights " "from the Modly Models page before generation." @@ -724,6 +750,16 @@ def get_active(self) -> BaseGenerator: gen.load() return gen + def model_status(self, model_id: str) -> dict: + gen = self.get_generator(model_id) + manifest = self._manifests[model_id] + return { + "id": model_id, + "name": manifest.get("name", gen.DISPLAY_NAME), + "downloaded": self._is_downloaded(model_id, gen), + "loaded": gen.is_loaded(), + } + def get_generator(self, model_id: str) -> BaseGenerator: self._assert_not_quarantined(model_id) if model_id not in self._generators: @@ -765,14 +801,7 @@ def _is_downloaded(self, model_id: str, gen: BaseGenerator) -> bool: return gen.is_downloaded() def active_status(self) -> dict: - gen = self._generators[self._active_id] - manifest = self._manifests[self._active_id] - return { - "id": self._active_id, - "name": manifest.get("name", gen.DISPLAY_NAME), - "downloaded": self._is_downloaded(self._active_id, gen), - "loaded": gen.is_loaded(), - } + return self.model_status(self._active_id) def all_status(self) -> list: result = [] diff --git a/api/services/generators/base.py b/api/services/generators/base.py index fd62ceef..0b5169a2 100644 --- a/api/services/generators/base.py +++ b/api/services/generators/base.py @@ -143,7 +143,6 @@ def is_loaded(self) -> bool: # Inference # ------------------------------------------------------------------ # - @abstractmethod def generate( self, image_bytes: bytes, @@ -157,7 +156,25 @@ def generate( progress_cb(percent: int, step_label: str) cancel_event: set this to interrupt generation between steps. """ - ... + raise NotImplementedError( + f"{type(self).__name__} does not implement legacy image generation" + ) + + def generate_artifact( + self, + input_kind: str, + artifact_path: Path, + params: dict, + progress_cb: Optional[Callable[[int, str], None]] = None, + cancel_event: Optional[threading.Event] = None, + ) -> Path: + """Generate from a validated typed artifact. + + New extensions should override this method. The default delegates to + ``generate`` with the canonical path so scene-capable extensions built + against the pre-release contract remain compatible. + """ + return self.generate(artifact_path, params, progress_cb, cancel_event) # type: ignore[arg-type] def _check_cancelled(self, cancel_event: Optional[threading.Event]) -> None: """Raises GenerationCancelled if cancel_event is set.""" diff --git a/api/services/scene_input.py b/api/services/scene_input.py new file mode 100644 index 00000000..c5395050 --- /dev/null +++ b/api/services/scene_input.py @@ -0,0 +1,138 @@ +"""Validation shared by the API and isolated model runner for scene inputs.""" +import json +import math +import re +import stat +from pathlib import Path, PurePosixPath, PureWindowsPath + +SCHEMA = "modly.scene-manifest.v1" +MANIFEST = "scene-manifest.json" +MAX_MANIFEST_BYTES = 1024 * 1024 +MAX_REFERENCES = 4096 +MAX_REFERENCED_FILE_BYTES = 16 * 1024**3 +MAX_REFERENCED_TOTAL_BYTES = 64 * 1024**3 + + +def _relative(value: str, *, allow_dot: bool = False) -> Path: + if not isinstance(value, str) or not value or value != value.strip() or "\x00" in value: + raise ValueError("Scene path must be a nonempty workspace-relative path") + value = value.replace("\\", "/") + if value == "." and allow_dot: + return Path(".") + if (PurePosixPath(value).is_absolute() or PureWindowsPath(value).is_absolute() + or re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", value) + or re.search(r"%(?:25|2e|2f|5c|00)", value, re.I) + or re.search(r"%(?![0-9a-f]{2})", value, re.I) + or any(part in ("", ".", "..") for part in value.split("/"))): + raise ValueError("Scene path must be a safe workspace-relative path") + return Path(*value.split("/")) + + +def _inside(path: Path, root: Path) -> Path: + try: + relative = path.relative_to(root) + except ValueError as exc: + raise ValueError("Scene path escapes its allowed root") from exc + current = root + for part in relative.parts: + current = current / part + try: + info = current.lstat() + except OSError as exc: + raise ValueError("Scene referenced path is missing or unreadable") from exc + is_reparse = bool(getattr(info, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if current.is_symlink() or is_reparse: + raise ValueError("Scene referenced path must not use symlinks or reparse points") + try: + resolved = path.resolve(strict=True) + except OSError as exc: + raise ValueError("Scene referenced path is missing or unreadable") from exc + if not resolved.is_relative_to(root): + raise ValueError("Scene path escapes its allowed root") + return resolved + + +def validate_scene_input(workspace: Path, scene_path: str) -> Path: + """Return a canonical manifest Path, rejecting traversal and symlink escapes. + + V1 sceneRoot is workspace-relative, except '.' denotes the manifest directory. + An asset's path and preview references are sceneRoot-relative; workspacePath + is always workspace-relative. Existence never determines which base applies. + """ + root = workspace.resolve(strict=True) + relative = _relative(scene_path) + requested = root / relative + if requested.name != MANIFEST: + if requested.suffix.lower() == ".json": + raise ValueError(f"Scene input accepts a directory or {MANIFEST}") + requested = requested / MANIFEST + try: + manifest_path = _inside(requested, root) + if not manifest_path.is_file(): + raise ValueError("Scene manifest is not a file") + if manifest_path.stat().st_size > MAX_MANIFEST_BYTES: + raise ValueError("Scene manifest exceeds the 1 MiB limit") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ValueError("Scene manifest is missing or invalid JSON") from exc + if not isinstance(manifest, dict) or manifest.get("schema") != SCHEMA: + raise ValueError(f"Scene manifest schema must be {SCHEMA}") + scene_root = _relative(manifest.get("sceneRoot"), allow_dot=True) + # v1 sceneRoot is workspace-relative; only '.' means the manifest's directory. + # Never select a base according to which unrelated path happens to exist. + scene_root_candidate = manifest_path.parent if scene_root == Path(".") else root / scene_root + root_path = _inside(scene_root_candidate, root) + if not root_path.is_dir(): + raise ValueError("Scene root is not a directory") + if not isinstance(manifest.get("assets"), list): + raise ValueError("Scene manifest assets must be an array") + if len(manifest["assets"]) > MAX_REFERENCES: + raise ValueError("Scene manifest contains too many asset references") + total_bytes = 0 + for asset in manifest["assets"]: + # Assets may be opaque extension metadata. Validate only references the + # host understands; never silently resolve a supplied path outside workspace. + if isinstance(asset, dict): + for field, base in (("workspacePath", root), ("path", root_path)): + if field not in asset: + continue + target = _inside(base / _relative(asset[field]), root) + if not target.is_file(): + raise ValueError(f"Scene asset {field} is not a file") + size = target.stat().st_size + if size > MAX_REFERENCED_FILE_BYTES: + raise ValueError("Scene referenced file exceeds the size limit") + total_bytes += size + if total_bytes > MAX_REFERENCED_TOTAL_BYTES: + raise ValueError("Scene referenced files exceed the total size limit") + preview = manifest.get("preview", {}) + if not isinstance(preview, dict): + raise ValueError("Scene preview must be an object") + for name in ("image", "video"): + if name in preview: + relative_preview = _relative(preview[name]) + target = _inside(root_path / relative_preview, root) + if not target.is_file(): + raise ValueError(f"Scene preview {name} is not a file") + view = manifest.get("initialView") + if view is not None: + if not isinstance(view, dict): + raise ValueError("Scene initialView must be an object") + for field in ("position", "target", "up"): + triple = view.get(field) + if triple is None and field == "up": + continue + if (not isinstance(triple, list) or len(triple) != 3 + or any(not isinstance(n, (int, float)) or isinstance(n, bool) + or not math.isfinite(n) for n in triple)): + raise ValueError(f"Scene initialView {field} must be a finite numeric triple") + if view["position"] == view["target"] or view.get("up") == [0, 0, 0]: + raise ValueError("Scene initialView has degenerate camera vectors") + return manifest_path + + +def revalidate_scene_manifest(workspace: Path, manifest_path: Path) -> Path: + """Recheck the file immediately before model invocation in the worker.""" + root = workspace.resolve(strict=True) + candidate = _inside(manifest_path, root) + return validate_scene_input(root, candidate.relative_to(root).as_posix()) diff --git a/api/tests/test_extension_process.py b/api/tests/test_extension_process.py index 348e293f..7db13e4a 100644 --- a/api/tests/test_extension_process.py +++ b/api/tests/test_extension_process.py @@ -2,6 +2,9 @@ import platform import queue import unittest +import json +import tempfile +from unittest.mock import patch from pathlib import Path from services.extension_process import ExtensionProcess, _venv_python @@ -12,6 +15,31 @@ def _make_proc() -> ExtensionProcess: class ExtensionProcessTests(unittest.TestCase): + def test_generation_envelope_pins_worker_model_id(self) -> None: + proc = _make_proc() + sent = [] + proc._send = sent.append + proc._receive_generation = lambda *args: Path("result.glb") + proc._generate_request({"image_b64": ""}, {}, None, None) + self.assertEqual(sent[0]["model_id"], "demo") + + def test_generate_artifact_sends_typed_scene_without_image_bytes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) / "workspace" + scene = workspace / "Workflows" / "room" + scene.mkdir(parents=True) + manifest = scene / "scene-manifest.json" + manifest.write_text(json.dumps({ + "schema": "modly.scene-manifest.v1", "sceneRoot": ".", "assets": [], + })) + proc = _make_proc() + calls = [] + proc._generate_request = lambda payload, params, progress, cancel: calls.append((payload, params)) or manifest + with patch("services.generator_registry.WORKSPACE_DIR", workspace): + result = proc.generate_artifact("scene", manifest, {"quality": "high"}) + self.assertEqual(result, manifest) + self.assertEqual(calls, [({"input": {"kind": "scene", "path": str(manifest.resolve())}}, {"quality": "high"})]) + def test_read_loop_writes_sentinel_to_own_queue_only(self) -> None: proc = _make_proc() diff --git a/api/tests/test_generation_router.py b/api/tests/test_generation_router.py index 20fdda94..5e03c241 100644 --- a/api/tests/test_generation_router.py +++ b/api/tests/test_generation_router.py @@ -49,6 +49,9 @@ def get_active(self) -> _FakeGenerator: def get_generator(self, model_id: str) -> _FakeGenerator: return self._gen + def get_manifest(self, model_id: str) -> dict: + return {"output": "mesh"} + def switch_model(self, model_id: str) -> None: pass diff --git a/api/tests/test_generator_registry.py b/api/tests/test_generator_registry.py index ff9d090c..a4814b9e 100644 --- a/api/tests/test_generator_registry.py +++ b/api/tests/test_generator_registry.py @@ -155,6 +155,53 @@ def test_legacy_generator_supports_eager_and_lazy_sibling_imports(self) -> None: self.registry.reload() self.assertNotIn(str(extension.resolve()), sys.path) + def test_scene_io_is_registered_but_capture_and_video_are_rejected(self) -> None: + for extension_id, input_kind in (("scene-io", "scene"), ("capture-io", "capture"), ("video-io", "video")): + extension = self._make_extension(extension_id) + manifest = { + "id": extension_id, "name": extension_id, "type": "model", + "generator_class": "TestGenerator", + "nodes": [{"id": "generate", "input": input_kind, "output": "scene"}], + } + (extension / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + (extension / "generator.py").write_text( + "from services.generators.base import BaseGenerator\n" + "class TestGenerator(BaseGenerator):\n" + " def load(self): self._model = object()\n" + " def generate(self, value, params, progress_cb=None, cancel_event=None): return self.outputs_dir\n", + encoding="utf-8", + ) + + self.registry.initialize() + self.assertEqual(self.registry.get_manifest("scene-io/generate")["input"], "scene") + self.assertIn("capture-io/generate", self.registry.load_errors()) + self.assertIn("video-io/generate", self.registry.load_errors()) + + def test_scene_input_rejects_multi_input_shapes_but_image_multi_can_output_scene(self) -> None: + cases = { + "scene-mixed": {"input": "scene", "inputs": ["scene", "text"], "output": "mesh"}, + "scene-array": {"input": "scene", "inputs": ["scene"], "output": "mesh"}, + "images-scene": {"input": "image", "inputs": ["image", "image"], "output": "scene"}, + } + for extension_id, node in cases.items(): + extension = self._make_extension(extension_id) + (extension / "manifest.json").write_text(json.dumps({ + "id": extension_id, "name": extension_id, "type": "model", + "generator_class": "TestGenerator", + "nodes": [{"id": "generate", **node}], + }), encoding="utf-8") + (extension / "generator.py").write_text( + "from services.generators.base import BaseGenerator\n" + "class TestGenerator(BaseGenerator):\n" + " def load(self): self._model = object()\n" + " def generate(self, value, params, progress_cb=None, cancel_event=None): return self.outputs_dir\n", + encoding="utf-8", + ) + self.registry.initialize() + self.assertIn("scene-mixed/generate", self.registry.load_errors()) + self.assertIn("scene-array/generate", self.registry.load_errors()) + self.assertIn("images-scene/generate", self.registry._generators) + def test_declared_sources_block_generation_even_when_generator_overrides_readiness(self) -> None: extension = self._make_extension("multi-source") manifest = { diff --git a/api/tests/test_runner.py b/api/tests/test_runner.py index 8fce3d31..a666f87a 100644 --- a/api/tests/test_runner.py +++ b/api/tests/test_runner.py @@ -5,6 +5,7 @@ import json import tempfile import importlib +from unittest.mock import patch from contextlib import redirect_stdout from pathlib import Path @@ -20,6 +21,30 @@ class RunnerTests(unittest.TestCase): + def test_decode_typed_scene_revalidates_worker_workspace_and_keeps_legacy_image(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) / "workspace" + scene = workspace / "Workflows" / "room" + scene.mkdir(parents=True) + manifest = scene / "scene-manifest.json" + manifest.write_text(json.dumps({ + "schema": "modly.scene-manifest.v1", "sceneRoot": ".", "assets": [], + })) + with patch.object(runner, "WORKSPACE_DIR", workspace): + typed = runner.decode_model_input({"input": {"kind": "scene", "path": str(manifest)}}) + self.assertEqual(typed.kind, "scene") + self.assertEqual(typed.path, manifest.resolve()) + self.assertEqual(runner.decode_model_input({"image_b64": "aW1hZ2U="}), b"image") + with self.assertRaises(ValueError): + runner.decode_model_input({"input": {"kind": "video", "path": str(manifest)}}) + + def test_runner_model_envelope_rejects_cross_node_dispatch(self) -> None: + manifest = {"id": "pixal3d"} + node = {"id": "worldsculpt"} + runner.validate_requested_model({"model_id": "pixal3d/worldsculpt"}, manifest, node) + with self.assertRaisesRegex(ValueError, "does not match"): + runner.validate_requested_model({"model_id": "pixal3d/generate"}, manifest, node) + def test_select_node_uses_model_dir_override(self) -> None: manifest = { "nodes": [ diff --git a/api/tests/test_scene_generation.py b/api/tests/test_scene_generation.py new file mode 100644 index 00000000..a804171a --- /dev/null +++ b/api/tests/test_scene_generation.py @@ -0,0 +1,115 @@ +import asyncio +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from fastapi import BackgroundTasks, HTTPException +from pydantic import ValidationError + +import routers.generation as generation +import services.generator_registry as registry +from schemas.generation import GenerateFromArtifactRequest + + +class _Registry: + def __init__(self): + self.switched = False + def get_generator(self, model_id): return object() + def get_manifest(self, model_id): return {"input": "scene"} + def switch_model(self, model_id): self.switched = True + + +class SceneGenerationTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.workspace = Path(self.tmp.name) / "workspace" + self.scene = self.workspace / "Workflows" / "room" + self.scene.mkdir(parents=True) + self.manifest = self.scene / "scene-manifest.json" + self.manifest.write_text(json.dumps({"schema": "modly.scene-manifest.v1", "sceneRoot": ".", "assets": []})) + self.registry = _Registry() + self.patches = [patch.object(generation, "generator_registry", self.registry), patch.object(registry, "WORKSPACE_DIR", self.workspace)] + for item in self.patches: item.start() + + def tearDown(self): + for item in reversed(self.patches): item.stop() + generation._jobs.clear(); generation._cancel_events.clear(); generation._cancelled.clear(); generation._completed_at.clear() + self.tmp.cleanup() + + def test_generic_route_queues_typed_scene_and_strips_reserved_params(self): + tasks = BackgroundTasks() + result = asyncio.run(generation.generate_from_artifact(GenerateFromArtifactRequest( + input_kind="scene", input_path="Workflows/room", model_id="demo/scene", + params={"artifact_path": "/etc/passwd", "input_kind": "image", "quality": "high"}, + ), tasks)) + queued = tasks.tasks[0] + self.assertEqual(queued.args[1].kind, "scene") + self.assertEqual(queued.args[1].path, self.manifest.resolve()) + self.assertEqual(queued.args[2]["scene_manifest_path"], str(self.manifest.resolve())) + self.assertNotIn("artifact_path", queued.args[2]) + self.assertNotIn("input_kind", queued.args[2]) + self.assertEqual(queued.args[5], "demo/scene") + self.assertEqual(result["job_id"], queued.args[0]) + + def test_generic_route_rejects_unsupported_kind_and_model_mismatch(self): + for kind in ("video", "capture", "image"): + with self.subTest(kind=kind), self.assertRaises(ValidationError): + GenerateFromArtifactRequest( + input_kind=kind, input_path="Workflows/room", model_id="demo/scene") + self.registry.get_manifest = lambda _model_id: {"input": "image"} + with self.assertRaises(HTTPException) as caught: + asyncio.run(generation.generate_from_artifact(GenerateFromArtifactRequest( + input_kind="scene", input_path="Workflows/room", model_id="demo/image"), BackgroundTasks())) + self.assertEqual(caught.exception.status_code, 400) + + def test_rejects_traversal_before_switch_or_queue(self): + with self.assertRaises(HTTPException): + asyncio.run(generation.generate_from_artifact(GenerateFromArtifactRequest( + input_kind="scene", input_path="../outside", model_id="demo/scene"), BackgroundTasks())) + self.assertFalse(self.registry.switched) + + def test_queued_scene_job_is_pinned_to_requested_model(self): + calls = [] + + class Generator: + outputs_dir = None + def is_loaded(self): return True + def generate_artifact(self, kind, path, params, progress_cb, cancel_event=None): + calls.append(("model-a", kind, path)) + output = Path(self.outputs_dir) / "result.glb" + output.write_bytes(b"glb") + return output + + generator = Generator() + registry_stub = type("Registry", (), { + "get_ready_generator": lambda self, model_id: generator if model_id == "demo/a" else (_ for _ in ()).throw(ValueError(f"Unknown model ID: {model_id}")), + "get_active": lambda self: (_ for _ in ()).throw(AssertionError("mutable active model must not be used")), + })() + job_id = "pinned-scene" + generation._jobs[job_id] = generation.JobStatus(job_id=job_id, status="pending", progress=0) + generation._cancel_events[job_id] = __import__("threading").Event() + with patch.object(generation, "generator_registry", registry_stub): + asyncio.run(generation._run_generation( + job_id, generation.TypedArtifactInput("scene", self.manifest.resolve()), {}, + "Workflows", "mesh", "demo/a", + )) + self.assertEqual(calls[0][0], "model-a") + self.assertEqual(generation._jobs[job_id].status, "done") + + def test_missing_pinned_model_fails_actionably(self): + registry_stub = type("Registry", (), { + "get_ready_generator": lambda self, model_id: (_ for _ in ()).throw(ValueError(f"Unknown model ID: {model_id}")), + "get_active": lambda self: (_ for _ in ()).throw(AssertionError("must not use active model")), + })() + job_id = "missing-scene" + generation._jobs[job_id] = generation.JobStatus(job_id=job_id, status="pending", progress=0) + generation._cancel_events[job_id] = __import__("threading").Event() + with patch.object(generation, "generator_registry", registry_stub): + asyncio.run(generation._run_generation( + job_id, generation.TypedArtifactInput("scene", self.manifest.resolve()), {}, + "Workflows", "mesh", "demo/missing", + )) + self.assertEqual(generation._jobs[job_id].status, "error") + self.assertIn("Unknown model ID: demo/missing", generation._jobs[job_id].error) diff --git a/api/tests/test_scene_input.py b/api/tests/test_scene_input.py new file mode 100644 index 00000000..43bd9946 --- /dev/null +++ b/api/tests/test_scene_input.py @@ -0,0 +1,67 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from services.scene_input import validate_scene_input, revalidate_scene_manifest + + +class SceneInputTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.workspace = Path(self.tmp.name) / "workspace" + self.scene = self.workspace / "Workflows" / "room" + self.scene.mkdir(parents=True) + (self.scene / "model.glb").write_bytes(b"mesh") + self.manifest = self.scene / "scene-manifest.json" + self.manifest.write_text(json.dumps({ + "schema": "modly.scene-manifest.v1", + "sceneRoot": ".", + "assets": [{"path": "model.glb"}], + })) + + def tearDown(self): + self.tmp.cleanup() + + def test_directory_and_manifest_are_canonical_paths(self): + expected = self.manifest.resolve() + self.assertEqual(validate_scene_input(self.workspace, "Workflows/room"), expected) + self.assertEqual(validate_scene_input(self.workspace, "Workflows/room/scene-manifest.json"), expected) + self.assertEqual(revalidate_scene_manifest(self.workspace, expected), expected) + + def test_rejects_traversal_absolute_encoded_and_non_manifest_json(self): + for path in ("../outside", "/etc/passwd", "C:/outside", "Workflows/room/../room", "Workflows/room/other.json", "Workflows/%2e%2e"): + with self.subTest(path=path), self.assertRaises(ValueError): + validate_scene_input(self.workspace, path) + + def test_rejects_symlinks_missing_assets_and_oversized_manifest(self): + outside = Path(self.tmp.name) / "outside" + outside.mkdir() + (outside / "scene-manifest.json").write_text(self.manifest.read_text()) + (self.workspace / "Workflows" / "link").symlink_to(outside, target_is_directory=True) + with self.assertRaises(ValueError): + validate_scene_input(self.workspace, "Workflows/link") + + data = json.loads(self.manifest.read_text()) + data["assets"] = [{"path": "missing.glb"}] + self.manifest.write_text(json.dumps(data)) + with self.assertRaises(ValueError): + validate_scene_input(self.workspace, "Workflows/room") + + self.manifest.write_text(" " * (1024 * 1024 + 1)) + with self.assertRaises(ValueError): + validate_scene_input(self.workspace, "Workflows/room") + + def test_rejects_malformed_manifest_and_asset_paths(self): + original = json.loads(self.manifest.read_text()) + for patch in ( + {"schema": "wrong"}, + {"assets": "bad"}, + {"assets": [{"path": "../escape.glb"}]}, + {"preview": {"image": None}}, + {"initialView": {"position": [0, 0, 0], "target": [0, 0, 0]}}, + ): + with self.subTest(patch=patch): + self.manifest.write_text(json.dumps({**original, **patch})) + with self.assertRaises(ValueError): + validate_scene_input(self.workspace, "Workflows/room") diff --git a/electron/main/artifact-registry-service.test.ts b/electron/main/artifact-registry-service.test.ts index 882120d4..276e90a1 100644 --- a/electron/main/artifact-registry-service.test.ts +++ b/electron/main/artifact-registry-service.test.ts @@ -64,6 +64,18 @@ test('lists Workflows and Exports assets while skipping hidden, cache, and inter assert.equal(result.success && result.entries.find((entry) => entry.workspacePath.endsWith('exported.ply'))?.openable, false) })) +test('registers a generated scene directory through its canonical manifest artifact', () => withWorkspace(async (workspaceDir) => { + await mkdir(path.join(workspaceDir, 'Workflows/world'), { recursive: true }) + await writeFile(path.join(workspaceDir, 'Workflows/world/scene-manifest.json'), JSON.stringify({ + schema: 'modly.scene-manifest.v1', sceneRoot: '.', assets: [], + })) + const result = await listWorkspaceAssetLibrary({ workspaceDir }) + assert.equal(result.success, true) + const scene = result.success && result.entries.find((entry) => entry.workspacePath === 'Workflows/world/scene-manifest.json') + assert.equal(scene && scene.capability, 'scene-manifest') + assert.equal(scene && scene.state, 'ready') +})) + test('reads and opens only safe GLB/GLTF workspace assets', () => withWorkspace(async (workspaceDir) => { await mkdir(path.join(workspaceDir, 'Workflows/checkpoints'), { recursive: true }) await mkdir(path.join(workspaceDir, 'Exports'), { recursive: true }) diff --git a/electron/main/artifact-registry-service.ts b/electron/main/artifact-registry-service.ts index 3c26de73..2acb657f 100644 --- a/electron/main/artifact-registry-service.ts +++ b/electron/main/artifact-registry-service.ts @@ -124,7 +124,7 @@ export function classifyAssetLibraryCandidate(candidate: AssetLibraryClassificat if (candidate.workspacePath.endsWith('.world.json')) { return { capability: 'generated-world', state: 'ready', previewKind: 'text', openable: false, nonOpenableReason: 'Generated worlds are list-only in this release.' } } - if (candidate.workspacePath.endsWith('.scene.json')) { + if (candidate.workspacePath.endsWith('.scene.json') || candidate.workspacePath.endsWith('/scene-manifest.json')) { return { capability: 'scene-manifest', state: 'ready', previewKind: 'text', openable: false, nonOpenableReason: 'Scene manifests are list-only in this release.' } } if (INTRINSIC_MOTION_EXTENSIONS.has(extension)) { @@ -153,6 +153,7 @@ function objectField(value: unknown): Record | undefined { function manifestCapabilityFor(workspacePath: string): 'generated-world' | 'scene-manifest' | undefined { if (workspacePath.endsWith('.world.json')) return 'generated-world' if (workspacePath.endsWith('.scene.json')) return 'scene-manifest' + if (workspacePath.endsWith('/scene-manifest.json')) return 'scene-manifest' return undefined } diff --git a/electron/main/extension-install-utils.test.mjs b/electron/main/extension-install-utils.test.mjs index 84139f9a..4dcf42da 100644 --- a/electron/main/extension-install-utils.test.mjs +++ b/electron/main/extension-install-utils.test.mjs @@ -85,6 +85,50 @@ test('validateInstallManifest accepts multi-source nodes and preserves legacy sh }, { hasEntryFile: () => false, hasGeneratorFile: () => true }, 'repository')) }) +test('validateInstallManifest accepts scene IO and rejects undeclared future artifact kinds', () => { + const mod = loadModule() + const files = { hasEntryFile: () => false, hasGeneratorFile: () => true } + assert.doesNotThrow(() => mod.validateInstallManifest({ + id: 'scene-model', generator_class: 'Generator', + nodes: [{ id: 'normalize', input: 'scene', output: 'scene' }], + }, files, 'repository')) + for (const input of ['capture', 'video']) { + assert.throws(() => mod.validateInstallManifest({ + id: 'future-model', generator_class: 'Generator', + nodes: [{ id: 'future', input, output: 'scene' }], + }, files, 'repository'), /supported artifact type/) + } +}) + +test('scene is model-only, single-input, while image-multi to scene stays valid', () => { + const mod = loadModule() + const modelFiles = { hasEntryFile: () => false, hasGeneratorFile: () => true } + const processFiles = { hasEntryFile: () => true, hasGeneratorFile: () => false } + for (const node of [ + { id: 'mixed', input: 'scene', inputs: ['scene', 'text'], output: 'mesh' }, + { id: 'duplicate', input: 'scene', inputs: ['scene', 'scene'], output: 'mesh' }, + { id: 'hidden', input: 'image', inputs: ['scene'], output: 'mesh' }, + ]) { + assert.throws(() => mod.validateInstallManifest({ id: 'bad', generator_class: 'Generator', nodes: [node] }, modelFiles, 'repository'), /scene.*single|single.*scene/i) + } + for (const node of [ + { id: 'input', input: 'scene', output: 'mesh' }, + { id: 'output', input: 'image', output: 'scene' }, + ]) { + assert.throws(() => mod.validateInstallManifest({ id: 'proc', type: 'process', entry: 'processor.js', nodes: [node] }, processFiles, 'repository'), /scene.*model|model.*scene/i) + } + assert.doesNotThrow(() => mod.validateInstallManifest({ + id: 'images-to-scene', generator_class: 'Generator', + nodes: [{ id: 'prepare', input: 'image', inputs: ['image', 'image'], output: 'scene' }], + }, modelFiles, 'repository')) + for (const output of ['scene', 'mesh']) { + assert.doesNotThrow(() => mod.validateInstallManifest({ + id: `scene-to-${output}`, generator_class: 'Generator', + nodes: [{ id: 'generate', input: 'scene', output }], + }, modelFiles, 'repository')) + } +}) + test('validateInstallManifest rejects malformed or process model_sources', () => { const mod = loadModule() const source = { diff --git a/electron/main/extension-install-utils.ts b/electron/main/extension-install-utils.ts index 05b965b0..41e9798c 100644 --- a/electron/main/extension-install-utils.ts +++ b/electron/main/extension-install-utils.ts @@ -10,7 +10,13 @@ export interface InstallManifest { entry?: string generator_class?: string model_sources?: unknown - nodes?: Array<{ id?: string; model_sources?: unknown } & ModelSourceNode> + nodes?: Array<{ + id?: string + input?: unknown + inputs?: unknown + output?: unknown + model_sources?: unknown + } & ModelSourceNode> } export interface ValidatedInstallManifest { @@ -27,6 +33,21 @@ export interface ExtensionReloadPayload { errors: Record } +export function assertSupportedSceneNodeShape( + kind: 'model' | 'process', + node: { id?: string; input?: unknown; inputs?: unknown; output?: unknown }, + declaredInputs: unknown[], + output: unknown, +): void { + const usesSceneInput = declaredInputs.includes('scene') + if (kind === 'process' && (usesSceneInput || output === 'scene')) { + throw new Error('manifest.json: scene input and output are supported only for model nodes') + } + if (kind === 'model' && usesSceneInput && (node.inputs !== undefined || node.input !== 'scene')) { + throw new Error(`manifest.json: ${node.id ?? 'node'} must declare scene as its single input field`) + } +} + export type IncompleteInstallRecoveryAction = | 'none' | 'remove-incomplete' @@ -45,11 +66,22 @@ export function validateInstallManifest( const isProcess = manifest.type === 'process' const entryFile = manifest.entry ?? 'processor.js' const nodes = Array.isArray(manifest.nodes) ? manifest.nodes.filter((node) => node?.id) : [] + const allowedIo = new Set(['image', 'text', 'mesh', 'audio', 'scene']) if (manifest.model_sources !== undefined) { throw new Error('manifest.json: model_sources must be declared on a model node') } for (const node of Array.isArray(manifest.nodes) ? manifest.nodes : []) { + const declaredInputs = node.inputs === undefined ? [node.input ?? 'image'] : node.inputs + if (!Array.isArray(declaredInputs) || declaredInputs.length === 0 + || declaredInputs.some((value) => typeof value !== 'string' || !allowedIo.has(value))) { + throw new Error(`manifest.json: ${node.id ?? 'node'}.input must use a supported artifact type`) + } + const output = node.output ?? 'mesh' + if (typeof output !== 'string' || !allowedIo.has(output)) { + throw new Error(`manifest.json: ${node.id ?? 'node'}.output must use a supported artifact type`) + } + assertSupportedSceneNodeShape(isProcess ? 'process' : 'model', node, declaredInputs, output) if (node.model_sources === undefined) continue if (isProcess) { throw new Error('manifest.json: model_sources is supported only for model nodes') diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 4fa24788..9e5131de 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -49,6 +49,7 @@ import { validateExtensionReloadPayload, validateExistingExtensionReplacement, validateInstallManifest, + assertSupportedSceneNodeShape, } from './extension-install-utils' import { beginExtensionRegistrationTransaction, @@ -844,10 +845,10 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe nodes?: { id: string name?: string - input?: 'mesh' | 'image' | 'text' | 'audio' - inputs?: ('mesh' | 'image' | 'text' | 'audio')[] + input?: 'mesh' | 'image' | 'text' | 'audio' | 'scene' + inputs?: ('mesh' | 'image' | 'text' | 'audio' | 'scene')[] input_labels?: string[] - output?: 'mesh' | 'image' | 'text' | 'audio' + output?: 'mesh' | 'image' | 'text' | 'audio' | 'scene' params_schema?: unknown[] param_defaults?: Record hf_repo?: string @@ -873,7 +874,15 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe if (parsed.model_sources !== undefined) { throw new Error('manifest.json: model_sources must be declared on a model node') } + const allowedIo = new Set(['image', 'text', 'mesh', 'audio', 'scene']) const nodes = (parsed.nodes ?? []).map(n => { + const declaredInputs = n.inputs ?? [n.input ?? 'image'] + for (const input of declaredInputs) { + if (!allowedIo.has(input)) throw new Error(`manifest.json: unsupported node input type "${input}"`) + } + const output = n.output ?? 'mesh' + if (!allowedIo.has(output)) throw new Error(`manifest.json: unsupported node output type "${output}"`) + assertSupportedSceneNodeShape(parsed.type === 'process' ? 'process' : 'model', n, declaredInputs, output) if (parsed.type === 'process' && n.model_sources !== undefined) { throw new Error('manifest.json: model_sources is supported only for model nodes') } diff --git a/src/areas/workflows/WorkflowsPage.tsx b/src/areas/workflows/WorkflowsPage.tsx index d15c06ff..32452774 100644 --- a/src/areas/workflows/WorkflowsPage.tsx +++ b/src/areas/workflows/WorkflowsPage.tsx @@ -27,6 +27,7 @@ import ImageNode from './nodes/ImageNode' import TextNode from './nodes/TextNode' import AddToSceneNode from './nodes/AddToSceneNode' import Load3DMeshNode from './nodes/Load3DMeshNode' +import LoadSceneNode from './nodes/LoadSceneNode' import PreviewImageNode from './nodes/PreviewImageNode' import ImagePreviewNode from './nodes/ImagePreviewNode' import WaitNode from './nodes/WaitNode' @@ -38,7 +39,7 @@ import WorkflowEdge from './nodes/WorkflowEdge' const DRAG_KEY = 'modly/extension-id' const DRAG_NODE_KEY = 'modly/node-type' -const NODE_TYPES = { extensionNode: ExtensionNode, imageNode: ImageNode, textNode: TextNode, outputNode: AddToSceneNode, meshNode: Load3DMeshNode, previewNode: PreviewImageNode, imagePreviewNode: ImagePreviewNode, waitNode: WaitNode, whileNode: WhileNode, forEachNode: ForEachNode } +const NODE_TYPES = { extensionNode: ExtensionNode, imageNode: ImageNode, textNode: TextNode, outputNode: AddToSceneNode, meshNode: Load3DMeshNode, sceneNode: LoadSceneNode, previewNode: PreviewImageNode, imagePreviewNode: ImagePreviewNode, waitNode: WaitNode, whileNode: WhileNode, forEachNode: ForEachNode } // Loop-container node types: resizable frames whose children form a loop body. // (For Each iterators are plain source nodes, not containers.) @@ -62,14 +63,15 @@ function findWhileContainerAt(nodes: Node[], pos: { x: number; y: number }): Nod // ─── IO badge ───────────────────────────────────────────────────────────────── -const IO_STYLES: Record<'image' | 'text' | 'mesh' | 'audio', string> = { +const IO_STYLES: Record<'image' | 'text' | 'mesh' | 'audio' | 'scene', string> = { audio: 'bg-emerald-500/15 text-emerald-400 border-emerald-500/25', image: 'bg-sky-500/15 text-sky-400 border-sky-500/25', mesh: 'bg-violet-500/15 text-violet-400 border-violet-500/25', text: 'bg-amber-500/15 text-amber-400 border-amber-500/25', + scene: 'bg-emerald-500/15 text-emerald-400 border-emerald-500/25', } -function IoBadge({ type }: { type: 'image' | 'text' | 'mesh' | 'audio' }) { +function IoBadge({ type }: { type: 'image' | 'text' | 'mesh' | 'audio' | 'scene' }) { return ( {type} @@ -99,6 +101,7 @@ const PANEL_BUILTIN_NODES = [ { type: 'imageNode', label: 'Image', color: '#38bdf8', icon: <> }, { type: 'textNode', label: 'Text', color: '#fbbf24', icon: <> }, { type: 'meshNode', label: 'Load 3D Mesh', color: '#a78bfa', icon: <> }, + { type: 'sceneNode', label: 'Load Scene', color: '#34d399', icon: <> }, { type: 'outputNode', label: 'Add to Scene', color: '#a78bfa', icon: <> }, { type: 'previewNode', label: 'Preview Views', color: '#38bdf8', icon: <> }, { type: 'imagePreviewNode', label: 'Preview Image', color: '#38bdf8', icon: <> }, @@ -346,6 +349,7 @@ const BUILTIN_NODES = [ { type: 'imageNode', label: 'Image', color: '#38bdf8', description: 'Image input' }, { type: 'textNode', label: 'Text', color: '#fbbf24', description: 'Text input' }, { type: 'meshNode', label: 'Load 3D Mesh', color: '#a78bfa', description: 'Load a 3D mesh file or use current model' }, + { type: 'sceneNode', label: 'Load Scene', color: '#34d399', description: 'Load and validate a workspace scene directory' }, { type: 'outputNode', label: 'Add to Scene', color: '#a78bfa', description: 'Output node — adds the mesh to the 3D scene' }, { type: 'previewNode', label: 'Preview Views', color: '#38bdf8', description: 'Displays multi-view image outputs in a 2×3 grid' }, { type: 'imagePreviewNode', label: 'Preview Image', color: '#38bdf8', description: 'Displays a single image output in the workflow' }, @@ -703,6 +707,7 @@ function getNodeOutputType(node: Node | undefined, allExts: WorkflowExtension[]) if (!node) return undefined if (node.type === 'imageNode') return 'image' if (node.type === 'meshNode') return 'mesh' + if (node.type === 'sceneNode') return 'scene' if (node.type === 'textNode') return 'text' if (node.type === 'imagePreviewNode') return 'image' return allExts.find((e) => e.id === (node.data as WFNodeData)?.extensionId)?.output @@ -1374,6 +1379,7 @@ const MINI_NODE_TINTS: Record = { imageNode: { fill: 'rgba(52,211,153,0.22)', stroke: '#34d399' }, textNode: { fill: 'rgba(52,211,153,0.22)', stroke: '#34d399' }, meshNode: { fill: 'rgba(52,211,153,0.22)', stroke: '#34d399' }, + sceneNode: { fill: 'rgba(52,211,153,0.22)', stroke: '#34d399' }, extensionNode: { fill: 'rgba(167,139,250,0.24)', stroke: '#a78bfa' }, outputNode: { fill: 'rgba(56,189,248,0.22)', stroke: '#38bdf8' }, previewNode: { fill: 'rgba(56,189,248,0.22)', stroke: '#38bdf8' }, diff --git a/src/areas/workflows/mockExtensions.ts b/src/areas/workflows/mockExtensions.ts index 2bbc8cc6..be727fea 100644 --- a/src/areas/workflows/mockExtensions.ts +++ b/src/areas/workflows/mockExtensions.ts @@ -10,10 +10,10 @@ export interface WorkflowExtension { nodeId: string // "node_id" name: string description: string - input: 'image' | 'text' | 'mesh' | 'audio' - inputs?: ('image' | 'text' | 'mesh' | 'audio')[] // multi-input; overrides input when set + input: 'image' | 'text' | 'mesh' | 'audio' | 'scene' + inputs?: ('image' | 'text' | 'mesh' | 'audio' | 'scene')[] // multi-input; overrides input when set inputLabels?: string[] // display labels per input slot - output: 'image' | 'text' | 'mesh' | 'audio' + output: 'image' | 'text' | 'mesh' | 'audio' | 'scene' params: ParamSchema[] builtin: boolean type: 'model' | 'process' diff --git a/src/areas/workflows/nodes/ExtensionNode.tsx b/src/areas/workflows/nodes/ExtensionNode.tsx index abe2f9f6..4be11a14 100644 --- a/src/areas/workflows/nodes/ExtensionNode.tsx +++ b/src/areas/workflows/nodes/ExtensionNode.tsx @@ -16,6 +16,7 @@ const HANDLE_COLOR: Record = { image: '#38bdf8', mesh: '#a78bfa', text: '#fbbf24', + scene: '#34d399', } const TAG_CLS: Record = { @@ -23,6 +24,7 @@ const TAG_CLS: Record = { image: 'border-sky-500/30 bg-sky-500/10 text-sky-400', mesh: 'border-violet-500/30 bg-violet-500/10 text-violet-400', text: 'border-amber-500/30 bg-amber-500/10 text-amber-400', + scene: 'border-emerald-500/30 bg-emerald-500/10 text-emerald-400', } // ─── Param control ──────────────────────────────────────────────────────────── diff --git a/src/areas/workflows/nodes/LoadSceneNode.tsx b/src/areas/workflows/nodes/LoadSceneNode.tsx new file mode 100644 index 00000000..2ac7dded --- /dev/null +++ b/src/areas/workflows/nodes/LoadSceneNode.tsx @@ -0,0 +1,134 @@ +import { useCallback, useLayoutEffect, useRef, useState } from 'react' +import { Handle, Position, useReactFlow } from '@xyflow/react' +import type { WFNodeData } from '@shared/types/electron.d' + +import BaseNode from './BaseNode' +import { resolveSceneSourceManifest } from '../workflowSceneSource' + +const OUTPUT_COLOR = '#34d399' + +async function validateAndPersistScenePath(args: { + id: string + data: WFNodeData + nextPath: string + updateNodeData: ReturnType['updateNodeData'] +}): Promise { + const settings = await window.electron.settings.get() + const resolution = await resolveSceneSourceManifest({ + scenePath: args.nextPath, + workspaceDir: settings.workspaceDir, + readFileBase64: window.electron.fs.readFileBase64, + }) + + if (!resolution.ok) { + args.updateNodeData(args.id, { + params: { + ...args.data.params, + path: args.nextPath, + manifestPath: undefined, + sceneRoot: undefined, + error: resolution.error, + }, + }) + return + } + + args.updateNodeData(args.id, { + params: { + ...args.data.params, + path: resolution.inputWorkspacePath, + manifestPath: resolution.manifestWorkspacePath, + sceneRoot: resolution.sceneRoot, + sourceKind: resolution.sourceKind, + error: undefined, + }, + }) +} + +export default function LoadSceneNode({ id, data, selected }: { id: string; data: WFNodeData; selected?: boolean }) { + const { updateNodeData } = useReactFlow() + const ioRowRef = useRef(null) + const [handleTop, setHandleTop] = useState('50%') + + useLayoutEffect(() => { + if (ioRowRef.current) { + const center = ioRowRef.current.offsetTop + ioRowRef.current.offsetHeight / 2 + setHandleTop(`${center}px`) + } + }, []) + + const scenePath = typeof data.params.path === 'string' ? data.params.path : '' + const manifestPath = typeof data.params.manifestPath === 'string' ? data.params.manifestPath : undefined + const sceneRoot = typeof data.params.sceneRoot === 'string' ? data.params.sceneRoot : undefined + const error = typeof data.params.error === 'string' ? data.params.error : undefined + + const browseDirectory = useCallback(async () => { + const path = await window.electron.fs.selectDirectory() + if (!path) return + await validateAndPersistScenePath({ id, data, nextPath: path, updateNodeData }) + }, [id, data, updateNodeData]) + + const validatePath = useCallback(async () => { + if (!scenePath.trim()) return + await validateAndPersistScenePath({ id, data, nextPath: scenePath, updateNodeData }) + }, [id, data, scenePath, updateNodeData]) + + return ( + + + + + + + } + subheader={ +
+ scene +
+ } + handles={ + + } + > +
+ updateNodeData(id, { params: { ...data.params, path: event.target.value } })} + className="nodrag w-full rounded-lg border border-zinc-700 bg-zinc-800 px-2.5 py-2 text-[10px] text-zinc-200 placeholder-zinc-600 focus:outline-none focus:border-emerald-500/40" + /> +
+ + +
+ {manifestPath ? ( +
+
Manifest: {manifestPath}
+ {sceneRoot &&
sceneRoot: {sceneRoot}
} +
+ ) : ( +
+ Loads an existing workspace scene manifest for downstream scene nodes. +
+ )} + {error &&
{error}
} +
+ + ) +} diff --git a/src/areas/workflows/preflight.test.mjs b/src/areas/workflows/preflight.test.mjs index 75b09d71..3b4df8aa 100644 --- a/src/areas/workflows/preflight.test.mjs +++ b/src/areas/workflows/preflight.test.mjs @@ -131,3 +131,38 @@ test('multi-input extension requires every declared input type', () => { assert.ok(!issues.some((i) => i.key === 'proc:missing:image')) assert.ok(issues.some((i) => i.key === 'proc:missing:text')) }) + +test('scene input requires a validated Load Scene source and rejects image wiring', () => { + const { validateWorkflowPreflight } = loadModule() + const model = { id: 'model', type: 'extensionNode', position: { x: 0, y: 0 }, data: { extensionId: 'pack/process-node' } } + const scene = { id: 'scene', type: 'sceneNode', position: { x: 0, y: 0 }, data: { params: { manifestPath: 'Workflows/room/scene-manifest.json' } } } + for (const output of ['scene', 'mesh']) { + const extension = ext({ input: 'scene', output, type: 'model' }) + assert.deepEqual(validateWorkflowPreflight(wf([scene, model], [{ id: 'scene-edge', source: 'scene', target: 'model' }]), [extension]), []) + } + + const extension = ext({ input: 'scene', output: 'scene', type: 'model' }) + const issues = validateWorkflowPreflight(wf([imageNode(), model], [{ id: 'image-edge', source: 'img', target: 'model' }]), [extension]) + assert.ok(issues.some((issue) => issue.key === 'model:missing:scene')) + assert.ok(issues.some((issue) => issue.key === 'model:type:image-edge')) +}) + +test('Load Scene must be validated before a workflow can run', () => { + const { validateWorkflowPreflight } = loadModule() + const scene = { id: 'scene', type: 'sceneNode', position: { x: 0, y: 0 }, data: { params: { path: 'Workflows/room' } } } + const issues = validateWorkflowPreflight(wf([scene], []), []) + assert.equal(issues[0].key, 'scene:scene-invalid') +}) + +test('renderer fails closed for unsupported process and mixed scene node shapes', () => { + const { validateWorkflowPreflight } = loadModule() + const scene = { id: 'scene', type: 'sceneNode', position: { x: 0, y: 0 }, data: { params: { manifestPath: 'Workflows/room/scene-manifest.json' } } } + const target = { id: 'target', type: 'extensionNode', position: { x: 0, y: 0 }, data: { extensionId: 'pack/process-node' } } + for (const extension of [ + ext({ input: 'scene', output: 'mesh', type: 'process' }), + ext({ input: 'scene', inputs: ['scene', 'text'], output: 'mesh', type: 'model' }), + ]) { + const issues = validateWorkflowPreflight(wf([scene, target], [{ id: 'e', source: 'scene', target: 'target' }]), [extension]) + assert.ok(issues.some((issue) => issue.key === 'target:unsupported-scene-shape')) + } +}) diff --git a/src/areas/workflows/preflight.ts b/src/areas/workflows/preflight.ts index 3985855c..df7d35ca 100644 --- a/src/areas/workflows/preflight.ts +++ b/src/areas/workflows/preflight.ts @@ -2,7 +2,7 @@ import type { Workflow, WFNode } from '@shared/types/electron.d' import { getWorkflowExtension, type WorkflowExtension } from './mockExtensions' import { isPassthrough, isBranchConsumer, resolveDataSource, nearestUpstreamWaits } from './nodeBehaviors' -type DataType = 'image' | 'text' | 'mesh' | 'audio' +type DataType = 'image' | 'text' | 'mesh' | 'audio' | 'scene' export interface WorkflowPreflightIssue { key: string @@ -14,6 +14,7 @@ function nodeLabel(node: WFNode, allExtensions: WorkflowExtension[]): string { if (node.type === 'imageNode') return 'Image' if (node.type === 'textNode') return 'Text' if (node.type === 'meshNode') return 'Load 3D Mesh' + if (node.type === 'sceneNode') return 'Load Scene' if (node.type === 'outputNode') return 'Add to Scene' if (node.type === 'previewNode') return 'Preview Views' if (node.type === 'imagePreviewNode') return 'Preview Image' @@ -28,6 +29,7 @@ function nodeLabel(node: WFNode, allExtensions: WorkflowExtension[]): string { } function formatType(type: DataType): string { + if (type === 'scene') return 'scene' if (type === 'mesh') return 'mesh' if (type === 'image') return 'image' if (type === 'audio') return 'audio' @@ -44,6 +46,7 @@ function getNodeOutputType(node: WFNode, allExtensions: WorkflowExtension[]): Da if (node.type === 'imageNode') return 'image' if (node.type === 'textNode') return 'text' if (node.type === 'meshNode' || node.type === 'outputNode') return 'mesh' + if (node.type === 'sceneNode') return 'scene' if (node.type === 'previewNode') return 'image' if (node.type === 'imagePreviewNode') return 'image' if (node.type === 'forEachNode') { @@ -96,6 +99,13 @@ export function validateWorkflowPreflight( }) } + if (node.type === 'sceneNode' && !((node.data.params?.manifestPath as string | undefined)?.trim())) { + pushIssue(issues, { + key: `${node.id}:scene-invalid`, nodeId: node.id, + message: 'Load Scene needs a validated scene directory.', + }) + } + // A node fed by two different Wait branches can't be scheduled into a single // branch — it would run before either branch produces its mesh. if ( @@ -121,6 +131,20 @@ export function validateWorkflowPreflight( continue } + const usesSceneInput = ext.input === 'scene' || ext.inputs?.includes('scene') === true + const unsupportedSceneShape = + (ext.type === 'process' && (usesSceneInput || ext.output === 'scene')) + || (ext.type === 'model' && usesSceneInput + && (ext.inputs !== undefined || ext.input !== 'scene')) + if (unsupportedSceneShape) { + pushIssue(issues, { + key: `${node.id}:unsupported-scene-shape`, + nodeId: node.id, + message: `${ext.name} uses an unsupported scene input or output declaration.`, + }) + continue + } + const incomingEdges = workflow.edges.filter((edge) => edge.target === node.id) const requiredTypes = [...new Set((ext.inputs ?? [ext.input]) as DataType[])] diff --git a/src/areas/workflows/workflowRunStore.ts b/src/areas/workflows/workflowRunStore.ts index d6e6d8fc..fe025b52 100644 --- a/src/areas/workflows/workflowRunStore.ts +++ b/src/areas/workflows/workflowRunStore.ts @@ -305,6 +305,14 @@ async function executeExtensionNode( selectedImagePath, selectedImageData } = ctx const ext = getWorkflowExtension(node.data.extensionId ?? '', allExtensions) + if (ext) { + const usesSceneInput = ext.input === 'scene' || ext.inputs?.includes('scene') === true + if ((ext.type === 'process' && (usesSceneInput || ext.output === 'scene')) + || (ext.type === 'model' && usesSceneInput + && (ext.inputs !== undefined || ext.input !== 'scene'))) { + throw new Error(`${ext.name} uses an unsupported scene input or output declaration`) + } + } // Freshest params at the moment the node starts (so loop iterations / Retry pick // up edits made while paused, not the values captured at run start). const liveParams = _liveParams.current.get(node.id) ?? node.data.params ?? {} @@ -317,6 +325,7 @@ async function executeExtensionNode( let nodeInputPath: string | undefined let nodeInputText: string | undefined let nodeInputMeshPath: string | undefined + let nodeInputScenePath: string | undefined // Per-slot texts for multi-text-input nodes (e.g. positive/negative prompts). // Indexed by target handle: input-0 → texts[0], input-1 → texts[1]. const nodeInputTexts: (string | undefined)[] = [] @@ -349,6 +358,8 @@ async function executeExtensionNode( if (!fp) continue if (inputTypes[i] === 'mesh') { nodeInputMeshPath = fp + } else if (inputTypes[i] === 'scene') { + nodeInputScenePath = fp } else if (inputTypes[i] === 'image') { if (!nodeInputPath) nodeInputPath = fp else extraImagePaths.push(fp) @@ -359,21 +370,24 @@ async function executeExtensionNode( const src = resolveSource(edge.source) if (src?.filePath !== undefined) nodeInputPath = src.filePath if (src?.text !== undefined && src.text.trim().length > 0) nodeInputText = src.text + if (src?.outputType === 'scene') nodeInputScenePath = src.filePath } } const isModelNode = ext?.type === 'model' if (isModelNode) { + const isSceneInput = ext?.inputs ? ext.inputs.includes('scene') : ext?.input === 'scene' const isTextInput = ext?.inputs ? ext.inputs.every((i) => i === 'text') : ext?.input === 'text' - const activeImagePath = isTextInput ? undefined : (nodeInputPath ?? selectedImagePath) - if (!isTextInput && !selectedImageData && (!activeImagePath || activeImagePath.trim().length === 0)) { + if (isSceneInput && !nodeInputScenePath) throw new Error(`${ext?.name ?? 'Model'} needs an incoming scene connection`) + const activeImagePath = (isTextInput || isSceneInput) ? undefined : (nodeInputPath ?? selectedImagePath) + if (!isTextInput && !isSceneInput && !selectedImageData && (!activeImagePath || activeImagePath.trim().length === 0)) { throw new Error('No input image selected for model node') } let blob: Blob let fname: string - if (isTextInput || (selectedImageData && nodeInputPath === undefined)) { + if (isTextInput || isSceneInput || (selectedImageData && nodeInputPath === undefined)) { const base64 = selectedImageData && nodeInputPath === undefined ? selectedImageData : 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==' // 1x1 transparent PNG @@ -406,21 +420,30 @@ async function executeExtensionNode( ) const effectiveParams = { ...schemaDefaults, ...liveParams } - const fd = new FormData() - fd.append('image', blob, fname) - fd.append('model_id', node.data.extensionId ?? '') - fd.append('collection', 'Workflows') - fd.append('remesh', 'none') - fd.append('enable_texture', 'false') - fd.append('texture_resolution', '1024') - fd.append('params', JSON.stringify({ ...effectiveParams, ...extraParams })) - setRunState((s) => ({ ...s, blockProgress: 5, blockStep: 'Submitting to model…' })) - - const { data } = await client.post<{ job_id: string }>( - '/generate/from-image', fd, - { headers: { 'Content-Type': 'multipart/form-data' } }, - ) + let submission: { data: { job_id: string } } + if (isSceneInput) { + const normalized = nodeInputScenePath!.replace(/\\/g, '/') + const inputPath = normalized.startsWith(`${workspaceDir}/`) + ? normalized.slice(workspaceDir.length + 1) + : normalized.replace(/^\/workspace\//, '') + submission = await client.post('/generate/from-artifact', { + input_kind: 'scene', input_path: inputPath, + model_id: node.data.extensionId ?? '', collection: 'Workflows', + params: { ...effectiveParams, ...extraParams }, + }) + } else { + const fd = new FormData() + fd.append('image', blob, fname) + fd.append('model_id', node.data.extensionId ?? '') + fd.append('collection', 'Workflows') + fd.append('remesh', 'none') + fd.append('enable_texture', 'false') + fd.append('texture_resolution', '1024') + fd.append('params', JSON.stringify({ ...effectiveParams, ...extraParams })) + submission = await client.post('/generate/from-image', fd, { headers: { 'Content-Type': 'multipart/form-data' } }) + } + const { data } = submission _activeJobId.current = data.job_id while (true) { @@ -599,7 +622,7 @@ export const useWorkflowRunStore = create((set, get) => { if (!outputUrl) { for (const [, o] of ctx.nodeOutputs) { if (o.filePath) { - if (o.outputType === 'audio') { + if (o.outputType === 'audio' || o.outputType === 'scene') { outputPath = o.filePath continue } @@ -807,6 +830,13 @@ export const useWorkflowRunStore = create((set, get) => { if (fp) nodeOutputs.set(node.id, { filePath: fp, outputType: 'mesh' }) } } + if (node.type === 'sceneNode') { + const manifestPath = node.data.params?.manifestPath as string | undefined + if (manifestPath) nodeOutputs.set(node.id, { + filePath: `${workspaceDir}/${manifestPath.replace(/^\/+/, '')}`, + outputType: 'scene', + }) + } } const ctx: RunContext = { diff --git a/src/areas/workflows/workflowSceneRun.test.mjs b/src/areas/workflows/workflowSceneRun.test.mjs new file mode 100644 index 00000000..a2c2b2d0 --- /dev/null +++ b/src/areas/workflows/workflowSceneRun.test.mjs @@ -0,0 +1,60 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { build } from 'esbuild' +import { createRequire } from 'node:module' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +const dir = mkdtempSync(join(tmpdir(), 'modly-scene-run-')) +const stub = (name, source) => { const path = join(dir, name); writeFileSync(path, source); return path } +const appStoreStub = stub('app.ts', ` +export const appState: any = { apiUrl: 'http://modly.test', currentJob: null, + setCurrentJob(value: any) { this.currentJob = value }, + updateCurrentJob(value: any) { this.currentJob = { ...(this.currentJob ?? {}), ...value } } } +export const useAppStore: any = (selector: any) => selector(appState) +useAppStore.getState = () => appState +`) +const axiosStub = stub('axios.ts', `const axios: any = { create: () => (globalThis as any).__sceneClient }; export default axios; export type AxiosInstance = any`) +const extStub = stub('ext.ts', `export const getWorkflowExtension = (id: string, all: any[]) => all.find((value) => value.id === id); export type WorkflowExtension = any`) +const notifyStub = stub('notify.ts', `export const showCompletionNotification = async () => {}`) +const aliases = new Map([ + ['axios', axiosStub], ['@shared/stores/appStore', appStoreStub], + ['./mockExtensions', extStub], ['@shared/utils/notification', notifyStub], +]) +const outfile = join(dir, 'store.cjs') +writeFileSync(outfile, (await build({ + entryPoints: [resolve('src/areas/workflows/workflowRunStore.ts')], bundle: true, + platform: 'node', format: 'cjs', write: false, + plugins: [{ name: 'aliases', setup(build) { build.onResolve({ filter: /.*/ }, (args) => aliases.has(args.path) ? { path: aliases.get(args.path) } : null) } }], +})).outputFiles[0].text) +const { useWorkflowRunStore } = createRequire(import.meta.url)(outfile) + +test('scene model uses typed artifact route and preserves scene output', async () => { + const posts = [] + globalThis.window = { electron: { + settings: { get: async () => ({ workspaceDir: '/workspace' }) }, + fs: { deleteDirectory: async () => ({ success: true }), listFiles: async () => [], readFileBase64: async () => { throw new Error('scene must not be read as image bytes') } }, + } } + globalThis.__sceneClient = { + post: async (url, body) => { posts.push({ url, body }); return { data: { job_id: 'scene-job' } } }, + get: async () => ({ data: { status: 'done', progress: 100, output_url: '/workspace/Workflows/result/scene-manifest.json' } }), + } + const workflow = { + id: 'wf', name: 'Scene', description: '', createdAt: '', updatedAt: '', + nodes: [ + { id: 'source', type: 'sceneNode', position: { x: 0, y: 0 }, data: { enabled: true, params: { manifestPath: 'Workflows/input/scene-manifest.json' } } }, + { id: 'model', type: 'extensionNode', position: { x: 1, y: 0 }, data: { enabled: true, extensionId: 'pixal/world', params: {} } }, + ], + edges: [{ id: 'e', source: 'source', target: 'model' }], + } + const extension = { id: 'pixal/world', name: 'World', type: 'model', input: 'scene', output: 'scene', params: [] } + await useWorkflowRunStore.getState().run(workflow, [extension]) + assert.equal(posts[0].url, '/generate/from-artifact') + assert.deepEqual(posts[0].body, { + input_kind: 'scene', input_path: 'Workflows/input/scene-manifest.json', + model_id: 'pixal/world', collection: 'Workflows', params: {}, + }) + assert.equal(useWorkflowRunStore.getState().runState.outputPath, '/workspace/Workflows/result/scene-manifest.json') + assert.equal(useWorkflowRunStore.getState().runState.outputUrl, undefined) +}) diff --git a/src/areas/workflows/workflowSceneSource.test.mjs b/src/areas/workflows/workflowSceneSource.test.mjs new file mode 100644 index 00000000..ad2a0280 --- /dev/null +++ b/src/areas/workflows/workflowSceneSource.test.mjs @@ -0,0 +1,29 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildSync } from 'esbuild' +import { createRequire } from 'node:module' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +const outfile = join(mkdtempSync(join(tmpdir(), 'modly-scene-source-')), 'scene.cjs') +writeFileSync(outfile, buildSync({ entryPoints: [resolve('src/areas/workflows/workflowSceneSource.ts')], bundle: true, platform: 'node', format: 'cjs', write: false }).outputFiles[0].text) +const { resolveSceneSourceManifest } = createRequire(import.meta.url)(outfile) +const encoded = Buffer.from(JSON.stringify({ schema: 'modly.scene-manifest.v1', sceneRoot: '.', assets: [] })).toString('base64') + +test('Load Scene resolves directory and manifest without image bytes', async () => { + for (const scenePath of ['Workflows/room', 'Workflows/room/scene-manifest.json']) { + const result = await resolveSceneSourceManifest({ scenePath, workspaceDir: '/workspace', readFileBase64: async () => encoded }) + assert.equal(result.ok, true) + assert.equal(result.manifestWorkspacePath, 'Workflows/room/scene-manifest.json') + } +}) + +test('Load Scene refuses unsafe paths before reading', async () => { + let reads = 0 + for (const scenePath of ['../outside', '/etc/passwd', 'C:/outside', 'Workflows/%2e%2e', 'Workflows/room/']) { + const result = await resolveSceneSourceManifest({ scenePath, workspaceDir: '/workspace', readFileBase64: async () => { reads++; return encoded } }) + assert.equal(result.ok, false, scenePath) + } + assert.equal(reads, 0) +}) diff --git a/src/areas/workflows/workflowSceneSource.ts b/src/areas/workflows/workflowSceneSource.ts new file mode 100644 index 00000000..5873b692 --- /dev/null +++ b/src/areas/workflows/workflowSceneSource.ts @@ -0,0 +1,205 @@ +import type { SceneArtifactManifestInitialView, SceneArtifactManifestPreview, SceneArtifactManifestV1 } from '../../shared/types/artifacts' + +export const SCENE_MANIFEST_FILE_NAME = 'scene-manifest.json' + +export type SceneSourceKind = 'manifest' | 'directory' + +export type ResolveSceneSourceSuccess = { + ok: true + sourceKind: SceneSourceKind + inputWorkspacePath: string + manifestWorkspacePath: string + manifestAbsolutePath: string + sceneRoot: string + manifest: SceneArtifactManifestV1 +} + +export type ResolveSceneSourceFailure = { + ok: false + error: string +} + +export type ResolveSceneSourceResult = ResolveSceneSourceSuccess | ResolveSceneSourceFailure + +type ResolveSceneSourceArgs = { + scenePath: string + workspaceDir: string + readFileBase64: (filePath: string) => Promise +} + +function isAbsolutePath(value: string): boolean { + return value.startsWith('/') || /^[A-Za-z]:\//.test(value) +} + +function trimTrailingSlashes(value: string): string { + return value.replace(/\/+$/, '') +} + +function isSafeRelativePath(value: unknown, allowDot = false): value is string { + if (typeof value !== 'string' || !value || value !== value.trim() || value.includes('\u0000')) return false + const normalized = value.replace(/\\/g, '/') + if (allowDot && normalized === '.') return true + if (isAbsolutePath(normalized) || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(normalized) + || /%(?:25|2e|2f|5c|00)/i.test(normalized) || /%(?![0-9a-f]{2})/i.test(normalized)) return false + return normalized.split('/').every((segment) => segment.length > 0 && segment !== '.' && segment !== '..') +} + +function normalizeWorkspaceRelativePath(value: string | undefined, workspaceDir: string): string | undefined { + const normalizedValue = value?.replace(/\\/g, '/') + if (!normalizedValue) return undefined + + const normalizedWorkspace = trimTrailingSlashes(workspaceDir.replace(/\\/g, '/')) + let relativePath: string | undefined + + if (normalizedValue.startsWith('/workspace/')) { + relativePath = normalizedValue.slice('/workspace/'.length) + } else if (normalizedValue === normalizedWorkspace) { + return undefined + } else if (normalizedValue.startsWith(`${normalizedWorkspace}/`)) { + relativePath = normalizedValue.slice(normalizedWorkspace.length + 1) + } else if (!isAbsolutePath(normalizedValue)) { + relativePath = normalizedValue + } + + if (!relativePath) return undefined + return isSafeRelativePath(relativePath) ? relativePath : undefined +} + +function resolveSceneSourceKind(inputWorkspacePath: string): SceneSourceKind | undefined { + if (inputWorkspacePath === SCENE_MANIFEST_FILE_NAME || inputWorkspacePath.endsWith(`/${SCENE_MANIFEST_FILE_NAME}`)) { + return 'manifest' + } + if (inputWorkspacePath.toLowerCase().endsWith('.json')) return undefined + return 'directory' +} + +function decodeBase64Utf8(base64: string): string { + const bytes = Uint8Array.from(atob(base64), (char) => char.charCodeAt(0)) + return new TextDecoder().decode(bytes) +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function isSafePreviewReference(value: unknown): value is string { + return isSafeRelativePath(value) +} + +function isSceneManifestPreview(value: unknown): value is SceneArtifactManifestPreview { + return isPlainObject(value) + && (!('image' in value) || isSafePreviewReference(value.image)) + && (!('video' in value) || isSafePreviewReference(value.video)) +} + +function isFiniteTriple(value: unknown): value is [number, number, number] { + return Array.isArray(value) && value.length === 3 + && value.every((component) => typeof component === 'number' && Number.isFinite(component)) +} + +function isSceneManifestInitialView(value: unknown): value is SceneArtifactManifestInitialView { + if (!isPlainObject(value)) return false + const { position, target, up } = value + if (!isFiniteTriple(position) || !isFiniteTriple(target)) return false + if (position.every((component, index) => component === target[index])) return false + return up === undefined || (isFiniteTriple(up) && up.some((component) => component !== 0)) +} + +function normalizeSceneRoot(sceneRoot: unknown): string | undefined { + return isSafeRelativePath(sceneRoot, true) ? sceneRoot.replace(/\\/g, '/') : undefined +} + +function validateSceneManifest(manifest: unknown): { ok: true; manifest: SceneArtifactManifestV1; sceneRoot: string } | { ok: false; error: string } { + if (!isPlainObject(manifest)) { + return { ok: false, error: 'Scene manifest must be a JSON object.' } + } + if (manifest.schema !== 'modly.scene-manifest.v1') { + return { ok: false, error: 'Scene manifest schema must be modly.scene-manifest.v1.' } + } + + const rawSceneRoot = manifest.sceneRoot + const sceneRoot = normalizeSceneRoot(rawSceneRoot) + if (typeof rawSceneRoot !== 'string' || !sceneRoot) { + return { ok: false, error: 'Scene manifest sceneRoot must be a safe relative path.' } + } + if (!Array.isArray(manifest.assets)) { + return { ok: false, error: 'Scene manifest assets must be an array.' } + } + if (manifest.assets.some((asset) => isPlainObject(asset) + && (('workspacePath' in asset && !isSafeRelativePath(asset.workspacePath)) + || ('path' in asset && !isSafeRelativePath(asset.path))))) { + return { ok: false, error: 'Scene manifest asset paths must be safe relative file references.' } + } + + const { preview, initialView, ...metadata } = manifest + if (preview !== undefined && !isPlainObject(preview)) { + return { ok: false, error: 'Scene manifest preview must be a JSON object.' } + } + if (preview !== undefined && !isSceneManifestPreview(preview)) { + return { ok: false, error: 'Scene manifest preview image/video must be safe relative file references.' } + } + if (initialView !== undefined && !isPlainObject(initialView)) { + return { ok: false, error: 'Scene manifest initialView must be a JSON object.' } + } + if (initialView !== undefined && !isSceneManifestInitialView(initialView)) { + return { ok: false, error: 'Scene manifest initialView requires distinct finite numeric position/target triples and optional non-zero finite up.' } + } + + return { + ok: true, + sceneRoot, + manifest: { + ...metadata, + schema: 'modly.scene-manifest.v1', + sceneRoot: rawSceneRoot, + assets: manifest.assets, + ...(preview !== undefined ? { preview } : {}), + ...(initialView !== undefined ? { initialView } : {}), + }, + } +} + +export async function resolveSceneSourceManifest(args: ResolveSceneSourceArgs): Promise { + const inputWorkspacePath = normalizeWorkspaceRelativePath(args.scenePath, args.workspaceDir) + if (!inputWorkspacePath) { + return { ok: false, error: 'Load Scene requires a safe workspace-relative scene path.' } + } + + const sourceKind = resolveSceneSourceKind(inputWorkspacePath) + if (!sourceKind) { + return { ok: false, error: `Load Scene accepts ${SCENE_MANIFEST_FILE_NAME} or a scene directory.` } + } + + const manifestWorkspacePath = sourceKind === 'manifest' + ? inputWorkspacePath + : `${inputWorkspacePath}/${SCENE_MANIFEST_FILE_NAME}` + const normalizedWorkspace = trimTrailingSlashes(args.workspaceDir.replace(/\\/g, '/')) + const manifestAbsolutePath = `${normalizedWorkspace}/${manifestWorkspacePath}` + + let manifestRaw: string + try { + manifestRaw = decodeBase64Utf8(await args.readFileBase64(manifestAbsolutePath)) + } catch (error) { + return { ok: false, error: `Unable to read scene manifest: ${String(error)}` } + } + + let parsed: unknown + try { + parsed = JSON.parse(manifestRaw) + } catch (error) { + return { ok: false, error: `Scene manifest is not valid JSON: ${String(error)}` } + } + + const validation = validateSceneManifest(parsed) + if (!validation.ok) return validation + + return { + ok: true, + sourceKind, + inputWorkspacePath, + manifestWorkspacePath, + manifestAbsolutePath, + sceneRoot: validation.sceneRoot, + manifest: validation.manifest, + } +} diff --git a/src/shared/stores/workflowsStore.ts b/src/shared/stores/workflowsStore.ts index b8753375..f6fb829c 100644 --- a/src/shared/stores/workflowsStore.ts +++ b/src/shared/stores/workflowsStore.ts @@ -98,7 +98,7 @@ interface LegacyWorkflow { // Source-only nodes have no target handle; sink-only nodes have no source handle. // An edge into/out of the wrong side can't resolve a handle and makes React Flow // warn ("Couldn't create edge for target handle id: null") on every render. -export const NODE_TYPES_WITHOUT_TARGET = new Set(['imageNode', 'textNode', 'meshNode', 'inputNode', 'forEachNode']) +export const NODE_TYPES_WITHOUT_TARGET = new Set(['imageNode', 'textNode', 'meshNode', 'sceneNode', 'inputNode', 'forEachNode']) export const NODE_TYPES_WITHOUT_SOURCE = new Set(['outputNode', 'previewNode']) function sanitizeEdges(nodes: WFNode[], edges: WFEdge[]): WFEdge[] { diff --git a/src/shared/types/artifacts.ts b/src/shared/types/artifacts.ts index a8dfc6eb..57c6b8d4 100644 --- a/src/shared/types/artifacts.ts +++ b/src/shared/types/artifacts.ts @@ -4,3 +4,17 @@ export interface ArtifactProvenance { source?: string [key: string]: unknown } + +export interface SceneArtifactManifestPreview { image?: string; video?: string } +export interface SceneArtifactManifestInitialView { + position: [number, number, number] + target: [number, number, number] + up?: [number, number, number] +} +export interface SceneArtifactManifestV1 { + schema: 'modly.scene-manifest.v1' + sceneRoot: string + assets: unknown[] + preview?: SceneArtifactManifestPreview + initialView?: SceneArtifactManifestInitialView +} diff --git a/src/shared/types/electron.d.ts b/src/shared/types/electron.d.ts index 5119a1d4..6d929962 100644 --- a/src/shared/types/electron.d.ts +++ b/src/shared/types/electron.d.ts @@ -14,10 +14,10 @@ import type { export interface ExtensionNode { id: string name: string - input: 'image' | 'text' | 'mesh' | 'audio' - inputs?: ('image' | 'text' | 'mesh' | 'audio')[] // multi-input nodes; overrides input when set + input: 'image' | 'text' | 'mesh' | 'audio' | 'scene' + inputs?: ('image' | 'text' | 'mesh' | 'audio' | 'scene')[] // multi-input nodes; overrides input when set inputLabels?: string[] // display labels per input slot (e.g. positive/negative) - output: 'image' | 'text' | 'mesh' | 'audio' + output: 'image' | 'text' | 'mesh' | 'audio' | 'scene' paramsSchema: ParamSchema[] paramDefaults?: Record hfRepo?: string From c60d93dfda0dfba9e281210676d21191928029d1 Mon Sep 17 00:00:00 2001 From: DrHepa Date: Mon, 21 Sep 2026 21:52:16 +0200 Subject: [PATCH 2/2] feat(workflows): support video model artifacts --- README.md | 13 +- api/README.md | 2 +- api/routers/generation.py | 22 ++- api/runner.py | 19 ++- api/schemas/generation.py | 4 +- api/services/artifact_input.py | 12 +- api/services/extension_process.py | 17 +- api/services/generator_registry.py | 14 +- api/services/generators/base.py | 4 + api/services/video_input.py | 145 ++++++++++++++++++ api/tests/test_extension_process.py | 22 +++ api/tests/test_generator_registry.py | 28 +++- api/tests/test_runner.py | 57 +++++++ api/tests/test_scene_generation.py | 65 +++++++- api/tests/test_video_input.py | 67 ++++++++ .../main/extension-install-utils.test.mjs | 23 ++- electron/main/extension-install-utils.ts | 28 +++- electron/main/ipc-handlers.ts | 27 +++- electron/main/video-import.test.mjs | 24 +++ electron/main/video-import.ts | 29 ++++ electron/preload/electron-api.ts | 2 + src/areas/workflows/WorkflowsPage.tsx | 12 +- src/areas/workflows/mockExtensions.ts | 4 +- src/areas/workflows/nodes/ExtensionNode.tsx | 2 + src/areas/workflows/nodes/LoadVideoNode.tsx | 46 ++++++ src/areas/workflows/nodes/WorkflowEdge.tsx | 6 + src/areas/workflows/preflight.test.mjs | 20 +++ src/areas/workflows/preflight.ts | 27 +++- src/areas/workflows/workflowRunStore.ts | 51 +++++- src/areas/workflows/workflowVideoRun.test.mjs | 31 ++++ .../workflows/workflowVideoSource.test.mjs | 20 +++ src/areas/workflows/workflowVideoSource.ts | 24 +++ src/shared/stores/workflowsStore.ts | 2 +- src/shared/types/electron.d.ts | 5 +- 34 files changed, 818 insertions(+), 56 deletions(-) create mode 100644 api/services/video_input.py create mode 100644 api/tests/test_video_input.py create mode 100644 electron/main/video-import.test.mjs create mode 100644 electron/main/video-import.ts create mode 100644 src/areas/workflows/nodes/LoadVideoNode.tsx create mode 100644 src/areas/workflows/workflowVideoRun.test.mjs create mode 100644 src/areas/workflows/workflowVideoSource.test.mjs create mode 100644 src/areas/workflows/workflowVideoSource.ts diff --git a/README.md b/README.md index 95a0b6ff..9a56bdcd 100644 --- a/README.md +++ b/README.md @@ -159,12 +159,21 @@ a workspace directory containing `scene-manifest.json` with schema Scene** workflow node to select and validate an existing scene directory. Scene-capable generators implement `generate_artifact(input_kind, artifact_path, ...)`; legacy image generators and `POST /generate/from-image` -remain unchanged. The generic `POST /generate/from-artifact` boundary currently -accepts only `scene`, leaving future artifact kinds to separate reviewed changes. +remain unchanged. The generic `POST /generate/from-artifact` boundary accepts +validated `scene` directories and `video` files without converting either to +fake image bytes. For this first contract, `scene` is model-only and must be declared as the single `input` value (not inside `inputs`); process and mixed-input scene nodes are rejected. Model nodes may still accept multiple images and produce a scene. +Video is model-input-only and must be declared exactly as `input: "video"`. +Video outputs, process video nodes, `inputs` arrays containing video, capture, +and heterogeneous video ports are rejected. Use **Load Video** to import a +durable copy under the workspace before connecting it to a video model node. +The host checks containment, regular-file status, extension, size, and container +signature, while full media decoding remains the extension's responsibility. +Accepted containers are MP4/M4V/MOV, WebM/Matroska, and AVI, up to 8 GiB. + ## Modly CLI diff --git a/api/README.md b/api/README.md index cdcd4d6b..2f30c3e4 100644 --- a/api/README.md +++ b/api/README.md @@ -29,7 +29,7 @@ uvicorn main:app --host 127.0.0.1 --port 8765 --reload | GET | `/model/status` | Model download / load status | | GET | `/model/download` | SSE stream of download progress | | POST | `/generate/from-image` | Start image-to-3D job | -| POST | `/generate/from-artifact` | Start a typed-artifact model job (`scene` only) | +| POST | `/generate/from-artifact` | Start a typed-artifact model job (`scene` or model-input `video`) | | GET | `/generate/status/{job_id}` | Poll job status | ## Model diff --git a/api/routers/generation.py b/api/routers/generation.py index 7c355014..add2fb44 100644 --- a/api/routers/generation.py +++ b/api/routers/generation.py @@ -142,7 +142,7 @@ async def generate_from_image( _RESERVED_ARTIFACT_PARAMS = { - "artifact_path", "input_kind", "input_path", "scene_path", "scene_manifest_path", + "artifact_path", "input_kind", "input_path", "scene_path", "scene_manifest_path", "video_path", } @@ -165,7 +165,8 @@ async def generate_from_artifact( raise HTTPException(400, str(exc)) from exc params = {k: v for k, v in request.params.items() if k not in _RESERVED_ARTIFACT_PARAMS} - params["scene_manifest_path"] = str(artifact.path) + if artifact.kind == "scene": + params["scene_manifest_path"] = str(artifact.path) collection = sanitize_collection(request.collection) job_id = str(uuid.uuid4()) _purge_old_jobs() @@ -277,11 +278,22 @@ def progress_cb(pct: int, step: str = "") -> None: from services.artifact_input import revalidate_artifact_input model_input = revalidate_artifact_input(registry.WORKSPACE_DIR, model_input) import inspect - supports_cancel = "cancel_event" in inspect.signature(gen.generate_artifact).parameters + artifact_parameters = inspect.signature(gen.generate_artifact).parameters + supports_cancel = "cancel_event" in artifact_parameters + supports_snapshot = "artifact_snapshot" in artifact_parameters + + def invoke_artifact(): + kwargs = {} + if supports_cancel: + kwargs["cancel_event"] = cancel_event + if supports_snapshot: + kwargs["artifact_snapshot"] = model_input.snapshot + return gen.generate_artifact( + model_input.kind, model_input.path, params, progress_cb, **kwargs + ) output_path = await loop.run_in_executor( None, - lambda: gen.generate_artifact(model_input.kind, model_input.path, params, progress_cb, cancel_event) - if supports_cancel else gen.generate_artifact(model_input.kind, model_input.path, params, progress_cb), + invoke_artifact, ) else: import inspect diff --git a/api/runner.py b/api/runner.py index 0dd17ddb..42104918 100644 --- a/api/runner.py +++ b/api/runner.py @@ -156,10 +156,20 @@ def decode_model_input(msg: dict): if "input" not in msg: return base64.b64decode(msg["image_b64"]) value = msg["input"] - if not isinstance(value, dict) or set(value) != {"kind", "path"}: + if not isinstance(value, dict): + raise ValueError("Typed artifact input must contain exactly kind and path") + kind = value.get("kind") + expected_fields = {"kind", "path", "snapshot"} if kind == "video" else {"kind", "path"} + if set(value) != expected_fields: + if kind == "video": + raise ValueError("Video artifact input requires a well-formed snapshot") raise ValueError("Typed artifact input must contain exactly kind and path") from services.artifact_input import TypedArtifactInput, revalidate_artifact_input - typed = TypedArtifactInput(kind=value.get("kind"), path=Path(value.get("path", ""))) + snapshot = None + if kind == "video": + from services.video_input import video_snapshot_from_dict + snapshot = video_snapshot_from_dict(value["snapshot"]) + typed = TypedArtifactInput(kind=kind, path=Path(value.get("path", "")), snapshot=snapshot) return revalidate_artifact_input(WORKSPACE_DIR, typed) @@ -235,9 +245,10 @@ def main() -> None: if hasattr(model_input, "kind"): if not isinstance(params, dict): raise ValueError("Model params must be an object") - reserved = {"artifact_path", "input_kind", "input_path", "scene_path", "scene_manifest_path"} + reserved = {"artifact_path", "input_kind", "input_path", "scene_path", "scene_manifest_path", "video_path"} params = {key: value for key, value in params.items() if key not in reserved} - params["scene_manifest_path"] = str(model_input.path) + if model_input.kind == "scene": + params["scene_manifest_path"] = str(model_input.path) if msg.get("outputs_dir"): gen.outputs_dir = Path(msg["outputs_dir"]) gen.outputs_dir.mkdir(parents=True, exist_ok=True) diff --git a/api/schemas/generation.py b/api/schemas/generation.py index 04c18c85..c6b80698 100644 --- a/api/schemas/generation.py +++ b/api/schemas/generation.py @@ -12,8 +12,8 @@ class JobStatus(BaseModel): class GenerateFromArtifactRequest(BaseModel): - """Generic typed-artifact request. Only scene is public in this release.""" - input_kind: Literal["scene"] + """Generic typed-artifact request for model-only scene and video inputs.""" + input_kind: Literal["scene", "video"] input_path: str model_id: str collection: str = "Workflows" diff --git a/api/services/artifact_input.py b/api/services/artifact_input.py index 165e29ec..11a018c8 100644 --- a/api/services/artifact_input.py +++ b/api/services/artifact_input.py @@ -3,14 +3,16 @@ from pathlib import Path from services.scene_input import validate_scene_input +from services.video_input import VideoSnapshot, validate_video_input -SUPPORTED_ARTIFACT_INPUTS = frozenset({"scene"}) +SUPPORTED_ARTIFACT_INPUTS = frozenset({"scene", "video"}) @dataclass(frozen=True) class TypedArtifactInput: kind: str path: Path + snapshot: VideoSnapshot | None = None def validate_artifact_input(workspace: Path, kind: str, input_path: str) -> TypedArtifactInput: @@ -18,6 +20,9 @@ def validate_artifact_input(workspace: Path, kind: str, input_path: str) -> Type raise ValueError(f"Unsupported artifact input kind: {kind}") if kind == "scene": return TypedArtifactInput(kind="scene", path=validate_scene_input(workspace, input_path)) + if kind == "video": + path, snapshot = validate_video_input(workspace, input_path) + return TypedArtifactInput(kind="video", path=path, snapshot=snapshot) raise ValueError(f"Unsupported artifact input kind: {kind}") @@ -26,4 +31,7 @@ def revalidate_artifact_input(workspace: Path, value: TypedArtifactInput) -> Typ relative = value.path.resolve(strict=True).relative_to(workspace.resolve(strict=True)) except (OSError, ValueError) as exc: raise ValueError("Artifact input is outside the workspace") from exc - return validate_artifact_input(workspace, value.kind, relative.as_posix()) + validated = validate_artifact_input(workspace, value.kind, relative.as_posix()) + if value.kind == "video" and value.snapshot is not None and validated.snapshot != value.snapshot: + raise ValueError("Video input changed after it was queued") + return validated diff --git a/api/services/extension_process.py b/api/services/extension_process.py index 3a39bfc5..5e5669fa 100644 --- a/api/services/extension_process.py +++ b/api/services/extension_process.py @@ -18,7 +18,10 @@ import threading import uuid from pathlib import Path -from typing import Callable, Optional +from typing import Callable, Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from services.video_input import VideoSnapshot _RUNNER_PATH = Path(__file__).parent.parent / "runner.py" _MISSING_MODULE_RE = re.compile(r"No module named ['\"]([^'\"]+)['\"]") @@ -386,16 +389,24 @@ def generate_artifact( params: dict, progress_cb: Optional[Callable[[int, str], None]] = None, cancel_event: Optional[threading.Event] = None, + artifact_snapshot: Optional["VideoSnapshot"] = None, ) -> Path: """Send a typed artifact envelope to the isolated runner.""" from services.artifact_input import TypedArtifactInput, revalidate_artifact_input from services.generator_registry import WORKSPACE_DIR + from services.video_input import video_snapshot_to_dict + snapshot_payload = (video_snapshot_to_dict(artifact_snapshot) + if input_kind == "video" else None) validated = revalidate_artifact_input( - WORKSPACE_DIR, TypedArtifactInput(kind=input_kind, path=artifact_path) + WORKSPACE_DIR, + TypedArtifactInput(kind=input_kind, path=artifact_path, snapshot=artifact_snapshot), ) + input_payload = {"kind": validated.kind, "path": str(validated.path)} + if validated.kind == "video": + input_payload["snapshot"] = snapshot_payload return self._generate_request( - {"input": {"kind": validated.kind, "path": str(validated.path)}}, + {"input": input_payload}, params, progress_cb, cancel_event, ) diff --git a/api/services/generator_registry.py b/api/services/generator_registry.py index de1750ca..e6270c7d 100644 --- a/api/services/generator_registry.py +++ b/api/services/generator_registry.py @@ -452,15 +452,16 @@ def _discover_extensions( node for node in raw_nodes if isinstance(node, dict) and node.get("id") ] - allowed_io = {"image", "text", "mesh", "audio", "scene"} + allowed_inputs = {"image", "text", "mesh", "audio", "scene", "video"} + allowed_outputs = {"image", "text", "mesh", "audio", "scene"} for node in nodes: declared_inputs = node.get("inputs") or [node.get("input", "image")] if (not isinstance(declared_inputs, list) - or any(value not in allowed_io for value in declared_inputs)): + or any(value not in allowed_inputs for value in declared_inputs)): raise ValueError( f'model node "{node.get("id", "unknown")}" has an unsupported input type' ) - if node.get("output", "mesh") not in allowed_io: + if node.get("output", "mesh") not in allowed_outputs: raise ValueError( f'model node "{node.get("id", "unknown")}" has an unsupported output type' ) @@ -471,6 +472,13 @@ def _discover_extensions( f'model node "{node.get("id", "unknown")}" must declare scene ' 'as its single input field' ) + if "video" in declared_inputs and ( + "inputs" in node or node.get("input", "image") != "video" + ): + raise ValueError( + f'model node "{node.get("id", "unknown")}" must declare video ' + 'as its single input field' + ) # Markers left while setup or runtime registration is unfinished: # the folder is not ready to be loaded. The readable manifest lets diff --git a/api/services/generators/base.py b/api/services/generators/base.py index 0b5169a2..3ce68e2d 100644 --- a/api/services/generators/base.py +++ b/api/services/generators/base.py @@ -174,6 +174,10 @@ def generate_artifact( ``generate`` with the canonical path so scene-capable extensions built against the pre-release contract remain compatible. """ + if input_kind != "scene": + raise NotImplementedError( + f"{type(self).__name__} does not implement typed artifact input '{input_kind}'" + ) return self.generate(artifact_path, params, progress_cb, cancel_event) # type: ignore[arg-type] def _check_cancelled(self, cancel_event: Optional[threading.Event]) -> None: diff --git a/api/services/video_input.py b/api/services/video_input.py new file mode 100644 index 00000000..d5c01e68 --- /dev/null +++ b/api/services/video_input.py @@ -0,0 +1,145 @@ +"""Secure validation for workspace video model inputs.""" +from dataclasses import dataclass +import hashlib +import os +import re +import stat +from pathlib import Path, PurePosixPath, PureWindowsPath + +MAX_VIDEO_BYTES = 8 * 1024**3 +SUPPORTED_VIDEO_EXTENSIONS = frozenset({".mp4", ".m4v", ".mov", ".webm", ".mkv", ".avi"}) +_HEADER_BYTES = 64 + + +@dataclass(frozen=True) +class VideoSnapshot: + size: int + mtime_ns: int + device: int + inode: int + header_sha256: str + + +_SNAPSHOT_FIELDS = frozenset({"size", "mtime_ns", "device", "inode", "header_sha256"}) + + +def video_snapshot_to_dict(snapshot: VideoSnapshot) -> dict: + """Serialize a validated snapshot for the isolated runner envelope.""" + if not isinstance(snapshot, VideoSnapshot): + raise ValueError("Video snapshot is required") + return { + "size": snapshot.size, + "mtime_ns": snapshot.mtime_ns, + "device": snapshot.device, + "inode": snapshot.inode, + "header_sha256": snapshot.header_sha256, + } + + +def video_snapshot_from_dict(value: object) -> VideoSnapshot: + """Strictly reconstruct a snapshot received across the process boundary.""" + if not isinstance(value, dict) or set(value) != _SNAPSHOT_FIELDS: + raise ValueError("Video snapshot must contain exactly the expected fields") + numeric = ("size", "mtime_ns", "device", "inode") + if any(isinstance(value[field], bool) or not isinstance(value[field], int) + for field in numeric): + raise ValueError("Video snapshot numeric fields must be integers") + if value["size"] <= 0 or any(value[field] < 0 for field in numeric[1:]): + raise ValueError("Video snapshot contains invalid numeric values") + digest = value["header_sha256"] + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise ValueError("Video snapshot contains an invalid header digest") + return VideoSnapshot( + size=value["size"], + mtime_ns=value["mtime_ns"], + device=value["device"], + inode=value["inode"], + header_sha256=digest, + ) + + +def _safe_relative(value: str) -> Path: + if not isinstance(value, str) or not value or value != value.strip() or "\x00" in value: + raise ValueError("Video path must be a nonempty workspace-relative path") + normalized = value.replace("\\", "/") + if (PurePosixPath(normalized).is_absolute() or PureWindowsPath(normalized).is_absolute() + or re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", normalized) + or re.search(r"%(?:25|2e|2f|5c|00)", normalized, re.I) + or re.search(r"%(?![0-9a-f]{2})", normalized, re.I) + or any(part in ("", ".", "..") for part in normalized.split("/"))): + raise ValueError("Video path must be a safe workspace-relative path") + return Path(*normalized.split("/")) + + +def _reject_link_components(path: Path, root: Path) -> None: + try: + relative = path.relative_to(root) + except ValueError as exc: + raise ValueError("Video path escapes the workspace") from exc + current = root + for part in relative.parts: + current = current / part + try: + info = current.lstat() + except OSError as exc: + raise ValueError("Video file is missing or unreadable") from exc + is_reparse = bool(getattr(info, "st_file_attributes", 0) + & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)) + if current.is_symlink() or is_reparse: + raise ValueError("Video path must not use symlinks or reparse points") + + +def _signature_matches(suffix: str, header: bytes) -> bool: + if suffix in {".mp4", ".m4v", ".mov"}: + return len(header) >= 12 and header[4:8] == b"ftyp" + if suffix in {".webm", ".mkv"}: + return header.startswith(b"\x1a\x45\xdf\xa3") + if suffix == ".avi": + return len(header) >= 12 and header[:4] == b"RIFF" and header[8:12] == b"AVI " + return False + + +def validate_video_input(workspace: Path, video_path: str) -> tuple[Path, VideoSnapshot]: + """Return a canonical file and stable snapshot without decoding the video.""" + root = workspace.resolve(strict=True) + candidate = root / _safe_relative(video_path) + _reject_link_components(candidate, root) + suffix = candidate.suffix.lower() + if suffix not in SUPPORTED_VIDEO_EXTENSIONS: + raise ValueError("Video input uses an unsupported file extension") + + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(candidate, flags) + except OSError as exc: + raise ValueError("Video file is missing or unreadable") from exc + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode): + raise ValueError("Video input must be a regular file") + if info.st_size <= 0: + raise ValueError("Video input must not be empty") + if info.st_size > MAX_VIDEO_BYTES: + raise ValueError("Video input exceeds the 8 GiB size limit") + header = os.read(descriptor, _HEADER_BYTES) + finally: + os.close(descriptor) + + try: + canonical = candidate.resolve(strict=True) + canonical.relative_to(root) + after = candidate.lstat() + except (OSError, ValueError) as exc: + raise ValueError("Video path escapes the workspace") from exc + if not stat.S_ISREG(after.st_mode) or (after.st_dev, after.st_ino) != (info.st_dev, info.st_ino): + raise ValueError("Video file changed while it was being validated") + if not _signature_matches(suffix, header): + raise ValueError("Video file signature does not match its extension") + snapshot = VideoSnapshot( + size=info.st_size, + mtime_ns=info.st_mtime_ns, + device=info.st_dev, + inode=info.st_ino, + header_sha256=hashlib.sha256(header).hexdigest(), + ) + return canonical, snapshot diff --git a/api/tests/test_extension_process.py b/api/tests/test_extension_process.py index 7db13e4a..e5bdf449 100644 --- a/api/tests/test_extension_process.py +++ b/api/tests/test_extension_process.py @@ -8,6 +8,7 @@ from pathlib import Path from services.extension_process import ExtensionProcess, _venv_python +from services.artifact_input import validate_artifact_input def _make_proc() -> ExtensionProcess: @@ -40,6 +41,27 @@ def test_generate_artifact_sends_typed_scene_without_image_bytes(self) -> None: self.assertEqual(result, manifest) self.assertEqual(calls, [({"input": {"kind": "scene", "path": str(manifest.resolve())}}, {"quality": "high"})]) + def test_generate_artifact_sends_canonical_video_without_image_bytes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) / "workspace" + video = workspace / "Workflows" / "clip.mp4" + video.parent.mkdir(parents=True) + video.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2") + proc = _make_proc() + calls = [] + proc._generate_request = lambda payload, params, progress, cancel: calls.append(payload) or video + queued = validate_artifact_input(workspace, "video", "Workflows/clip.mp4") + with patch("services.generator_registry.WORKSPACE_DIR", workspace): + proc.generate_artifact("video", video, {}, artifact_snapshot=queued.snapshot) + self.assertEqual(calls[0]["input"]["kind"], "video") + self.assertEqual(calls[0]["input"]["path"], str(video.resolve())) + self.assertEqual(calls[0]["input"]["snapshot"]["size"], len(video.read_bytes())) + + def test_generate_artifact_requires_original_video_snapshot(self) -> None: + proc = _make_proc() + with self.assertRaisesRegex(ValueError, "snapshot"): + proc.generate_artifact("video", Path("clip.mp4"), {}) + def test_read_loop_writes_sentinel_to_own_queue_only(self) -> None: proc = _make_proc() diff --git a/api/tests/test_generator_registry.py b/api/tests/test_generator_registry.py index a4814b9e..177a58da 100644 --- a/api/tests/test_generator_registry.py +++ b/api/tests/test_generator_registry.py @@ -155,13 +155,13 @@ def test_legacy_generator_supports_eager_and_lazy_sibling_imports(self) -> None: self.registry.reload() self.assertNotIn(str(extension.resolve()), sys.path) - def test_scene_io_is_registered_but_capture_and_video_are_rejected(self) -> None: - for extension_id, input_kind in (("scene-io", "scene"), ("capture-io", "capture"), ("video-io", "video")): + def test_scene_and_video_inputs_are_registered_but_capture_is_rejected(self) -> None: + for extension_id, input_kind, output_kind in (("scene-io", "scene", "scene"), ("capture-io", "capture", "mesh"), ("video-io", "video", "mesh")): extension = self._make_extension(extension_id) manifest = { "id": extension_id, "name": extension_id, "type": "model", "generator_class": "TestGenerator", - "nodes": [{"id": "generate", "input": input_kind, "output": "scene"}], + "nodes": [{"id": "generate", "input": input_kind, "output": output_kind}], } (extension / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") (extension / "generator.py").write_text( @@ -174,8 +174,28 @@ def test_scene_io_is_registered_but_capture_and_video_are_rejected(self) -> None self.registry.initialize() self.assertEqual(self.registry.get_manifest("scene-io/generate")["input"], "scene") + self.assertEqual(self.registry.get_manifest("video-io/generate")["input"], "video") self.assertIn("capture-io/generate", self.registry.load_errors()) - self.assertIn("video-io/generate", self.registry.load_errors()) + + def test_video_input_rejects_arrays_and_video_output(self) -> None: + cases = { + "video-array": {"input": "video", "inputs": ["video"], "output": "mesh"}, + "video-mixed": {"input": "video", "inputs": ["video", "text"], "output": "mesh"}, + "video-output": {"input": "image", "output": "video"}, + } + for extension_id, node in cases.items(): + extension = self._make_extension(extension_id) + (extension / "manifest.json").write_text(json.dumps({ + "id": extension_id, "name": extension_id, "type": "model", + "generator_class": "TestGenerator", "nodes": [{"id": "generate", **node}], + }), encoding="utf-8") + (extension / "generator.py").write_text( + "from services.generators.base import BaseGenerator\nclass TestGenerator(BaseGenerator):\n pass\n", + encoding="utf-8", + ) + self.registry.initialize() + for extension_id in cases: + self.assertIn(f"{extension_id}/generate", self.registry.load_errors()) def test_scene_input_rejects_multi_input_shapes_but_image_multi_can_output_scene(self) -> None: cases = { diff --git a/api/tests/test_runner.py b/api/tests/test_runner.py index a666f87a..4a1f660f 100644 --- a/api/tests/test_runner.py +++ b/api/tests/test_runner.py @@ -8,6 +8,9 @@ from unittest.mock import patch from contextlib import redirect_stdout from pathlib import Path +from services.generators.base import BaseGenerator +from services.artifact_input import validate_artifact_input +from services.video_input import video_snapshot_to_dict _tmp_ext_dir = tempfile.mkdtemp(prefix="modly-runner-test-") @@ -21,6 +24,15 @@ class RunnerTests(unittest.TestCase): + def test_legacy_generator_fails_actionably_for_video_artifact(self) -> None: + class Legacy(BaseGenerator): + def load(self): pass + def generate(self, image_bytes, params, progress_cb=None, cancel_event=None): return Path("result.glb") + with tempfile.TemporaryDirectory() as tmp: + generator = Legacy(Path(tmp), Path(tmp)) + with self.assertRaisesRegex(NotImplementedError, "typed artifact input 'video'"): + generator.generate_artifact("video", Path(tmp) / "clip.mp4", {}) + def test_decode_typed_scene_revalidates_worker_workspace_and_keeps_legacy_image(self) -> None: with tempfile.TemporaryDirectory() as tmp: workspace = Path(tmp) / "workspace" @@ -45,6 +57,51 @@ def test_runner_model_envelope_rejects_cross_node_dispatch(self) -> None: with self.assertRaisesRegex(ValueError, "does not match"): runner.validate_requested_model({"model_id": "pixal3d/generate"}, manifest, node) + def test_runner_decodes_and_revalidates_video_artifact(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) / "workspace" + video = workspace / "Workflows" / "clip.mp4" + video.parent.mkdir(parents=True) + video.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2") + queued = validate_artifact_input(workspace, "video", "Workflows/clip.mp4") + with patch.object(runner, "WORKSPACE_DIR", workspace): + typed = runner.decode_model_input({"input": { + "kind": "video", "path": str(video), + "snapshot": video_snapshot_to_dict(queued.snapshot), + }}) + self.assertEqual(typed.kind, "video") + self.assertEqual(typed.path, video.resolve()) + + def test_runner_rejects_swapped_video_using_queued_snapshot(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) / "workspace" + video = workspace / "Workflows" / "clip.mp4" + video.parent.mkdir(parents=True) + video.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2") + queued = validate_artifact_input(workspace, "video", "Workflows/clip.mp4") + replacement = video.with_suffix(".replacement") + replacement.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2changed") + os.replace(replacement, video) + with patch.object(runner, "WORKSPACE_DIR", workspace), self.assertRaisesRegex(ValueError, "changed"): + runner.decode_model_input({"input": { + "kind": "video", "path": str(video), + "snapshot": video_snapshot_to_dict(queued.snapshot), + }}) + + def test_runner_requires_well_formed_video_snapshot(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + workspace = Path(tmp) / "workspace" + video = workspace / "Workflows" / "clip.mp4" + video.parent.mkdir(parents=True) + video.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2") + for input_value in ( + {"kind": "video", "path": str(video)}, + {"kind": "video", "path": str(video), "snapshot": {}}, + {"kind": "video", "path": str(video), "snapshot": {"size": True}}, + ): + with self.subTest(input_value=input_value), patch.object(runner, "WORKSPACE_DIR", workspace), self.assertRaisesRegex(ValueError, "snapshot"): + runner.decode_model_input({"input": input_value}) + def test_select_node_uses_model_dir_override(self) -> None: manifest = { "nodes": [ diff --git a/api/tests/test_scene_generation.py b/api/tests/test_scene_generation.py index a804171a..d7e4a1fe 100644 --- a/api/tests/test_scene_generation.py +++ b/api/tests/test_scene_generation.py @@ -11,6 +11,20 @@ import routers.generation as generation import services.generator_registry as registry from schemas.generation import GenerateFromArtifactRequest +from services.artifact_input import validate_artifact_input + + +def _run_generation(coro): + """Keep unit tests deterministic without exercising the platform executor.""" + class ImmediateLoop: + async def run_in_executor(self, _executor, callback): + return callback() + + async def invoke(): + with patch.object(generation.asyncio, "get_running_loop", return_value=ImmediateLoop()): + return await coro + + return asyncio.run(invoke()) class _Registry: @@ -54,7 +68,7 @@ def test_generic_route_queues_typed_scene_and_strips_reserved_params(self): self.assertEqual(result["job_id"], queued.args[0]) def test_generic_route_rejects_unsupported_kind_and_model_mismatch(self): - for kind in ("video", "capture", "image"): + for kind in ("capture", "image"): with self.subTest(kind=kind), self.assertRaises(ValidationError): GenerateFromArtifactRequest( input_kind=kind, input_path="Workflows/room", model_id="demo/scene") @@ -64,6 +78,22 @@ def test_generic_route_rejects_unsupported_kind_and_model_mismatch(self): input_kind="scene", input_path="Workflows/room", model_id="demo/image"), BackgroundTasks())) self.assertEqual(caught.exception.status_code, 400) + def test_generic_route_queues_video_without_forgeable_transport_params(self): + video = self.workspace / "Workflows" / "clip.mp4" + video.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2") + self.registry.get_manifest = lambda _model_id: {"input": "video", "output": "mesh"} + tasks = BackgroundTasks() + asyncio.run(generation.generate_from_artifact(GenerateFromArtifactRequest( + input_kind="video", input_path="Workflows/clip.mp4", model_id="demo/video", + params={"video_path": "/etc/passwd", "input_path": "fake", "quality": "high"}, + ), tasks)) + queued = tasks.tasks[0] + self.assertEqual(queued.args[1].kind, "video") + self.assertEqual(queued.args[1].path, video.resolve()) + self.assertNotIn("video_path", queued.args[2]) + self.assertNotIn("input_path", queued.args[2]) + self.assertEqual(queued.args[5], "demo/video") + def test_rejects_traversal_before_switch_or_queue(self): with self.assertRaises(HTTPException): asyncio.run(generation.generate_from_artifact(GenerateFromArtifactRequest( @@ -91,7 +121,7 @@ def generate_artifact(self, kind, path, params, progress_cb, cancel_event=None): generation._jobs[job_id] = generation.JobStatus(job_id=job_id, status="pending", progress=0) generation._cancel_events[job_id] = __import__("threading").Event() with patch.object(generation, "generator_registry", registry_stub): - asyncio.run(generation._run_generation( + _run_generation(generation._run_generation( job_id, generation.TypedArtifactInput("scene", self.manifest.resolve()), {}, "Workflows", "mesh", "demo/a", )) @@ -107,9 +137,38 @@ def test_missing_pinned_model_fails_actionably(self): generation._jobs[job_id] = generation.JobStatus(job_id=job_id, status="pending", progress=0) generation._cancel_events[job_id] = __import__("threading").Event() with patch.object(generation, "generator_registry", registry_stub): - asyncio.run(generation._run_generation( + _run_generation(generation._run_generation( job_id, generation.TypedArtifactInput("scene", self.manifest.resolve()), {}, "Workflows", "mesh", "demo/missing", )) self.assertEqual(generation._jobs[job_id].status, "error") self.assertIn("Unknown model ID: demo/missing", generation._jobs[job_id].error) + + def test_video_job_stays_pinned_and_receives_cancellation_event(self): + video = self.workspace / "Workflows" / "clip.mp4" + video.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2") + artifact = validate_artifact_input(self.workspace, "video", "Workflows/clip.mp4") + calls = [] + class Generator: + outputs_dir = None + def generate_artifact(self, kind, path, params, progress_cb, cancel_event=None, + artifact_snapshot=None): + calls.append((kind, path, cancel_event, artifact_snapshot)) + output = Path(self.outputs_dir) / "result.glb" + output.write_bytes(b"glb") + return output + generator = Generator() + registry_stub = type("Registry", (), { + "get_ready_generator": lambda self, model_id: generator if model_id == "demo/video" else None, + "get_active": lambda self: (_ for _ in ()).throw(AssertionError("active model must not be used")), + })() + job_id = "pinned-video" + generation._jobs[job_id] = generation.JobStatus(job_id=job_id, status="pending", progress=0) + generation._cancel_events[job_id] = __import__("threading").Event() + with patch.object(generation, "generator_registry", registry_stub): + _run_generation(generation._run_generation( + job_id, artifact, {}, "Workflows", "mesh", "demo/video" + )) + self.assertEqual(calls[0][:2], ("video", video.resolve())) + self.assertIs(calls[0][2], generation._cancel_events[job_id]) + self.assertEqual(calls[0][3], artifact.snapshot) diff --git a/api/tests/test_video_input.py b/api/tests/test_video_input.py new file mode 100644 index 00000000..784e0068 --- /dev/null +++ b/api/tests/test_video_input.py @@ -0,0 +1,67 @@ +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from services.artifact_input import revalidate_artifact_input, validate_artifact_input +from services.video_input import validate_video_input + + +MP4 = b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2" + + +class VideoInputTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.workspace = Path(self.tmp.name) / "workspace" + self.video = self.workspace / "Workflows" / "Videos" / "clip.mp4" + self.video.parent.mkdir(parents=True) + self.video.write_bytes(MP4) + + def tearDown(self): + self.tmp.cleanup() + + def test_accepts_supported_video_signatures_as_canonical_regular_files(self): + samples = { + "clip.mp4": MP4, + "clip.mov": b"\x00\x00\x00\x14ftypqt \x00\x00\x00\x00", + "clip.webm": b"\x1aE\xdf\xa3\x9fB\x86\x81\x01", + "clip.mkv": b"\x1aE\xdf\xa3\x9fB\x86\x81\x01", + "clip.avi": b"RIFF\x10\x00\x00\x00AVI LIST", + } + for name, payload in samples.items(): + path = self.video.parent / name + path.write_bytes(payload) + with self.subTest(name=name): + self.assertEqual(validate_video_input(self.workspace, f"Workflows/Videos/{name}")[0], path.resolve()) + + def test_rejects_traversal_absolute_encoded_symlink_and_non_regular_paths(self): + outside = Path(self.tmp.name) / "outside.mp4" + outside.write_bytes(MP4) + (self.video.parent / "link.mp4").symlink_to(outside) + for value in ("../outside.mp4", "/etc/passwd", "C:/outside.mp4", "Workflows/%2e%2e/out.mp4", "Workflows/Videos/link.mp4", "Workflows/Videos"): + with self.subTest(value=value), self.assertRaises(ValueError): + validate_video_input(self.workspace, value) + + def test_rejects_extension_magic_mismatch_empty_and_oversized_files(self): + bad = self.video.parent / "bad.mp4" + bad.write_bytes(b"not a video") + empty = self.video.parent / "empty.webm" + empty.write_bytes(b"") + unsupported = self.video.parent / "clip.exe" + unsupported.write_bytes(MP4) + for value in (bad, empty, unsupported): + with self.subTest(value=value.name), self.assertRaises(ValueError): + validate_video_input(self.workspace, value.relative_to(self.workspace).as_posix()) + with patch("services.video_input.MAX_VIDEO_BYTES", len(MP4) - 1): + with self.assertRaisesRegex(ValueError, "size limit"): + validate_video_input(self.workspace, "Workflows/Videos/clip.mp4") + + def test_snapshot_detects_replacement_before_inference(self): + artifact = validate_artifact_input(self.workspace, "video", "Workflows/Videos/clip.mp4") + replacement = self.video.with_suffix(".replacement") + replacement.write_bytes(MP4 + b"changed") + os.replace(replacement, self.video) + with self.assertRaisesRegex(ValueError, "changed"): + revalidate_artifact_input(self.workspace, artifact) diff --git a/electron/main/extension-install-utils.test.mjs b/electron/main/extension-install-utils.test.mjs index 4dcf42da..3cfbf57a 100644 --- a/electron/main/extension-install-utils.test.mjs +++ b/electron/main/extension-install-utils.test.mjs @@ -92,7 +92,7 @@ test('validateInstallManifest accepts scene IO and rejects undeclared future art id: 'scene-model', generator_class: 'Generator', nodes: [{ id: 'normalize', input: 'scene', output: 'scene' }], }, files, 'repository')) - for (const input of ['capture', 'video']) { + for (const input of ['capture']) { assert.throws(() => mod.validateInstallManifest({ id: 'future-model', generator_class: 'Generator', nodes: [{ id: 'future', input, output: 'scene' }], @@ -129,6 +129,27 @@ test('scene is model-only, single-input, while image-multi to scene stays valid' } }) +test('video is model-input-only and must use the singular input field', () => { + const mod = loadModule() + const modelFiles = { hasEntryFile: () => false, hasGeneratorFile: () => true } + const processFiles = { hasEntryFile: () => true, hasGeneratorFile: () => false } + assert.doesNotThrow(() => mod.validateInstallManifest({ + id: 'video-model', generator_class: 'Generator', + nodes: [{ id: 'generate', input: 'video', output: 'mesh' }], + }, modelFiles, 'repository')) + for (const node of [ + { id: 'array', input: 'video', inputs: ['video'], output: 'mesh' }, + { id: 'mixed', input: 'video', inputs: ['video', 'text'], output: 'mesh' }, + { id: 'hidden', input: 'image', inputs: ['video'], output: 'mesh' }, + { id: 'output', input: 'image', output: 'video' }, + ]) { + assert.throws(() => mod.validateInstallManifest({ id: 'bad', generator_class: 'Generator', nodes: [node] }, modelFiles, 'repository'), /video/i) + } + assert.throws(() => mod.validateInstallManifest({ + id: 'process', type: 'process', entry: 'processor.js', nodes: [{ id: 'run', input: 'video', output: 'mesh' }], + }, processFiles, 'repository'), /video.*model|model.*video/i) +}) + test('validateInstallManifest rejects malformed or process model_sources', () => { const mod = loadModule() const source = { diff --git a/electron/main/extension-install-utils.ts b/electron/main/extension-install-utils.ts index 41e9798c..d5508441 100644 --- a/electron/main/extension-install-utils.ts +++ b/electron/main/extension-install-utils.ts @@ -48,6 +48,24 @@ export function assertSupportedSceneNodeShape( } } +export function assertSupportedVideoNodeShape( + kind: 'model' | 'process', + node: { id?: string; input?: unknown; inputs?: unknown; output?: unknown }, + declaredInputs: unknown[], + output: unknown, +): void { + const usesVideoInput = declaredInputs.includes('video') + if (output === 'video') { + throw new Error('manifest.json: video is supported only as a model input') + } + if (kind === 'process' && usesVideoInput) { + throw new Error('manifest.json: video input is supported only for model nodes') + } + if (kind === 'model' && usesVideoInput && (node.inputs !== undefined || node.input !== 'video')) { + throw new Error(`manifest.json: ${node.id ?? 'node'} must declare video as its single input field`) + } +} + export type IncompleteInstallRecoveryAction = | 'none' | 'remove-incomplete' @@ -66,7 +84,8 @@ export function validateInstallManifest( const isProcess = manifest.type === 'process' const entryFile = manifest.entry ?? 'processor.js' const nodes = Array.isArray(manifest.nodes) ? manifest.nodes.filter((node) => node?.id) : [] - const allowedIo = new Set(['image', 'text', 'mesh', 'audio', 'scene']) + const allowedInputs = new Set(['image', 'text', 'mesh', 'audio', 'scene', 'video']) + const allowedOutputs = new Set(['image', 'text', 'mesh', 'audio', 'scene']) if (manifest.model_sources !== undefined) { throw new Error('manifest.json: model_sources must be declared on a model node') @@ -74,14 +93,15 @@ export function validateInstallManifest( for (const node of Array.isArray(manifest.nodes) ? manifest.nodes : []) { const declaredInputs = node.inputs === undefined ? [node.input ?? 'image'] : node.inputs if (!Array.isArray(declaredInputs) || declaredInputs.length === 0 - || declaredInputs.some((value) => typeof value !== 'string' || !allowedIo.has(value))) { + || declaredInputs.some((value) => typeof value !== 'string' || !allowedInputs.has(value))) { throw new Error(`manifest.json: ${node.id ?? 'node'}.input must use a supported artifact type`) } const output = node.output ?? 'mesh' - if (typeof output !== 'string' || !allowedIo.has(output)) { + assertSupportedSceneNodeShape(isProcess ? 'process' : 'model', node, declaredInputs, output) + assertSupportedVideoNodeShape(isProcess ? 'process' : 'model', node, declaredInputs, output) + if (typeof output !== 'string' || !allowedOutputs.has(output)) { throw new Error(`manifest.json: ${node.id ?? 'node'}.output must use a supported artifact type`) } - assertSupportedSceneNodeShape(isProcess ? 'process' : 'model', node, declaredInputs, output) if (node.model_sources === undefined) continue if (isProcess) { throw new Error('manifest.json: model_sources is supported only for model nodes') diff --git a/electron/main/ipc-handlers.ts b/electron/main/ipc-handlers.ts index 9e5131de..9ce0ff72 100644 --- a/electron/main/ipc-handlers.ts +++ b/electron/main/ipc-handlers.ts @@ -9,6 +9,7 @@ import * as tar from 'tar' import * as os from 'os' import { promisify } from 'util' import { PythonBridge, API_BASE_URL } from './python-bridge' +import { importVideoToWorkspace } from './video-import' import { isModelDownloaded, listDownloadedModels, @@ -50,6 +51,7 @@ import { validateExistingExtensionReplacement, validateInstallManifest, assertSupportedSceneNodeShape, + assertSupportedVideoNodeShape, } from './extension-install-utils' import { beginExtensionRegistrationTransaction, @@ -284,6 +286,19 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe return result.canceled ? null : result.filePaths[0] }) + ipcMain.handle('fs:selectVideo', async () => { + const win = getWindow() + if (!win) return null + const result = await dialog.showOpenDialog(win, { + title: 'Import a video', + filters: [{ name: 'Videos', extensions: ['mp4', 'm4v', 'mov', 'webm', 'mkv', 'avi'] }], + properties: ['openFile'], + }) + if (result.canceled) return null + const workspaceDir = getSettings(app.getPath('userData')).workspaceDir + return importVideoToWorkspace(result.filePaths[0], workspaceDir) + }) + ipcMain.handle('fs:selectMeshFile', async () => { const win = getWindow() if (!win) return null @@ -845,8 +860,8 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe nodes?: { id: string name?: string - input?: 'mesh' | 'image' | 'text' | 'audio' | 'scene' - inputs?: ('mesh' | 'image' | 'text' | 'audio' | 'scene')[] + input?: 'mesh' | 'image' | 'text' | 'audio' | 'scene' | 'video' + inputs?: ('mesh' | 'image' | 'text' | 'audio' | 'scene' | 'video')[] input_labels?: string[] output?: 'mesh' | 'image' | 'text' | 'audio' | 'scene' params_schema?: unknown[] @@ -874,15 +889,17 @@ export function setupIpcHandlers(pythonBridge: PythonBridge, getWindow: WindowGe if (parsed.model_sources !== undefined) { throw new Error('manifest.json: model_sources must be declared on a model node') } - const allowedIo = new Set(['image', 'text', 'mesh', 'audio', 'scene']) + const allowedInputs = new Set(['image', 'text', 'mesh', 'audio', 'scene', 'video']) + const allowedOutputs = new Set(['image', 'text', 'mesh', 'audio', 'scene']) const nodes = (parsed.nodes ?? []).map(n => { const declaredInputs = n.inputs ?? [n.input ?? 'image'] for (const input of declaredInputs) { - if (!allowedIo.has(input)) throw new Error(`manifest.json: unsupported node input type "${input}"`) + if (!allowedInputs.has(input)) throw new Error(`manifest.json: unsupported node input type "${input}"`) } const output = n.output ?? 'mesh' - if (!allowedIo.has(output)) throw new Error(`manifest.json: unsupported node output type "${output}"`) assertSupportedSceneNodeShape(parsed.type === 'process' ? 'process' : 'model', n, declaredInputs, output) + assertSupportedVideoNodeShape(parsed.type === 'process' ? 'process' : 'model', n, declaredInputs, output) + if (!allowedOutputs.has(output)) throw new Error(`manifest.json: unsupported node output type "${output}"`) if (parsed.type === 'process' && n.model_sources !== undefined) { throw new Error('manifest.json: model_sources is supported only for model nodes') } diff --git a/electron/main/video-import.test.mjs b/electron/main/video-import.test.mjs new file mode 100644 index 00000000..2b619f94 --- /dev/null +++ b/electron/main/video-import.test.mjs @@ -0,0 +1,24 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildSync } from 'esbuild' +import { createRequire } from 'node:module' +import { mkdtempSync, readFileSync, symlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +const outfile = join(mkdtempSync(join(tmpdir(), 'modly-video-import-build-')), 'import.cjs') +writeFileSync(outfile, buildSync({ entryPoints: [resolve('electron/main/video-import.ts')], bundle: true, platform: 'node', format: 'cjs', write: false }).outputFiles[0].text) +const { importVideoToWorkspace } = createRequire(import.meta.url)(outfile) + +test('video import creates a durable workspace copy and refuses symlink sources', async () => { + const root = mkdtempSync(join(tmpdir(), 'modly-video-import-')) + const source = join(root, 'source clip.mp4') + const payload = Buffer.from('\x00\x00\x00\x18ftypisom') + writeFileSync(source, payload) + const result = await importVideoToWorkspace(source, join(root, 'workspace')) + assert.match(result.workspacePath, /^Workflows\/Imported Videos\/[0-9a-f-]+-source_clip\.mp4$/) + assert.deepEqual(readFileSync(result.absolutePath), payload) + const link = join(root, 'linked.mp4') + symlinkSync(source, link) + await assert.rejects(importVideoToWorkspace(link, join(root, 'workspace')), /regular file/) +}) diff --git a/electron/main/video-import.ts b/electron/main/video-import.ts new file mode 100644 index 00000000..fbf9c800 --- /dev/null +++ b/electron/main/video-import.ts @@ -0,0 +1,29 @@ +import { randomUUID } from 'node:crypto' +import { copyFile, lstat, mkdir } from 'node:fs/promises' +import { basename, extname, join } from 'node:path' + +const MAX_VIDEO_BYTES = 8 * 1024 ** 3 +const VIDEO_EXTENSIONS = new Set(['.mp4', '.m4v', '.mov', '.webm', '.mkv', '.avi']) + +export interface ImportedWorkspaceVideo { + workspacePath: string + absolutePath: string +} + +export async function importVideoToWorkspace(source: string, workspaceDir: string): Promise { + const info = await lstat(source) + if (!info.isFile() || info.isSymbolicLink()) throw new Error('Selected video must be a regular file') + if (info.size <= 0 || info.size > MAX_VIDEO_BYTES) throw new Error('Selected video must be between 1 byte and 8 GiB') + const extension = extname(source).toLowerCase() + if (!VIDEO_EXTENSIONS.has(extension)) throw new Error('Selected video uses an unsupported extension') + + const importDir = join(workspaceDir, 'Workflows', 'Imported Videos') + await mkdir(importDir, { recursive: true }) + const safeName = basename(source).replace(/[^a-zA-Z0-9._-]+/g, '_') + const fileName = `${randomUUID()}-${safeName}` + const absolutePath = join(importDir, fileName) + await copyFile(source, absolutePath) + const copied = await lstat(absolutePath) + if (!copied.isFile() || copied.size !== info.size) throw new Error('Imported video copy could not be verified') + return { workspacePath: `Workflows/Imported Videos/${fileName}`, absolutePath } +} diff --git a/electron/preload/electron-api.ts b/electron/preload/electron-api.ts index af0b1265..68d83ac6 100644 --- a/electron/preload/electron-api.ts +++ b/electron/preload/electron-api.ts @@ -75,6 +75,8 @@ export function createElectronApi(ipcRenderer: IpcRendererLike, webFrame: WebFra fs: { selectImage: (): Promise => ipcRenderer.invoke('fs:selectImage') as Promise, + selectVideo: (): Promise<{ workspacePath: string; absolutePath: string } | null> => + ipcRenderer.invoke('fs:selectVideo') as Promise<{ workspacePath: string; absolutePath: string } | null>, selectMeshFile: (): Promise => ipcRenderer.invoke('fs:selectMeshFile') as Promise, saveModel: (defaultName: string): Promise => diff --git a/src/areas/workflows/WorkflowsPage.tsx b/src/areas/workflows/WorkflowsPage.tsx index 32452774..7eff2fc0 100644 --- a/src/areas/workflows/WorkflowsPage.tsx +++ b/src/areas/workflows/WorkflowsPage.tsx @@ -28,6 +28,7 @@ import TextNode from './nodes/TextNode' import AddToSceneNode from './nodes/AddToSceneNode' import Load3DMeshNode from './nodes/Load3DMeshNode' import LoadSceneNode from './nodes/LoadSceneNode' +import LoadVideoNode from './nodes/LoadVideoNode' import PreviewImageNode from './nodes/PreviewImageNode' import ImagePreviewNode from './nodes/ImagePreviewNode' import WaitNode from './nodes/WaitNode' @@ -39,7 +40,7 @@ import WorkflowEdge from './nodes/WorkflowEdge' const DRAG_KEY = 'modly/extension-id' const DRAG_NODE_KEY = 'modly/node-type' -const NODE_TYPES = { extensionNode: ExtensionNode, imageNode: ImageNode, textNode: TextNode, outputNode: AddToSceneNode, meshNode: Load3DMeshNode, sceneNode: LoadSceneNode, previewNode: PreviewImageNode, imagePreviewNode: ImagePreviewNode, waitNode: WaitNode, whileNode: WhileNode, forEachNode: ForEachNode } +const NODE_TYPES = { extensionNode: ExtensionNode, imageNode: ImageNode, textNode: TextNode, outputNode: AddToSceneNode, meshNode: Load3DMeshNode, sceneNode: LoadSceneNode, videoNode: LoadVideoNode, previewNode: PreviewImageNode, imagePreviewNode: ImagePreviewNode, waitNode: WaitNode, whileNode: WhileNode, forEachNode: ForEachNode } // Loop-container node types: resizable frames whose children form a loop body. // (For Each iterators are plain source nodes, not containers.) @@ -63,15 +64,16 @@ function findWhileContainerAt(nodes: Node[], pos: { x: number; y: number }): Nod // ─── IO badge ───────────────────────────────────────────────────────────────── -const IO_STYLES: Record<'image' | 'text' | 'mesh' | 'audio' | 'scene', string> = { +const IO_STYLES: Record<'image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video', string> = { audio: 'bg-emerald-500/15 text-emerald-400 border-emerald-500/25', image: 'bg-sky-500/15 text-sky-400 border-sky-500/25', mesh: 'bg-violet-500/15 text-violet-400 border-violet-500/25', text: 'bg-amber-500/15 text-amber-400 border-amber-500/25', scene: 'bg-emerald-500/15 text-emerald-400 border-emerald-500/25', + video: 'bg-pink-500/15 text-pink-400 border-pink-500/25', } -function IoBadge({ type }: { type: 'image' | 'text' | 'mesh' | 'audio' | 'scene' }) { +function IoBadge({ type }: { type: 'image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video' }) { return ( {type} @@ -102,6 +104,7 @@ const PANEL_BUILTIN_NODES = [ { type: 'textNode', label: 'Text', color: '#fbbf24', icon: <> }, { type: 'meshNode', label: 'Load 3D Mesh', color: '#a78bfa', icon: <> }, { type: 'sceneNode', label: 'Load Scene', color: '#34d399', icon: <> }, + { type: 'videoNode', label: 'Load Video', color: '#f472b6', icon: <> }, { type: 'outputNode', label: 'Add to Scene', color: '#a78bfa', icon: <> }, { type: 'previewNode', label: 'Preview Views', color: '#38bdf8', icon: <> }, { type: 'imagePreviewNode', label: 'Preview Image', color: '#38bdf8', icon: <> }, @@ -350,6 +353,7 @@ const BUILTIN_NODES = [ { type: 'textNode', label: 'Text', color: '#fbbf24', description: 'Text input' }, { type: 'meshNode', label: 'Load 3D Mesh', color: '#a78bfa', description: 'Load a 3D mesh file or use current model' }, { type: 'sceneNode', label: 'Load Scene', color: '#34d399', description: 'Load and validate a workspace scene directory' }, + { type: 'videoNode', label: 'Load Video', color: '#f472b6', description: 'Import a durable workspace video file' }, { type: 'outputNode', label: 'Add to Scene', color: '#a78bfa', description: 'Output node — adds the mesh to the 3D scene' }, { type: 'previewNode', label: 'Preview Views', color: '#38bdf8', description: 'Displays multi-view image outputs in a 2×3 grid' }, { type: 'imagePreviewNode', label: 'Preview Image', color: '#38bdf8', description: 'Displays a single image output in the workflow' }, @@ -708,6 +712,7 @@ function getNodeOutputType(node: Node | undefined, allExts: WorkflowExtension[]) if (node.type === 'imageNode') return 'image' if (node.type === 'meshNode') return 'mesh' if (node.type === 'sceneNode') return 'scene' + if (node.type === 'videoNode') return 'video' if (node.type === 'textNode') return 'text' if (node.type === 'imagePreviewNode') return 'image' return allExts.find((e) => e.id === (node.data as WFNodeData)?.extensionId)?.output @@ -1380,6 +1385,7 @@ const MINI_NODE_TINTS: Record = { textNode: { fill: 'rgba(52,211,153,0.22)', stroke: '#34d399' }, meshNode: { fill: 'rgba(52,211,153,0.22)', stroke: '#34d399' }, sceneNode: { fill: 'rgba(52,211,153,0.22)', stroke: '#34d399' }, + videoNode: { fill: 'rgba(244,114,182,0.22)', stroke: '#f472b6' }, extensionNode: { fill: 'rgba(167,139,250,0.24)', stroke: '#a78bfa' }, outputNode: { fill: 'rgba(56,189,248,0.22)', stroke: '#38bdf8' }, previewNode: { fill: 'rgba(56,189,248,0.22)', stroke: '#38bdf8' }, diff --git a/src/areas/workflows/mockExtensions.ts b/src/areas/workflows/mockExtensions.ts index be727fea..75723d2b 100644 --- a/src/areas/workflows/mockExtensions.ts +++ b/src/areas/workflows/mockExtensions.ts @@ -10,8 +10,8 @@ export interface WorkflowExtension { nodeId: string // "node_id" name: string description: string - input: 'image' | 'text' | 'mesh' | 'audio' | 'scene' - inputs?: ('image' | 'text' | 'mesh' | 'audio' | 'scene')[] // multi-input; overrides input when set + input: 'image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video' + inputs?: ('image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video')[] // multi-input; overrides input when set inputLabels?: string[] // display labels per input slot output: 'image' | 'text' | 'mesh' | 'audio' | 'scene' params: ParamSchema[] diff --git a/src/areas/workflows/nodes/ExtensionNode.tsx b/src/areas/workflows/nodes/ExtensionNode.tsx index 4be11a14..533b6621 100644 --- a/src/areas/workflows/nodes/ExtensionNode.tsx +++ b/src/areas/workflows/nodes/ExtensionNode.tsx @@ -17,6 +17,7 @@ const HANDLE_COLOR: Record = { mesh: '#a78bfa', text: '#fbbf24', scene: '#34d399', + video: '#f472b6', } const TAG_CLS: Record = { @@ -25,6 +26,7 @@ const TAG_CLS: Record = { mesh: 'border-violet-500/30 bg-violet-500/10 text-violet-400', text: 'border-amber-500/30 bg-amber-500/10 text-amber-400', scene: 'border-emerald-500/30 bg-emerald-500/10 text-emerald-400', + video: 'border-pink-500/30 bg-pink-500/10 text-pink-400', } // ─── Param control ──────────────────────────────────────────────────────────── diff --git a/src/areas/workflows/nodes/LoadVideoNode.tsx b/src/areas/workflows/nodes/LoadVideoNode.tsx new file mode 100644 index 00000000..4a52046f --- /dev/null +++ b/src/areas/workflows/nodes/LoadVideoNode.tsx @@ -0,0 +1,46 @@ +import { useCallback, useLayoutEffect, useRef, useState } from 'react' +import { Handle, Position, useReactFlow } from '@xyflow/react' +import type { WFNodeData } from '@shared/types/electron.d' + +import BaseNode from './BaseNode' +import { normalizeVideoSource } from '../workflowVideoSource' + +const OUTPUT_COLOR = '#f472b6' + +export default function LoadVideoNode({ id, data, selected }: { id: string; data: WFNodeData; selected?: boolean }) { + const { updateNodeData } = useReactFlow() + const ioRowRef = useRef(null) + const [handleTop, setHandleTop] = useState('50%') + useLayoutEffect(() => { + if (ioRowRef.current) setHandleTop(`${ioRowRef.current.offsetTop + ioRowRef.current.offsetHeight / 2}px`) + }, []) + + const workspacePath = typeof data.params.workspacePath === 'string' ? data.params.workspacePath : '' + const error = typeof data.params.error === 'string' ? data.params.error : undefined + + const browse = useCallback(async () => { + const selectedVideo = await window.electron.fs.selectVideo() + if (!selectedVideo) return + const settings = await window.electron.settings.get() + const normalized = normalizeVideoSource(selectedVideo.workspacePath, settings.workspaceDir) + updateNodeData(id, { params: normalized + ? { ...data.params, workspacePath: normalized.workspacePath, error: undefined } + : { ...data.params, workspacePath: undefined, absolutePath: undefined, error: 'Video import did not return a safe workspace path.' } }) + }, [id, data, updateNodeData]) + + return ( + } + subheader={
video
} + handles={} + > +
+ +
+ {workspacePath || 'Imports a durable copy into the workspace for downstream video model nodes.'} +
+ {error &&
{error}
} +
+ + ) +} diff --git a/src/areas/workflows/nodes/WorkflowEdge.tsx b/src/areas/workflows/nodes/WorkflowEdge.tsx index 90d32931..3f37ff57 100644 --- a/src/areas/workflows/nodes/WorkflowEdge.tsx +++ b/src/areas/workflows/nodes/WorkflowEdge.tsx @@ -8,6 +8,8 @@ const HANDLE_COLOR: Record = { image: '#38bdf8', mesh: '#a78bfa', text: '#fbbf24', + scene: '#34d399', + video: '#f472b6', } export default function WorkflowEdge({ @@ -33,6 +35,10 @@ export default function WorkflowEdge({ ? HANDLE_COLOR.text : sourceNode?.type === 'meshNode' ? HANDLE_COLOR.mesh + : sourceNode?.type === 'sceneNode' + ? HANDLE_COLOR.scene + : sourceNode?.type === 'videoNode' + ? HANDLE_COLOR.video : (HANDLE_COLOR[allExtensions.find((e) => e.id === sourceNode?.data?.extensionId)?.output ?? ''] ?? '#52525b') // For multi-input nodes pick the color of the specific connected handle diff --git a/src/areas/workflows/preflight.test.mjs b/src/areas/workflows/preflight.test.mjs index 3b4df8aa..bcad3967 100644 --- a/src/areas/workflows/preflight.test.mjs +++ b/src/areas/workflows/preflight.test.mjs @@ -166,3 +166,23 @@ test('renderer fails closed for unsupported process and mixed scene node shapes' assert.ok(issues.some((issue) => issue.key === 'target:unsupported-scene-shape')) } }) + +test('video input accepts only a validated Load Video source and fails closed on unsupported shapes', () => { + const { validateWorkflowPreflight } = loadModule() + const video = { id: 'video', type: 'videoNode', position: { x: 0, y: 0 }, data: { params: { workspacePath: 'Workflows/Videos/clip.mp4' } } } + const target = { id: 'target', type: 'extensionNode', position: { x: 0, y: 0 }, data: { extensionId: 'pack/process-node' } } + const valid = ext({ input: 'video', output: 'mesh', type: 'model' }) + assert.deepEqual(validateWorkflowPreflight(wf([video, target], [{ id: 'e', source: 'video', target: 'target' }]), [valid]), []) + for (const extension of [ + ext({ input: 'video', output: 'mesh', type: 'process' }), + ext({ input: 'video', inputs: ['video'], output: 'mesh', type: 'model' }), + ext({ input: 'image', output: 'video', type: 'model' }), + ]) { + const issues = validateWorkflowPreflight(wf([video, target], [{ id: 'e', source: 'video', target: 'target' }]), [extension]) + assert.ok(issues.some((issue) => issue.key === 'target:unsupported-video-shape')) + } + for (const params of [{ path: '/tmp/clip.mp4' }, { workspacePath: '../clip.mp4' }, { workspacePath: 'Workflows/clip.exe' }]) { + const invalid = { ...video, data: { params } } + assert.ok(validateWorkflowPreflight(wf([invalid], []), []).some((issue) => issue.key === 'video:video-invalid')) + } +}) diff --git a/src/areas/workflows/preflight.ts b/src/areas/workflows/preflight.ts index df7d35ca..b3ddeddc 100644 --- a/src/areas/workflows/preflight.ts +++ b/src/areas/workflows/preflight.ts @@ -1,8 +1,9 @@ import type { Workflow, WFNode } from '@shared/types/electron.d' import { getWorkflowExtension, type WorkflowExtension } from './mockExtensions' import { isPassthrough, isBranchConsumer, resolveDataSource, nearestUpstreamWaits } from './nodeBehaviors' +import { normalizeVideoSource } from './workflowVideoSource' -type DataType = 'image' | 'text' | 'mesh' | 'audio' | 'scene' +type DataType = 'image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video' export interface WorkflowPreflightIssue { key: string @@ -15,6 +16,7 @@ function nodeLabel(node: WFNode, allExtensions: WorkflowExtension[]): string { if (node.type === 'textNode') return 'Text' if (node.type === 'meshNode') return 'Load 3D Mesh' if (node.type === 'sceneNode') return 'Load Scene' + if (node.type === 'videoNode') return 'Load Video' if (node.type === 'outputNode') return 'Add to Scene' if (node.type === 'previewNode') return 'Preview Views' if (node.type === 'imagePreviewNode') return 'Preview Image' @@ -30,6 +32,7 @@ function nodeLabel(node: WFNode, allExtensions: WorkflowExtension[]): string { function formatType(type: DataType): string { if (type === 'scene') return 'scene' + if (type === 'video') return 'video' if (type === 'mesh') return 'mesh' if (type === 'image') return 'image' if (type === 'audio') return 'audio' @@ -47,6 +50,7 @@ function getNodeOutputType(node: WFNode, allExtensions: WorkflowExtension[]): Da if (node.type === 'textNode') return 'text' if (node.type === 'meshNode' || node.type === 'outputNode') return 'mesh' if (node.type === 'sceneNode') return 'scene' + if (node.type === 'videoNode') return 'video' if (node.type === 'previewNode') return 'image' if (node.type === 'imagePreviewNode') return 'image' if (node.type === 'forEachNode') { @@ -105,6 +109,14 @@ export function validateWorkflowPreflight( message: 'Load Scene needs a validated scene directory.', }) } + if (node.type === 'videoNode' && !normalizeVideoSource( + node.data.params?.workspacePath as string | undefined, '/workspace', + )) { + pushIssue(issues, { + key: `${node.id}:video-invalid`, nodeId: node.id, + message: 'Load Video needs an imported workspace video file.', + }) + } // A node fed by two different Wait branches can't be scheduled into a single // branch — it would run before either branch produces its mesh. @@ -144,6 +156,19 @@ export function validateWorkflowPreflight( }) continue } + const usesVideoInput = ext.input === 'video' || ext.inputs?.includes('video') === true + const unsupportedVideoShape = ext.output === 'video' + || (ext.type === 'process' && usesVideoInput) + || (ext.type === 'model' && usesVideoInput + && (ext.inputs !== undefined || ext.input !== 'video')) + if (unsupportedVideoShape) { + pushIssue(issues, { + key: `${node.id}:unsupported-video-shape`, + nodeId: node.id, + message: `${ext.name} uses an unsupported video input or output declaration.`, + }) + continue + } const incomingEdges = workflow.edges.filter((edge) => edge.target === node.id) const requiredTypes = [...new Set((ext.inputs ?? [ext.input]) as DataType[])] diff --git a/src/areas/workflows/workflowRunStore.ts b/src/areas/workflows/workflowRunStore.ts index fe025b52..87458875 100644 --- a/src/areas/workflows/workflowRunStore.ts +++ b/src/areas/workflows/workflowRunStore.ts @@ -6,6 +6,7 @@ import { showCompletionNotification } from '@shared/utils/notification' import type { WorkflowExtension } from './mockExtensions' import type { Workflow, WFNode, WFEdge } from '@shared/types/electron.d' import { isBranchStarter, isSceneOutput, resolveDataSource, reachesSceneOutput, nearestUpstreamWaits } from './nodeBehaviors' +import { normalizeVideoSource } from './workflowVideoSource' // ─── Types ──────────────────────────────────────────────────────────────────── @@ -307,11 +308,17 @@ async function executeExtensionNode( const ext = getWorkflowExtension(node.data.extensionId ?? '', allExtensions) if (ext) { const usesSceneInput = ext.input === 'scene' || ext.inputs?.includes('scene') === true + const usesVideoInput = ext.input === 'video' || ext.inputs?.includes('video') === true if ((ext.type === 'process' && (usesSceneInput || ext.output === 'scene')) || (ext.type === 'model' && usesSceneInput && (ext.inputs !== undefined || ext.input !== 'scene'))) { throw new Error(`${ext.name} uses an unsupported scene input or output declaration`) } + if (ext.output === 'video' || (ext.type === 'process' && usesVideoInput) + || (ext.type === 'model' && usesVideoInput + && (ext.inputs !== undefined || ext.input !== 'video'))) { + throw new Error(`${ext.name} uses an unsupported video input or output declaration`) + } } // Freshest params at the moment the node starts (so loop iterations / Retry pick // up edits made while paused, not the values captured at run start). @@ -326,6 +333,7 @@ async function executeExtensionNode( let nodeInputText: string | undefined let nodeInputMeshPath: string | undefined let nodeInputScenePath: string | undefined + let nodeInputVideoPath: string | undefined // Per-slot texts for multi-text-input nodes (e.g. positive/negative prompts). // Indexed by target handle: input-0 → texts[0], input-1 → texts[1]. const nodeInputTexts: (string | undefined)[] = [] @@ -360,6 +368,8 @@ async function executeExtensionNode( nodeInputMeshPath = fp } else if (inputTypes[i] === 'scene') { nodeInputScenePath = fp + } else if (inputTypes[i] === 'video') { + nodeInputVideoPath = fp } else if (inputTypes[i] === 'image') { if (!nodeInputPath) nodeInputPath = fp else extraImagePaths.push(fp) @@ -371,6 +381,7 @@ async function executeExtensionNode( if (src?.filePath !== undefined) nodeInputPath = src.filePath if (src?.text !== undefined && src.text.trim().length > 0) nodeInputText = src.text if (src?.outputType === 'scene') nodeInputScenePath = src.filePath + if (src?.outputType === 'video') nodeInputVideoPath = src.filePath } } @@ -378,16 +389,20 @@ async function executeExtensionNode( if (isModelNode) { const isSceneInput = ext?.inputs ? ext.inputs.includes('scene') : ext?.input === 'scene' + const isVideoInput = ext?.inputs ? ext.inputs.includes('video') : ext?.input === 'video' const isTextInput = ext?.inputs ? ext.inputs.every((i) => i === 'text') : ext?.input === 'text' if (isSceneInput && !nodeInputScenePath) throw new Error(`${ext?.name ?? 'Model'} needs an incoming scene connection`) - const activeImagePath = (isTextInput || isSceneInput) ? undefined : (nodeInputPath ?? selectedImagePath) - if (!isTextInput && !isSceneInput && !selectedImageData && (!activeImagePath || activeImagePath.trim().length === 0)) { + if (isVideoInput && !nodeInputVideoPath) throw new Error(`${ext?.name ?? 'Model'} needs an incoming video connection`) + const activeImagePath = (isTextInput || isSceneInput || isVideoInput) ? undefined : (nodeInputPath ?? selectedImagePath) + if (!isTextInput && !isSceneInput && !isVideoInput && !selectedImageData && (!activeImagePath || activeImagePath.trim().length === 0)) { throw new Error('No input image selected for model node') } - let blob: Blob - let fname: string - if (isTextInput || isSceneInput || (selectedImageData && nodeInputPath === undefined)) { + let blob: Blob | undefined + let fname: string | undefined + if (isSceneInput || isVideoInput) { + // Typed artifacts cross the dedicated JSON boundary; never manufacture image bytes. + } else if (isTextInput || (selectedImageData && nodeInputPath === undefined)) { const base64 = selectedImageData && nodeInputPath === undefined ? selectedImageData : 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==' // 1x1 transparent PNG @@ -424,17 +439,29 @@ async function executeExtensionNode( let submission: { data: { job_id: string } } if (isSceneInput) { const normalized = nodeInputScenePath!.replace(/\\/g, '/') - const inputPath = normalized.startsWith(`${workspaceDir}/`) - ? normalized.slice(workspaceDir.length + 1) + const workspaceRoot = workspaceDir.replace(/\\/g, '/').replace(/\/+$/, '') + const inputPath = normalized.startsWith(`${workspaceRoot}/`) + ? normalized.slice(workspaceRoot.length + 1) : normalized.replace(/^\/workspace\//, '') submission = await client.post('/generate/from-artifact', { input_kind: 'scene', input_path: inputPath, model_id: node.data.extensionId ?? '', collection: 'Workflows', params: { ...effectiveParams, ...extraParams }, }) + } else if (isVideoInput) { + const normalized = nodeInputVideoPath!.replace(/\\/g, '/') + const workspaceRoot = workspaceDir.replace(/\\/g, '/').replace(/\/+$/, '') + const inputPath = normalized.startsWith(`${workspaceRoot}/`) + ? normalized.slice(workspaceRoot.length + 1) + : normalized.replace(/^\/workspace\//, '') + submission = await client.post('/generate/from-artifact', { + input_kind: 'video', input_path: inputPath, + model_id: node.data.extensionId ?? '', collection: 'Workflows', + params: { ...effectiveParams, ...extraParams }, + }) } else { const fd = new FormData() - fd.append('image', blob, fname) + fd.append('image', blob!, fname!) fd.append('model_id', node.data.extensionId ?? '') fd.append('collection', 'Workflows') fd.append('remesh', 'none') @@ -837,6 +864,14 @@ export const useWorkflowRunStore = create((set, get) => { outputType: 'scene', }) } + if (node.type === 'videoNode') { + const workspacePath = node.data.params?.workspacePath as string | undefined + const video = normalizeVideoSource(workspacePath, workspaceDir) + if (video) nodeOutputs.set(node.id, { + filePath: video.absolutePath, + outputType: 'video', + }) + } } const ctx: RunContext = { diff --git a/src/areas/workflows/workflowVideoRun.test.mjs b/src/areas/workflows/workflowVideoRun.test.mjs new file mode 100644 index 00000000..2422a286 --- /dev/null +++ b/src/areas/workflows/workflowVideoRun.test.mjs @@ -0,0 +1,31 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { build } from 'esbuild' +import { createRequire } from 'node:module' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +const dir = mkdtempSync(join(tmpdir(), 'modly-video-run-')) +const stub = (name, source) => { const path = join(dir, name); writeFileSync(path, source); return path } +const aliases = new Map([ + ['axios', stub('axios.ts', `const axios: any = { create: () => (globalThis as any).__client }; export default axios; export type AxiosInstance = any`)], + ['@shared/stores/appStore', stub('app.ts', `export const state: any = { apiUrl: 'x', setCurrentJob() {}, updateCurrentJob() {} }; export const useAppStore: any = (s: any) => s(state); useAppStore.getState = () => state`)], + ['./mockExtensions', stub('ext.ts', `export const getWorkflowExtension = (id: string, all: any[]) => all.find((x) => x.id === id); export type WorkflowExtension = any`)], + ['@shared/utils/notification', stub('notify.ts', `export const showCompletionNotification = async () => {}`)], +]) +const outfile = join(dir, 'store.cjs') +writeFileSync(outfile, (await build({ entryPoints: [resolve('src/areas/workflows/workflowRunStore.ts')], bundle: true, platform: 'node', format: 'cjs', write: false, plugins: [{ name: 'aliases', setup(build) { build.onResolve({ filter: /.*/ }, (args) => aliases.has(args.path) ? { path: aliases.get(args.path) } : null) } }] })).outputFiles[0].text) +const { useWorkflowRunStore } = createRequire(import.meta.url)(outfile) + +test('video model receives a typed artifact path without fake image bytes', async () => { + const posts = [] + globalThis.window = { electron: { settings: { get: async () => ({ workspaceDir: 'C:\\MODLY\\workspace' }) }, fs: { deleteDirectory: async () => ({ success: true }), listFiles: async () => [], readFileBase64: async () => { throw new Error('video must not be read as image bytes') } } } } + globalThis.__client = { post: async (url, body) => { posts.push({ url, body }); return { data: { job_id: 'video-job' } } }, get: async () => ({ data: { status: 'done', progress: 100, output_url: '/workspace/Workflows/result.glb' } }) } + const workflow = { id: 'wf', name: 'Video', description: '', createdAt: '', updatedAt: '', nodes: [ + { id: 'source', type: 'videoNode', position: { x: 0, y: 0 }, data: { enabled: true, params: { workspacePath: 'Workflows/Videos/clip.mp4' } } }, + { id: 'model', type: 'extensionNode', position: { x: 1, y: 0 }, data: { enabled: true, extensionId: 'demo/video', params: {} } }, + ], edges: [{ id: 'e', source: 'source', target: 'model' }] } + await useWorkflowRunStore.getState().run(workflow, [{ id: 'demo/video', name: 'Video', type: 'model', input: 'video', output: 'mesh', params: [] }]) + assert.deepEqual(posts[0], { url: '/generate/from-artifact', body: { input_kind: 'video', input_path: 'Workflows/Videos/clip.mp4', model_id: 'demo/video', collection: 'Workflows', params: {} } }) +}) diff --git a/src/areas/workflows/workflowVideoSource.test.mjs b/src/areas/workflows/workflowVideoSource.test.mjs new file mode 100644 index 00000000..c8f7aa90 --- /dev/null +++ b/src/areas/workflows/workflowVideoSource.test.mjs @@ -0,0 +1,20 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildSync } from 'esbuild' +import { createRequire } from 'node:module' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' + +const outfile = join(mkdtempSync(join(tmpdir(), 'modly-video-source-')), 'video.cjs') +writeFileSync(outfile, buildSync({ entryPoints: [resolve('src/areas/workflows/workflowVideoSource.ts')], bundle: true, platform: 'node', format: 'cjs', write: false }).outputFiles[0].text) +const { normalizeVideoSource } = createRequire(import.meta.url)(outfile) + +test('Load Video accepts durable workspace video paths only', () => { + assert.deepEqual(normalizeVideoSource('Workflows/Videos/clip.mp4', '/workspace'), { + workspacePath: 'Workflows/Videos/clip.mp4', absolutePath: '/workspace/Workflows/Videos/clip.mp4', + }) + for (const value of ['../clip.mp4', '/tmp/clip.mp4', 'C:/clip.mp4', 'Workflows/%2e%2e/clip.mp4', 'Workflows/clip.exe']) { + assert.equal(normalizeVideoSource(value, '/workspace'), undefined, value) + } +}) diff --git a/src/areas/workflows/workflowVideoSource.ts b/src/areas/workflows/workflowVideoSource.ts new file mode 100644 index 00000000..8fe6f9a2 --- /dev/null +++ b/src/areas/workflows/workflowVideoSource.ts @@ -0,0 +1,24 @@ +const VIDEO_EXTENSIONS = new Set(['mp4', 'm4v', 'mov', 'webm', 'mkv', 'avi']) + +function isAbsolutePath(value: string): boolean { + return value.startsWith('/') || /^[A-Za-z]:\//.test(value) || value.startsWith('//') +} + +function isSafeRelativePath(value: string): boolean { + if (!value || value !== value.trim() || value.includes('\u0000')) return false + if (isAbsolutePath(value) || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(value) + || /%(?:25|2e|2f|5c|00)/i.test(value) || /%(?![0-9a-f]{2})/i.test(value)) return false + return value.split('/').every((part) => part.length > 0 && part !== '.' && part !== '..') +} + +export function normalizeVideoSource( + rawPath: string | undefined, + workspaceDir: string, +): { workspacePath: string; absolutePath: string } | undefined { + const workspacePath = rawPath?.replace(/\\/g, '/') + if (!workspacePath || !isSafeRelativePath(workspacePath)) return undefined + const extension = workspacePath.split('.').pop()?.toLowerCase() + if (!extension || !VIDEO_EXTENSIONS.has(extension)) return undefined + const root = workspaceDir.replace(/\\/g, '/').replace(/\/+$/, '') + return { workspacePath, absolutePath: `${root}/${workspacePath}` } +} diff --git a/src/shared/stores/workflowsStore.ts b/src/shared/stores/workflowsStore.ts index f6fb829c..0579f3e3 100644 --- a/src/shared/stores/workflowsStore.ts +++ b/src/shared/stores/workflowsStore.ts @@ -98,7 +98,7 @@ interface LegacyWorkflow { // Source-only nodes have no target handle; sink-only nodes have no source handle. // An edge into/out of the wrong side can't resolve a handle and makes React Flow // warn ("Couldn't create edge for target handle id: null") on every render. -export const NODE_TYPES_WITHOUT_TARGET = new Set(['imageNode', 'textNode', 'meshNode', 'sceneNode', 'inputNode', 'forEachNode']) +export const NODE_TYPES_WITHOUT_TARGET = new Set(['imageNode', 'textNode', 'meshNode', 'sceneNode', 'videoNode', 'inputNode', 'forEachNode']) export const NODE_TYPES_WITHOUT_SOURCE = new Set(['outputNode', 'previewNode']) function sanitizeEdges(nodes: WFNode[], edges: WFEdge[]): WFEdge[] { diff --git a/src/shared/types/electron.d.ts b/src/shared/types/electron.d.ts index 6d929962..77baf080 100644 --- a/src/shared/types/electron.d.ts +++ b/src/shared/types/electron.d.ts @@ -14,8 +14,8 @@ import type { export interface ExtensionNode { id: string name: string - input: 'image' | 'text' | 'mesh' | 'audio' | 'scene' - inputs?: ('image' | 'text' | 'mesh' | 'audio' | 'scene')[] // multi-input nodes; overrides input when set + input: 'image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video' + inputs?: ('image' | 'text' | 'mesh' | 'audio' | 'scene' | 'video')[] // multi-input nodes; overrides input when set inputLabels?: string[] // display labels per input slot (e.g. positive/negative) output: 'image' | 'text' | 'mesh' | 'audio' | 'scene' paramsSchema: ParamSchema[] @@ -185,6 +185,7 @@ declare global { } fs: { selectImage: () => Promise + selectVideo: () => Promise<{ workspacePath: string; absolutePath: string } | null> selectMeshFile: () => Promise saveModel: (defaultName: string) => Promise readFileBase64: (filePath: string) => Promise