feat(workflows): support scene model artifacts

README.md

@@ -153,6 +153,18 @@ original behavior.
 ## Workflows
 Start with a basic workflow first. For example, on the "Workflows" tab, try: Image -> Generate Mesh -> Add to Scene. Make sure there is a connection between each of the steps. Go to the "Generate" tab, make sure the workflow is selected, then click on "Generate 3D Model". Click on "Settings/Logs/Errors" to see any issues.
 
+Model extensions may also declare `scene` as a node input or output. A scene is
+a workspace directory containing `scene-manifest.json` with schema
+`modly.scene-manifest.v1`; it is not an arbitrary JSON file. Use the **Load
+Scene** workflow node to select and validate an existing scene directory.
+Scene-capable generators implement `generate_artifact(input_kind,
+artifact_path, ...)`; legacy image generators and `POST /generate/from-image`
+remain unchanged. The generic `POST /generate/from-artifact` boundary currently
+accepts only `scene`, leaving future artifact kinds to separate reviewed changes.
+For this first contract, `scene` is model-only and must be declared as the single
+`input` value (not inside `inputs`); process and mixed-input scene nodes are rejected.
+Model nodes may still accept multiple images and produce a scene.
+
 
 ## Modly CLI
 

api/README.md

@@ -29,6 +29,7 @@ uvicorn main:app --host 127.0.0.1 --port 8765 --reload
 | GET | `/model/status` | Model download / load status |
 | GET | `/model/download` | SSE stream of download progress |
 | POST | `/generate/from-image` | Start image-to-3D job |
+| POST | `/generate/from-artifact` | Start a typed-artifact model job (`scene` only) |
 | GET | `/generate/status/{job_id}` | Poll job status |
 
 ## Model

api/routers/generation.py

@@ -4,7 +4,8 @@
 import time
 import traceback
 import uuid
-from typing import Dict
+from pathlib import Path
+from typing import Dict, Optional, Union
 from fastapi import APIRouter, File, Form, UploadFile, HTTPException, BackgroundTasks
 from services.generators.base import smooth_progress, GenerationCancelled
 
@@ -14,7 +15,8 @@
 # binding captured at import would keep writing output to the old directory.
 import services.generator_registry as registry
 from services.generator_registry import generator_registry
-from schemas.generation import JobStatus
+from schemas.generation import GenerateFromArtifactRequest, JobStatus
+from services.artifact_input import TypedArtifactInput, validate_artifact_input
 
 router = APIRouter(tags=["generation"])
 
@@ -105,6 +107,7 @@ async def generate_from_image(
     # Verify the requested model exists in the registry
     try:
         generator_registry.get_generator(model_id)
+        output_kind = generator_registry.get_manifest(model_id).get("output", "mesh")
     except ValueError as e:
         raise HTTPException(400, str(e))
 
@@ -131,11 +134,49 @@ async def generate_from_image(
     _jobs[job_id] = job
     _cancel_events[job_id] = threading.Event()
 
-    background_tasks.add_task(_run_generation, job_id, image_bytes, full_params, collection)
+    background_tasks.add_task(
+        _run_generation, job_id, image_bytes, full_params, collection, output_kind, model_id
+    )
 
     return {"job_id": job_id}
 
 
+_RESERVED_ARTIFACT_PARAMS = {
+    "artifact_path", "input_kind", "input_path", "scene_path", "scene_manifest_path",
+}
+
+
+@router.post("/from-artifact")
+async def generate_from_artifact(
+    request: GenerateFromArtifactRequest,
+    background_tasks: BackgroundTasks,
+):
+    """Queue a validated typed artifact without serializing it as image bytes."""
+    try:
+        manifest = generator_registry.get_manifest(request.model_id)
+    except (KeyError, ValueError) as exc:
+        raise HTTPException(400, str(exc)) from exc
+    declared = manifest.get("inputs") or [manifest.get("input", "image")]
+    if request.input_kind not in declared:
+        raise HTTPException(400, f"Model {request.model_id} does not accept {request.input_kind} input")
+    try:
+        artifact = validate_artifact_input(registry.WORKSPACE_DIR, request.input_kind, request.input_path)
+    except ValueError as exc:
+        raise HTTPException(400, str(exc)) from exc
+
+    params = {k: v for k, v in request.params.items() if k not in _RESERVED_ARTIFACT_PARAMS}
+    params["scene_manifest_path"] = str(artifact.path)
+    collection = sanitize_collection(request.collection)
+    job_id = str(uuid.uuid4())
+    _purge_old_jobs()
+    _jobs[job_id] = JobStatus(job_id=job_id, status="pending", progress=0)
+    _cancel_events[job_id] = threading.Event()
+    background_tasks.add_task(
+        _run_generation, job_id, artifact, params, collection, manifest.get("output", "mesh"), request.model_id
+    )
+    return {"job_id": job_id}
+
+
 
 @router.get("/status/{job_id}")
 async def job_status(job_id: str):
@@ -170,7 +211,14 @@ async def cancel_job(job_id: str):
     return {"cancelled": True}
 
 
-async def _run_generation(job_id: str, image_bytes: bytes, params: dict, collection: str = "Default") -> None:
+async def _run_generation(
+    job_id: str,
+    model_input: Union[bytes, TypedArtifactInput],
+    params: dict,
+    collection: str = "Default",
+    output_kind: str = "mesh",
+    model_id: Optional[str] = None,
+) -> None:
     job = _jobs[job_id]
     job.status = "running"
 
@@ -189,8 +237,14 @@ def progress_cb(pct: int, step: str = "") -> None:
         # Check if the model needs to be loaded BEFORE calling get_active(),
         # because get_active() loads the model in a blocking manner.
         # active_status() is an instantaneous operation (simple dict lookup).
-        if not generator_registry.active_status()["loaded"]:
-            active = generator_registry.active_status()
+        get_generator = (lambda: generator_registry.get_ready_generator(model_id)) \
+            if model_id is not None else generator_registry.get_active
+        status_reader = getattr(generator_registry, "model_status", None)
+        status = (status_reader(model_id) if model_id is not None and status_reader
+                  else generator_registry.active_status() if model_id is None
+                  else {"name": model_id, "downloaded": True, "loaded": False})
+        if not status["loaded"]:
+            active = status
             model_name = active['name']
             init_label = f"Downloading {model_name}…" if not active['downloaded'] else f"Loading {model_name}…"
             progress_cb(0, init_label)
@@ -202,11 +256,11 @@ def progress_cb(pct: int, step: str = "") -> None:
             )
             load_thread.start()
             try:
-                gen = await loop.run_in_executor(None, generator_registry.get_active)
+                gen = await loop.run_in_executor(None, get_generator)
             finally:
                 stop_load_evt.set()
         else:
-            gen = await loop.run_in_executor(None, generator_registry.get_active)
+            gen = await loop.run_in_executor(None, get_generator)
 
         if job_id in _cancelled:
             return
@@ -217,18 +271,39 @@ def progress_cb(pct: int, step: str = "") -> None:
         gen.outputs_dir = coll_dir
 
         cancel_event = _cancel_events.get(job_id)
-        import inspect
-        supports_cancel = "cancel_event" in inspect.signature(gen.generate).parameters
-        output_path = await loop.run_in_executor(
-            None,
-            lambda: gen.generate(image_bytes, params, progress_cb, cancel_event)
-                    if supports_cancel
-                    else gen.generate(image_bytes, params, progress_cb),
-        )
+        if isinstance(model_input, TypedArtifactInput):
+            # Revalidate just before crossing the inference boundary. The
+            # subprocess runner repeats this check inside the worker.
+            from services.artifact_input import revalidate_artifact_input
+            model_input = revalidate_artifact_input(registry.WORKSPACE_DIR, model_input)
+            import inspect
+            supports_cancel = "cancel_event" in inspect.signature(gen.generate_artifact).parameters
+            output_path = await loop.run_in_executor(
+                None,
+                lambda: gen.generate_artifact(model_input.kind, model_input.path, params, progress_cb, cancel_event)
+                        if supports_cancel else gen.generate_artifact(model_input.kind, model_input.path, params, progress_cb),
+            )
+        else:
+            import inspect
+            supports_cancel = "cancel_event" in inspect.signature(gen.generate).parameters
+            output_path = await loop.run_in_executor(
+                None,
+                lambda: gen.generate(model_input, params, progress_cb, cancel_event)
+                        if supports_cancel else gen.generate(model_input, params, progress_cb),
+            )
 
         if job_id in _cancelled:
             return
 
+        output_path = Path(output_path).resolve(strict=True)
+        if output_kind == "scene":
+            from services.scene_input import validate_scene_input
+            try:
+                output_relative = output_path.relative_to(registry.WORKSPACE_DIR.resolve())
+                output_path = validate_scene_input(registry.WORKSPACE_DIR, output_relative.as_posix())
+            except (OSError, ValueError) as exc:
+                raise ValueError("Generated scene output is not a valid workspace scene") from exc
+
         job.status   = "done"
         job.progress = 100
         _completed_at[job_id] = time.monotonic()

api/runner.py

@@ -151,6 +151,32 @@ def _apply_manifest_metadata(gen, manifest: dict, node: dict) -> None:
     gen._params_schema = node.get("params_schema") or manifest.get("params_schema", [])
 
 
+def decode_model_input(msg: dict):
+    """Decode legacy image bytes or revalidate a typed artifact in the worker."""
+    if "input" not in msg:
+        return base64.b64decode(msg["image_b64"])
+    value = msg["input"]
+    if not isinstance(value, dict) or set(value) != {"kind", "path"}:
+        raise ValueError("Typed artifact input must contain exactly kind and path")
+    from services.artifact_input import TypedArtifactInput, revalidate_artifact_input
+    typed = TypedArtifactInput(kind=value.get("kind"), path=Path(value.get("path", "")))
+    return revalidate_artifact_input(WORKSPACE_DIR, typed)
+
+
+def validate_requested_model(msg: dict, manifest: dict, node: dict) -> None:
+    """Reject requests routed to a worker for a different manifest node."""
+    requested = msg.get("model_id")
+    if requested is None:  # Backward compatibility with already-running legacy hosts.
+        return
+    expected = manifest["id"]
+    if node.get("id"):
+        expected = f"{expected}/{node['id']}"
+    if requested != expected:
+        raise ValueError(
+            f"Generation request model '{requested}' does not match worker model '{expected}'"
+        )
+
+
 # ------------------------------------------------------------------ #
 # Main loop
 # ------------------------------------------------------------------ #
@@ -201,10 +227,17 @@ def main() -> None:
 
             # ---- generate --------------------------------------------
             elif action == "generate":
+                validate_requested_model(msg, manifest, node)
                 cancel_evt = threading.Event()
                 _cancel[rid] = cancel_evt
-                image_bytes  = base64.b64decode(msg["image_b64"])
+                model_input  = decode_model_input(msg)
                 params       = msg.get("params", {})
+                if hasattr(model_input, "kind"):
+                    if not isinstance(params, dict):
+                        raise ValueError("Model params must be an object")
+                    reserved = {"artifact_path", "input_kind", "input_path", "scene_path", "scene_manifest_path"}
+                    params = {key: value for key, value in params.items() if key not in reserved}
+                    params["scene_manifest_path"] = str(model_input.path)
                 if msg.get("outputs_dir"):
                     gen.outputs_dir = Path(msg["outputs_dir"])
                     gen.outputs_dir.mkdir(parents=True, exist_ok=True)
@@ -217,7 +250,20 @@ def progress_cb(pct: int, step: str = "") -> None:
                         send({"type": "log", "level": "warning",
                               "message": ("Model was not loaded (earlier setup failure?); "
                                           "reloaded before generating.")})
-                    output_path = gen.generate(image_bytes, params, progress_cb, cancel_evt)
+                    if hasattr(model_input, "kind"):
+                        output_path = gen.generate_artifact(
+                            model_input.kind, model_input.path, params, progress_cb, cancel_evt
+                        )
+                    else:
+                        output_path = gen.generate(model_input, params, progress_cb, cancel_evt)
+                    if node.get("output") == "scene":
+                        from services.scene_input import validate_scene_input
+                        resolved_output = Path(output_path).resolve(strict=True)
+                        try:
+                            relative_output = resolved_output.relative_to(WORKSPACE_DIR.resolve(strict=True))
+                        except (OSError, ValueError) as exc:
+                            raise ValueError("Generated scene output is outside the workspace") from exc
+                        output_path = validate_scene_input(WORKSPACE_DIR, relative_output.as_posix())
                     send({"type": "done", "id": rid, "output_path": str(output_path)})
                 except Exception as exc:
                     # Detect GenerationCancelled by name to avoid import issues

api/schemas/generation.py

@@ -1,5 +1,5 @@
-from typing import Literal, Optional
-from pydantic import BaseModel
+from typing import Any, Literal, Optional
+from pydantic import BaseModel, Field
 
 
 class JobStatus(BaseModel):
@@ -9,3 +9,12 @@ class JobStatus(BaseModel):
     step: Optional[str] = None    # Human-readable current step
     output_url: Optional[str] = None
     error: Optional[str] = None
+
+
+class GenerateFromArtifactRequest(BaseModel):
+    """Generic typed-artifact request. Only scene is public in this release."""
+    input_kind: Literal["scene"]
+    input_path: str
+    model_id: str
+    collection: str = "Workflows"
+    params: dict[str, Any] = Field(default_factory=dict)

api/services/artifact_input.py

@@ -0,0 +1,29 @@
+"""Typed model artifact inputs shared by the API, worker bridge, and runner."""
+from dataclasses import dataclass
+from pathlib import Path
+
+from services.scene_input import validate_scene_input
+
+SUPPORTED_ARTIFACT_INPUTS = frozenset({"scene"})
+
+
+@dataclass(frozen=True)
+class TypedArtifactInput:
+    kind: str
+    path: Path
+
+
+def validate_artifact_input(workspace: Path, kind: str, input_path: str) -> TypedArtifactInput:
+    if kind not in SUPPORTED_ARTIFACT_INPUTS:
+        raise ValueError(f"Unsupported artifact input kind: {kind}")
+    if kind == "scene":
+        return TypedArtifactInput(kind="scene", path=validate_scene_input(workspace, input_path))
+    raise ValueError(f"Unsupported artifact input kind: {kind}")
+
+
+def revalidate_artifact_input(workspace: Path, value: TypedArtifactInput) -> TypedArtifactInput:
+    try:
+        relative = value.path.resolve(strict=True).relative_to(workspace.resolve(strict=True))
+    except (OSError, ValueError) as exc:
+        raise ValueError("Artifact input is outside the workspace") from exc
+    return validate_artifact_input(workspace, value.kind, relative.as_posix())

api/services/extension_process.py

@@ -291,16 +291,18 @@ def generate(
         progress_cb: Optional[Callable[[int, str], None]] = None,
         cancel_event: Optional[threading.Event] = None,
     ) -> Path:
-        from services.generators.base import GenerationCancelled
+        return self._generate_request(
+            {"image_b64": base64.b64encode(image_bytes).decode()},
+            params, progress_cb, cancel_event,
+        )
 
-        req_id = str(uuid.uuid4())
-        self._send({
-            "action":      "generate",
-            "id":          req_id,
-            "image_b64":   base64.b64encode(image_bytes).decode(),
-            "params":      params,
-            "outputs_dir": str(self.outputs_dir) if self.outputs_dir else None,
-        })
+    def _receive_generation(
+        self,
+        req_id: str,
+        progress_cb: Optional[Callable[[int, str], None]],
+        cancel_event: Optional[threading.Event],
+    ) -> Path:
+        from services.generators.base import GenerationCancelled
 
         # Grace period after sending a cooperative cancel before hard-killing
         # the subprocess. Long enough to let generators that check cancel_event
@@ -377,6 +379,41 @@ def generate(
             elif t == "log":
                 print(f"[{self.MODEL_ID}] {msg.get('message', '')}", file=sys.stderr)
 
+    def generate_artifact(
+        self,
+        input_kind: str,
+        artifact_path: Path,
+        params: dict,
+        progress_cb: Optional[Callable[[int, str], None]] = None,
+        cancel_event: Optional[threading.Event] = None,
+    ) -> Path:
+        """Send a typed artifact envelope to the isolated runner."""
+        from services.artifact_input import TypedArtifactInput, revalidate_artifact_input
+        from services.generator_registry import WORKSPACE_DIR
+
+        validated = revalidate_artifact_input(
+            WORKSPACE_DIR, TypedArtifactInput(kind=input_kind, path=artifact_path)
+        )
+        return self._generate_request(
+            {"input": {"kind": validated.kind, "path": str(validated.path)}},
+            params, progress_cb, cancel_event,
+        )
+
+    def _generate_request(
+        self,
+        input_payload: dict,
+        params: dict,
+        progress_cb: Optional[Callable[[int, str], None]],
+        cancel_event: Optional[threading.Event],
+    ) -> Path:
+        req_id = str(uuid.uuid4())
+        self._send({
+            "action": "generate", "id": req_id, "model_id": self.MODEL_ID,
+            **input_payload, "params": params,
+            "outputs_dir": str(self.outputs_dir) if self.outputs_dir else None,
+        })
+        return self._receive_generation(req_id, progress_cb, cancel_event)
+
     def params_schema(self) -> list:
         return self._params_schema