From 41b01393bc29dd48cfb552d132b8b3a3371ca5ae Mon Sep 17 00:00:00 2001 From: ltx-desktop-bot Date: Fri, 14 Aug 2026 12:11:35 +0000 Subject: [PATCH] Sync from internal - 2026-08-14 --- NOTICES.md | 1 + backend/_routes/settings.py | 3 + backend/api_model_specs.py | 204 +++++- backend/api_types.py | 95 ++- backend/app_handler.py | 27 +- backend/frame_math.py | 19 + backend/handlers/download_handler.py | 45 +- backend/handlers/extend_handler.py | 22 +- backend/handlers/ic_lora_handler.py | 21 +- backend/handlers/models_handler.py | 207 ++++-- backend/handlers/pipelines_handler.py | 110 ++- .../handlers/prompt_enhancement_handler.py | 61 +- backend/handlers/retake_handler.py | 19 + backend/handlers/text_handler.py | 62 +- backend/handlers/video_generation_handler.py | 264 +++++--- backend/ltx2_server.py | 11 +- backend/pyproject.toml | 54 +- backend/runtime_config/lora_catalog.json | 28 + .../ltx_api_text_encoder_ids.py | 10 + backend/runtime_config/ltx_capabilities.py | 228 +++++++ backend/runtime_config/ltx_runtime_paths.py | 65 ++ .../runtime_config/model_download_specs.py | 357 ++++++++-- backend/services/a2v_pipeline/a2v_pipeline.py | 4 + .../a2v_pipeline/distilled_a2v_pipeline.py | 16 +- .../services/a2v_pipeline/ltx_a2v_pipeline.py | 36 +- .../fast_video_pipeline.py | 7 +- .../ltx_fast_video_pipeline.py | 74 +- .../ic_lora_pipeline/ic_lora_pipeline.py | 4 + .../ic_lora_pipeline/ltx_ic_lora_pipeline.py | 40 +- .../services/ltx_api_client/ltx_api_client.py | 6 +- .../ltx_api_client/ltx_api_client_impl.py | 8 +- backend/services/ltx_pipeline_common.py | 98 ++- .../services/patches/diffvae_decode_vram.py | 129 ++++ .../patches/diffvae_mps_tiling_budget.py | 144 ++++ .../services/patches/ic_lora_stage2_lora.py | 57 +- .../services/patches/natten_libnatten_gate.py | 44 ++ .../patches/safetensors_metadata_fix.py | 49 +- .../services/prompt_enhancement/__init__.py | 2 + .../prompt_enhancement/system_prompt.py | 17 + .../ltx_prompt_enhancer_pipeline.py | 64 +- .../retake_pipeline/ltx_retake_pipeline.py | 87 ++- .../retake_pipeline/retake_pipeline.py | 3 + backend/services/services_utils.py | 3 + .../services/text_encoder/ltx_text_encoder.py | 36 +- backend/services/text_encoder/text_encoder.py | 9 +- backend/state/app_settings.py | 13 +- backend/state/app_state_types.py | 14 +- backend/tests/conftest.py | 49 +- backend/tests/fakes/services.py | 86 ++- backend/tests/test_api_calls.py | 110 ++- backend/tests/test_diffusion_stage_cache.py | 6 +- backend/tests/test_diffvae_decode_vram.py | 159 +++++ .../tests/test_diffvae_mps_tiling_budget.py | 104 +++ backend/tests/test_generation.py | 639 +++++++++++++++++- backend/tests/test_health.py | 1 + backend/tests/test_ic_lora.py | 46 +- backend/tests/test_logging_policy.py | 5 +- backend/tests/test_lora_catalog.py | 25 + backend/tests/test_ltx_api_client.py | 18 +- backend/tests/test_ltx_capabilities.py | 118 ++++ backend/tests/test_ltx_runtime_paths.py | 122 ++++ backend/tests/test_model_download_specs.py | 188 +++++- backend/tests/test_models.py | 434 ++++++++++-- backend/tests/test_natten_libnatten_gate.py | 64 ++ backend/tests/test_prompt_enhancement.py | 163 ++++- backend/tests/test_response_models.py | 1 + backend/tests/test_settings.py | 63 +- backend/tests/test_state_actions.py | 36 +- backend/uv.lock | 267 ++++++-- frontend/App.tsx | 8 +- frontend/components/AssetPreviewModal.tsx | 158 +++++ frontend/components/FirstRunSetup.tsx | 218 ++++-- frontend/components/HfModelAccessGate.tsx | 98 +++ frontend/components/ICLoraPanel.tsx | 30 +- frontend/components/LtxUpgradePrompt.tsx | 55 +- frontend/components/SettingsModal.tsx | 359 ++++++---- frontend/components/SettingsPanel.tsx | 18 +- .../components/settings/BaseModelSection.tsx | 79 ++- frontend/contexts/AppSettingsContext.tsx | 19 +- frontend/generated/backend-openapi.json | 428 +++++++++++- frontend/generated/backend-openapi.ts | 128 +++- frontend/hooks/use-extend.ts | 3 + frontend/hooks/use-generation.ts | 7 +- frontend/hooks/use-hf-auth.ts | 2 +- frontend/hooks/use-hf-model-access.ts | 31 +- .../hooks/use-prompt-enhancer-provider.ts | 16 +- frontend/hooks/use-retake.ts | 18 + .../hooks/use-video-generation-model-specs.ts | 6 +- frontend/lib/generation-recovery-importers.ts | 11 +- frontend/lib/video-generation-model-specs.ts | 142 ++-- frontend/lib/video-resolution.ts | 25 +- frontend/types/project-model.ts | 6 +- frontend/views/GenSpace.tsx | 339 +++++----- frontend/views/editor/ClipContextMenu.tsx | 12 +- frontend/views/editor/ClipPropertiesPanel.tsx | 6 +- .../VideoEditorTimelineEditingPanel.tsx | 18 +- frontend/views/editor/editor-selectors.ts | 22 +- frontend/views/editor/editor-state.ts | 1 + frontend/views/editor/useRegeneration.ts | 7 +- package.json | 2 +- scripts/prepare-python.ps1 | 1 + scripts/prepare-python.sh | 2 +- 102 files changed, 6654 insertions(+), 1202 deletions(-) create mode 100644 backend/runtime_config/ltx_api_text_encoder_ids.py create mode 100644 backend/runtime_config/ltx_capabilities.py create mode 100644 backend/runtime_config/ltx_runtime_paths.py create mode 100644 backend/services/patches/diffvae_decode_vram.py create mode 100644 backend/services/patches/diffvae_mps_tiling_budget.py create mode 100644 backend/services/patches/natten_libnatten_gate.py create mode 100644 backend/tests/test_diffvae_decode_vram.py create mode 100644 backend/tests/test_diffvae_mps_tiling_budget.py create mode 100644 backend/tests/test_ltx_capabilities.py create mode 100644 backend/tests/test_ltx_runtime_paths.py create mode 100644 backend/tests/test_natten_libnatten_gate.py create mode 100644 frontend/components/AssetPreviewModal.tsx create mode 100644 frontend/components/HfModelAccessGate.tsx diff --git a/NOTICES.md b/NOTICES.md index 72fcbf0fe..62320b238 100644 --- a/NOTICES.md +++ b/NOTICES.md @@ -58,6 +58,7 @@ used by LTX Desktop. - **transformers** — Copyright (c) Hugging Face — Apache License 2.0 - **sentencepiece** — Copyright (c) Google LLC — Apache License 2.0 - **sageattention** — Copyright (c) Jintao Zhang et al. — Apache License 2.0 +- **natten** — Copyright (c) Ali Hassani, Steven Walton, et al. (SHI Labs) — Apache License 2.0 - **opencv-python-headless** — Copyright (c) OpenCV team — Apache License 2.0 - **fastapi** — Copyright (c) Sebastian Ramirez — MIT License - **uvicorn** — Copyright (c) Encode OSS Ltd. — BSD 3-Clause License diff --git a/backend/_routes/settings.py b/backend/_routes/settings.py index 468b9c5b6..fe4b2f3cd 100644 --- a/backend/_routes/settings.py +++ b/backend/_routes/settings.py @@ -42,4 +42,7 @@ def route_post_settings( ", ".join(sorted(changed_roots)) if changed_roots else "none", ) + if "use_conv_vae" in changed_roots: + handler.pipelines.unload_gpu_pipeline() + return StatusResponse(status="ok") diff --git a/backend/api_model_specs.py b/backend/api_model_specs.py index a7408cf8c..8e93f35d3 100644 --- a/backend/api_model_specs.py +++ b/backend/api_model_specs.py @@ -5,6 +5,8 @@ from api_types import ( GenerateVideoModelsSpecsResponse, GenerateVideoRequest, + LTXLocalModelId, + LTXOfferingCapabilitiesSpec, LTXVideoGenerationModelSpecItem, LTXVideoGenerationResolutionSpec, LTXVideoGenerationSpec, @@ -13,8 +15,20 @@ LTXVideoGenPipeline, LTXVideoGenResolution, ) +from runtime_config.ltx_capabilities import LtxOfferingCapabilities, api_caps, effective_local_caps from runtime_config.model_download_specs import get_latest_ltx_model_id, get_ltx_model_spec +# The concrete ltxv-api model id each Desktop-facing pipeline maps to when forced onto +# the API backend. SSOT for this mapping — video_generation_handler and the +# retake/extend handlers all import it from here. +FORCED_API_MODEL_MAP: dict[str, str] = { + "fast": "ltx-2-3-fast", + "pro": "ltx-2-3-pro", + "fast-2.5": "ltx-2-5-fast", + "pro-2.5": "ltx-2-5-pro", +} + + def _resolution_spec( *, fps_to_durations: dict[LTXVideoGenFps, tuple[LTXVideoGenDuration, ...]], @@ -58,7 +72,58 @@ def _resolution_spec( }, ), }, + # No A2V envelope: ltxv-api audio-to-video does not accept ltx-2-3-fast. + ), + ), + ( + "pro", + LTXVideoGenerationSpec( + display_name="LTX-2.3 Pro (API)", + supported_resolutions_durations={ + "1080p": _resolution_spec( + fps_to_durations={ + 24: (6, 8, 10), + 25: (6, 8, 10), + 48: (6, 8, 10), + 50: (6, 8, 10), + }, + ), + "1440p": _resolution_spec( + fps_to_durations={ + 24: (6, 8, 10), + 25: (6, 8, 10), + 48: (6, 8, 10), + 50: (6, 8, 10), + }, + ), + "2160p": _resolution_spec( + fps_to_durations={ + 24: (6, 8, 10), + 25: (6, 8, 10), + 48: (6, 8, 10), + 50: (6, 8, 10), + }, + ), + }, a2v_supported_resolutions_durations={ + "1080p": _resolution_spec( + fps_to_durations={ + 24: (6, 8, 10), + 25: (6, 8, 10), + 48: (6, 8, 10), + 50: (6, 8, 10), + }, + ), + }, + ), + ), + # ltx-2-5-fast: t2v/i2v duration envelope matches API Fast. A2V is 1080p + # (ltxv-api MAX_AUDIO_SECONDS for this model). + ( + "fast-2.5", + LTXVideoGenerationSpec( + display_name="LTX-2.5 Fast (API)", + supported_resolutions_durations={ "1080p": _resolution_spec( fps_to_durations={ 24: (6, 8, 10, 12, 14, 16, 18, 20), @@ -67,13 +132,39 @@ def _resolution_spec( 50: (6, 8, 10), }, ), + "1440p": _resolution_spec( + fps_to_durations={ + 24: (6, 8, 10), + 25: (6, 8, 10), + 48: (6, 8, 10), + 50: (6, 8, 10), + }, + ), + "2160p": _resolution_spec( + fps_to_durations={ + 24: (6, 8, 10), + 25: (6, 8, 10), + 48: (6, 8, 10), + 50: (6, 8, 10), + }, + ), + }, + a2v_supported_resolutions_durations={ + "1080p": _resolution_spec( + fps_to_durations={ + 24: (6, 8, 10), + 25: (6, 8, 10), + 48: (6, 8, 10), + 50: (6, 8, 10), + }, + ), }, ), ), ( - "pro", + "pro-2.5", LTXVideoGenerationSpec( - display_name="LTX-2.3 Pro (API)", + display_name="LTX-2.5 Pro (API)", supported_resolutions_durations={ "1080p": _resolution_spec( fps_to_durations={ @@ -115,27 +206,69 @@ def _resolution_spec( ) -def _pairs_to_items( - pairs: tuple[tuple[LTXVideoGenPipeline, LTXVideoGenerationSpec], ...], +def _capabilities_spec(caps: LtxOfferingCapabilities) -> LTXOfferingCapabilitiesSpec: + return LTXOfferingCapabilitiesSpec( + t2v=caps.t2v, + i2v=caps.i2v, + a2v=caps.a2v, + ic_lora=caps.ic_lora, + retake=caps.retake, + extend=caps.extend, + user_loras=caps.user_loras, + camera_motion=caps.camera_motion, + auto_duration=caps.auto_duration, + ) + + +def _item_with_caps( + pipeline: LTXVideoGenPipeline, + spec: LTXVideoGenerationSpec, + caps: LtxOfferingCapabilities, +) -> LTXVideoGenerationModelSpecItem: + return LTXVideoGenerationModelSpecItem( + pipeline=pipeline, + spec=spec.model_copy( + update={ + "capabilities": _capabilities_spec(caps), + "a2v_supported_resolutions_durations": ( + spec.a2v_supported_resolutions_durations if caps.a2v else None + ), + } + ), + ) + + +def get_local_video_generation_model_specs( + model_id: LTXLocalModelId | None = None, + *, + duration_head_ready: bool = False, ) -> list[LTXVideoGenerationModelSpecItem]: + resolved_id = model_id or get_latest_ltx_model_id() + local_model_spec = get_ltx_model_spec(resolved_id) + caps = effective_local_caps(resolved_id, duration_head_ready=duration_head_ready) return [ - LTXVideoGenerationModelSpecItem(pipeline=pipeline, spec=spec) - for pipeline, spec in pairs + _item_with_caps(pipeline, spec, caps) + for pipeline, spec in local_model_spec.supported_pipelines ] -def get_local_video_generation_model_specs() -> list[LTXVideoGenerationModelSpecItem]: - local_model_spec = get_ltx_model_spec(get_latest_ltx_model_id()) - return _pairs_to_items(local_model_spec.supported_pipelines) - - def get_api_video_generation_model_specs() -> list[LTXVideoGenerationModelSpecItem]: - return _pairs_to_items(ltx_api_model_specs) + # API Auto duration is a cloud capability. Do not gate it on the local DurationHead file. + return [ + _item_with_caps(pipeline, spec, api_caps(pipeline)) + for pipeline, spec in ltx_api_model_specs + ] -def build_generate_video_model_specs_response() -> GenerateVideoModelsSpecsResponse: +def build_generate_video_model_specs_response( + local_model_id: LTXLocalModelId | None = None, + *, + duration_head_ready: bool = False, +) -> GenerateVideoModelsSpecsResponse: return GenerateVideoModelsSpecsResponse( - local_models=get_local_video_generation_model_specs(), + local_models=get_local_video_generation_model_specs( + local_model_id, duration_head_ready=duration_head_ready + ), api_models=get_api_video_generation_model_specs(), ) @@ -146,7 +279,12 @@ def _get_resolution_spec( resolution: LTXVideoGenResolution, is_a2v: bool, ) -> LTXVideoGenerationResolutionSpec | None: - if is_a2v and item.spec.a2v_supported_resolutions_durations is not None: + if is_a2v: + # A pipeline with no a2v spec doesn't support audio-conditioned generation at + # all — must not fall back to the plain (non-a2v) matrix, or an unsupported + # pipeline looks valid here and only fails downstream at the LTX API. + if item.spec.a2v_supported_resolutions_durations is None: + return None resolution_map = item.spec.a2v_supported_resolutions_durations else: resolution_map = item.spec.supported_resolutions_durations @@ -161,12 +299,36 @@ def get_supported_durations( return list(resolution_spec.fps_to_durations.get(fps, [])) +def supported_duration_range( + item: LTXVideoGenerationModelSpecItem, + *, + resolution: LTXVideoGenResolution, + fps: LTXVideoGenFps, + is_a2v: bool = False, +) -> tuple[LTXVideoGenDuration, LTXVideoGenDuration]: + resolution_spec = _get_resolution_spec(item, resolution=resolution, is_a2v=is_a2v) + if resolution_spec is None: + raise KeyError(resolution) + durations = get_supported_durations(resolution_spec, fps=fps) + if not durations: + raise KeyError(fps) + return min(durations), max(durations) + + def validate_generate_video_request( req: GenerateVideoRequest, *, use_api_specs: bool, + local_model_id: LTXLocalModelId | None = None, + duration_head_ready: bool = False, ) -> str | None: - items = get_api_video_generation_model_specs() if use_api_specs else get_local_video_generation_model_specs() + items = ( + get_api_video_generation_model_specs() + if use_api_specs + else get_local_video_generation_model_specs( + local_model_id, duration_head_ready=duration_head_ready + ) + ) item = next((candidate for candidate in items if candidate.pipeline == req.model), None) generation_backend = "api" if use_api_specs else "local" generation_mode = "audio-to-video" if req.audioPath is not None else "image-to-video" if req.imagePath is not None else "text-to-video" @@ -190,6 +352,16 @@ def validate_generate_video_request( f"for pipeline '{req.model}' at resolution '{req.resolution}'" ) + if req.duration is None: + if req.audioPath is not None: + return "Automatic duration cannot be combined with audio-to-video" + if item.spec.capabilities is None or not item.spec.capabilities.auto_duration: + return ( + f"Automatic duration is not supported for {generation_backend} " + f"pipeline '{req.model}'" + ) + return None + supported_durations = get_supported_durations(resolution_spec, fps=req.fps) if req.duration not in supported_durations: return ( diff --git a/backend/api_types.py b/backend/api_types.py index 7defc671e..b14dbef2a 100644 --- a/backend/api_types.py +++ b/backend/api_types.py @@ -6,7 +6,7 @@ from typing import Annotated from typing import Literal, NamedTuple, TypeAlias -from pydantic import BaseModel, ConfigDict, Field, JsonValue, StringConstraints, model_validator +from pydantic import BaseModel, ConfigDict, Field, JsonValue, StringConstraints, field_validator, model_validator NonEmptyPrompt = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] ModelCheckpointID = Literal[ @@ -15,13 +15,25 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo", ] -LTXLocalModelId = Literal["ltx-2.3-22b-distilled-1.1", "ltx-2.3-22b-distilled"] +LTXLocalModelId = Literal[ + "ltx-2.5-22b-distilled", + "ltx-2.3-22b-distilled-1.1", + "ltx-2.3-22b-distilled", +] class ImageConditioningInput(NamedTuple): @@ -251,6 +263,10 @@ class ActiveDownloadResponse(BaseModel): class LtxDownloadRecommendationResponse(BaseModel): status: Literal["download"] cps_to_download: list[ModelCheckpointID] + # Checkpoints left out of cps_to_download only because an LTX API key covers what they do. + # Offered as an opt-in so a user who wants to generate offline can take the download now + # instead of discovering later that the key is the only thing making generation work. + optional_cp_ids: list[ModelCheckpointID] = [] class LtxUpgradeRecommendationResponse(BaseModel): @@ -259,6 +275,10 @@ class LtxUpgradeRecommendationResponse(BaseModel): upgrade_message: str | None = None cps_to_download: list[ModelCheckpointID] cps_to_delete: list[ModelCheckpointID] + # True when deleting the old bundle would also remove built-in Union Control IC-LoRA + # (2.3 → 2.5). The upgrade UI defaults "delete old" off in that case so users don't + # silently lose the easy path back to depth/canny/pose control. + loses_built_in_control: bool = False class LtxOkRecommendationResponse(BaseModel): @@ -276,12 +296,29 @@ class ImageGenRecommendationResponse(BaseModel): class LtxIcLoraRecommendationResponse(BaseModel): cps_to_download: list[ModelCheckpointID] + # False when the active model has no built-in Union Control IC-LoRA (LTX 2.5): distinct from + # "supported but already downloaded" (both have empty cps_to_download). + supported: bool = True class TextEncoderRecommendationResponse(BaseModel): cp_to_download: ModelCheckpointID | None expected_size_bytes: int expected_size_gb: float + # False when the active model can't be encoded by the LTX API, making the local encoder + # mandatory instead of one of two interchangeable options. + api_encoding_supported: bool + ltx_version_label: str + # False when no generative checkpoint local Enhance can run is downloaded, so Enhance + # needs Gemini. True if the preferred enhancer *or* a fallback (Gemma 3 on 2.5) is present. + local_enhancement_supported: bool + # The separate generative checkpoint local Enhance prefers, when the encoder can only encode + # (2.5). None when the encoder enhances too (2.3), so there's no extra download to offer. + local_enhancer_cp: ModelCheckpointID | None + local_enhancer_expected_size_gb: float | None + # The checkpoint Enhance will actually load, after E2B-then-Gemma-3 fallback. None if local + # Enhance cannot run. Equal to local_enhancer_cp when the preferred extra model is present. + active_local_enhancer_cp: ModelCheckpointID | None class LtxModelVersionItem(BaseModel): @@ -303,7 +340,7 @@ class SetActiveLtxModelRequest(BaseModel): model_id: LTXLocalModelId -CheckpointRole = Literal["base", "upscaler", "text_encoder", "image", "support"] +CheckpointRole = Literal["base", "upscaler", "text_encoder", "vae", "image", "support"] class CheckpointDescriptor(BaseModel): @@ -342,17 +379,32 @@ class LtxInsufficientFundsErrorResponse(BaseModel): LTXVideoGenResolution: TypeAlias = Literal["540p", "720p", "1080p", "1440p", "2160p"] LTXVideoGenDuration: TypeAlias = Literal[5, 6, 8, 10, 12, 14, 16, 18, 20] LTXVideoGenFps: TypeAlias = Literal[24, 25, 48, 50] -LTXVideoGenPipeline: TypeAlias = Literal["fast", "pro"] +LTXVideoGenPipeline: TypeAlias = Literal["fast", "pro", "fast-2.5", "pro-2.5"] class LTXVideoGenerationResolutionSpec(BaseModel): fps_to_durations: dict[LTXVideoGenFps, list[LTXVideoGenDuration]] +class LTXOfferingCapabilitiesSpec(BaseModel): + """Feature flags for one local model or API pipeline. Pixel maps stay backend-only.""" + + t2v: bool + i2v: bool + a2v: bool + ic_lora: bool + retake: bool + extend: bool + user_loras: bool + camera_motion: bool + auto_duration: bool + + class LTXVideoGenerationSpec(BaseModel): display_name: str supported_resolutions_durations: dict[LTXVideoGenResolution, LTXVideoGenerationResolutionSpec] a2v_supported_resolutions_durations: dict[LTXVideoGenResolution, LTXVideoGenerationResolutionSpec] | None = None + capabilities: LTXOfferingCapabilitiesSpec | None = None class LTXVideoGenerationModelSpecItem(BaseModel): @@ -393,7 +445,8 @@ class GenerateVideoRequest(BaseModel): model: LTXVideoGenPipeline = "fast" cameraMotion: VideoCameraMotion = "none" negativePrompt: str = "" - duration: LTXVideoGenDuration = 5 + # None = automatic duration (API 2.5 t2v/i2v only): the worker picks length from the prompt. + duration: LTXVideoGenDuration | None = 5 fps: LTXVideoGenFps = 24 audio: bool = False imagePath: str | None = None @@ -466,6 +519,10 @@ def _validate_input_image_mode(self) -> "SuggestGapPromptRequest": RetakeMode: TypeAlias = Literal["replace_audio_and_video", "replace_video", "replace_audio"] +# ltxv-api /v1/retake and /v2/extend accept ltx-2-pro / ltx-2-3-pro. Desktop maps +# those to pipeline "pro" — narrower than LTXVideoGenPipeline on purpose. +RetakeExtendModel: TypeAlias = Literal["pro"] + class TargetResolution(BaseModel): """Desired output resolution for a local generation. The backend corrects it to the @@ -486,6 +543,8 @@ class RetakeRequest(BaseModel): prompt: str = "" mode: RetakeMode = "replace_audio_and_video" resolution: TargetResolution | None = None + # API-mode only; ignored for local generation (there is no local model choice). + model: RetakeExtendModel = "pro" ExtendMode: TypeAlias = Literal["start", "end"] @@ -501,6 +560,8 @@ class ExtendRequest(BaseModel): prompt: str = "" mode: ExtendMode = "end" resolution: TargetResolution | None = None + # API-mode only; ignored for local generation (there is no local model choice). + model: RetakeExtendModel = "pro" # Extend returns the same shapes as retake (video file, remote payload, or cancelled). @@ -623,6 +684,10 @@ class IcLoraGenerateRequest(BaseModel): # instruction blocks by purpose without string-matching display titles. "tips" also covers # the "Notes" title; "summary" covers "What it does". InstructionKind: TypeAlias = Literal["summary", "prompting", "tips", "input"] +# Families a catalog adapter is known to run on. Distinct from `base_model` (what it was +# trained on). 2.3 adapters often run on 2.5; that is recorded here after validation, not +# inferred from the training tag. +LtxCatalogModelFamily: TypeAlias = Literal["LTX-2.3", "LTX-2.5"] # Where a trigger word/phrase must appear in the prompt. Not set (and not applicable) when # `prompt_template` is present, since the template's placeholder position already encodes it. TriggerPlacement: TypeAlias = Literal["first_token", "anywhere"] @@ -756,6 +821,10 @@ def _empty_tags() -> list[str]: return [] +def _default_supported_models() -> list[LtxCatalogModelFamily]: + return ["LTX-2.3", "LTX-2.5"] + + class LoraCatalogItem(BaseModel): """Base catalog entry — used as-is for a plain LoRA.""" model_config = ConfigDict(strict=True) @@ -772,7 +841,11 @@ class LoraCatalogItem(BaseModel): media: MediaSpec | None = None # ISO-8601 date the model was created at the source (e.g. HuggingFace). created_at: str | None = None + # What the adapter was trained on. Not a compatibility list — see `supported_models`. base_model: str | None = None + # Families this adapter is known to run on. Currently both, so 2.5 testing can + # use the full catalog; trim after validation. + supported_models: list[LtxCatalogModelFamily] = Field(default_factory=_default_supported_models) tags: list[str] = Field(default_factory=_empty_tags) # Trigger phrase to include in the prompt if the LoRA needs one (e.g. "ADD WATER"). trigger: str | None = None @@ -794,6 +867,18 @@ class LoraCatalogItem(BaseModel): # Enforced for the catalog IC-LoRA path; plain-LoRA t2v enforcement is not wired yet. allows_empty_prompt: bool = False + def supports_family(self, family: LtxCatalogModelFamily) -> bool: + return family in self.supported_models + + @field_validator("supported_models") + @classmethod + def _check_supported_models(cls, value: list[LtxCatalogModelFamily]) -> list[LtxCatalogModelFamily]: + if not value: + raise ValueError("supported_models must not be empty") + if len(value) != len(set(value)): + raise ValueError("supported_models must be unique") + return value + @model_validator(mode="after") def _check_trigger_placement(self) -> "LoraCatalogItem": if self.prompt_template is not None: diff --git a/backend/app_handler.py b/backend/app_handler.py index f9e1647eb..9a7f6b331 100644 --- a/backend/app_handler.py +++ b/backend/app_handler.py @@ -171,12 +171,27 @@ def __init__( self.generation = GenerationHandler(state=self.state, lock=self._lock, config=config) + # Before video generation: local text encoding has no server-side rewrite step, so the + # generation path runs this enhancer itself. + self.prompt_enhancement = PromptEnhancementHandler( + state=self.state, + lock=self._lock, + generation_handler=self.generation, + pipelines_handler=self.pipelines, + text_handler=self.text, + lora_catalog_provider=lora_catalog_provider, + prompt_enhancer_pipeline_class=prompt_enhancer_pipeline_class, + gemini_pipeline=GeminiPromptEnhancerPipeline(http), + config=config, + ) + self.video_generation = VideoGenerationHandler( state=self.state, lock=self._lock, generation_handler=self.generation, pipelines_handler=self.pipelines, text_handler=self.text, + prompt_enhancement_handler=self.prompt_enhancement, ltx_api_client=ltx_api_client, config=config, ) @@ -238,18 +253,6 @@ def __init__( config=config, ) - self.prompt_enhancement = PromptEnhancementHandler( - state=self.state, - lock=self._lock, - generation_handler=self.generation, - pipelines_handler=self.pipelines, - text_handler=self.text, - lora_catalog_provider=lora_catalog_provider, - prompt_enhancer_pipeline_class=prompt_enhancer_pipeline_class, - gemini_pipeline=GeminiPromptEnhancerPipeline(http), - config=config, - ) - self.downloads.cleanup_downloading_dir() self.load_persistent_state(default_settings) diff --git a/backend/frame_math.py b/backend/frame_math.py index cafab6d9f..94f7071d7 100644 --- a/backend/frame_math.py +++ b/backend/frame_math.py @@ -2,6 +2,25 @@ from __future__ import annotations +from dataclasses import dataclass + + +@dataclass(frozen=True) +class AutoDurationSpec: + """Ask DurationHead to pick length, clamped to this envelope (seconds).""" + + min_seconds: float + max_seconds: float + + +def snap_up_to_multiple(n: int, multiple: int) -> int: + """Round ``n`` up to the next multiple of ``multiple``. + + For pixel dimensions on a VAE/latent grid, where rounding down loses picture: a value + already on the grid is returned unchanged. + """ + return -(-n // multiple) * multiple + def snap_to_frame_grid(n: int, *, floor: int = 9) -> int: """Snap a frame count DOWN to the largest valid (n - 1) % 8 == 0 count, clamped to ``floor``. diff --git a/backend/handlers/download_handler.py b/backend/handlers/download_handler.py index d52d34feb..b3070a71f 100644 --- a/backend/handlers/download_handler.py +++ b/backend/handlers/download_handler.py @@ -8,6 +8,7 @@ from collections.abc import Callable, Iterable from threading import RLock from typing import TYPE_CHECKING +from pathlib import Path from uuid import uuid4 import requests as http_requests @@ -24,7 +25,7 @@ ModelCheckpointID, ) from handlers.base import StateHandlerBase, with_state_lock -from handlers.hf_auth_utils import optional_hf_token +from handlers.hf_auth_utils import optional_hf_token, require_hf_token from handlers.models_handler import ModelsHandler from runtime_config.model_download_specs import ( ALL_MODEL_CP_IDS, @@ -227,13 +228,28 @@ def _download_to_staging(self, cp_id: ModelCheckpointID, hf_token: str | None) - token=hf_token, ) else: + staging_root = resolve_downloading_dir(self.models_dir) self._model_downloader.download_file( repo_id=spec.repo_id, - filename=spec.name, - local_dir=str(resolve_downloading_path(self.models_dir, cp_id)), + filename=spec.download_filename, + local_dir=str(staging_root), on_progress=progress_cb, token=hf_token, ) + # Nested HF paths land under staging_root/ (e.g. vae/foo.safetensors + # for LTX 2.5); move to the spec's local relative_path before commit (e.g. + # ltx-2.5/foo.safetensors). + downloaded = staging_root / Path(spec.download_filename) + target = resolve_downloading_target_path(self.models_dir, cp_id) + if downloaded != target and downloaded.exists(): + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists(): + target.unlink() + downloaded.replace(target) + parent = downloaded.parent + while parent != staging_root and parent.exists() and not any(parent.iterdir()): + parent.rmdir() + parent = parent.parent def _commit_staged_checkpoint(self, cp_id: ModelCheckpointID) -> bool: src = resolve_downloading_target_path(self.models_dir, cp_id) @@ -281,9 +297,8 @@ def _download_worker(self, cp_ids: tuple[ModelCheckpointID, ...], *, atomic_comm self.finish_download() return - # Base/bundled checkpoints are public — never require sign-in. Attach the in-app token - # if the user happens to be signed in, otherwise download anonymously. (Gated *catalog* - # entries are handled separately in lora_catalog_handler and do require auth.) + # Most bundled checkpoints are public and download anonymously; attach the in-app token + # when signed in. Gated specs (LTX 2.5) are rejected up front in start_model_download. hf_token = optional_hf_token(self.state, self._lock) try: @@ -327,6 +342,9 @@ def start_model_download(self, *, download_type: str, cp_ids: set[ModelCheckpoin else: raise HTTPError(400, "INVALID_DOWNLOAD_REQUEST") + if any(get_model_cp_spec(cp_id).gated for cp_id in ordered_cp_ids): + require_hf_token(self.state, self._lock) + # Check-and-set in one lock acquisition (RLock is reentrant, so start_download's own # lock nests fine) — otherwise two concurrent calls both pass the guard and the second # clobbers the first's session, racing on the same staging dir. @@ -343,14 +361,21 @@ def start_model_download(self, *, download_type: str, cp_ids: set[ModelCheckpoin return session_id def check_model_access(self, cp_ids: set[ModelCheckpointID]) -> CheckModelAccessResponse: + gated_repo_ids = { + spec.repo_id for spec in map(get_model_cp_spec, cp_ids) if spec.gated + } repo_ids = {get_model_cp_spec(cp_id).repo_id for cp_id in cp_ids} - # Bundled checkpoints are public, so when the user isn't signed in there's nothing to - # verify — treat them all as authorized. When signed in, do a real per-repo check (covers - # any future gated checkpoint without forcing sign-in for the common public case). + # Signed out there is no token to verify with: public repos download fine, gated ones + # would 401 mid-transfer, so report them up front rather than letting the download start. hf_token = optional_hf_token(self.state, self._lock) if hf_token is None: - return CheckModelAccessResponse(access={repo_id: "authorized" for repo_id in repo_ids}) + return CheckModelAccessResponse( + access={ + repo_id: "not_authorized" if repo_id in gated_repo_ids else "authorized" + for repo_id in repo_ids + } + ) access: dict[str, ModelAccessStatus] = {} for repo_id in sorted(repo_ids): diff --git a/backend/handlers/extend_handler.py b/backend/handlers/extend_handler.py index 22c67be50..74e3d66c0 100644 --- a/backend/handlers/extend_handler.py +++ b/backend/handlers/extend_handler.py @@ -17,11 +17,13 @@ ExtendRequest, ExtendResponse, RetakeCancelledResponse, + RetakeExtendModel, RetakePayloadResponse, RetakeVideoResponse, TargetResolution, ) from _routes._errors import HTTPError +from api_model_specs import FORCED_API_MODEL_MAP from handlers.base import StateHandlerBase from handlers.generation_handler import GenerationHandler from handlers.pipelines_handler import PipelinesHandler @@ -33,6 +35,8 @@ resolve_target_resolution, validate_source_video_path, ) +from runtime_config.ltx_capabilities import local_caps, supports +from runtime_config.model_download_specs import resolve_active_ltx_model_id from runtime_config.runtime_config import RuntimeConfig from services.ltx_api_client.ltx_api_client import LTXAPIClientError from services.interfaces import LTXAPIClient @@ -80,7 +84,21 @@ def run(self, req: ExtendRequest) -> ExtendResponse: ): # The cloud preserves source resolution (no resolution param); resolution # selection is local-only. - return self._run_api_extend(video_file=video_file, duration=duration, prompt=prompt, mode=mode) + return self._run_api_extend( + video_file=video_file, duration=duration, prompt=prompt, mode=mode, model=req.model, + ) + + model_id = resolve_active_ltx_model_id( + self.models_dir, self.state.app_settings.active_ltx_model_id + ) + if model_id is None: + raise HTTPError(409, "NO_DOWNLOADED_LTX_MODEL") + if not supports(local_caps(model_id), "extend"): + raise HTTPError( + 409, + "Extend is not supported for the active LTX model.", + code="UNSUPPORTED_EXTEND", + ) return self._run_local_extend( video_file=video_file, duration=duration, prompt=prompt, mode=mode, resolution=req.resolution @@ -93,6 +111,7 @@ def _run_api_extend( duration: float, prompt: str, mode: ExtendMode, + model: RetakeExtendModel, ) -> ExtendResponse: api_key = self.state.app_settings.ltx_api_key if not api_key: @@ -117,6 +136,7 @@ def _run_api_extend( duration=duration, prompt=prompt, mode=mode, + model=FORCED_API_MODEL_MAP[model], ) if result.video_bytes is not None: diff --git a/backend/handlers/ic_lora_handler.py b/backend/handlers/ic_lora_handler.py index f09191df4..3c21f0660 100644 --- a/backend/handlers/ic_lora_handler.py +++ b/backend/handlers/ic_lora_handler.py @@ -30,12 +30,13 @@ from handlers.pipelines_handler import PipelinesHandler from handlers.text_handler import TextHandler from ic_lora_preprocessing import MediaArtifact, OutpaintParams, PreprocessingContext, run_preprocessing +from runtime_config.ltx_capabilities import local_caps, supports from runtime_config.model_download_specs import ( DEPTH_PROCESSOR_CP_ID, find_installed_ic_lora_path, - get_downloaded_ltx_model_id, get_existing_cp_path, get_ltx_model_spec, + resolve_active_ltx_model_id, resolve_ic_lora_path, ) from runtime_config.models_scanner import is_ic_lora_file, resolve_lora_ref @@ -131,10 +132,26 @@ def _build_conditioning_frame( raise HTTPError(400, f"Unsupported conditioning_type: {conditioning_type}") def _require_ic_lora_model_paths(self, conditioning_type: ConditioningType) -> tuple[Path, Path | None]: - model_id = get_downloaded_ltx_model_id(self.models_dir) + model_id = resolve_active_ltx_model_id( + self.models_dir, self.state.app_settings.active_ltx_model_id + ) if model_id is None: raise HTTPError(409, "NO_DOWNLOADED_LTX_MODEL") + if not supports(local_caps(model_id), "ic_lora"): + raise HTTPError( + 409, + "Built-in control IC-LoRA is not available for the active LTX model. " + "Switch to an LTX 2.3 local model to use depth/canny control.", + code="UNSUPPORTED_IC_LORA", + ) ic_loras_spec = get_ltx_model_spec(model_id).ic_loras_spec + if ic_loras_spec is None: + raise HTTPError( + 409, + "Built-in control IC-LoRA is not available for the active LTX model. " + "Switch to an LTX 2.3 local model to use depth/canny control.", + code="UNSUPPORTED_IC_LORA", + ) depth_model_path: Path | None = None match conditioning_type: case "canny": diff --git a/backend/handlers/models_handler.py b/backend/handlers/models_handler.py index 6589f6082..4eaa25bee 100644 --- a/backend/handlers/models_handler.py +++ b/backend/handlers/models_handler.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from dataclasses import dataclass from threading import RLock from typing import TYPE_CHECKING @@ -28,6 +29,7 @@ from handlers.base import StateHandlerBase from handlers.settings_handler import SettingsHandler from runtime_config.models_scanner import scan_models_dir +from runtime_config.ltx_capabilities import local_caps, supports from runtime_config.model_download_specs import ( ALL_LTX_LOCAL_MODEL_IDS, ALL_MODEL_CP_IDS, @@ -44,6 +46,9 @@ get_model_cp_spec, is_cp_downloaded, resolve_active_ltx_model_id, + resolve_downloaded_prompt_enhancer_cp, + selected_video_vae_cp, + unused_video_vae_cp, delete_cp_path, ) @@ -51,6 +56,8 @@ from runtime_config.runtime_config import RuntimeConfig from state.app_state_types import AppState +logger = logging.getLogger(__name__) + @dataclass(frozen=True, slots=True) class ResolvedUpgradeDownload: @@ -78,8 +85,15 @@ def _ensure_local_model_mode(self) -> None: raise HTTPError(409, "LOCAL_MODEL_RECOMMENDATIONS_DISABLED_IN_FORCE_API_MODE") def _current_downloaded_ltx_model_id(self) -> LTXLocalModelId | None: + # Upgrade / "do we already have latest" still keys off whatever transformers are on disk + # (newest-first), not the user-selected active version. return get_downloaded_ltx_model_id(self.models_dir) + def _current_active_ltx_model_id(self) -> LTXLocalModelId | None: + return resolve_active_ltx_model_id( + self.models_dir, self.state.app_settings.active_ltx_model_id + ) + def _has_api_key(self) -> bool: return bool(self.state.app_settings.ltx_api_key.strip()) @@ -90,15 +104,20 @@ def get_downloaded_checkpoints(self) -> set[ModelCheckpointID]: return {cp_id for cp_id in ALL_MODEL_CP_IDS if self.is_cp_downloaded(cp_id)} def _cp_role(self, cp_id: ModelCheckpointID) -> CheckpointRole: - # A base transformer of ANY version is "base" — not just the latest — so an older - # version's checkpoint isn't misclassified as a generic "support" model. - if any(cp_id == get_ltx_model_spec(model_id).model_cp for model_id in ALL_LTX_LOCAL_MODEL_IDS): - return "base" - spec = get_ltx_model_spec(get_latest_ltx_model_id()) - if cp_id == spec.upscale_cp: - return "upscaler" - if cp_id == spec.text_encoder_cp: - return "text_encoder" + # Walk every version, not only latest: otherwise a 2.5 VAE (or an older upscaler/TE) + # falls through to "support" and first-run tooltips call it a depth/edges/pose model. + for model_id in ALL_LTX_LOCAL_MODEL_IDS: + spec = get_ltx_model_spec(model_id) + if cp_id == spec.model_cp: + return "base" + if cp_id == spec.upscale_cp: + return "upscaler" + if cp_id == spec.text_encoder_cp: + return "text_encoder" + if cp_id in (spec.video_vae_cp, spec.video_vae_conv_cp, spec.audio_vae_cp): + return "vae" + if cp_id == spec.duration_head_cp: + return "support" if cp_id == IMG_GEN_MODEL_CP_ID: return "image" return "support" @@ -122,13 +141,45 @@ def describe_checkpoints(self, cp_ids: list[ModelCheckpointID]) -> DescribeCheck ) return DescribeCheckpointsResponse(checkpoints=descriptors) + def _use_conv_vae(self) -> bool: + from state.app_settings import resolved_use_conv_vae + + return resolved_use_conv_vae(self.state.app_settings) + def _get_required_ltx_cp_ids(self, model_id: LTXLocalModelId) -> set[ModelCheckpointID]: spec = get_ltx_model_spec(model_id) required: set[ModelCheckpointID] = {spec.model_cp, spec.upscale_cp} - if not self._has_api_key(): + selected_vae = selected_video_vae_cp(spec, use_conv_vae=self._use_conv_vae()) + if selected_vae is not None: + required.add(selected_vae) + # Conv VAE is always part of the 2.5 download set so Fast decode can be turned on + # later without a hidden extra fetch. DiffVAE stays selected-only (Mac default). + if spec.video_vae_conv_cp is not None: + required.add(spec.video_vae_conv_cp) + if spec.audio_vae_cp is not None: + required.add(spec.audio_vae_cp) + if spec.duration_head_cp is not None: + required.add(spec.duration_head_cp) + if not self._has_api_key() or not spec.supports_api_text_encoding: required.add(spec.text_encoder_cp) return required + def _get_optional_ltx_cp_ids(self, model_id: LTXLocalModelId) -> set[ModelCheckpointID]: + """Missing checkpoints the user can still choose to download. + + Includes the unused 2.5 DiffVAE when Fast decode is on, and any text encoder an + LTX API key excused. Never overlaps the required set. + """ + spec = get_ltx_model_spec(model_id) + optional: set[ModelCheckpointID] = set() + unused_vae = unused_video_vae_cp(spec, use_conv_vae=self._use_conv_vae()) + if unused_vae is not None: + optional.add(unused_vae) + if self._has_api_key() and spec.supports_api_text_encoding: + optional.add(spec.text_encoder_cp) + optional -= self._get_required_ltx_cp_ids(model_id) + return self._get_missing_cp_ids(optional) + def _get_missing_cp_ids(self, cp_ids: set[ModelCheckpointID]) -> set[ModelCheckpointID]: return {cp_id for cp_id in cp_ids if not self.is_cp_downloaded(cp_id)} @@ -138,6 +189,21 @@ def _get_upgrade_message(self, current_model_id: LTXLocalModelId, target_model_i return None return relevance.upgrade_messages.get(current_model_id) + def _maybe_add_upgrade_companion( + self, + cp_ids: set[ModelCheckpointID], + *, + current_cp: ModelCheckpointID | None, + target_cp: ModelCheckpointID | None, + ) -> None: + if ( + target_cp is not None + and current_cp != target_cp + and (current_cp is None or self.is_cp_downloaded(current_cp)) + and not self.is_cp_downloaded(target_cp) + ): + cp_ids.add(target_cp) + def _get_upgrade_dependency_downloads( self, current_model_id: LTXLocalModelId, @@ -147,34 +213,38 @@ def _get_upgrade_dependency_downloads( target_spec = get_ltx_model_spec(target_model_id) cp_ids: set[ModelCheckpointID] = {target_spec.model_cp} - if ( - current_spec.upscale_cp != target_spec.upscale_cp - and self.is_cp_downloaded(current_spec.upscale_cp) - and not self.is_cp_downloaded(target_spec.upscale_cp) - ): - cp_ids.add(target_spec.upscale_cp) - - if ( - current_spec.text_encoder_cp != target_spec.text_encoder_cp - and self.is_cp_downloaded(current_spec.text_encoder_cp) - and not self.is_cp_downloaded(target_spec.text_encoder_cp) - ): - cp_ids.add(target_spec.text_encoder_cp) - - current_ic_loras_spec = current_spec.ic_loras_spec - target_ic_loras_spec = target_spec.ic_loras_spec - ic_lora_pairs: tuple[tuple[ModelCheckpointID, ModelCheckpointID], ...] = ( - (current_ic_loras_spec.depth_cp, target_ic_loras_spec.depth_cp), - (current_ic_loras_spec.canny_cp, target_ic_loras_spec.canny_cp), - (current_ic_loras_spec.pose_cp, target_ic_loras_spec.pose_cp), + self._maybe_add_upgrade_companion( + cp_ids, current_cp=current_spec.upscale_cp, target_cp=target_spec.upscale_cp + ) + # Same rule as a fresh install: an LTX API key that can encode this version makes the + # text encoder optional, so don't force it onto the upgrade download. + if not self._has_api_key() or not target_spec.supports_api_text_encoding: + self._maybe_add_upgrade_companion( + cp_ids, current_cp=current_spec.text_encoder_cp, target_cp=target_spec.text_encoder_cp + ) + self._maybe_add_upgrade_companion( + cp_ids, current_cp=current_spec.video_vae_cp, target_cp=target_spec.video_vae_cp + ) + self._maybe_add_upgrade_companion( + cp_ids, current_cp=current_spec.video_vae_conv_cp, target_cp=target_spec.video_vae_conv_cp + ) + self._maybe_add_upgrade_companion( + cp_ids, current_cp=current_spec.audio_vae_cp, target_cp=target_spec.audio_vae_cp + ) + self._maybe_add_upgrade_companion( + cp_ids, current_cp=current_spec.duration_head_cp, target_cp=target_spec.duration_head_cp ) - for current_cp_id, target_cp_id in ic_lora_pairs: - if ( - current_cp_id != target_cp_id - and self.is_cp_downloaded(current_cp_id) - and not self.is_cp_downloaded(target_cp_id) - ): - cp_ids.add(target_cp_id) + + current_ic = current_spec.ic_loras_spec + target_ic = target_spec.ic_loras_spec + if current_ic is not None and target_ic is not None: + ic_lora_pairs: tuple[tuple[ModelCheckpointID, ModelCheckpointID], ...] = ( + (current_ic.depth_cp, target_ic.depth_cp), + (current_ic.canny_cp, target_ic.canny_cp), + (current_ic.pose_cp, target_ic.pose_cp), + ) + for current_cp_id, target_cp_id in ic_lora_pairs: + self._maybe_add_upgrade_companion(cp_ids, current_cp=current_cp_id, target_cp=target_cp_id) return cp_ids @@ -201,7 +271,11 @@ def get_ltx_recommendation(self) -> LtxRecommendationResponse: cps_to_download = self._ordered_cp_ids( self._get_missing_cp_ids(self._get_required_ltx_cp_ids(latest_model_id)) ) - return LtxDownloadRecommendationResponse(status="download", cps_to_download=cps_to_download) + return LtxDownloadRecommendationResponse( + status="download", + cps_to_download=cps_to_download, + optional_cp_ids=self._ordered_cp_ids(self._get_optional_ltx_cp_ids(latest_model_id)), + ) # A required checkpoint for the current model can be missing even when its base # transformer is present — e.g. a hotfixed shared companion (the 2x upscaler) that @@ -212,7 +286,11 @@ def get_ltx_recommendation(self) -> LtxRecommendationResponse: self._get_missing_cp_ids(self._get_required_ltx_cp_ids(current_model_id)) ) if missing_current: - return LtxDownloadRecommendationResponse(status="download", cps_to_download=missing_current) + return LtxDownloadRecommendationResponse( + status="download", + cps_to_download=missing_current, + optional_cp_ids=self._ordered_cp_ids(self._get_optional_ltx_cp_ids(current_model_id)), + ) if current_model_id == latest_model_id: return LtxOkRecommendationResponse(status="ok") @@ -223,12 +301,20 @@ def get_ltx_recommendation(self) -> LtxRecommendationResponse: cps_to_delete = self._ordered_cp_ids( self._get_upgrade_delete_cp_ids(current_model_id, latest_model_id) ) + current_spec = get_ltx_model_spec(current_model_id) + target_spec = get_ltx_model_spec(latest_model_id) + loses_control = ( + current_spec.ic_loras_spec is not None + and target_spec.ic_loras_spec is None + and bool(set(cps_to_delete) & set(get_ic_loras_cp_ids(current_spec.ic_loras_spec))) + ) return LtxUpgradeRecommendationResponse( status="upgrade", ltx_model_id=latest_model_id, upgrade_message=self._get_upgrade_message(current_model_id, latest_model_id), cps_to_download=cps_to_download, cps_to_delete=cps_to_delete, + loses_built_in_control=loses_control, ) def get_img_gen_recommendation(self) -> ImageGenRecommendationResponse: @@ -267,24 +353,47 @@ def _require_downloaded_ltx_model_id(self) -> LTXLocalModelId: raise HTTPError(409, "NO_DOWNLOADED_LTX_MODEL") return model_id + def _require_active_ltx_model_id(self) -> LTXLocalModelId: + model_id = self._current_active_ltx_model_id() + if model_id is None: + raise HTTPError(409, "NO_DOWNLOADED_LTX_MODEL") + return model_id + def get_ltx_ic_lora_recommendation(self) -> LtxIcLoraRecommendationResponse: self._ensure_local_model_mode() - model_id = self._require_downloaded_ltx_model_id() + model_id = self._require_active_ltx_model_id() spec = get_ltx_model_spec(model_id) + if not supports(local_caps(model_id), "ic_lora") or spec.ic_loras_spec is None: + return LtxIcLoraRecommendationResponse(cps_to_download=[], supported=False) required_cp_ids: set[ModelCheckpointID] = set(get_ic_loras_cp_ids(spec.ic_loras_spec)) required_cp_ids.add(DEPTH_PROCESSOR_CP_ID) cp_ids = self._get_missing_cp_ids(required_cp_ids) - return LtxIcLoraRecommendationResponse(cps_to_download=self._ordered_cp_ids(cp_ids)) + return LtxIcLoraRecommendationResponse( + cps_to_download=self._ordered_cp_ids(cp_ids), + supported=True, + ) def get_text_encoder_recommendation(self) -> TextEncoderRecommendationResponse: self._ensure_local_model_mode() - model_id = self._require_downloaded_ltx_model_id() - cp_id = get_ltx_model_spec(model_id).text_encoder_cp + model_id = self._require_active_ltx_model_id() + ltx_spec = get_ltx_model_spec(model_id) + cp_id = ltx_spec.text_encoder_cp spec = get_model_cp_spec(cp_id) + enhancer_cp = ltx_spec.prompt_enhancer_cp + active_enhancer_cp = resolve_downloaded_prompt_enhancer_cp(self.models_dir, ltx_spec) return TextEncoderRecommendationResponse( cp_to_download=None if self.is_cp_downloaded(cp_id) else cp_id, expected_size_bytes=spec.expected_size_bytes, expected_size_gb=round(spec.expected_size_bytes / (1024**3), 1), + api_encoding_supported=ltx_spec.supports_api_text_encoding, + ltx_version_label=ltx_spec.version_label, + local_enhancement_supported=active_enhancer_cp is not None, + local_enhancer_cp=enhancer_cp, + local_enhancer_expected_size_gb=( + None if enhancer_cp is None + else round(get_model_cp_spec(enhancer_cp).expected_size_bytes / (1024**3), 1) + ), + active_local_enhancer_cp=active_enhancer_cp, ) def resolve_upgrade_download(self, requested_cp_ids: set[ModelCheckpointID]) -> ResolvedUpgradeDownload: @@ -339,7 +448,8 @@ def delete_checkpoints(self, cp_ids: set[ModelCheckpointID]) -> None: protected = self.get_protected_cp_ids() if cp_ids & protected: raise HTTPError(409, "DELETE_PROTECTED_CHECKPOINT") - for cp_id in cp_ids: + for cp_id in self._ordered_cp_ids(cp_ids): + logger.info("Deleting checkpoint %s from %s", cp_id, self.models_dir) delete_cp_path(self.models_dir, cp_id) def list_ltx_versions(self) -> LtxModelVersionsResponse: @@ -349,18 +459,23 @@ def list_ltx_versions(self) -> LtxModelVersionsResponse: items: list[LtxModelVersionItem] = [] for model_id in ALL_LTX_LOCAL_MODEL_IDS: spec = get_ltx_model_spec(model_id) - cp_spec = get_model_cp_spec(spec.model_cp) missing = self._get_missing_cp_ids(self._get_required_ltx_cp_ids(model_id)) # "installed" must mean runnable (transformer + companions), matching the bundle # set_active requires — otherwise a partial install reports installed yet 409s on # activation, and BaseModelSection hides the Download button that would repair it. installed = not missing + # Sum the required bundle so Settings doesn't imply "42 GB" when VAEs + upscaler + # (and the TE when no API key covers it) are also part of the install. + size_bytes = sum( + get_model_cp_spec(cp_id).expected_size_bytes + for cp_id in self._get_required_ltx_cp_ids(model_id) + ) items.append( LtxModelVersionItem( model_id=model_id, label=spec.version_label, model_cp=spec.model_cp, - size_bytes=cp_spec.expected_size_bytes, + size_bytes=size_bytes, installed=installed, active=model_id == active, is_newest=model_id == latest, diff --git a/backend/handlers/pipelines_handler.py b/backend/handlers/pipelines_handler.py index 1be340ccc..aff78e8ad 100644 --- a/backend/handlers/pipelines_handler.py +++ b/backend/handlers/pipelines_handler.py @@ -13,7 +13,6 @@ from runtime_config.model_download_specs import ( IMG_GEN_MODEL_CP_ID, get_existing_cp_path, - get_ltx_model_spec, resolve_active_ltx_model_id, ) from runtime_config.runtime_policy import streaming_prefetch_count_for_mode @@ -75,6 +74,18 @@ def __init__( self._retake_pipeline_class = retake_pipeline_class self._runtime_device = get_device_type(self.config.device) + def _resolve_ltx_paths(self, model_id: LTXLocalModelId, gemma_root: str | None): + from runtime_config.ltx_runtime_paths import ResolvedLtxModelPaths, resolve_ltx_runtime_paths + from state.app_settings import resolved_use_conv_vae + + paths: ResolvedLtxModelPaths = resolve_ltx_runtime_paths( + self.models_dir, + model_id, + gemma_root=gemma_root, + use_conv_vae=resolved_use_conv_vae(self.state.app_settings), + ) + return paths + def _ensure_no_running_generation(self) -> None: match self.state.active_generation: case GpuGeneration(state=GenerationRunning()) if self.state.gpu_slot is not None: @@ -148,24 +159,27 @@ def _create_video_pipeline( ) -> VideoPipelineState: gemma_root = self._text_handler.resolve_gemma_root() model_id = self._require_downloaded_ltx_model_id() - spec = get_ltx_model_spec(model_id) - checkpoint_path = str(get_existing_cp_path(self.models_dir, spec.model_cp)) - upsampler_path = str(get_existing_cp_path(self.models_dir, spec.upscale_cp)) + paths = self._resolve_ltx_paths(model_id, gemma_root) pipeline = self._fast_video_pipeline_class.create( - checkpoint_path, - gemma_root, - upsampler_path, + paths.checkpoint_path, + paths.gemma_root, + paths.upsampler_path, self.config.device, streaming_prefetch_count_for_mode(self.config.local_generations_mode), loras=loras or [], + video_vae_path=paths.video_vae_path, + audio_vae_path=paths.audio_vae_path, + duration_head_path=paths.duration_head_path, ) state = VideoPipelineState( pipeline=pipeline, is_compiled=False, + ltx_model_id=model_id, loras=tuple(loras) if loras else (), gemma_root=gemma_root, + video_vae_path=paths.video_vae_path, ) return self._compile_if_enabled(state) @@ -275,6 +289,10 @@ def load_gpu_pipeline( requested_loras = tuple(loras) if loras else () requested_gemma_root = self._text_handler.resolve_gemma_root() + requested_model_id = self._require_downloaded_ltx_model_id() + requested_video_vae_path = self._resolve_ltx_paths( + requested_model_id, requested_gemma_root + ).video_vae_path state: VideoPipelineState | None = None with self._lock: if self._pipeline_matches_model_type(model_type): @@ -282,8 +300,10 @@ def load_gpu_pipeline( case GpuSlot( active_pipeline=VideoPipelineState() as existing_state ) if ( - existing_state.loras == requested_loras + existing_state.ltx_model_id == requested_model_id + and existing_state.loras == requested_loras and existing_state.gemma_root == requested_gemma_root + and existing_state.video_vae_path == requested_video_vae_path ): state = existing_state case _: @@ -307,6 +327,8 @@ def load_ic_lora( self._install_text_patches_if_needed() gemma_root = self._text_handler.resolve_gemma_root() + model_id = self._require_downloaded_ltx_model_id() + paths = self._resolve_ltx_paths(model_id, gemma_root) with self._lock: match self.state.gpu_slot: case GpuSlot( @@ -315,29 +337,34 @@ def load_ic_lora( depth_model_path=current_depth_model_path, lora_strength=current_lora_strength, gemma_root=current_gemma_root, + ltx_model_id=current_model_id, + video_vae_path=current_video_vae_path, ) as state ) if ( current_lora_path == lora_path and current_depth_model_path == depth_model_path and current_lora_strength == lora_strength and current_gemma_root == gemma_root + and current_model_id == model_id + and current_video_vae_path == paths.video_vae_path ): return state case _: pass self._evict_gpu_pipeline_for_swap() - model_id = self._require_downloaded_ltx_model_id() - model_spec = get_ltx_model_spec(model_id) pipeline = self._ic_lora_pipeline_class.create( - str(get_existing_cp_path(self.models_dir, model_spec.model_cp)), - gemma_root, - str(get_existing_cp_path(self.models_dir, model_spec.upscale_cp)), + paths.checkpoint_path, + paths.gemma_root, + paths.upsampler_path, lora_path, self.config.device, streaming_prefetch_count_for_mode(self.config.local_generations_mode), lora_strength, + video_vae_path=paths.video_vae_path, + audio_vae_path=paths.audio_vae_path, + duration_head_path=paths.duration_head_path, ) depth_pipeline = ( self._depth_processor_pipeline_class.create(depth_model_path, self.config.device) @@ -349,8 +376,10 @@ def load_ic_lora( lora_path=lora_path, depth_pipeline=depth_pipeline, depth_model_path=depth_model_path, + ltx_model_id=model_id, lora_strength=lora_strength, gemma_root=gemma_root, + video_vae_path=paths.video_vae_path, ) with self._lock: @@ -363,28 +392,40 @@ def load_a2v_pipeline(self, loras: list[tuple[str, float]] | None = None) -> A2V requested_loras = tuple(loras) if loras else () gemma_root = self._text_handler.resolve_gemma_root() + model_id = self._require_downloaded_ltx_model_id() + paths = self._resolve_ltx_paths(model_id, gemma_root) with self._lock: match self.state.gpu_slot: case GpuSlot(active_pipeline=A2VPipelineState() as state) if ( - state.loras == requested_loras and state.gemma_root == gemma_root + state.ltx_model_id == model_id + and state.loras == requested_loras + and state.gemma_root == gemma_root + and state.video_vae_path == paths.video_vae_path ): return state case _: pass self._evict_gpu_pipeline_for_swap() - model_id = self._require_downloaded_ltx_model_id() - model_spec = get_ltx_model_spec(model_id) pipeline = self._a2v_pipeline_class.create( - str(get_existing_cp_path(self.models_dir, model_spec.model_cp)), - gemma_root, - str(get_existing_cp_path(self.models_dir, model_spec.upscale_cp)), + paths.checkpoint_path, + paths.gemma_root, + paths.upsampler_path, self.config.device, streaming_prefetch_count_for_mode(self.config.local_generations_mode), loras=loras or [], + video_vae_path=paths.video_vae_path, + audio_vae_path=paths.audio_vae_path, + duration_head_path=paths.duration_head_path, + ) + state = A2VPipelineState( + pipeline=pipeline, + ltx_model_id=model_id, + loras=requested_loras, + gemma_root=gemma_root, + video_vae_path=paths.video_vae_path, ) - state = A2VPipelineState(pipeline=pipeline, loras=requested_loras, gemma_root=gemma_root) with self._lock: self.state.gpu_slot = GpuSlot(active_pipeline=state) @@ -396,17 +437,25 @@ def load_retake_pipeline(self, *, distilled: bool = True) -> RetakePipelineState quantized = device_supports_fp8(self.config.device) gemma_root = self._text_handler.resolve_gemma_root() + model_id = self._require_downloaded_ltx_model_id() + paths = self._resolve_ltx_paths(model_id, gemma_root) with self._lock: match self.state.gpu_slot: case GpuSlot( active_pipeline=RetakePipelineState( - distilled=current_distilled, quantized=current_quantized, gemma_root=current_gemma_root + distilled=current_distilled, + quantized=current_quantized, + gemma_root=current_gemma_root, + ltx_model_id=current_model_id, + video_vae_path=current_video_vae_path, ) as state ) if ( current_distilled == distilled and current_quantized == quantized and current_gemma_root == gemma_root + and current_model_id == model_id + and current_video_vae_path == paths.video_vae_path ): return state case _: @@ -416,20 +465,25 @@ def load_retake_pipeline(self, *, distilled: bool = True) -> RetakePipelineState from ltx_core.quantization.fp8_cast import build_policy as build_fp8_cast_policy - model_id = self._require_downloaded_ltx_model_id() - model_spec = get_ltx_model_spec(model_id) - checkpoint_path = str(get_existing_cp_path(self.models_dir, model_spec.model_cp)) - quantization = build_fp8_cast_policy(checkpoint_path) if quantized else None + quantization = build_fp8_cast_policy(paths.checkpoint_path) if quantized else None pipeline = self._retake_pipeline_class.create( - checkpoint_path=checkpoint_path, - gemma_root=gemma_root, + checkpoint_path=paths.checkpoint_path, + gemma_root=paths.gemma_root, device=self.config.device, streaming_prefetch_count=streaming_prefetch_count_for_mode(self.config.local_generations_mode), loras=[], quantization=quantization, + video_vae_path=paths.video_vae_path, + audio_vae_path=paths.audio_vae_path, + duration_head_path=paths.duration_head_path, ) state = RetakePipelineState( - pipeline=pipeline, distilled=distilled, quantized=quantized, gemma_root=gemma_root + pipeline=pipeline, + distilled=distilled, + quantized=quantized, + ltx_model_id=model_id, + gemma_root=gemma_root, + video_vae_path=paths.video_vae_path, ) with self._lock: diff --git a/backend/handlers/prompt_enhancement_handler.py b/backend/handlers/prompt_enhancement_handler.py index 2e24f6cb1..330019315 100644 --- a/backend/handlers/prompt_enhancement_handler.py +++ b/backend/handlers/prompt_enhancement_handler.py @@ -18,6 +18,7 @@ from services.interfaces import PromptEnhancerPipeline from services.lora_catalog import LoraCatalogProvider from services.prompt_enhancement import ( + build_audio_visual_caption_system_prompt, build_conditioning_system_prompt, build_ic_lora_enhancement_system_prompt, build_image_edit_system_prompt, @@ -80,7 +81,7 @@ def enhance(self, req: EnhancePromptRequest) -> EnhancePromptResponse: with self._generation.reserved_generation_start(): gemma_root: str | None = None if req.provider == "local": - gemma_root = self._text_handler.resolve_gemma_root_if_downloaded() + gemma_root = self._text_handler.resolve_prompt_enhancer_root_if_downloaded() if gemma_root is None: raise HTTPError(409, "LOCAL_TEXT_ENCODER_NOT_AVAILABLE") elif not self.state.app_settings.gemini_api_key: @@ -129,7 +130,63 @@ def _resolve_and_enhance(self, req: EnhancePromptRequest, gemma_root: str | None system_prompt = build_conditioning_system_prompt(req.conditioningType) return self._run_free_rewrite(req, system_prompt, gemma_root) - return self._run_free_rewrite(req, None, gemma_root) + return self._run_free_rewrite(req, self._default_video_system_prompt(req), gemma_root) + + def _default_video_system_prompt(self, req: EnhancePromptRequest) -> str | None: + return self._video_system_prompt(t2v=req.imagePath is None) + + def _video_system_prompt(self, *, t2v: bool) -> str | None: + """The active model's own caption style, or None to keep each provider's default. + + Only the audio-visual generations (2.5) need this: their captions cover the soundscape, + which neither the generic Gemini fallback nor a 2.3-era prompt asks for. + """ + spec = self._text_handler.active_ltx_model_spec() + if spec is None or not spec.wants_audio_visual_captions: + return None + return build_audio_visual_caption_system_prompt(t2v=t2v) + + def enhance_for_generation(self, prompt: str, *, image_path: str | None) -> str: + """Rewrite ``prompt`` on the local enhancer for a generation that's already started. + + Only for the local text-encoding path: API encoding enhances server-side inside the + same call, so it never gets here. Deliberately not `enhance()` — the caller already + holds the generation slot, and there's no provider choice to make, only "is the + enhancer on disk". + + Never raises: enhancement is a quality step, so a missing checkpoint or a failed + rewrite degrades to the prompt as typed rather than failing the generation. + """ + if not prompt.strip(): + return prompt + gemma_root = self._text_handler.resolve_prompt_enhancer_root_if_downloaded() + if gemma_root is None: + logger.info("Skipping automatic enhancement: no local prompt enhancer downloaded") + return prompt + + try: + pipeline = self._load_prompt_enhancer_pipeline(gemma_root) + system_prompt = self._video_system_prompt(t2v=image_path is None) + seed = self._random_seed() + if image_path is not None: + enhanced = pipeline.enhance_i2v( + prompt, image_path, system_prompt=system_prompt, seed=seed + ) + else: + enhanced = pipeline.enhance_t2v(prompt, system_prompt=system_prompt, seed=seed) + except Exception: + logger.warning("Automatic local enhancement failed; using the prompt as typed", exc_info=True) + return prompt + + if not enhanced.strip(): + return prompt + logger.info( + "Enhanced prompt locally for generation (%d -> %d chars): %s", + len(prompt), + len(enhanced), + enhanced, + ) + return enhanced def _enhance_loras( self, loras: list[LoraCatalogItem], req: EnhancePromptRequest, gemma_root: str | None diff --git a/backend/handlers/retake_handler.py b/backend/handlers/retake_handler.py index d6c5abd13..9666beee2 100644 --- a/backend/handlers/retake_handler.py +++ b/backend/handlers/retake_handler.py @@ -9,6 +9,7 @@ from api_types import ( RetakeCancelledResponse, + RetakeExtendModel, RetakeMode, RetakePayloadResponse, RetakeRequest, @@ -17,6 +18,7 @@ TargetResolution, ) from _routes._errors import HTTPError +from api_model_specs import FORCED_API_MODEL_MAP from handlers.base import StateHandlerBase from handlers.generation_handler import GenerationHandler from handlers.pipelines_handler import PipelinesHandler @@ -27,6 +29,8 @@ resolve_target_resolution, validate_source_video_path, ) +from runtime_config.ltx_capabilities import local_caps, supports +from runtime_config.model_download_specs import resolve_active_ltx_model_id from runtime_config.runtime_config import RuntimeConfig from services.ltx_api_client.ltx_api_client import LTXAPIClientError from services.interfaces import LTXAPIClient @@ -73,6 +77,19 @@ def run(self, req: RetakeRequest) -> RetakeResponse: duration=duration, prompt=prompt, mode=mode, + model=req.model, + ) + + model_id = resolve_active_ltx_model_id( + self.models_dir, self.state.app_settings.active_ltx_model_id + ) + if model_id is None: + raise HTTPError(409, "NO_DOWNLOADED_LTX_MODEL") + if not supports(local_caps(model_id), "retake"): + raise HTTPError( + 409, + "Retake is not supported for the active LTX model.", + code="UNSUPPORTED_RETAKE", ) return self._run_local_retake( @@ -92,6 +109,7 @@ def _run_api_retake( duration: float, prompt: str, mode: RetakeMode, + model: RetakeExtendModel, ) -> RetakeResponse: api_key = self.state.app_settings.ltx_api_key if not api_key: @@ -119,6 +137,7 @@ def _run_api_retake( duration=duration, prompt=prompt, mode=mode, + model=FORCED_API_MODEL_MAP[model], ) if result.video_bytes is not None: diff --git a/backend/handlers/text_handler.py b/backend/handlers/text_handler.py index 94c31e70b..1e5eaa172 100644 --- a/backend/handlers/text_handler.py +++ b/backend/handlers/text_handler.py @@ -6,12 +6,16 @@ from typing import TYPE_CHECKING from _routes._errors import HTTPError +from api_types import LTXLocalModelId from handlers.base import StateHandlerBase, with_state_lock from runtime_config.model_download_specs import ( - get_downloaded_ltx_model_id, + LTXLocalModelSpec, get_existing_cp_path, get_ltx_model_spec, + get_model_cp_spec, is_cp_downloaded, + resolve_active_ltx_model_id, + resolve_downloaded_prompt_enhancer_cp, ) from state.app_state_types import AppState, TextEncodingResult @@ -23,6 +27,15 @@ class TextHandler(StateHandlerBase): def __init__(self, state: AppState, lock: RLock, config: RuntimeConfig) -> None: super().__init__(state, lock, config) + def _active_ltx_model_id(self) -> LTXLocalModelId | None: + return resolve_active_ltx_model_id( + self.models_dir, self.state.app_settings.active_ltx_model_id + ) + + def active_ltx_model_spec(self) -> LTXLocalModelSpec | None: + model_id = self._active_ltx_model_id() + return None if model_id is None else get_ltx_model_spec(model_id) + @with_state_lock def _get_cached_prompt(self, prompt: str, enhance_prompt: bool) -> TextEncodingResult | None: te = self.state.text_encoder @@ -65,14 +78,11 @@ def should_use_local_encoding(self) -> bool: """ settings = self.state.app_settings.model_copy(deep=True) api_available = bool(settings.ltx_api_key) - model_id = get_downloaded_ltx_model_id(self.models_dir) - local_available = ( - False - if model_id is None - else is_cp_downloaded(self.models_dir, get_ltx_model_spec(model_id).text_encoder_cp) - ) + spec = self.active_ltx_model_spec() + local_available = spec is not None and is_cp_downloaded(self.models_dir, spec.text_encoder_cp) + api_usable = api_available and (spec is None or spec.supports_api_text_encoding) - if api_available and local_available: + if api_usable and local_available: return settings.use_local_text_encoder # setting is tiebreaker return local_available # use whichever is available @@ -85,12 +95,15 @@ def prepare_text_encoding(self, prompt: str, enhance_prompt: bool) -> None: """ settings = self.state.app_settings.model_copy(deep=True) api_available = bool(settings.ltx_api_key) - model_id = get_downloaded_ltx_model_id(self.models_dir) - local_available = ( - False - if model_id is None - else is_cp_downloaded(self.models_dir, get_ltx_model_spec(model_id).text_encoder_cp) - ) + spec = self.active_ltx_model_spec() + local_available = spec is not None and is_cp_downloaded(self.models_dir, spec.text_encoder_cp) + + if spec is not None and not spec.supports_api_text_encoding and not local_available: + raise RuntimeError( + f"TEXT_ENCODING_NOT_CONFIGURED: LTX {spec.version_label} requires the local text " + f"encoder ({get_model_cp_spec(spec.text_encoder_cp).description}); an LTX API key " + "cannot encode prompts for this version. Download it in Settings." + ) if not api_available and not local_available: raise RuntimeError( @@ -111,24 +124,27 @@ def prepare_text_encoding(self, prompt: str, enhance_prompt: bool) -> None: def resolve_gemma_root(self) -> str | None: if not self.should_use_local_encoding(): return None - model_id = get_downloaded_ltx_model_id(self.models_dir) + model_id = self._active_ltx_model_id() if model_id is None: return None return str(get_existing_cp_path(self.models_dir, get_ltx_model_spec(model_id).text_encoder_cp)) - def resolve_gemma_root_if_downloaded(self) -> str | None: + def resolve_prompt_enhancer_root_if_downloaded(self) -> str | None: """Like `resolve_gemma_root`, but answers "is the checkpoint present" rather than "should generation prefer local text encoding" — `should_use_local_encoding()`'s API-key tiebreaker (which defaults to API when both are available) answers a different question than local Enhance availability, and gates on a setting the Enhance UI never shows. The frontend's own local-availability check (`getTextEncoderRecommendation`) already uses checkpoint presence alone; this mirrors that for the backend gate. + + Resolves the enhancer rather than the encoder, which differ on 2.5. Prefers + Gemma 4 E2B when downloaded, then Gemma 3 from a 2.3 install. """ - model_id = get_downloaded_ltx_model_id(self.models_dir) - if model_id is None: + spec = self.active_ltx_model_spec() + if spec is None: return None - cp_id = get_ltx_model_spec(model_id).text_encoder_cp - if not is_cp_downloaded(self.models_dir, cp_id): + cp_id = resolve_downloaded_prompt_enhancer_cp(self.models_dir, spec) + if cp_id is None: return None return str(get_existing_cp_path(self.models_dir, cp_id)) @@ -159,15 +175,17 @@ def _prepare_api_embeddings(self, prompt: str, enhance_prompt: bool) -> TextEnco if te is None: return None - model_id = get_downloaded_ltx_model_id(self.models_dir) + model_id = self._active_ltx_model_id() if model_id is None: raise HTTPError(409, "NO_DOWNLOADED_LTX_MODEL") + model_spec = get_ltx_model_spec(model_id) encoded = te.service.encode_via_api( prompt=prompt, api_key=settings.ltx_api_key, - checkpoint_path=str(get_existing_cp_path(self.models_dir, get_ltx_model_spec(model_id).model_cp)), + checkpoint_path=str(get_existing_cp_path(self.models_dir, model_spec.model_cp)), enhance_prompt=enhance_prompt, + api_model_id=model_spec.api_text_encoder_model_id, ) if encoded is not None: self._cache_prompt(prompt, enhance_prompt, encoded) diff --git a/backend/handlers/video_generation_handler.py b/backend/handlers/video_generation_handler.py index f0c861341..4f0711e75 100644 --- a/backend/handlers/video_generation_handler.py +++ b/backend/handlers/video_generation_handler.py @@ -5,7 +5,7 @@ import logging import os -from frame_math import compute_num_frames +from frame_math import AutoDurationSpec, compute_num_frames, snap_up_to_multiple import tempfile import time import uuid @@ -24,19 +24,27 @@ GenerateVideoResponse, ImageConditioningInput, LoraEntry, + LTXLocalModelId, + LTXVideoGenResolution, VideoCameraMotion, ) +from runtime_config.ltx_capabilities import LtxAspectRatio, api_caps, local_caps, pixels_for, supports from runtime_config.models_scanner import resolve_lora_ref from _routes._errors import HTTPError from api_model_specs import ( + FORCED_API_MODEL_MAP, build_generate_video_model_specs_response, + get_local_video_generation_model_specs, + supported_duration_range, validate_generate_video_request, ) from handlers.base import StateHandlerBase from server_utils.heartbeat import log_heartbeat from handlers.generation_handler import GenerationHandler from handlers.pipelines_handler import PipelinesHandler +from handlers.prompt_enhancement_handler import PromptEnhancementHandler from handlers.text_handler import TextHandler +from runtime_config.model_download_specs import is_duration_head_ready, resolve_active_ltx_model_id from server_utils.media_validation import ( normalize_optional_path, validate_audio_file, @@ -52,15 +60,23 @@ logger = logging.getLogger(__name__) -FORCED_API_MODEL_MAP: dict[str, str] = { - "fast": "ltx-2-3-fast", - "pro": "ltx-2-3-pro", -} -FORCED_API_RESOLUTION_MAP: dict[str, dict[str, str]] = { - "1080p": {"16:9": "1920x1080", "9:16": "1080x1920"}, - "1440p": {"16:9": "2560x1440", "9:16": "1440x2560"}, - "2160p": {"16:9": "3840x2160", "9:16": "2160x3840"}, -} + +def _wxh(size: tuple[int, int]) -> str: + return f"{size[0]}x{size[1]}" + + +def _forced_api_resolution_map() -> dict[str, dict[str, str]]: + caps = api_caps("fast") + return { + resolution: { + "16:9": _wxh(pixels_for(caps, resolution, "16:9")), + "9:16": _wxh(pixels_for(caps, resolution, "9:16")), + } + for resolution in caps.resolution_pixels_16_9 + } + + +FORCED_API_RESOLUTION_MAP: dict[str, dict[str, str]] = _forced_api_resolution_map() FORCED_API_ALLOWED_ASPECT_RATIOS = {"16:9", "9:16"} _LTX_INSUFFICIENT_FUNDS_MESSAGE = "Your LTX API credits are insufficient for this generation. Buy more credits and try again." @@ -73,6 +89,7 @@ def __init__( generation_handler: GenerationHandler, pipelines_handler: PipelinesHandler, text_handler: TextHandler, + prompt_enhancement_handler: PromptEnhancementHandler, ltx_api_client: LTXAPIClient, config: RuntimeConfig, ) -> None: @@ -80,17 +97,76 @@ def __init__( self._generation = generation_handler self._pipelines = pipelines_handler self._text = text_handler + self._prompt_enhancement = prompt_enhancement_handler self._ltx_api_client = ltx_api_client + def _resolve_prompt_enhancement( + self, prompt: str, *, image_path: str | None + ) -> tuple[str, bool]: + """Apply the enhancer setting, returning ``(prompt, enhance_via_api)``. + + The two encoding paths enhance in different places: API encoding rewrites server-side + inside the same /prompt-embedding call, so it only needs the flag forwarded. Local + encoding has no such step, so the rewrite happens here — without it the model sees the + prompt as typed, which for a version captioned in 150-220 word audio-visual paragraphs + (2.5) lands far outside its training distribution and it invents the rest. + + Must be called before the pipeline is loaded and before start_generation: the enhancer + needs the VRAM a resident pipeline holds, and PipelinesHandler refuses to evict one + while a generation is running. + """ + settings = self.state.app_settings + enabled = ( + settings.prompt_enhancer_enabled_i2v if image_path is not None + else settings.prompt_enhancer_enabled_t2v + ) + if not enabled: + return prompt, False + if not self._text.should_use_local_encoding(): + return prompt, True + return self._prompt_enhancement.enhance_for_generation(prompt, image_path=image_path), False + + def _active_ltx_model_id(self) -> LTXLocalModelId | None: + return resolve_active_ltx_model_id( + self.models_dir, self.state.app_settings.active_ltx_model_id + ) + + def _duration_head_ready(self) -> bool: + model_id = self._active_ltx_model_id() + return model_id is not None and is_duration_head_ready(self.models_dir, model_id) + + def _local_pixels( + self, + resolution: LTXVideoGenResolution, + aspect: LtxAspectRatio, + *, + invalid_code: str, + ) -> tuple[int, int]: + model_id = self._active_ltx_model_id() + if model_id is None: + raise HTTPError(409, "NO_DOWNLOADED_LTX_MODEL") + try: + return pixels_for(local_caps(model_id), resolution, aspect) + except KeyError as exc: + raise HTTPError(400, invalid_code) from exc + def get_model_specs(self) -> GenerateVideoModelsSpecsResponse: - return build_generate_video_model_specs_response() + return build_generate_video_model_specs_response( + local_model_id=self._active_ltx_model_id(), + duration_head_ready=self._duration_head_ready(), + ) def generate(self, req: GenerateVideoRequest) -> GenerateVideoResponse: use_api_specs = should_video_generate_with_ltx_api( force_api_generations=self.config.force_api_generations, settings=self.state.app_settings, ) - validation_error = validate_generate_video_request(req, use_api_specs=use_api_specs) + validation_error = validate_generate_video_request( + req, + use_api_specs=use_api_specs, + local_model_id=None if use_api_specs else self._active_ltx_model_id(), + duration_head_ready=False if use_api_specs else self._duration_head_ready(), + ) if validation_error is not None: raise HTTPError(422, validation_error, code="INVALID_VIDEO_GENERATION_SPEC") @@ -105,33 +181,38 @@ def generate(self, req: GenerateVideoRequest) -> GenerateVideoResponse: audio_path = normalize_optional_path(req.audioPath) if audio_path: + if duration is None: + raise HTTPError( + 422, + "Automatic duration cannot be combined with audio-to-video", + code="INVALID_VIDEO_GENERATION_SPEC", + ) return self._generate_a2v(req, duration, fps, audio_path=audio_path) logger.info("Resolution %s - using fast pipeline", resolution) - RESOLUTION_MAP_16_9: dict[str, tuple[int, int]] = { - "540p": (960, 544), - "720p": (1280, 704), - "1080p": (1920, 1088), - } - - def get_16_9_size(res: str) -> tuple[int, int]: - size = RESOLUTION_MAP_16_9.get(res) - if size is None: - raise HTTPError(400, "INVALID_LOCAL_RESOLUTION") - return size - - def get_9_16_size(res: str) -> tuple[int, int]: - w, h = get_16_9_size(res) - return h, w - - match req.aspectRatio: - case "9:16": - width, height = get_9_16_size(resolution) - case "16:9": - width, height = get_16_9_size(resolution) + width, height = self._local_pixels( + resolution, req.aspectRatio, invalid_code="INVALID_LOCAL_RESOLUTION" + ) - num_frames = self._compute_num_frames(duration, fps) + if duration is None: + item = next( + candidate + for candidate in get_local_video_generation_model_specs( + self._active_ltx_model_id(), + duration_head_ready=self._duration_head_ready(), + ) + if candidate.pipeline == req.model + ) + min_seconds, max_seconds = supported_duration_range( + item, resolution=resolution, fps=fps + ) + num_frames: int | AutoDurationSpec = AutoDurationSpec( + min_seconds=float(min_seconds), + max_seconds=float(max_seconds), + ) + else: + num_frames = self._compute_num_frames(duration, fps) image = None image_path = normalize_optional_path(req.imagePath) @@ -143,12 +224,20 @@ def get_9_16_size(res: str) -> tuple[int, int]: seed = req.seed if req.seed is not None else self._resolve_seed() loras = self._resolve_loras(req.loras) + # Before the pipeline loads and before the generation is marked running: local + # enhancement needs the VRAM a resident pipeline holds, and evicting a pipeline is + # refused once a generation is running. + prompt, enhance_via_api = self._resolve_prompt_enhancement( + req.prompt, image_path=image_path + ) + try: self._pipelines.load_gpu_pipeline("fast", loras=loras) self._generation.start_generation(generation_id) output_path = self.generate_video( - prompt=req.prompt, + prompt=prompt, + enhance_via_api=enhance_via_api, image=image, height=height, width=width, @@ -175,6 +264,16 @@ def get_9_16_size(res: str) -> tuple[int, int]: raise HTTPError(500, str(e)) from e def _resolve_loras(self, loras: list[LoraEntry]) -> list[tuple[str, float]]: + if loras: + model_id = self._active_ltx_model_id() + if model_id is None: + raise HTTPError(409, "NO_DOWNLOADED_LTX_MODEL") + if not supports(local_caps(model_id), "user_loras"): + raise HTTPError( + 409, + "User LoRAs are not supported for the active LTX model.", + code="UNSUPPORTED_USER_LORAS", + ) try: return [(str(resolve_lora_ref(self.models_dir, e.ref)), e.scale) for e in loras] except ValueError as exc: @@ -183,10 +282,11 @@ def _resolve_loras(self, loras: list[LoraEntry]) -> list[tuple[str, float]]: def generate_video( self, prompt: str, + enhance_via_api: bool, image: Image.Image | None, height: int, width: int, - num_frames: int, + num_frames: int | AutoDurationSpec, fps: float, seed: int, camera_motion: VideoCameraMotion, @@ -195,23 +295,18 @@ def generate_video( ) -> str: t_total_start = time.perf_counter() gen_mode = "i2v" if image is not None else "t2v" - logger.info("[%s] Generation started (model=fast, %dx%d, %d frames, %d fps)", gen_mode, width, height, num_frames, int(fps)) + frames_log = ( + f"auto {num_frames.min_seconds:g}-{num_frames.max_seconds:g}s" + if isinstance(num_frames, AutoDurationSpec) + else f"{num_frames} frames" + ) + logger.info("[%s] Generation started (model=fast, %dx%d, %s, %d fps)", gen_mode, width, height, frames_log, int(fps)) if self._generation.is_generation_cancelled(): raise RuntimeError("Generation was cancelled") total_steps = 8 - self._generation.update_progress("loading_model", 5, 0, total_steps) - t_load_start = time.perf_counter() - pipeline_state = self._pipelines.load_gpu_pipeline("fast", loras=loras) - t_load_end = time.perf_counter() - logger.info("[%s] Pipeline load: %.2fs", gen_mode, t_load_end - t_load_start) - - self._generation.update_progress("encoding_text", 10, 0, total_steps) - - enhanced_prompt = prompt + self.config.camera_motion_prompts.get(camera_motion, "") - images: list[ImageConditioningInput] = [] temp_image_path: str | None = None if image is not None: @@ -221,24 +316,31 @@ def generate_video( output_path = self._make_output_path() + # Appended after any rewrite the caller already applied, so the enhancer can't + # paraphrase the camera directive away. + enhanced_prompt = prompt + self.config.camera_motion_prompts.get(camera_motion, "") + try: - settings = self.state.app_settings - use_api_encoding = not self._text.should_use_local_encoding() - if image is not None: - enhance = use_api_encoding and settings.prompt_enhancer_enabled_i2v - else: - enhance = use_api_encoding and settings.prompt_enhancer_enabled_t2v + self._generation.update_progress("loading_model", 5, 0, total_steps) + t_load_start = time.perf_counter() + pipeline_state = self._pipelines.load_gpu_pipeline("fast", loras=loras) + t_load_end = time.perf_counter() + logger.info("[%s] Pipeline load: %.2fs", gen_mode, t_load_end - t_load_start) - encoding_method = "api" if use_api_encoding else "local" + self._generation.update_progress("encoding_text", 10, 0, total_steps) + encoding_method = "api" if not self._text.should_use_local_encoding() else "local" t_text_start = time.perf_counter() - self._text.prepare_text_encoding(enhanced_prompt, enhance_prompt=enhance) + self._text.prepare_text_encoding(enhanced_prompt, enhance_prompt=enhance_via_api) t_text_end = time.perf_counter() logger.info("[%s] Text encoding (%s): %.2fs", gen_mode, encoding_method, t_text_end - t_text_start) self._generation.update_progress("inference", 15, 0, total_steps) - height = round(height / 64) * 64 - width = round(width / 64) * 64 + # Guard for the /64 two-stage grid. Half-way values round up: Python's round() is + # half-to-even, which turned a 544 height into 512 and silently shipped a frame 32px + # shorter (and off its stated aspect ratio) rather than the nearest legal size. + height = snap_up_to_multiple(height, 64) + width = snap_up_to_multiple(width, 64) t_inference_start = time.perf_counter() with log_heartbeat(f"{gen_mode} inference"): @@ -275,20 +377,21 @@ def generate_video( def _generate_a2v( self, req: GenerateVideoRequest, duration: int, fps: int, *, audio_path: str ) -> GenerateVideoResponse: + model_id = self._active_ltx_model_id() + if model_id is None: + raise HTTPError(409, "NO_DOWNLOADED_LTX_MODEL") + if not supports(local_caps(model_id), "a2v"): + raise HTTPError( + 409, + "Audio-to-video is not supported for the active LTX model.", + code="UNSUPPORTED_A2V", + ) validated_audio_path = validate_audio_file(audio_path) audio_path_str = str(validated_audio_path) - RESOLUTION_MAP: dict[str, tuple[int, int]] = { - "540p": (960, 576), - "720p": (1280, 704), - "1080p": (1920, 1088), - } - size = RESOLUTION_MAP.get(req.resolution) - if size is None: - raise HTTPError(400, "INVALID_LOCAL_A2V_RESOLUTION") - width, height = size - if req.aspectRatio == "9:16": - width, height = height, width + width, height = self._local_pixels( + req.resolution, req.aspectRatio, invalid_code="INVALID_LOCAL_A2V_RESOLUTION" + ) num_frames = self._compute_num_frames(duration, fps) @@ -304,10 +407,6 @@ def _generate_a2v( generation_id = self._make_generation_id() try: - a2v_state = self._pipelines.load_a2v_pipeline(loras=loras) - self._generation.start_generation(generation_id) - - enhanced_prompt = req.prompt + self.config.camera_motion_prompts.get(req.cameraMotion, "") neg = req.negativePrompt if req.negativePrompt else self.config.default_negative_prompt images: list[ImageConditioningInput] = [] @@ -316,17 +415,20 @@ def _generate_a2v( image.save(temp_image_path) images = [ImageConditioningInput(path=temp_image_path, frame_idx=0, strength=1.0)] + # Same ordering rule as the fast path: enhance before the pipeline takes the GPU + # (and so before start_generation, which requires a pipeline to already be loaded). + a2v_base_prompt, a2v_enhance = self._resolve_prompt_enhancement( + req.prompt, image_path=image_path + ) + enhanced_prompt = a2v_base_prompt + self.config.camera_motion_prompts.get(req.cameraMotion, "") + + a2v_state = self._pipelines.load_a2v_pipeline(loras=loras) + self._generation.start_generation(generation_id) + output_path = self._make_output_path() total_steps = 11 # distilled: 8 steps (stage 1) + 3 steps (stage 2) - a2v_settings = self.state.app_settings - a2v_use_api = not self._text.should_use_local_encoding() - if image is not None: - a2v_enhance = a2v_use_api and a2v_settings.prompt_enhancer_enabled_i2v - else: - a2v_enhance = a2v_use_api and a2v_settings.prompt_enhancer_enabled_t2v - self._generation.update_progress("loading_model", 5, 0, total_steps) self._generation.update_progress("encoding_text", 10, 0, total_steps) self._text.prepare_text_encoding(enhanced_prompt, enhance_prompt=a2v_enhance) @@ -490,7 +592,7 @@ def _generate_forced_api(self, req: GenerateVideoRequest) -> GenerateVideoRespon image_uri=image_uri, model=api_model_id, resolution=api_resolution, - duration=float(duration), + duration=None if duration is None else float(duration), fps=float(fps), generate_audio=generate_audio, camera_motion=req.cameraMotion, @@ -507,7 +609,7 @@ def _generate_forced_api(self, req: GenerateVideoRequest) -> GenerateVideoRespon prompt=prompt, model=api_model_id, resolution=api_resolution, - duration=float(duration), + duration=None if duration is None else float(duration), fps=float(fps), generate_audio=generate_audio, camera_motion=req.cameraMotion, diff --git a/backend/ltx2_server.py b/backend/ltx2_server.py index 4878ec412..3db1031c0 100644 --- a/backend/ltx2_server.py +++ b/backend/ltx2_server.py @@ -52,6 +52,12 @@ del _ic_lora_stage2_lora import services.patches.diffusion_stage_cache as _diffusion_stage_cache # pyright: ignore[reportUnusedImport] # EXPERIMENTAL: remove once DiffusionStage caches/reuses identical builds upstream del _diffusion_stage_cache +import services.patches.diffvae_mps_tiling_budget as _diffvae_mps_tiling_budget # pyright: ignore[reportUnusedImport] # Remove once ltx-pipelines queries MPS/unified free memory +del _diffvae_mps_tiling_budget +import services.patches.diffvae_decode_vram as _diffvae_decode_vram # pyright: ignore[reportUnusedImport] # Remove once ltx-pipelines offloads the transformer before DiffVAE decode +del _diffvae_decode_vram +import services.patches.natten_libnatten_gate as _natten_libnatten_gate # pyright: ignore[reportUnusedImport] # Remove once ltx-core natten_available checks HAS_LIBNATTEN +del _natten_libnatten_gate from state.app_settings import AppSettings @@ -257,7 +263,7 @@ def _resolve_local_generations_mode() -> LocalGenerationMode: DEFAULT_NEGATIVE_PROMPT = """blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of field""" -HF_OAUTH_CLIENT_ID = "a8189e14-9246-4f19-bd6a-a307bdcb9276" +HF_OAUTH_CLIENT_ID = "508843e5-65c3-4565-aeed-fb17e94cf394" runtime_config = RuntimeConfig( device=DEVICE, @@ -315,6 +321,9 @@ def log_hardware_info() -> None: if gpu.get_mps_available(): avail = gpu.get_available_ram_gb() gpu_line += f" | Available RAM: {avail if avail is not None else '?'} GB" + logger.info( + "LTX 2.5 decode uses eager SDPA on Mac (no Triton; slower than Linux/Windows)." + ) logger.info(gpu_line) logger.info(f"SageAttention: {'enabled' if use_sage_attention else 'disabled'}") logger.info(f"Python: {sys.version.split()[0]} | Torch: {torch.__version__}") diff --git a/backend/pyproject.toml b/backend/pyproject.toml index e73a43142..258fb07b8 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -11,23 +11,31 @@ dependencies = [ # torch to 2.11.x keeps torch+torchaudio in lockstep. win/linux use the cu128 index below. "torch>=2.3.0; sys_platform != 'darwin'", "torch>=2.3.0,<2.12; sys_platform == 'darwin'", + # transformers' Gemma 4 processor imports torchvision.transforms.v2 at module scope, so + # LTX 2.5's text encoder fails to build without it (ImportError on Gemma4UnifiedProcessor). + "torchvision>=0.18.0", "huggingface-hub>=0.23.0", "tqdm>=4.66.0", "pynvml>=11.5.0; sys_platform != 'darwin'", "psutil>=5.9.0", "pydantic>=2.7.0", - # LTX-2 inference dependencies - "ltx-core", - "ltx-pipelines", + # LTX-2 inference dependencies (v1.2.0 is the 2.5-capable release). + "ltx-core==1.2.0", + "ltx-pipelines==1.2.0", "diffusers>=0.36.0", "ftfy>=6.0.0", "imageio>=2.37.2", "imageio-ffmpeg>=0.6.0", "peft>=0.13.2", "protobuf>=3.20.0", - "transformers>=4.52,<5", + # ltx-core 1.2 / Gemma 4 need transformers 5.8+; 5.15 breaks Gemma 4 config access. + "transformers>=5.8.0,<5.15", "sentencepiece>=0.1.99", "sageattention>=1.0.0; sys_platform != 'darwin'", + # Official NATTEN wheels are Linux-only. GCS-hosted Windows wheel matches + # the embed runtime (cp313, torch 2.10.0+cu128). Do not use ltx-core's + # natten extra (0.21.7+torch2130cu132). Mac has no CUDA NATTEN. + "natten==0.21.6; sys_platform == 'win32'", # ninja is required (on PATH) for torch to JIT-build mps-sdpa's zero-copy # `mpsgraph_zc` attention backend on Apple Silicon. Without it, mps-sdpa # silently falls back to the pyobjc `mpsgraph` backend, which leaks Metal @@ -50,15 +58,47 @@ explicit = true torch = [ { index = "pytorch-cu128", marker = "sys_platform == 'win32' or sys_platform == 'linux'" }, ] -torchaudio = { index = "pytorch-cu128" } +torchaudio = [ + { index = "pytorch-cu128", marker = "sys_platform == 'win32' or sys_platform == 'linux'" }, +] +torchvision = [ + { index = "pytorch-cu128", marker = "sys_platform == 'win32' or sys_platform == 'linux'" }, +] # PyPI's sageattention caps at 1.0.6 (Nov 2024, predates Blackwell/RTX 50xx support) and # can't host per-torch/CUDA wheel variants anyway. woct0rdho's Windows fork ships prebuilt # 2.2.0 wheels with bundled sm120 kernels for CUDA>=12.8 — matches our torch2.10.0+cu128 pin. # Linux keeps resolving sageattention from PyPI (no reported Blackwell crashes there yet). sageattention = { url = "https://github.com/woct0rdho/SageAttention/releases/download/v2.2.0-windows.post5/sageattention-2.2.0+cu128torch2.10.0andhigher.post5-cp310-abi3-win_amd64.whl", marker = "sys_platform == 'win32'" } +natten = { url = "https://storage.googleapis.com/ltx-desktop-artifacts/wheels/natten-0.21.6+torch2100cu128-cp313-cp313-win_amd64.whl", marker = "sys_platform == 'win32'" } diffusers = { git = "https://github.com/huggingface/diffusers.git", rev = "01de02e8b4f2cc91df4f3e91cb6535ebcbeb490c" } -ltx-core = { git = "https://github.com/Lightricks/LTX-2.git", subdirectory = "packages/ltx-core", rev = "9377758131b1ffde4b7f766804590a6617bf2ab9" } -ltx-pipelines = { git = "https://github.com/Lightricks/LTX-2.git", subdirectory = "packages/ltx-pipelines", rev = "9377758131b1ffde4b7f766804590a6617bf2ab9" } +# Official LTX-2 v1.2.0 packages (not on PyPI). Same git+subdirectory pattern as +# 2.3; the tag is the 2.5-capable release. We do not take ltx-core's own +# tool.uv.sources (cu132) — this app's pytorch-cu128 pin above owns torch. +ltx-core = { git = "https://github.com/Lightricks/LTX-2.git", subdirectory = "packages/ltx-core", tag = "v1.2.0" } +ltx-pipelines = { git = "https://github.com/Lightricks/LTX-2.git", subdirectory = "packages/ltx-pipelines", tag = "v1.2.0" } + +# v1.2.0's library pyproject pins torch to a cu132 index. Static metadata lets uv +# resolve without reading that table, so this app's cu128 pin above stays in charge. +[[tool.uv.dependency-metadata]] +name = "ltx-core" +version = "1.2.0" +requires-dist = [ + "torch~=2.7", + "torchaudio", + "einops", + "numpy", + "av", + "transformers>=5.8.0,<5.15", + "safetensors", + "accelerate", + "scipy>=1.14", + "mps-sdpa>=0.2.0; sys_platform == 'darwin' and platform_machine == 'arm64'", +] + +[[tool.uv.dependency-metadata]] +name = "ltx-pipelines" +version = "1.2.0" +requires-dist = ["ltx-core", "av", "tqdm", "pillow", "openimageio", "cloudpickle>=3.1"] [project.optional-dependencies] test = ["pytest>=8.0", "requests>=2.31", "httpx>=0.27"] diff --git a/backend/runtime_config/lora_catalog.json b/backend/runtime_config/lora_catalog.json index 542bbc9ba..2448e082e 100644 --- a/backend/runtime_config/lora_catalog.json +++ b/backend/runtime_config/lora_catalog.json @@ -27,6 +27,7 @@ "url": "https://huggingface.co/chsengni/ltx2.3-fpv-motion" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "created_at": "2026-06-08", "tags": [ "motion" @@ -82,6 +83,7 @@ "url": "https://huggingface.co/lovis93/crt-animation-terminal-ltx-2.3-lora" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "tags": [ "style" ], @@ -143,6 +145,7 @@ "url": "https://huggingface.co/mxturbo/Openwheel-motorsports-Cockpit-T-Cam-LTX2.3" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "created_at": "2026-05-28", "tags": [ "style" @@ -194,6 +197,7 @@ "affiliation": "community" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "tags": [ "style" ], @@ -250,6 +254,7 @@ "affiliation": "community" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "tags": [ "style" ], @@ -306,6 +311,7 @@ "affiliation": "community" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "tags": [ "style" ], @@ -362,6 +368,7 @@ "affiliation": "community" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "tags": [ "style" ], @@ -418,6 +425,7 @@ "affiliation": "community" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "tags": [ "style" ], @@ -474,6 +482,7 @@ "affiliation": "community" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "tags": [ "style" ], @@ -530,6 +539,7 @@ "affiliation": "community" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "tags": [ "style" ], @@ -586,6 +596,7 @@ "affiliation": "community" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "tags": [ "enhance" ], @@ -640,6 +651,7 @@ "affiliation": "community" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "tags": [ "style" ], @@ -700,6 +712,7 @@ "url": "https://github.com/Lightricks/LTX-2/blob/main/LICENSE" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "created_at": "2026-07-06", "tags": [ "style" @@ -765,6 +778,7 @@ "url": "https://github.com/Lightricks/LTX-2/blob/main/LICENSE" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "created_at": "2026-07-06", "tags": [ "motion" @@ -818,6 +832,7 @@ "ic_loras": [ { "id": "clean-plate", + "supported_models": ["LTX-2.3", "LTX-2.5"], "name": "Clean Plate", "description": "Remove people and vehicles from a video and reconstruct the static background.", "tags": [ @@ -884,6 +899,7 @@ }, { "id": "3d-render-to-real", + "supported_models": ["LTX-2.3", "LTX-2.5"], "name": "3D Render to Photoreal", "description": "Turn a rough 3D viewport render (Blender blockout, game-engine viewport) into a photorealistic, cinematic video.", "tags": [ @@ -959,6 +975,7 @@ }, { "id": "day-to-night", + "supported_models": ["LTX-2.3", "LTX-2.5"], "name": "Day to Night", "description": "Relight a daytime shot to look like it was filmed at night.", "tags": [ @@ -1029,6 +1046,7 @@ }, { "id": "colorization", + "supported_models": ["LTX-2.3", "LTX-2.5"], "name": "Colorization", "description": "Add realistic color to a grayscale video.", "tags": [ @@ -1107,6 +1125,7 @@ }, { "id": "decompression", + "supported_models": ["LTX-2.3", "LTX-2.5"], "name": "Decompression", "description": "Remove compression artifacts and restore a degraded video.", "tags": [ @@ -1185,6 +1204,7 @@ }, { "id": "deblur", + "supported_models": ["LTX-2.3", "LTX-2.5"], "name": "Deblur", "description": "Sharpen and remove blur from a video.", "tags": [ @@ -1263,6 +1283,7 @@ }, { "id": "cross-eyed", + "supported_models": ["LTX-2.3", "LTX-2.5"], "name": "Cross-Eyed", "description": "Turn a close-up portrait's eyes permanently crossed (inward), keeping expression and framing.", "tags": [ @@ -1324,6 +1345,7 @@ }, { "id": "water-simulation", + "supported_models": ["LTX-2.3", "LTX-2.5"], "name": "Water Simulation", "description": "Add simulated water / fluid VFX to a scene.", "tags": [ @@ -1401,6 +1423,7 @@ }, { "id": "ingredients", + "supported_models": ["LTX-2.3", "LTX-2.5"], "name": "Ingredients", "description": "Keep a subject consistent across a video from a reference sheet.", "tags": [ @@ -1501,6 +1524,7 @@ }, { "id": "instant-shave", + "supported_models": ["LTX-2.3", "LTX-2.5"], "name": "Instant Shave", "description": "Remove a beard / facial hair from a person in a video.", "tags": [ @@ -1573,6 +1597,7 @@ }, { "id": "outpainting", + "supported_models": ["LTX-2.3", "LTX-2.5"], "name": "Outpainting", "description": "Extend a video beyond its frame \u2014 widen it, change the aspect ratio, or fill new canvas around the shot.", "tags": [ @@ -1679,6 +1704,7 @@ "url": "https://huggingface.co/OmerHagage/ltx2-greenscreen-avatar-ic-lora-vertical-v1" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "created_at": "2026-05-13", "input": { "kind": "video" @@ -1719,6 +1745,7 @@ }, { "id": "upscale", + "supported_models": ["LTX-2.3", "LTX-2.5"], "name": "Upscale (Refine)", "description": "Generative second-pass refinement \u2014 turn a soft / low-res clip into a cleaner, more detailed one.", "tags": [ @@ -1817,6 +1844,7 @@ "url": "https://huggingface.co/Cseti/LTX2.3-22B_IC-LoRA-CrossView-Prompt" }, "base_model": "LTX-2.3", + "supported_models": ["LTX-2.3", "LTX-2.5"], "created_at": "2026-07-11", "input": { "kind": "video" diff --git a/backend/runtime_config/ltx_api_text_encoder_ids.py b/backend/runtime_config/ltx_api_text_encoder_ids.py new file mode 100644 index 000000000..ac335a31a --- /dev/null +++ b/backend/runtime_config/ltx_api_text_encoder_ids.py @@ -0,0 +1,10 @@ +"""API text-encoder model ids used by /v1/prompt-embedding. + +These values are the worker `encrypted_wandb_properties` payload (Fernet), not a +user secret. Split 2.5 checkpoints omit this header, so generation falls back +to this override. Rotate the blob when serving rotates the key. +""" + +# Worker key for LTX 2.5 Fast prompt-embedding. Do not replace with a plaintext +# id unless the worker is verified to accept one. +LTX_2_5_API_TEXT_ENCODER_MODEL_ID = "gAAAAABqcenRC9mecBlpR24xY8xWrCgERvgD5LK80sJw1J9WuSbi_J_SYXw4L3UCrCBciI5T6KJ7o5ZOld2hM6rYbs6R7_XftiN6MghrdbNt1bMZ_boxWcZ-HFvWQYgkAhJGjqLUPLyiCzDf4Idi1UQ8JebKcdK2qTxMCAEDIqLEAg_IwfPHd93YPHzT11gURnQ9vWEkJU385nRu8SRnI1ubKyGx78FmPSa8CRdh3I0zrJawmi0On2BnehplBzKU-Ub2IZaGKOP5pN9yLPW6fjgHi28d20hlNu5SoQl2fD1QnJBYNPbd6YIWYgmL7dYZzZFNLLpwYsXaY48Kh_uPY8p_wALCu_Vl2ONcEvy9spFRW5tRn0YaCs8SuIrv_8MaEO8xG0s5vLmsRfhA29yvhUIqOAuTo13vuqCKlVXv9gm17wjzkK2zcPzE_e26KUGyQepqLVHDWWxxZyPtXiWIkdOwFyzdTYlEjegEMmpMoT2dk-A-AOLdmx6QSpNd7oHcgoS78OsMO-hKG9pdjpmA6ryn3YwvAYumiFuTFM9ELKph2LEdFCyMUFp4mBaLTIxpROEc9b6xSRTQOYxSvzMkPsuJYYsdJXHB4MiDYGmzNoghIpFNABiv_i2Q8H4b4M5wi8PuYLQICiKbYCEOvlF6U_9TlIZpWf3Pt7DU5ufsCHAhYmUmSYQDerlNISYojFqOBRzyik3tqExlDdrIr51bn83YyJSnvC_q88K06dtwC4CrmVyGI0QDMcGowxLqDvdrsfYwxBLBUY_No1D6qA8_SyxlMchRF0rOzUJ_gI3MJJGeQ9loWIb-oG_RveHXQsWTP82wy7TmlOxi1BkplatMPR07qAINEYGBMApZ7ZuMWAV70c80tevyZkSCBu4OY3wMZmVHDQHIO4NM4_261A8e7ISg7fP6mJIkBehc9yGvZuE3eGQK1PD6USYjbabs-TVsLo1ZUn5_5kCn8cMCnyT2djYi3zG2CaFTfBmUSsF9ubXzk4BLDY3b1B9V4RtQf0cOUMSOvBDqi2Rvv0YXMxZcS4OdQDdUxzGC5_XekkM_HWITsyazl8jx713Dk_5fV-Z3n0_pjSVLVZUVr1DRnPkUsSDUpzq9zZ6G-i2c2AoUA2T1LRvZCKs8inqz5Y-6WJJP2R9IMpG22pjWc_Rpg2Fdd1SDnd9umy_AR-VNu2ZP0IC13MpvheyoQnPfIln7yZVupdvitfIeFKaibatgmWgVqtEevP_zavu-MnmDBpKd2mTBAPHD7X9QJXkZUtgebPjJQ3j9SQrJTIS899O4QZxC0r4h9bzpM7bEsKGO6KrnCsMISE5KgNGccPATxnOTuhQznjCfT763DpQz7MJxI8PRHEo2aR4b1uSjILrNMLczMIOx6GZPAxvPHM7HjBQZQSZThiRhhqh1kJk47w13vN2VNy0XSIwPjVPj84Af_yxB6K-2BiLQtn6ndhjDlTJ-nPBhLXltfnfKKksUt_LzYl2pd76i15oUCJrs52SNAyeN_D2uEcQ0FiSUB4u-hiYilpg_dtV2N2Jx9DmeewTcJEY9VCQr7Ccvp__EgJaDLFgFCPMPMAp_BsgUbJJIQEDGhjYJLypOIpBqJgwfIprvn7P7dVTseNL9MrZEgYI4IbABj6hVHsPGDBtTXiTU9PtTrLBiMAlJNE4Kdkd99YJuTWqLdps-rIfyagNoeJ5ItPzmDvy0oeDyyl93dLRpHtwyRtvBUB3IWGknVfX7Gj8GzfnvmIGw85Rh000AhtIIp8zH7WXIxNc8SF5zQZM06_LVePt77Ag58ihsADiRJZEniUlTGSGynZNr3Rrv4tAZUIkhIIUVGVNejtLQhLLIwY_TsblO_6H2cBb4Ep-2T4vnNk1zD0pwMDQXoHy_U3aG00FOgsSje4sTbqGOvUeO7PCEhZ8a9OoDXoNV_Myi" diff --git a/backend/runtime_config/ltx_capabilities.py b/backend/runtime_config/ltx_capabilities.py new file mode 100644 index 000000000..beebbeae2 --- /dev/null +++ b/backend/runtime_config/ltx_capabilities.py @@ -0,0 +1,228 @@ +"""Desktop LTX offering capabilities: feature flags + 16:9 pixel maps. + +Duration/fps envelopes stay on LTXVideoGenerationSpec. This matrix is the +feature/pixel SSOT those specs do not have. Local 2.5 is the on-device distilled +offering; its flags are independent of the API Fast rows. +""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +from typing import Literal, assert_never + +from api_types import LTXLocalModelId, LTXVideoGenPipeline, LTXVideoGenResolution + +LtxCapabilityFeature = Literal[ + "t2v", + "i2v", + "a2v", + "ic_lora", + "retake", + "extend", + "user_loras", + "camera_motion", + "auto_duration", +] +LtxAspectRatio = Literal["16:9", "9:16"] + + +@dataclass(frozen=True) +class LtxOfferingCapabilities: + t2v: bool + i2v: bool + a2v: bool + ic_lora: bool + retake: bool + extend: bool + user_loras: bool + camera_motion: bool + # t2v/i2v only: send duration=null and the cloud worker picks length from the prompt. + auto_duration: bool + # Label → (width, height) for 16:9; 9:16 is swapped at pixels_for(). + resolution_pixels_16_9: dict[LTXVideoGenResolution, tuple[int, int]] + + +# Shared local Fast sizes except 540p, which is version-specific: 2.3 is 960×544 +# (off 16:9 on the /64 two-stage grid); 2.5 is 1024×576. +_LOCAL_720P_1080P: dict[LTXVideoGenResolution, tuple[int, int]] = { + "720p": (1280, 704), + "1080p": (1920, 1088), +} + +_LOCAL_2_3_PIXELS_16_9: dict[LTXVideoGenResolution, tuple[int, int]] = { + "540p": (960, 544), + **_LOCAL_720P_1080P, +} + +_LOCAL_2_5_PIXELS_16_9: dict[LTXVideoGenResolution, tuple[int, int]] = { + "540p": (1024, 576), + **_LOCAL_720P_1080P, +} + +_API_PIXELS_16_9: dict[LTXVideoGenResolution, tuple[int, int]] = { + "1080p": (1920, 1080), + "1440p": (2560, 1440), + "2160p": (3840, 2160), +} + +_LOCAL_2_3 = LtxOfferingCapabilities( + t2v=True, + i2v=True, + a2v=True, + ic_lora=True, + retake=True, + extend=True, + user_loras=True, + camera_motion=True, + auto_duration=False, + resolution_pixels_16_9=_LOCAL_2_3_PIXELS_16_9, +) + +# DistilledA2V is wired for local 2.5. Auto duration is DurationHead on the +# distilled checkpoint (t2v/i2v; A2V length comes from the audio). Advertised +# only when those weights are on disk — see effective_local_caps(). +_LOCAL_2_5 = LtxOfferingCapabilities( + t2v=True, + i2v=True, + a2v=True, + ic_lora=True, + retake=False, + extend=False, + user_loras=True, + camera_motion=True, + auto_duration=True, + resolution_pixels_16_9=_LOCAL_2_5_PIXELS_16_9, +) + +# API rows follow ltxv-api handlers. camera_motion is a named LoRA on the tia2v +# stack, not a Desktop capability on Fast. +_API_FAST = LtxOfferingCapabilities( + t2v=True, + i2v=True, + a2v=False, + ic_lora=False, + retake=False, + extend=False, + user_loras=False, + camera_motion=False, + auto_duration=False, + resolution_pixels_16_9=_API_PIXELS_16_9, +) + +_API_FAST_2_5 = LtxOfferingCapabilities( + t2v=True, + i2v=True, + a2v=True, + ic_lora=False, + retake=False, + extend=False, + user_loras=False, + camera_motion=False, + auto_duration=True, + resolution_pixels_16_9=_API_PIXELS_16_9, +) + +_API_PRO = LtxOfferingCapabilities( + t2v=True, + i2v=True, + a2v=True, + ic_lora=False, + retake=True, + extend=True, + user_loras=False, + camera_motion=True, + auto_duration=False, + resolution_pixels_16_9=_API_PIXELS_16_9, +) + +# ltxv-api retake/extend accept ltx-2-pro / ltx-2-3-pro. Auto duration is on both +# 2.5 API variants (t2v/i2v duration=null). +_API_PRO_2_5 = LtxOfferingCapabilities( + t2v=True, + i2v=True, + a2v=True, + ic_lora=False, + retake=False, + extend=False, + user_loras=False, + camera_motion=True, + auto_duration=True, + resolution_pixels_16_9=_API_PIXELS_16_9, +) + + +def local_caps(model_id: LTXLocalModelId) -> LtxOfferingCapabilities: + match model_id: + case "ltx-2.5-22b-distilled": + return _LOCAL_2_5 + case "ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1": + return _LOCAL_2_3 + case _: + assert_never(model_id) + + +def effective_local_caps( + model_id: LTXLocalModelId, + *, + duration_head_ready: bool, +) -> LtxOfferingCapabilities: + """Static offering flags, with Auto duration on only when DurationHead weights are on disk. + + Local-only. API 2.5 Auto duration is independent of this file. + """ + caps = local_caps(model_id) + if caps.auto_duration and not duration_head_ready: + return replace(caps, auto_duration=False) + return caps + + +def api_caps(pipeline: LTXVideoGenPipeline) -> LtxOfferingCapabilities: + match pipeline: + case "fast": + return _API_FAST + case "fast-2.5": + return _API_FAST_2_5 + case "pro": + return _API_PRO + case "pro-2.5": + return _API_PRO_2_5 + case _: + assert_never(pipeline) + + +def supports(caps: LtxOfferingCapabilities, feature: LtxCapabilityFeature) -> bool: + match feature: + case "t2v": + return caps.t2v + case "i2v": + return caps.i2v + case "a2v": + return caps.a2v + case "ic_lora": + return caps.ic_lora + case "retake": + return caps.retake + case "extend": + return caps.extend + case "user_loras": + return caps.user_loras + case "camera_motion": + return caps.camera_motion + case "auto_duration": + return caps.auto_duration + case _: + assert_never(feature) + + +def pixels_for( + caps: LtxOfferingCapabilities, + resolution: LTXVideoGenResolution, + aspect: LtxAspectRatio, +) -> tuple[int, int]: + size = caps.resolution_pixels_16_9.get(resolution) + if size is None: + raise KeyError(resolution) + width, height = size + if aspect == "9:16": + return height, width + return width, height diff --git a/backend/runtime_config/ltx_runtime_paths.py b/backend/runtime_config/ltx_runtime_paths.py new file mode 100644 index 000000000..177467e6a --- /dev/null +++ b/backend/runtime_config/ltx_runtime_paths.py @@ -0,0 +1,65 @@ +"""Resolve LTX model component paths for pipeline construction.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from api_types import LTXLocalModelId +from runtime_config.model_download_specs import ( + get_existing_cp_path, + get_ltx_model_spec, + is_cp_downloaded, + selected_video_vae_cp, +) + + +@dataclass(frozen=True, slots=True) +class ResolvedLtxModelPaths: + """Filesystem paths for the active local LTX model bundle.""" + + checkpoint_path: str + upsampler_path: str + gemma_root: str | None + video_vae_path: str | None + audio_vae_path: str | None + duration_head_path: str | None + + +def resolve_ltx_runtime_paths( + models_dir: Path, + model_id: LTXLocalModelId, + *, + gemma_root: str | None, + use_conv_vae: bool, +) -> ResolvedLtxModelPaths: + """Resolve transformer/upscaler/(optional)VAE paths for ``model_id``. + + ``gemma_root`` is passed in from the text handler (may be None in API-encode mode). + Split VAEs are required for 2.5 and omitted for monolith 2.3. DurationHead is + a small split-pack file on 2.5; missing it still lets explicit-duration + generation run (AutoDuration then fails in the pipeline). + + ``use_conv_vae`` selects the 2.5 conv VAE (fast decode) vs DiffVAE. The decoder + class is chosen by vendored ltx-core from checkpoint metadata — Desktop only + swaps the path. Missing the selected file is an error; do not fall back. + """ + spec = get_ltx_model_spec(model_id) + video_vae_path: str | None = None + audio_vae_path: str | None = None + duration_head_path: str | None = None + video_vae_cp = selected_video_vae_cp(spec, use_conv_vae=use_conv_vae) + if video_vae_cp is not None: + video_vae_path = str(get_existing_cp_path(models_dir, video_vae_cp)) + if spec.audio_vae_cp is not None: + audio_vae_path = str(get_existing_cp_path(models_dir, spec.audio_vae_cp)) + if spec.duration_head_cp is not None and is_cp_downloaded(models_dir, spec.duration_head_cp): + duration_head_path = str(get_existing_cp_path(models_dir, spec.duration_head_cp)) + return ResolvedLtxModelPaths( + checkpoint_path=str(get_existing_cp_path(models_dir, spec.model_cp)), + upsampler_path=str(get_existing_cp_path(models_dir, spec.upscale_cp)), + gemma_root=gemma_root, + video_vae_path=video_vae_path, + audio_vae_path=audio_vae_path, + duration_head_path=duration_head_path, + ) diff --git a/backend/runtime_config/model_download_specs.py b/backend/runtime_config/model_download_specs.py index e807cb1c9..be514ec16 100644 --- a/backend/runtime_config/model_download_specs.py +++ b/backend/runtime_config/model_download_specs.py @@ -13,13 +13,18 @@ LTXVideoGenDuration, LTXVideoGenFps, LTXVideoGenPipeline, + LTXVideoGenResolution, LTXVideoGenerationResolutionSpec, LTXVideoGenerationSpec, ModelCheckpointID, ) +from runtime_config.ltx_api_text_encoder_ids import LTX_2_5_API_TEXT_ENCODER_MODEL_ID logger = logging.getLogger(__name__) +# 2.5-native weights live under this family dir. 2.3 and shared tooling stay at models_root. +# 2.5.1 patches use new filenames in the same dir — not a nested ltx-2.5.1/. +LTX_2_5_FAMILY_DIR = Path("ltx-2.5") ALL_MODEL_CP_IDS = cast(tuple[ModelCheckpointID, ...], get_args(ModelCheckpointID)) ALL_LTX_LOCAL_MODEL_IDS = cast(tuple[LTXLocalModelId, ...], get_args(LTXLocalModelId)) @@ -32,11 +37,21 @@ class ModelCheckpointSpec: is_folder: bool repo_id: str description: str + # Hugging Face path inside the repo. Defaults to the local basename when None + # (2.3-style root files). Set for nested 2.5 Comfy-aligned paths. + repo_filename: str | None = None + # Repo requires an accepted HF license + auth token; downloading signed out 401s. + gated: bool = False @property def name(self) -> str: return self.relative_path.name + @property + def download_filename(self) -> str: + """Filename argument for ``hf_hub_download`` (may include subdirectories).""" + return self.repo_filename if self.repo_filename is not None else self.name + @dataclass(frozen=True, slots=True) class LTXLocalModelDeprecated: @@ -63,10 +78,33 @@ class LTXLocalModelSpec: model_cp: ModelCheckpointID upscale_cp: ModelCheckpointID text_encoder_cp: ModelCheckpointID - ic_loras_spec: LtxIcLorasSpec + # None for monolith 2.3 (VAEs live inside the transformer checkpoint). + video_vae_cp: ModelCheckpointID | None + # Optional 2.5 conv-VAE (faster decode). None on 2.3. + video_vae_conv_cp: ModelCheckpointID | None + audio_vae_cp: ModelCheckpointID | None + # None for monolith 2.3 (DurationHead lives inside the fat checkpoint). Split 2.5 + # ships it as model_patches/ltx-2.5-duration-head-bf16.safetensors. + duration_head_cp: ModelCheckpointID | None + # None when the model has no built-in Union Control IC-LoRA (LTX 2.5 today). + ic_loras_spec: LtxIcLorasSpec | None relevance: LTXLocalModelRelevance supported_pipelines: tuple[tuple[LTXVideoGenPipeline, LTXVideoGenerationSpec], ...] version_label: str + supports_api_text_encoding: bool = True + # Overrides encrypted_wandb_properties for published checkpoints that omit it. + api_text_encoder_model_id: str | None = None + # True when the model was captioned as audio-visual: enhancement must describe the soundscape + # (and quote dialogue) as well as the visuals, or the prompt lands outside the training + # distribution and the model improvises the missing audio — typically as someone speaking. + wants_audio_visual_captions: bool = False + # A second, generative checkpoint that local Enhance prefers because ``text_encoder_cp`` + # isn't allowed to do that job. 2.3 encodes with stock Gemma 3 (an instruct model that + # rewrites prompts), so it leaves this None. 2.5 encodes with an LTX fine-tune that cannot + # generate, so this is Gemma 4 E2B — the preferred enhancer, not the only one: Gemma 3 + # already on disk from a 2.3 install is a valid fallback (ltx-pipelines names both). + # Optional: missing both only costs local Enhance, never generation. + prompt_enhancer_cp: ModelCheckpointID | None = None # The single newest model the app should recommend/upgrade to. Exactly one spec sets this # True (enforced in _validate_ltx_specs) so "latest" is explicit, not tuple-order-dependent. is_latest: bool = False @@ -89,28 +127,43 @@ def _local_resolution_spec( PERSON_DETECTOR_CP_ID: ModelCheckpointID = "yolox-l-torchscript" POSE_PROCESSOR_CP_ID: ModelCheckpointID = "dw-ll-ucoco-384-bs5" -_DISTILLED_PIPELINES: tuple[tuple[LTXVideoGenPipeline, LTXVideoGenerationSpec], ...] = ( +_FAST_LOCAL_RESOLUTIONS_DURATIONS: dict[LTXVideoGenResolution, LTXVideoGenerationResolutionSpec] = { + "540p": _local_resolution_spec( + fps_to_durations={ + 24: (5, 6, 8, 10, 20), + }, + ), + "720p": _local_resolution_spec( + fps_to_durations={ + 24: (5, 6, 8, 10), + }, + ), + "1080p": _local_resolution_spec( + fps_to_durations={ + 24: (5,), + }, + ), +} + +_DISTILLED_PIPELINES_2_3: tuple[tuple[LTXVideoGenPipeline, LTXVideoGenerationSpec], ...] = ( ( "fast", LTXVideoGenerationSpec( display_name="LTX 2.3 Fast", - supported_resolutions_durations={ - "540p": _local_resolution_spec( - fps_to_durations={ - 24: (5, 6, 8, 10, 20), - }, - ), - "720p": _local_resolution_spec( - fps_to_durations={ - 24: (5, 6, 8, 10), - }, - ), - "1080p": _local_resolution_spec( - fps_to_durations={ - 24: (5,), - }, - ), - }, + supported_resolutions_durations=_FAST_LOCAL_RESOLUTIONS_DURATIONS, + # DistilledA2VPipeline supports A2V at the same envelope as t2v/i2v. + a2v_supported_resolutions_durations=_FAST_LOCAL_RESOLUTIONS_DURATIONS, + ), + ), +) + +_DISTILLED_PIPELINES_2_5: tuple[tuple[LTXVideoGenPipeline, LTXVideoGenerationSpec], ...] = ( + ( + "fast", + LTXVideoGenerationSpec( + display_name="LTX 2.5 Fast", + supported_resolutions_durations=_FAST_LOCAL_RESOLUTIONS_DURATIONS, + a2v_supported_resolutions_durations=_FAST_LOCAL_RESOLUTIONS_DURATIONS, ), ), ) @@ -161,6 +214,66 @@ def get_model_cp_spec(cp_id: ModelCheckpointID) -> ModelCheckpointSpec: repo_id="Lightricks/LTX-2.3-22b-IC-LoRA-Union-Control", description="Union IC-LoRA control model", ) + case "ltx-2.5-22b-distilled": + return ModelCheckpointSpec( + relative_path=LTX_2_5_FAMILY_DIR / "ltx-2.5-22b-distilled-transformer-bf16.safetensors", + expected_size_bytes=42_000_000_000, + is_folder=False, + repo_id="Lightricks/LTX-2.5", + description="LTX 2.5 distilled transformer", + repo_filename="diffusion_models/ltx-2.5-22b-distilled-transformer-bf16.safetensors", + gated=True, + ) + case "ltx-2.5-spatial-upscaler-x2-1.0": + return ModelCheckpointSpec( + relative_path=LTX_2_5_FAMILY_DIR / "ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors", + expected_size_bytes=995_000_000, + is_folder=False, + repo_id="Lightricks/LTX-2.5", + description="LTX 2.5 2x spatial upscaler", + repo_filename="latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors", + gated=True, + ) + case "ltx-2.5-video-vae": + return ModelCheckpointSpec( + relative_path=LTX_2_5_FAMILY_DIR / "ltx-2.5-video-vae-bf16.safetensors", + expected_size_bytes=1_470_000_000, + is_folder=False, + repo_id="Lightricks/LTX-2.5", + description="LTX 2.5 video VAE", + repo_filename="vae/ltx-2.5-video-vae-bf16.safetensors", + gated=True, + ) + case "ltx-2.5-video-vae-conv": + return ModelCheckpointSpec( + relative_path=LTX_2_5_FAMILY_DIR / "ltx-2.5-video-vae-conv-bf16.safetensors", + expected_size_bytes=1_200_000_000, + is_folder=False, + repo_id="Lightricks/LTX-2.5", + description="LTX 2.5 conv video VAE (fast decode)", + repo_filename="vae/ltx-2.5-video-vae-conv-bf16.safetensors", + gated=True, + ) + case "ltx-2.5-audio-vae": + return ModelCheckpointSpec( + relative_path=LTX_2_5_FAMILY_DIR / "ltx-2.5-audio-vae-bf16.safetensors", + expected_size_bytes=365_000_000, + is_folder=False, + repo_id="Lightricks/LTX-2.5", + description="LTX 2.5 audio VAE", + repo_filename="vae/ltx-2.5-audio-vae-bf16.safetensors", + gated=True, + ) + case "ltx-2.5-duration-head": + return ModelCheckpointSpec( + relative_path=LTX_2_5_FAMILY_DIR / "ltx-2.5-duration-head-bf16.safetensors", + expected_size_bytes=8_000_000, + is_folder=False, + repo_id="Lightricks/LTX-2.5", + description="LTX 2.5 duration head (auto duration)", + repo_filename="model_patches/ltx-2.5-duration-head-bf16.safetensors", + gated=True, + ) case "dpt-hybrid-midas": return ModelCheckpointSpec( relative_path=Path("dpt-hybrid-midas"), @@ -191,7 +304,25 @@ def get_model_cp_spec(cp_id: ModelCheckpointID) -> ModelCheckpointSpec: expected_size_bytes=25_000_000_000, is_folder=True, repo_id="Lightricks/gemma-3-12b-it-qat-q4_0-unquantized", - description="Gemma text encoder (bfloat16)", + description="Gemma 3 text encoder (folder)", + ) + case "gemma4-12b-with-proj-ltx-2.5": + return ModelCheckpointSpec( + relative_path=LTX_2_5_FAMILY_DIR / "gemma4-12b-with-proj-ltx-2.5-bf16.safetensors", + expected_size_bytes=26_300_000_000, + is_folder=False, + repo_id="Lightricks/LTX-2.5", + description="Gemma 4 text encoder with LTX 2.5 projections", + repo_filename="text_encoders/gemma4-12b-with-proj-ltx-2.5-bf16.safetensors", + gated=True, + ) + case "gemma-4-e2b-it": + return ModelCheckpointSpec( + relative_path=Path("gemma-4-E2B-it"), + expected_size_bytes=10_278_849_571, + is_folder=True, + repo_id="google/gemma-4-E2B-it", + description="Gemma 4 E2B instruct prompt enhancer (folder)", ) case "z-image-turbo": return ModelCheckpointSpec( @@ -219,31 +350,67 @@ def get_model_cp_spec(cp_id: ModelCheckpointID) -> ModelCheckpointSpec: "Keeps fine detail consistent through to the last frame." ) +_DISTILLED_2_5_WHATS_NEW = ( + "Upgrades the local model to LTX 2.5 distilled with improved fidelity and motion.\n" + "Uses the official split checkpoint pack (transformer, Gemma 4 text encoder, video/audio VAEs).\n" + "Keeps the same local Fast resolution and duration options." +) + def get_ltx_model_spec(model_id: LTXLocalModelId) -> LTXLocalModelSpec: match model_id: + case "ltx-2.5-22b-distilled": + return LTXLocalModelSpec( + model_cp="ltx-2.5-22b-distilled", + upscale_cp="ltx-2.5-spatial-upscaler-x2-1.0", + text_encoder_cp="gemma4-12b-with-proj-ltx-2.5", + video_vae_cp="ltx-2.5-video-vae", + video_vae_conv_cp="ltx-2.5-video-vae-conv", + audio_vae_cp="ltx-2.5-audio-vae", + duration_head_cp="ltx-2.5-duration-head", + ic_loras_spec=None, + relevance=LTXLocalModelRelevant( + upgrade_messages={ + "ltx-2.3-22b-distilled": _DISTILLED_2_5_WHATS_NEW, + "ltx-2.3-22b-distilled-1.1": _DISTILLED_2_5_WHATS_NEW, + }, + ), + supported_pipelines=_DISTILLED_PIPELINES_2_5, + version_label="2.5", + api_text_encoder_model_id=LTX_2_5_API_TEXT_ENCODER_MODEL_ID, + wants_audio_visual_captions=True, + prompt_enhancer_cp="gemma-4-e2b-it", + is_latest=True, + ) case "ltx-2.3-22b-distilled-1.1": return LTXLocalModelSpec( model_cp="ltx-2.3-22b-distilled-1.1", upscale_cp="ltx-2.3-spatial-upscaler-x2-1.1", text_encoder_cp="gemma-3-12b-it-qat-q4_0-unquantized", + video_vae_cp=None, + video_vae_conv_cp=None, + audio_vae_cp=None, + duration_head_cp=None, ic_loras_spec=_DISTILLED_IC_LORAS, relevance=LTXLocalModelRelevant( upgrade_messages={"ltx-2.3-22b-distilled": _DISTILLED_1_1_WHATS_NEW}, ), - supported_pipelines=_DISTILLED_PIPELINES, - version_label="1.1", - is_latest=True, + supported_pipelines=_DISTILLED_PIPELINES_2_3, + version_label="2.3", ) case "ltx-2.3-22b-distilled": return LTXLocalModelSpec( model_cp="ltx-2.3-22b-distilled", upscale_cp="ltx-2.3-spatial-upscaler-x2-1.1", text_encoder_cp="gemma-3-12b-it-qat-q4_0-unquantized", + video_vae_cp=None, + video_vae_conv_cp=None, + audio_vae_cp=None, + duration_head_cp=None, ic_loras_spec=_DISTILLED_IC_LORAS, relevance=LTXLocalModelRelevant(upgrade_messages={}), - supported_pipelines=_DISTILLED_PIPELINES, - version_label="1.0", + supported_pipelines=_DISTILLED_PIPELINES_2_3, + version_label="2.3 (1.0)", ) case _: assert_never(model_id) @@ -270,18 +437,71 @@ def get_ltx_model_id_for_cp(cp_id: ModelCheckpointID) -> LTXLocalModelId | None: return None -def get_ic_loras_cp_ids(ic_loras_spec: LtxIcLorasSpec) -> tuple[ModelCheckpointID, ...]: +def get_local_prompt_enhancer_cp(spec: LTXLocalModelSpec) -> ModelCheckpointID: + """Preferred checkpoint to download for local Enhance — the encoder itself unless split out.""" + return spec.prompt_enhancer_cp if spec.prompt_enhancer_cp is not None else spec.text_encoder_cp + + +def local_prompt_enhancer_candidates(spec: LTXLocalModelSpec) -> tuple[ModelCheckpointID, ...]: + """Preference order of checkpoints that can run local Enhance for ``spec``. + + 2.3's encoder enhances itself. 2.5 prefers Gemma 4 E2B, then Gemma 3 if a 2.3 + install already put it on disk. E2B cannot encode 2.3 (hidden-size / gemma3 check). + """ + preferred = get_local_prompt_enhancer_cp(spec) + if spec.prompt_enhancer_cp is None: + return (preferred,) + gemma3 = get_ltx_model_spec("ltx-2.3-22b-distilled-1.1").text_encoder_cp + if gemma3 == preferred: + return (preferred,) + return (preferred, gemma3) + + +def resolve_downloaded_prompt_enhancer_cp( + models_dir: Path, spec: LTXLocalModelSpec +) -> ModelCheckpointID | None: + """First candidate already on disk, or None if local Enhance cannot run.""" + for cp_id in local_prompt_enhancer_candidates(spec): + if is_cp_downloaded(models_dir, cp_id): + return cp_id + return None + + +def selected_video_vae_cp(spec: LTXLocalModelSpec, *, use_conv_vae: bool) -> ModelCheckpointID | None: + """Video VAE checkpoint the current Fast decode setting should load, or None for 2.3 monolith.""" + if spec.video_vae_cp is None: + return None + if use_conv_vae and spec.video_vae_conv_cp is not None: + return spec.video_vae_conv_cp + return spec.video_vae_cp + + +def unused_video_vae_cp(spec: LTXLocalModelSpec, *, use_conv_vae: bool) -> ModelCheckpointID | None: + """The other 2.5 video VAE, when both exist. Optional download / not required to run.""" + if spec.video_vae_cp is None or spec.video_vae_conv_cp is None: + return None + return spec.video_vae_cp if use_conv_vae else spec.video_vae_conv_cp + + +def get_ic_loras_cp_ids(ic_loras_spec: LtxIcLorasSpec | None) -> tuple[ModelCheckpointID, ...]: + if ic_loras_spec is None: + return () return tuple(dict.fromkeys((ic_loras_spec.depth_cp, ic_loras_spec.canny_cp, ic_loras_spec.pose_cp))) def get_ltx_model_cp_ids(model_id: LTXLocalModelId) -> tuple[ModelCheckpointID, ...]: spec = get_ltx_model_spec(model_id) - return ( - spec.model_cp, - spec.upscale_cp, - spec.text_encoder_cp, - *get_ic_loras_cp_ids(spec.ic_loras_spec), - ) + cps: list[ModelCheckpointID] = [spec.model_cp, spec.upscale_cp, spec.text_encoder_cp] + if spec.video_vae_cp is not None: + cps.append(spec.video_vae_cp) + if spec.video_vae_conv_cp is not None: + cps.append(spec.video_vae_conv_cp) + if spec.audio_vae_cp is not None: + cps.append(spec.audio_vae_cp) + if spec.duration_head_cp is not None: + cps.append(spec.duration_head_cp) + cps.extend(get_ic_loras_cp_ids(spec.ic_loras_spec)) + return tuple(cps) def _normalized_relative_path(cp_id: ModelCheckpointID) -> Path: @@ -299,9 +519,27 @@ def _normalized_relative_path(cp_id: ModelCheckpointID) -> Path: def resolve_model_path(models_dir: Path, cp_id: ModelCheckpointID) -> Path: + """Canonical on-disk location. New downloads always write here.""" return models_dir / _normalized_relative_path(cp_id) +def _legacy_root_fallback_path(models_dir: Path, cp_id: ModelCheckpointID) -> Path | None: + """Flat models_root copy of a 2.5-family file downloaded before the family dir existed. + + Shared 2.3 / tooling checkpoints are not in ltx-2.5/ and have no fallback. + """ + relative = _normalized_relative_path(cp_id) + if len(relative.parts) < 2 or relative.parts[0] != LTX_2_5_FAMILY_DIR.name: + return None + return models_dir / relative.name + + +def _cp_on_disk(path: Path, spec: ModelCheckpointSpec) -> bool: + if spec.is_folder: + return path.exists() and any(path.iterdir()) + return path.exists() + + def resolve_downloading_dir(models_dir: Path) -> Path: return models_dir / ".downloading" @@ -323,23 +561,31 @@ def resolve_downloading_path(models_dir: Path, cp_id: ModelCheckpointID) -> Path def is_cp_downloaded(models_dir: Path, cp_id: ModelCheckpointID) -> bool: - path = resolve_model_path(models_dir, cp_id) spec = get_model_cp_spec(cp_id) - if spec.is_folder: - return path.exists() and any(path.iterdir()) - return path.exists() + if _cp_on_disk(resolve_model_path(models_dir, cp_id), spec): + return True + fallback = _legacy_root_fallback_path(models_dir, cp_id) + return fallback is not None and _cp_on_disk(fallback, spec) -def get_existing_cp_path(models_dir: Path, cp_id: ModelCheckpointID) -> Path: - path = resolve_model_path(models_dir, cp_id) - if not is_cp_downloaded(models_dir, cp_id): - raise FileNotFoundError(f"Checkpoint not found: {cp_id} at {path}") - return path +def is_duration_head_ready(models_dir: Path, model_id: LTXLocalModelId) -> bool: + """True when this local offering has DurationHead weights on disk (required for Auto duration).""" + spec = get_ltx_model_spec(model_id) + return spec.duration_head_cp is not None and is_cp_downloaded(models_dir, spec.duration_head_cp) -def delete_cp_path(models_dir: Path, cp_id: ModelCheckpointID) -> None: - path = resolve_model_path(models_dir, cp_id) +def get_existing_cp_path(models_dir: Path, cp_id: ModelCheckpointID) -> Path: spec = get_model_cp_spec(cp_id) + canonical = resolve_model_path(models_dir, cp_id) + if _cp_on_disk(canonical, spec): + return canonical + fallback = _legacy_root_fallback_path(models_dir, cp_id) + if fallback is not None and _cp_on_disk(fallback, spec): + return fallback + raise FileNotFoundError(f"Checkpoint not found: {cp_id} at {canonical}") + + +def _remove_cp_path(path: Path, spec: ModelCheckpointSpec) -> None: if spec.is_folder: if path.exists(): import shutil @@ -349,6 +595,14 @@ def delete_cp_path(models_dir: Path, cp_id: ModelCheckpointID) -> None: path.unlink(missing_ok=True) +def delete_cp_path(models_dir: Path, cp_id: ModelCheckpointID) -> None: + spec = get_model_cp_spec(cp_id) + _remove_cp_path(resolve_model_path(models_dir, cp_id), spec) + fallback = _legacy_root_fallback_path(models_dir, cp_id) + if fallback is not None: + _remove_cp_path(fallback, spec) + + def get_downloaded_ltx_model_id(models_dir: Path) -> LTXLocalModelId | None: downloaded: list[LTXLocalModelId] = [] for model_id in ALL_LTX_LOCAL_MODEL_IDS: @@ -376,11 +630,22 @@ def get_downloaded_ltx_model_id(models_dir: Path) -> LTXLocalModelId | None: def _ltx_generation_bundle_on_disk(models_dir: Path, model_id: LTXLocalModelId) -> bool: """True when the always-required generation checkpoints for ``model_id`` are present. - Transformer + upscaler are required regardless of settings; the text encoder is - optional (an LTX API key encodes prompts instead), so it isn't checked here. + Transformer + upscaler (+ split VAEs for 2.5) are required regardless of settings. The text + encoder is optional when the LTX API can encode prompts for the model. """ spec = get_ltx_model_spec(model_id) - return is_cp_downloaded(models_dir, spec.model_cp) and is_cp_downloaded(models_dir, spec.upscale_cp) + if not is_cp_downloaded(models_dir, spec.model_cp) or not is_cp_downloaded(models_dir, spec.upscale_cp): + return False + if not spec.supports_api_text_encoding and not is_cp_downloaded(models_dir, spec.text_encoder_cp): + return False + if spec.video_vae_cp is not None: + has_diff = is_cp_downloaded(models_dir, spec.video_vae_cp) + has_conv = spec.video_vae_conv_cp is not None and is_cp_downloaded(models_dir, spec.video_vae_conv_cp) + if not has_diff and not has_conv: + return False + if spec.audio_vae_cp is not None and not is_cp_downloaded(models_dir, spec.audio_vae_cp): + return False + return True def resolve_active_ltx_model_id( diff --git a/backend/services/a2v_pipeline/a2v_pipeline.py b/backend/services/a2v_pipeline/a2v_pipeline.py index 9eaab633c..a5272abb7 100644 --- a/backend/services/a2v_pipeline/a2v_pipeline.py +++ b/backend/services/a2v_pipeline/a2v_pipeline.py @@ -19,6 +19,10 @@ def create( device: torch.device, streaming_prefetch_count: int | None, loras: list[tuple[str, float]] | None = None, + *, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> "A2VPipeline": ... def generate( diff --git a/backend/services/a2v_pipeline/distilled_a2v_pipeline.py b/backend/services/a2v_pipeline/distilled_a2v_pipeline.py index 632e77996..fa8d4d702 100644 --- a/backend/services/a2v_pipeline/distilled_a2v_pipeline.py +++ b/backend/services/a2v_pipeline/distilled_a2v_pipeline.py @@ -17,6 +17,7 @@ if TYPE_CHECKING: from ltx_core.loader.primitives import LoraPathStrengthAndSDOps + from ltx_pipelines.utils.model_paths import ModelPaths class DistilledA2VPipeline: @@ -29,8 +30,7 @@ class DistilledA2VPipeline: def __init__( self, - distilled_checkpoint_path: str, - gemma_root: str, + model_paths: ModelPaths, spatial_upsampler_path: str, loras: Sequence[LoraPathStrengthAndSDOps] | None = None, device: torch.device | None = None, @@ -55,16 +55,16 @@ def __init__( offload_mode = offload_mode_for_prefetch_count(streaming_prefetch_count, device) self.prompt_encoder = PromptEncoder( - distilled_checkpoint_path, gemma_root, self.dtype, device, offload_mode=offload_mode, + model_paths, self.dtype, device, offload_mode=offload_mode, ) self.image_conditioner = ImageConditioner( - distilled_checkpoint_path, self.dtype, device, + model_paths.video_vae(), self.dtype, device, ) self.audio_conditioner = AudioConditioner( - distilled_checkpoint_path, self.dtype, device, + model_paths.audio_vae(), self.dtype, device, ) self.stage = DiffusionStage.from_checkpoint( # type: ignore[reportUnknownMemberType] - distilled_checkpoint_path, + model_paths.transformer(), self.dtype, device, loras=tuple(loras) if loras else (), @@ -72,10 +72,10 @@ def __init__( offload_mode=offload_mode, ) self.upsampler = VideoUpsampler( - distilled_checkpoint_path, spatial_upsampler_path, self.dtype, device, + model_paths.video_vae(), spatial_upsampler_path, self.dtype, device, ) self.video_decoder = VideoDecoder( - distilled_checkpoint_path, self.dtype, device, + model_paths.video_vae(), self.dtype, device, ) @torch.inference_mode() diff --git a/backend/services/a2v_pipeline/ltx_a2v_pipeline.py b/backend/services/a2v_pipeline/ltx_a2v_pipeline.py index 821dc75fd..fbc37bae4 100644 --- a/backend/services/a2v_pipeline/ltx_a2v_pipeline.py +++ b/backend/services/a2v_pipeline/ltx_a2v_pipeline.py @@ -3,12 +3,16 @@ from __future__ import annotations from collections.abc import Iterator -from typing import cast import torch from api_types import ImageConditioningInput -from services.ltx_pipeline_common import default_tiling_config, encode_video_output, video_chunks_number +from services.ltx_pipeline_common import ( + build_model_paths, + encode_video_output, + resolve_tiling_config, + video_chunks_number, +) from services.services_utils import AudioOrNone, TilingConfigType, device_supports_fp8 @@ -21,6 +25,10 @@ def create( device: torch.device, streaming_prefetch_count: int | None, loras: list[tuple[str, float]] | None = None, + *, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> "LTXa2vPipeline": return LTXa2vPipeline( checkpoint_path=checkpoint_path, @@ -29,6 +37,9 @@ def create( device=device, streaming_prefetch_count=streaming_prefetch_count, loras=loras or [], + video_vae_path=video_vae_path, + audio_vae_path=audio_vae_path, + duration_head_path=duration_head_path, ) def __init__( @@ -39,6 +50,10 @@ def __init__( device: torch.device, streaming_prefetch_count: int | None, loras: list[tuple[str, float]] | None = None, + *, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> None: from ltx_core.loader.primitives import LoraPathStrengthAndSDOps from ltx_core.loader.sd_ops import LTXV_LORA_COMFY_RENAMING_MAP @@ -52,8 +67,13 @@ def __init__( ] self.pipeline = DistilledA2VPipeline( - distilled_checkpoint_path=checkpoint_path, - gemma_root=cast(str, gemma_root), + model_paths=build_model_paths( + checkpoint_path, + gemma_root, + video_vae_path=video_vae_path, + audio_vae_path=audio_vae_path, + duration_head_path=duration_head_path, + ), spatial_upsampler_path=upsampler_path, loras=lora_entries, device=device, @@ -108,7 +128,13 @@ def generate( audio_max_duration: float | None, output_path: str, ) -> None: - tiling_config = default_tiling_config() + tiling_config = resolve_tiling_config( + self.pipeline.video_decoder.checkpoint_path, + height=height, + width=width, + num_frames=num_frames, + device=self.pipeline.device, + ) video, audio = self._run_inference( prompt=prompt, negative_prompt=negative_prompt, diff --git a/backend/services/fast_video_pipeline/fast_video_pipeline.py b/backend/services/fast_video_pipeline/fast_video_pipeline.py index dfdb437ed..1b2777f4f 100644 --- a/backend/services/fast_video_pipeline/fast_video_pipeline.py +++ b/backend/services/fast_video_pipeline/fast_video_pipeline.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, ClassVar, Literal, Protocol from api_types import ImageConditioningInput +from frame_math import AutoDurationSpec if TYPE_CHECKING: import torch @@ -21,6 +22,10 @@ def create( device: torch.device, streaming_prefetch_count: int | None, loras: list[tuple[str, float]] | None = None, + *, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> "FastVideoPipeline": ... @@ -30,7 +35,7 @@ def generate( seed: int, height: int, width: int, - num_frames: int, + num_frames: int | AutoDurationSpec, frame_rate: float, images: list[ImageConditioningInput], output_path: str, diff --git a/backend/services/fast_video_pipeline/ltx_fast_video_pipeline.py b/backend/services/fast_video_pipeline/ltx_fast_video_pipeline.py index 80d145545..d06649ad6 100644 --- a/backend/services/fast_video_pipeline/ltx_fast_video_pipeline.py +++ b/backend/services/fast_video_pipeline/ltx_fast_video_pipeline.py @@ -4,18 +4,20 @@ from collections.abc import Iterator import os -from typing import Final, cast +from typing import Final import torch from api_types import ImageConditioningInput +from frame_math import AutoDurationSpec from services.ltx_pipeline_common import ( - default_tiling_config, + auto_tiling_config, + build_model_paths, encode_video_output, offload_mode_for_prefetch_count, video_chunks_number, ) -from services.services_utils import AudioOrNone, TilingConfigType, device_supports_fp8 +from services.services_utils import AudioOrNone, PipelineTilingType, TilingConfigType, device_supports_fp8 class LTXFastVideoPipeline: @@ -29,6 +31,10 @@ def create( device: torch.device, streaming_prefetch_count: int | None, loras: list[tuple[str, float]] | None = None, + *, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> "LTXFastVideoPipeline": return LTXFastVideoPipeline( checkpoint_path=checkpoint_path, @@ -37,6 +43,9 @@ def create( device=device, streaming_prefetch_count=streaming_prefetch_count, loras=loras or [], + video_vae_path=video_vae_path, + audio_vae_path=audio_vae_path, + duration_head_path=duration_head_path, ) def __init__( @@ -47,6 +56,10 @@ def __init__( device: torch.device, streaming_prefetch_count: int | None, loras: list[tuple[str, float]] | None = None, + *, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> None: from ltx_core.loader.primitives import LoraPathStrengthAndSDOps from ltx_core.loader.sd_ops import LTXV_LORA_COMFY_RENAMING_MAP @@ -56,6 +69,9 @@ def __init__( self._checkpoint_path = checkpoint_path self._gemma_root = gemma_root self._upsampler_path = upsampler_path + self._video_vae_path = video_vae_path + self._audio_vae_path = audio_vae_path + self._duration_head_path = duration_head_path self._device = device self._offload_mode = offload_mode_for_prefetch_count(streaming_prefetch_count, device) self._quantization = build_fp8_cast_policy(checkpoint_path) if device_supports_fp8(device) else None @@ -67,8 +83,13 @@ def __init__( ] self.pipeline = DistilledPipeline( - distilled_checkpoint_path=checkpoint_path, - gemma_root=cast(str, gemma_root), + model_paths=build_model_paths( + checkpoint_path, + gemma_root, + video_vae_path=video_vae_path, + audio_vae_path=audio_vae_path, + duration_head_path=duration_head_path, + ), spatial_upsampler_path=upsampler_path, loras=lora_entries, device=device, @@ -82,23 +103,31 @@ def _run_inference( seed: int, height: int, width: int, - num_frames: int, + num_frames: int | AutoDurationSpec, frame_rate: float, images: list[ImageConditioningInput], - tiling_config: TilingConfigType, - ) -> tuple[torch.Tensor | Iterator[torch.Tensor], AudioOrNone]: + tiling_config: PipelineTilingType, + ) -> tuple[torch.Tensor | Iterator[torch.Tensor], AudioOrNone, int, TilingConfigType | None]: from ltx_pipelines.utils.args import ImageConditioningInput as _LtxImageInput + from ltx_pipelines.utils.types import AutoDuration - return self.pipeline( + pipeline_num_frames: int | AutoDuration = ( + AutoDuration(min_seconds=num_frames.min_seconds, max_seconds=num_frames.max_seconds) + if isinstance(num_frames, AutoDurationSpec) + else num_frames + ) + + video, audio, resolved_frames, resolved_tiling = self.pipeline( prompt=prompt, seed=seed, height=height, width=width, - num_frames=num_frames, + num_frames=pipeline_num_frames, frame_rate=frame_rate, images=[_LtxImageInput(img.path, img.frame_idx, img.strength) for img in images], tiling_config=tiling_config, ) + return video, audio, resolved_frames, resolved_tiling @torch.inference_mode() def generate( @@ -107,13 +136,12 @@ def generate( seed: int, height: int, width: int, - num_frames: int, + num_frames: int | AutoDurationSpec, frame_rate: float, images: list[ImageConditioningInput], output_path: str, ) -> None: - tiling_config = default_tiling_config() - video, audio = self._run_inference( + video, audio, resolved_frames, resolved_tiling = self._run_inference( prompt=prompt, seed=seed, height=height, @@ -121,18 +149,17 @@ def generate( num_frames=num_frames, frame_rate=frame_rate, images=images, - tiling_config=tiling_config, + tiling_config=auto_tiling_config(), ) - chunks = video_chunks_number(num_frames, tiling_config) + chunks = video_chunks_number(resolved_frames, resolved_tiling) encode_video_output(video=video, audio=audio, fps=int(frame_rate), output_path=output_path, video_chunks_number_value=chunks) @torch.inference_mode() def warmup(self, output_path: str) -> None: warmup_frames = 9 - tiling_config = default_tiling_config() try: - video, audio = self._run_inference( + video, audio, resolved_frames, resolved_tiling = self._run_inference( prompt="test warmup", seed=42, height=256, @@ -140,9 +167,9 @@ def warmup(self, output_path: str) -> None: num_frames=warmup_frames, frame_rate=8, images=[], - tiling_config=tiling_config, + tiling_config=auto_tiling_config(), ) - chunks = video_chunks_number(warmup_frames, tiling_config) + chunks = video_chunks_number(resolved_frames, resolved_tiling) encode_video_output(video=video, audio=audio, fps=8, output_path=output_path, video_chunks_number_value=chunks) finally: if os.path.exists(output_path): @@ -160,8 +187,13 @@ def compile_transformer(self) -> None: ] self.pipeline = DistilledPipeline( - distilled_checkpoint_path=self._checkpoint_path, - gemma_root=cast(str, self._gemma_root), + model_paths=build_model_paths( + self._checkpoint_path, + self._gemma_root, + video_vae_path=self._video_vae_path, + audio_vae_path=self._audio_vae_path, + duration_head_path=self._duration_head_path, + ), spatial_upsampler_path=self._upsampler_path, loras=lora_entries, device=self._device, diff --git a/backend/services/ic_lora_pipeline/ic_lora_pipeline.py b/backend/services/ic_lora_pipeline/ic_lora_pipeline.py index 036d347ec..bc0afc513 100644 --- a/backend/services/ic_lora_pipeline/ic_lora_pipeline.py +++ b/backend/services/ic_lora_pipeline/ic_lora_pipeline.py @@ -20,6 +20,10 @@ def create( device: torch.device, streaming_prefetch_count: int | None, lora_strength: float = 1.0, + *, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> "IcLoraPipeline": ... diff --git a/backend/services/ic_lora_pipeline/ltx_ic_lora_pipeline.py b/backend/services/ic_lora_pipeline/ltx_ic_lora_pipeline.py index 79427ae72..fcd140f3f 100644 --- a/backend/services/ic_lora_pipeline/ltx_ic_lora_pipeline.py +++ b/backend/services/ic_lora_pipeline/ltx_ic_lora_pipeline.py @@ -8,7 +8,6 @@ import struct from collections.abc import Iterator from pathlib import Path -from typing import cast import numpy as np import torch @@ -16,12 +15,13 @@ from api_types import ImageConditioningInput from services.ltx_pipeline_common import ( - default_tiling_config, + auto_tiling_config, + build_model_paths, encode_video_output, offload_mode_for_prefetch_count, video_chunks_number, ) -from services.services_utils import AudioOrNone, TilingConfigType, device_supports_fp8 +from services.services_utils import AudioOrNone, PipelineTilingType, TilingConfigType, device_supports_fp8 logger = logging.getLogger(__name__) @@ -36,6 +36,10 @@ def create( device: torch.device, streaming_prefetch_count: int | None, lora_strength: float = 1.0, + *, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> "LTXIcLoraPipeline": return LTXIcLoraPipeline( checkpoint_path=checkpoint_path, @@ -45,6 +49,9 @@ def create( device=device, streaming_prefetch_count=streaming_prefetch_count, lora_strength=lora_strength, + video_vae_path=video_vae_path, + audio_vae_path=audio_vae_path, + duration_head_path=duration_head_path, ) def __init__( @@ -56,6 +63,10 @@ def __init__( device: torch.device, streaming_prefetch_count: int | None, lora_strength: float = 1.0, + *, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> None: from ltx_core.loader.primitives import LoraPathStrengthAndSDOps from ltx_core.loader.sd_ops import LTXV_LORA_COMFY_RENAMING_MAP @@ -70,9 +81,14 @@ def __init__( self._quantization = build_fp8_cast_policy(checkpoint_path) if device_supports_fp8(device) else None offload_mode = offload_mode_for_prefetch_count(streaming_prefetch_count, device) self.pipeline = ICLoraPipeline( - distilled_checkpoint_path=checkpoint_path, + model_paths=build_model_paths( + checkpoint_path, + gemma_root, + video_vae_path=video_vae_path, + audio_vae_path=audio_vae_path, + duration_head_path=duration_head_path, + ), spatial_upsampler_path=upsampler_path, - gemma_root=cast(str, gemma_root), loras=[lora_entry], device=device, quantization=self._quantization, @@ -156,13 +172,13 @@ def _run_inference( frame_rate: float, images: list[ImageConditioningInput], video_conditioning: list[tuple[str, float]], - tiling_config: TilingConfigType, + tiling_config: PipelineTilingType, skip_stage_2: bool, conditioning_attention_mask: torch.Tensor | None, - ) -> tuple[torch.Tensor | Iterator[torch.Tensor], AudioOrNone]: + ) -> tuple[torch.Tensor | Iterator[torch.Tensor], AudioOrNone, TilingConfigType | None]: from ltx_pipelines.utils.args import ImageConditioningInput as _LtxImageInput - return self.pipeline( + video, audio, resolved_tiling = self.pipeline( prompt=prompt, seed=seed, height=height, @@ -175,6 +191,7 @@ def _run_inference( skip_stage_2=skip_stage_2, conditioning_attention_mask=conditioning_attention_mask, ) + return video, audio, resolved_tiling def _load_mask_tensor(self, path: str, num_frames: int) -> torch.Tensor: """Load an outpaint mask video as a (1, 1, F, H, W) float tensor in [0, 1]. @@ -278,9 +295,8 @@ def generate( "reduce frames (shorter clip / lower FPS) or RES FACTOR.", free_gb, ) - tiling_config = default_tiling_config() try: - video, audio = self._run_inference( + video, audio, resolved_tiling = self._run_inference( prompt=prompt, seed=seed, height=height, @@ -289,7 +305,7 @@ def generate( frame_rate=frame_rate, images=images, video_conditioning=video_conditioning, - tiling_config=tiling_config, + tiling_config=auto_tiling_config(), skip_stage_2=skip_stage_2, conditioning_attention_mask=conditioning_attention_mask, ) @@ -315,7 +331,7 @@ def generate( audio.sampling_rate, audio.waveform.shape[0], a_dur, v_dur, "" if abs(a_dur - v_dur) < 0.1 else " MISMATCH", ) - chunks = video_chunks_number(num_frames, tiling_config) + chunks = video_chunks_number(num_frames, resolved_tiling) # round(), not int(): avoids truncating e.g. 23.976 -> 23 (encode_video int()s again). encode_video_output(video=video, audio=audio, fps=round(frame_rate), output_path=output_path, video_chunks_number_value=chunks) except torch.cuda.OutOfMemoryError: diff --git a/backend/services/ltx_api_client/ltx_api_client.py b/backend/services/ltx_api_client/ltx_api_client.py index c5f77e800..5a453609b 100644 --- a/backend/services/ltx_api_client/ltx_api_client.py +++ b/backend/services/ltx_api_client/ltx_api_client.py @@ -50,7 +50,7 @@ def generate_text_to_video( prompt: str, model: str, resolution: str, - duration: float, + duration: float | None, fps: float, generate_audio: bool, camera_motion: VideoCameraMotion = "none", @@ -65,7 +65,7 @@ def generate_image_to_video( image_uri: str, model: str, resolution: str, - duration: float, + duration: float | None, fps: float, generate_audio: bool, camera_motion: VideoCameraMotion = "none", @@ -93,6 +93,7 @@ def retake( duration: float, prompt: str, mode: RetakeMode, + model: str, ) -> LTXRetakeResult: ... @@ -104,5 +105,6 @@ def extend( duration: float, prompt: str, mode: ExtendMode, + model: str, ) -> LTXRetakeResult: ... diff --git a/backend/services/ltx_api_client/ltx_api_client_impl.py b/backend/services/ltx_api_client/ltx_api_client_impl.py index 312a6a9c1..2098d375f 100644 --- a/backend/services/ltx_api_client/ltx_api_client_impl.py +++ b/backend/services/ltx_api_client/ltx_api_client_impl.py @@ -164,7 +164,7 @@ def generate_text_to_video( prompt: str, model: str, resolution: str, - duration: float, + duration: float | None, fps: float, generate_audio: bool, camera_motion: VideoCameraMotion = "none", @@ -191,7 +191,7 @@ def generate_image_to_video( image_uri: str, model: str, resolution: str, - duration: float, + duration: float | None, fps: float, generate_audio: bool, camera_motion: VideoCameraMotion = "none", @@ -241,6 +241,7 @@ def retake( duration: float, prompt: str, mode: RetakeMode, + model: str, ) -> LTXRetakeResult: return self._run_video_edit( api_key=api_key, @@ -251,6 +252,7 @@ def retake( "start_time": float(start_time), "duration": float(duration), "mode": mode, + "model": model, }, prompt=prompt, ) @@ -263,6 +265,7 @@ def extend( duration: float, prompt: str, mode: ExtendMode, + model: str, ) -> LTXRetakeResult: # Extend uses the async v2 endpoint (submit → poll → download): a 12s Pro extend # takes minutes, which the sync v1 endpoint can't hold open without the connection @@ -275,6 +278,7 @@ def extend( edit_payload={ "duration": float(duration), "mode": mode, + "model": model, }, prompt=prompt, ) diff --git a/backend/services/ltx_pipeline_common.py b/backend/services/ltx_pipeline_common.py index a95738163..5b1bff1d2 100644 --- a/backend/services/ltx_pipeline_common.py +++ b/backend/services/ltx_pipeline_common.py @@ -8,17 +8,104 @@ import torch from api_types import ImageConditioningInput -from services.services_utils import AudioOrNone, TilingConfigType, device_supports_fp8 +from services.services_utils import AudioOrNone, PipelineTilingType, TilingConfigType, device_supports_fp8 if TYPE_CHECKING: from ltx_core.components.guiders import MultiModalGuiderParams + from ltx_pipelines.utils.model_paths import ModelPaths from ltx_pipelines.utils.types import OffloadMode -def default_tiling_config() -> TilingConfigType: - from ltx_core.model.video_vae import TilingConfig +def auto_tiling_config() -> PipelineTilingType: + """Let the pipeline derive decode tiling from the VAE it will decode with. - return TilingConfig.default() + A conv VAE (2.3 monolith) and a diffusion VAE (2.5 split) need different tile + overlaps, so a fixed layout that one accepts the other rejects. + """ + from ltx_core.model.video_vae import AUTO_TILING + + return AUTO_TILING + + +def host_available_bytes() -> int: + """Currently available system RAM in bytes (unified memory on Apple Silicon).""" + import psutil + + return int(psutil.virtual_memory().available) + + +def diffvae_activation_budget_bytes(device: torch.device | None = None) -> int: + """Bytes DiffVAE decode tiling may treat as free activation memory. + + ltx-pipelines only queries the CUDA allocator. On MPS/CPU that path yields 0, + so AUTO_TILING raises ``Cannot fit a DiffVAE decode tile`` before decode. + CUDA keeps the upstream allocator budget; everywhere else uses available RAM. + """ + if device is not None and device.type == "cuda" and torch.cuda.is_available(): + from ltx_core.devices import cuda_activation_budget_bytes + + return int(cuda_activation_budget_bytes(device)) + return host_available_bytes() + + +def resolve_diffvae_free_bytes(device: torch.device | None, free_bytes: int | None) -> int | None: + """Fill a DiffVAE tiling budget when upstream would treat non-CUDA as 0.""" + if free_bytes is not None and free_bytes > 0: + return free_bytes + if device is not None and device.type == "cuda": + return free_bytes + return host_available_bytes() + + +def resolve_tiling_config( + vae_checkpoint_path: str, + *, + height: int, + width: int, + num_frames: int, + device: torch.device | None = None, +) -> TilingConfigType: + """Same recommendation ``AUTO_TILING`` resolves to, for pipelines that decode themselves.""" + from ltx_pipelines.utils.helpers import get_device, tiling_config_for_vae + + if device is None: + device = get_device() + return tiling_config_for_vae( + vae_checkpoint_path, + height=height, + width=width, + num_frames=num_frames, + device=device, + free_bytes=diffvae_activation_budget_bytes(device), + ) + + +def build_model_paths( + checkpoint_path: str, + gemma_root: str | None, + *, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, +) -> ModelPaths: + """Build ``ModelPaths`` for monolith (2.3) or split (2.5) checkpoint layouts. + + When both VAE paths are provided, uses ``from_split`` (LTX 2.5). Otherwise uses + ``from_monolith`` where the fat checkpoint also supplies the VAEs and DurationHead. + Split 2.5 DurationHead is a separate safetensors; omit ``duration_head_path`` and + AutoDuration fails closed in the pipeline. + """ + from ltx_pipelines.utils.model_paths import ModelPaths + + if video_vae_path is not None and audio_vae_path is not None: + return ModelPaths.from_split( + transformer_path=checkpoint_path, + text_encoder_path=gemma_root, + video_vae_path=video_vae_path, + audio_vae_path=audio_vae_path, + duration_head_path=duration_head_path, + ) + return ModelPaths.from_monolith(checkpoint_path, gemma_root, video_vae_path=video_vae_path) def default_guiders() -> tuple[MultiModalGuiderParams, MultiModalGuiderParams]: @@ -101,9 +188,10 @@ def __init__( self.device = device self.dtype = torch.bfloat16 + model_paths = build_model_paths(checkpoint_path, gemma_root) self.prompt_encoder = PromptEncoder( - checkpoint_path, gemma_root or "", self.dtype, device, + model_paths, self.dtype, device, ) self.image_conditioner = ImageConditioner( checkpoint_path, self.dtype, device, diff --git a/backend/services/patches/diffvae_decode_vram.py b/backend/services/patches/diffvae_decode_vram.py new file mode 100644 index 000000000..beaa974e8 --- /dev/null +++ b/backend/services/patches/diffvae_decode_vram.py @@ -0,0 +1,129 @@ +"""Free denoise weights before DiffVAE decode so 32 GB CUDA can finish. + +Full-resident 2.5 keeps the ~23 GiB fp8 transformer on GPU through decode +(``diffusion_stage_cache`` holds it; even without the cache, CUDA reserved +memory stays at device capacity). DiffVAE then builds on ``free=0`` and +neighborhood attention never returns — 32 GB Windows hung past 6 minutes. + +``DistilledPipeline`` also resolves ``AUTO_TILING`` *before* denoise, while the +GPU is almost empty (~28 GiB free). Decode then runs after evict with ~6 GiB +driver-free — or, after a clean ``empty_cache``, ``mem_get_info`` reports ~30 GiB +free. Planning 1024×576 against that 30 GiB budget picks a near-full-frame tile; +the first NA slab then drives ``reserved`` to capacity and hangs. Cap the tile +budget so 540p always splits. + +``DistilledPipeline`` calls ``VideoDecoder`` only after both denoise stages, +so evicting here cannot interrupt a mid-denoise checkout. + +Remove once ltx-pipelines offloads the transformer before DiffVAE decode and +resolves AUTO_TILING against that post-offload budget. + +Usage: + import services.patches.diffvae_decode_vram # noqa: F401 +""" + +from __future__ import annotations + +import logging +from typing import Any + +import torch +from ltx_core.devices import cuda_activation_budget_bytes +from ltx_core.model.video_vae.model_configurator import is_diffusion_video_vae +from ltx_core.types import VideoLatentShape +from ltx_pipelines.utils.blocks import VideoDecoder +from ltx_pipelines.utils.helpers import cleanup_memory, tiling_config_for_vae + +from services.patches import diffusion_stage_cache + +logger = logging.getLogger(__name__) + +# After evict, Windows ``mem_get_info`` can still report ~30 GiB free. AUTO_TILING +# then covers 540p in one tile. 8 GiB is above the ~2.4 GiB min 540p/5s tile. +_CUDA_DIFFVAE_TILE_BUDGET_CAP_BYTES = 8 * 1024**3 + +_orig_video_decoder_call = VideoDecoder.__call__ + + +def _cuda_tile_budget_bytes(measured: int) -> int: + return min(max(int(measured), 0), _CUDA_DIFFVAE_TILE_BUDGET_CAP_BYTES) + + +def _release_denoise_weights() -> None: + diffusion_stage_cache.evict() + cleanup_memory() + logger.info("Freed resident transformer before DiffVAE decode") + + +def _pixel_shape_from_latent(latent: torch.Tensor) -> tuple[int, int, int]: + """Pixel (height, width, frames) from a transformer latent ``(B, C, T, H, W)``. + + Uses ``VIDEO_SCALE_FACTORS`` (8×32×32), not DiffVAE ``pixel_scale``. The latter + is stage-4-feature → pixel (8 spatial) and would report 540p as 256×144. + """ + pixels = VideoLatentShape.from_torch_shape(latent.shape).upscale() + return int(pixels.height), int(pixels.width), int(pixels.frames) + + +def _replace_tiling_arg( + args: tuple[Any, ...], kwargs: dict[str, Any], tiling_config: Any +) -> tuple[tuple[Any, ...], dict[str, Any]]: + if "tiling_config" in kwargs: + return args, {**kwargs, "tiling_config": tiling_config} + if len(args) >= 2: + return (args[0], tiling_config, *args[2:]), kwargs + return args, {**kwargs, "tiling_config": tiling_config} + + +def _with_post_evict_cuda_tiling( + decoder: Any, args: tuple[Any, ...], kwargs: dict[str, Any] +) -> tuple[tuple[Any, ...], dict[str, Any]]: + """Re-resolve DiffVAE AUTO tiles against the CUDA budget after transformer evict.""" + device = getattr(decoder, "_device", None) + if not isinstance(device, torch.device) or device.type != "cuda": + return args, kwargs + latent = kwargs.get("latent", args[0] if args else None) + if not isinstance(latent, torch.Tensor) or latent.ndim != 5: + return args, kwargs + checkpoint = getattr(decoder, "checkpoint_path", None) + if not isinstance(checkpoint, str): + checkpoint = getattr(decoder, "_checkpoint_path", None) + if not isinstance(checkpoint, str) or not is_diffusion_video_vae(checkpoint): + return args, kwargs + + height, width, num_frames = _pixel_shape_from_latent(latent) + measured = int(cuda_activation_budget_bytes(device)) + budget = _cuda_tile_budget_bytes(measured) + recommend_kwargs: dict[str, Any] = { + "height": height, + "width": width, + "num_frames": num_frames, + "device": device, + "free_bytes": budget, + } + optimization = getattr(decoder, "diffvae_optimization", None) + if optimization is not None: + recommend_kwargs["diffvae_optimization"] = optimization + tiling = tiling_config_for_vae(checkpoint, **recommend_kwargs) + logger.info( + "DiffVAE CUDA tiling after evict: measured=%.1f GiB budget=%.1f GiB " + "%sx%sx%s t/h/w tiles=%s/%s/%s", + measured / 1024**3, + budget / 1024**3, + width, + height, + num_frames, + getattr(getattr(tiling, "frames", None), "tile_size", "?"), + getattr(getattr(tiling, "height", None), "tile_size", "?"), + getattr(getattr(tiling, "width", None), "tile_size", "?"), + ) + return _replace_tiling_arg(args, kwargs, tiling) + + +def _patched_video_decoder_call(self: VideoDecoder, *args: Any, **kwargs: Any) -> Any: + _release_denoise_weights() + args, kwargs = _with_post_evict_cuda_tiling(self, args, kwargs) + return _orig_video_decoder_call(self, *args, **kwargs) + + +VideoDecoder.__call__ = _patched_video_decoder_call # type: ignore[method-assign] diff --git a/backend/services/patches/diffvae_mps_tiling_budget.py b/backend/services/patches/diffvae_mps_tiling_budget.py new file mode 100644 index 000000000..991a2958a --- /dev/null +++ b/backend/services/patches/diffvae_mps_tiling_budget.py @@ -0,0 +1,144 @@ +"""Monkey-patch: DiffVAE AUTO_TILING on non-CUDA devices. + +``ltx_pipelines.utils.helpers.tiling_config_for_vae`` (and +``DiffusionVideoDecoder.recommended_tiling_config``) only query the CUDA +allocator. On MPS that yields ``free_bytes=0`` → ``usable_bytes=0`` → +``Cannot fit a DiffVAE decode tile under the memory budget`` before decode +starts. The min tile for a 1024×576×121 job is ~2.4 GiB; the Mac has the RAM, +upstream just never counts it. + +Filling that budget lets AUTO_TILING pick a single full-width stage-5 tile +(1024 on 540p). Eager ``K=11`` neighborhood attention on MPS then silently +zeros the tail frames. After the budget is filled, this patch forces a 2-way +width split when MPS would otherwise decode in one width tile. + +``DistilledPipeline`` resolves ``AUTO_TILING`` internally and never goes through +Desktop's ``resolve_tiling_config``, so callers cannot pass a budget. + +Remove once ltx-pipelines/ltx-core query MPS/unified free memory themselves +and refuse full-width eager NA tiles on MPS. + +Usage: + import services.patches.diffvae_mps_tiling_budget # noqa: F401 +""" + +from __future__ import annotations + +import inspect +import logging +from dataclasses import replace +from typing import Any + +import torch +from ltx_core.model.video_vae.diffusion_video_decoder import DiffusionVideoDecoder +from ltx_core.tiling import DimensionSizeConfig, split_by_size +from ltx_pipelines.utils import helpers + +from services.ltx_pipeline_common import resolve_diffvae_free_bytes + +logger = logging.getLogger(__name__) + +_logged_budget = False +_WIDTH_ALIGN = 32 + + +def _width_tile_count(tile_size: int, overlap: int, width: int) -> int: + if tile_size <= 0 or tile_size >= width or overlap >= tile_size: + return 1 + return len(split_by_size(tile_size, overlap)(width).intervals) + + +def _ensure_mps_width_split(config: Any, *, width: object, device: torch.device) -> Any: + """Force n_w >= 2 on MPS when AUTO_TILING would cover the full frame in one tile.""" + if device.type != "mps" or not isinstance(width, int) or width < 1: + return config + width_axis = getattr(config, "width", None) + if width_axis is None: + return config + current = int(getattr(width_axis, "tile_size", 0) or 0) + overlap = int(getattr(width_axis, "overlap", 0) or 0) + if overlap <= 0: + height_axis = getattr(config, "height", None) + overlap = int(getattr(height_axis, "overlap", 0) or 0) if height_axis is not None else 0 + if overlap <= 0: + return config + n_w = _width_tile_count(current if current > 0 else width, overlap, width) + if n_w >= 2: + return config + min_size = max(overlap + _WIDTH_ALIGN, 2 * overlap) + if width <= min_size: + return config + raw = (width + overlap) // 2 + tile = ((raw + _WIDTH_ALIGN - 1) // _WIDTH_ALIGN) * _WIDTH_ALIGN + tile = min(max(tile, min_size), width - _WIDTH_ALIGN) + if tile <= overlap or tile >= width: + return config + new_n = _width_tile_count(tile, overlap, width) + if new_n < 2: + return config + logger.info("DiffVAE MPS width clamp: %s → %s (n_w %s → %s)", current or width, tile, n_w, new_n) + return replace(config, width=DimensionSizeConfig(tile_size=tile, overlap=overlap)) + + +assert "free_bytes" in inspect.signature(helpers.tiling_config_for_vae).parameters, ( + "tiling_config_for_vae missing free_bytes — patch needs updating." +) +assert hasattr(DiffusionVideoDecoder, "recommended_tiling_config"), ( + "DiffusionVideoDecoder.recommended_tiling_config not found — patch needs updating." +) +assert "free_bytes" in inspect.signature(DiffusionVideoDecoder.recommended_tiling_config).parameters, ( + "recommended_tiling_config missing free_bytes — patch needs updating." +) + +_orig_tiling_config_for_vae = helpers.tiling_config_for_vae +_orig_recommended_tiling_config = DiffusionVideoDecoder.recommended_tiling_config + + +def _maybe_log_injected_budget(device: torch.device, free_bytes: int) -> None: + global _logged_budget + if _logged_budget: + return + _logged_budget = True + logger.info( + "DiffVAE tiling uses available system RAM on %s: %.1f GiB " + "(upstream non-CUDA budget is 0).", + device.type, + free_bytes / (1024**3), + ) + + +def _patched_tiling_config_for_vae(*args: Any, **kwargs: Any) -> Any: + device = kwargs.get("device") + if not isinstance(device, torch.device): + device = helpers.get_device() + kwargs["device"] = device + raw = kwargs.get("free_bytes") + incoming = raw if isinstance(raw, int) else None + filled = resolve_diffvae_free_bytes(device, incoming) + if (incoming is None or incoming == 0) and filled is not None and filled > 0 and device.type != "cuda": + _maybe_log_injected_budget(device, filled) + kwargs["free_bytes"] = filled + return _ensure_mps_width_split( + _orig_tiling_config_for_vae(*args, **kwargs), + width=kwargs.get("width"), + device=device, + ) + + +def _patched_recommended_tiling_config(self: DiffusionVideoDecoder, *args: Any, **kwargs: Any) -> Any: + device = next(self.parameters()).device + raw = kwargs.get("free_bytes") + incoming = raw if isinstance(raw, int) else None + filled = resolve_diffvae_free_bytes(device, incoming) + if (incoming is None or incoming == 0) and filled is not None and filled > 0 and device.type != "cuda": + _maybe_log_injected_budget(device, filled) + kwargs["free_bytes"] = filled + return _ensure_mps_width_split( + _orig_recommended_tiling_config(self, *args, **kwargs), + width=kwargs.get("width"), + device=device, + ) + + +helpers.tiling_config_for_vae = _patched_tiling_config_for_vae +DiffusionVideoDecoder.recommended_tiling_config = _patched_recommended_tiling_config # type: ignore[method-assign] diff --git a/backend/services/patches/ic_lora_stage2_lora.py b/backend/services/patches/ic_lora_stage2_lora.py index 2a74ebfd9..89f36d920 100644 --- a/backend/services/patches/ic_lora_stage2_lora.py +++ b/backend/services/patches/ic_lora_stage2_lora.py @@ -60,14 +60,15 @@ import torch from ltx_core.components.noisers import GaussianNoiser -from ltx_core.model.video_vae import TilingConfig +from ltx_core.model.video_vae import AUTO_TILING, AutoTiling, TileSizeConfig, TilingConfig from ltx_core.model.video_vae.video_vae import VideoEncoder from ltx_core.types import Audio, VideoPixelShape from ltx_pipelines.ic_lora import ICLoraPipeline from ltx_pipelines.utils.args import ImageConditioningInput from ltx_pipelines.utils.constants import DISTILLED_SIGMAS, STAGE_2_DISTILLED_SIGMAS from ltx_pipelines.utils.denoisers import SimpleDenoiser -from ltx_pipelines.utils.helpers import assert_resolution, combined_image_conditionings +from ltx_pipelines.utils.helpers import assert_resolution, combined_image_conditionings, ensure_tiling_config, tiling_scale_factors_for_vae +from ltx_pipelines.utils.media_io import HDRColorSpace from ltx_pipelines.utils.types import ModalitySpec from services.patches import diffusion_stage_cache @@ -79,7 +80,7 @@ _encode_state = threading.local() -def _should_tile(shape: torch.Size, cfg: TilingConfig) -> bool: +def _should_tile(shape: torch.Size, cfg: TileSizeConfig) -> bool: """True when ``tiled_encode`` would split this input into more than one tile. Below the tile size in every dimension a single tile == the un-tiled forward, so we @@ -87,11 +88,11 @@ def _should_tile(shape: torch.Size, cfg: TilingConfig) -> bool: (B, C, F, H, W). """ f, h, w = shape[-3], shape[-2], shape[-1] - sc = getattr(cfg, "spatial_config", None) - tc = getattr(cfg, "temporal_config", None) - if sc is not None and (h > sc.tile_size_in_pixels or w > sc.tile_size_in_pixels): + if cfg.height.is_tiled() and h > cfg.height.tile_size: return True - if tc is not None and f > tc.tile_size_in_frames: + if cfg.width.is_tiled() and w > cfg.width.tile_size: + return True + if cfg.frames.is_tiled() and f > cfg.frames.tile_size: return True return False @@ -101,12 +102,12 @@ def _scoped_tiling_forward(self: VideoEncoder, sample: torch.Tensor) -> torch.Te # self.forward) and small inputs pass straight through to the original forward. if not getattr(_encode_state, "tile", False) or getattr(_encode_state, "in_tiled", False): return _orig_encoder_forward(self, sample) - cfg = TilingConfig.default() + cfg = TileSizeConfig.default() if not _should_tile(sample.shape, cfg): return _orig_encoder_forward(self, sample) logger.info( "[ic-lora] tiling conditioning VAE encode %s (tile %dpx) to avoid VRAM blow-up", - tuple(sample.shape), cfg.spatial_config.tile_size_in_pixels, + tuple(sample.shape), cfg.height.tile_size, ) _encode_state.in_tiled = True try: @@ -168,13 +169,16 @@ def _patched_call( # noqa: PLR0913 images: list[ImageConditioningInput], video_conditioning: list[tuple[str, float]], enhance_prompt: bool = False, - tiling_config: TilingConfig | None = None, + enhance_static_cache: bool = False, + vae_dtype: torch.dtype | None = None, + tiling_config: TilingConfig | AutoTiling | None = AUTO_TILING, conditioning_attention_strength: float = 1.0, skip_stage_2: bool = False, conditioning_attention_mask: torch.Tensor | None = None, stage_1_sigmas: torch.Tensor = DISTILLED_SIGMAS, stage_2_sigmas: torch.Tensor = STAGE_2_DISTILLED_SIGMAS, -) -> tuple[Iterator[torch.Tensor], Audio]: + color_space: HDRColorSpace | None = None, +) -> tuple[Iterator[torch.Tensor], Audio, TilingConfig | None]: """Copy of ICLoraPipeline.__call__ (pinned upstream rev) + IC-LoRA-in-stage-2 (PATCH).""" use_lora_in_stage_2 = getattr(self, "use_lora_in_stage_2", False) @@ -186,6 +190,7 @@ def _patched_call( # noqa: PLR0913 # bumped per call here. If VRAM pressure resurfaces on use_lora_in_stage_2, the fix is a # construction-time offload_mode override for this pipeline, not a per-call one. + images = self.image_conditioner.resolve_crf(images) assert_resolution(height=height, width=width, is_two_stage=True) if not (0.0 <= conditioning_attention_strength <= 1.0): raise ValueError( @@ -194,15 +199,28 @@ def _patched_call( # noqa: PLR0913 generator = torch.Generator(device=self.device).manual_seed(seed) noiser = GaussianNoiser(generator=generator) + if vae_dtype is None: + vae_dtype = self.dtype (ctx_p,) = self.prompt_encoder( [prompt], enhance_first_prompt=enhance_prompt, + enhance_static_cache=enhance_static_cache, enhance_prompt_image=images[0][0] if len(images) > 0 else None, enhance_prompt_seed=seed, ) video_context, audio_context = ctx_p.video_encoding, ctx_p.audio_encoding + scale_factors = tiling_scale_factors_for_vae(self.video_decoder.checkpoint_path) + tiling_config = ensure_tiling_config( + tiling_config, + scale_factors=scale_factors, + vae_checkpoint_path=self.video_decoder.checkpoint_path, + video_shape=VideoPixelShape(batch=1, frames=num_frames, height=height, width=width, fps=frame_rate), + diffvae_optimization=self.video_decoder.diffvae_optimization, + device=self.device, + ) + # Stage 1: Initial low resolution video generation. stage_1_output_shape = VideoPixelShape( batch=1, @@ -225,6 +243,7 @@ def _patched_call( # noqa: PLR0913 num_frames=num_frames, conditioning_attention_strength=conditioning_attention_strength, conditioning_attention_mask=conditioning_attention_mask, + color_space=color_space, ) ) @@ -259,9 +278,9 @@ def _patched_call( # noqa: PLR0913 if skip_stage_2: # Skip Stage 2: Decode directly from Stage 1 output at half resolution logging.info("[IC-LoRA] Skipping Stage 2 (--skip-stage-2 enabled)") - decoded_video = self.video_decoder(video_state.latent, tiling_config, generator) + decoded_video = self.video_decoder(video_state.latent, tiling_config, generator, dtype=vae_dtype) decoded_audio = self.audio_decoder(audio_state.latent) - return decoded_video, decoded_audio + return decoded_video, decoded_audio, tiling_config # Stage 2: Upsample and refine the video at higher resolution. upscaled_video_latent = self.upsampler(video_state.latent[:1]) @@ -297,6 +316,7 @@ def _patched_call( # noqa: PLR0913 num_frames=num_frames, conditioning_attention_strength=conditioning_attention_strength, conditioning_attention_mask=conditioning_attention_mask, + color_space=color_space, ) ) else: @@ -308,6 +328,7 @@ def _patched_call( # noqa: PLR0913 video_encoder=enc, dtype=self.dtype, device=self.device, + color_space=color_space, ) ) @@ -332,18 +353,18 @@ def _patched_call( # noqa: PLR0913 ), ) - decoded_video = self.video_decoder(video_state.latent, tiling_config, generator) + decoded_video = self.video_decoder(video_state.latent, tiling_config, generator, dtype=vae_dtype) decoded_audio = self.audio_decoder(audio_state.latent) - return decoded_video, decoded_audio + return decoded_video, decoded_audio, tiling_config ICLoraPipeline.__call__ = _patched_call # type: ignore[method-assign] if __name__ == "__main__": - cfg = TilingConfig.default() - sp = cfg.spatial_config.tile_size_in_pixels - fr = cfg.temporal_config.tile_size_in_frames + cfg = TileSizeConfig.default() + sp = cfg.height.tile_size + fr = cfg.frames.tile_size assert not _should_tile(torch.Size([1, 3, fr, sp, sp]), cfg), "exactly one tile -> no tiling" assert _should_tile(torch.Size([1, 3, fr, sp, sp + 1]), cfg), "wider than a tile -> tile" assert _should_tile(torch.Size([1, 3, fr + 1, sp, sp]), cfg), "more frames than a tile -> tile" diff --git a/backend/services/patches/natten_libnatten_gate.py b/backend/services/patches/natten_libnatten_gate.py new file mode 100644 index 000000000..34f9209ad --- /dev/null +++ b/backend/services/patches/natten_libnatten_gate.py @@ -0,0 +1,44 @@ +"""Treat Flex-only natten as missing so DiffVAE can fall back to Triton. + +ltx-core's ``natten_available()`` is True after ``import natten`` succeeds. +CHUNKED_EAGER then pins ``cutlass-fna``, which needs compiled ``libnatten``. +A PyPI Flex-Attention wheel imports fine, ``HAS_LIBNATTEN`` is False, and +decode crashes instead of taking the Triton/eager remap in ``apply.py``. + +Our Windows GCS wheel has ``HAS_LIBNATTEN=True``. This gate keeps that path +and fails closed for Flex-only installs. + +Remove once ltx-core's ``natten_available()`` checks ``natten.HAS_LIBNATTEN``. + +Usage: + import services.patches.natten_libnatten_gate # noqa: F401 +""" + +from __future__ import annotations + +import logging +from typing import Any + +from ltx_core.model.video_vae.transformer import attention as na_mod + +logger = logging.getLogger(__name__) + + +def _has_libnatten() -> bool: + try: + import natten + except ImportError: + return False + return bool(getattr(natten, "HAS_LIBNATTEN", False)) + + +def _gate_natten_available(attention_mod: Any, *, has_libnatten: bool) -> None: + if attention_mod._NATTEN_AVAILABLE and not has_libnatten: + logger.warning( + "natten imported without libnatten; " + "DiffVAE will fall back to Triton/eager instead of cutlass-fna" + ) + attention_mod._NATTEN_AVAILABLE = False + + +_gate_natten_available(na_mod, has_libnatten=_has_libnatten()) diff --git a/backend/services/patches/safetensors_metadata_fix.py b/backend/services/patches/safetensors_metadata_fix.py index 08bae4fcc..925a19b71 100644 --- a/backend/services/patches/safetensors_metadata_fix.py +++ b/backend/services/patches/safetensors_metadata_fix.py @@ -34,10 +34,22 @@ def _read_safetensors_metadata(path: str) -> dict[str, str] | None: def _patched_model_metadata(self: SafetensorsModelStateDictLoader, path: str) -> dict: + """Full ``__metadata__`` dict with JSON-encoded values parsed, mirroring upstream. + + Callers index into it themselves (``config``, ``model_version``, + ``gemma_source_checkpoint``), so returning only ``config`` silently hides the + sibling keys. + """ meta = _read_safetensors_metadata(path) - if meta is None or "config" not in meta: + if meta is None: return {} - return json.loads(meta["config"]) + parsed: dict[str, object] = {} + for key, value in meta.items(): + try: + parsed[key] = json.loads(value) + except json.JSONDecodeError: + parsed[key] = value + return parsed assert hasattr(SafetensorsModelStateDictLoader, "metadata") and callable( @@ -78,15 +90,19 @@ def _patched_read_lora_reference_downscale_factor(lora_path: str) -> int: setattr(_ic_lora_module, _DOWNSCALE_FN, _patched_read_lora_reference_downscale_factor) -# --- Patch 3: ltx_pipelines.utils.constants.detect_params --- +# --- Patch 3: ltx_pipelines.utils.constants.detect_model_version --- +# Only the metadata read is replaced; the version -> params mapping stays upstream's +# (``detect_params`` calls this by module global), so new generations keep their own defaults. +import ltx_pipelines.distilled as _distilled_module import ltx_pipelines.utils.constants as _constants_module +from ltx_core.loader.helpers import parse_model_version -_original_detect_params = _constants_module.detect_params +_DETECT_VERSION_FN = "detect_model_version" -def _patched_detect_params(checkpoint_path: str) -> object: +def _patched_detect_model_version(checkpoint_path: str) -> tuple[int, ...]: import logging logger = logging.getLogger(__name__) @@ -94,20 +110,23 @@ def _patched_detect_params(checkpoint_path: str) -> object: meta = _read_safetensors_metadata(checkpoint_path) or {} version = meta.get("model_version", "") except Exception: - logger.warning("Could not read checkpoint metadata from %s, using defaults", checkpoint_path) - return _constants_module.LTX_2_PARAMS - - if version.startswith(_constants_module._LTX_2_3_MODEL_VERSION_PREFIX): - return _constants_module.LTX_2_3_PARAMS + logger.warning("Could not read checkpoint metadata from %s, treating it as unversioned", checkpoint_path) + return () - logger.info("Using LTX_2_PARAMS for checkpoint (version=%s)", version or "unknown") - return _constants_module.LTX_2_PARAMS + # Pre-release tags come both dot- and hyphen-separated ("2.3.rc1", "2.4-rc2"). + parsed = parse_model_version(version.replace("-", ".")) + logger.info("Checkpoint declares model_version=%s (parsed as %s)", version or "unknown", parsed) + return parsed -assert hasattr(_constants_module, "detect_params"), ( - "ltx_pipelines.utils.constants.detect_params not found — patch needs updating." +assert hasattr(_constants_module, _DETECT_VERSION_FN), ( + f"ltx_pipelines.utils.constants.{_DETECT_VERSION_FN} not found — patch needs updating." ) -_constants_module.detect_params = _patched_detect_params +setattr(_constants_module, _DETECT_VERSION_FN, _patched_detect_model_version) +# distilled.py binds the name via `from ...constants import ...`, so its module-local +# reference (the sampler-selection call site) must be patched too. +if hasattr(_distilled_module, _DETECT_VERSION_FN): + setattr(_distilled_module, _DETECT_VERSION_FN, _patched_detect_model_version) # --- Patch 4: services.text_encoder.ltx_text_encoder.TextHandler.get_model_id_from_checkpoint --- diff --git a/backend/services/prompt_enhancement/__init__.py b/backend/services/prompt_enhancement/__init__.py index a921305a5..ec04f8293 100644 --- a/backend/services/prompt_enhancement/__init__.py +++ b/backend/services/prompt_enhancement/__init__.py @@ -1,4 +1,5 @@ from services.prompt_enhancement.system_prompt import ( + build_audio_visual_caption_system_prompt, build_conditioning_system_prompt, build_default_free_rewrite_system_prompt, build_ic_lora_enhancement_system_prompt, @@ -14,6 +15,7 @@ ) __all__ = [ + "build_audio_visual_caption_system_prompt", "build_conditioning_system_prompt", "build_default_free_rewrite_system_prompt", "build_ic_lora_enhancement_system_prompt", diff --git a/backend/services/prompt_enhancement/system_prompt.py b/backend/services/prompt_enhancement/system_prompt.py index dcb9c44f0..0b1886ac0 100644 --- a/backend/services/prompt_enhancement/system_prompt.py +++ b/backend/services/prompt_enhancement/system_prompt.py @@ -82,6 +82,23 @@ def build_default_free_rewrite_system_prompt() -> str: ) +def build_audio_visual_caption_system_prompt(*, t2v: bool) -> str: + """The captioning instructions LTX 2.5 itself ships with, for any enhancer provider. + + Read from ltx-core rather than restated here so it tracks the model: these describe the exact + caption style the checkpoint was trained on (single 150-220 word paragraph, shot type + camera + motion + viewpoint, and a full soundscape including quoted dialogue). The local Gemma enhancer + picks this up on its own from the encoder's model type; providers with no model-side default — + Gemini — would otherwise get the visual-only generic fallback and drop the audio half. + """ + from ltx_core.text_encoders.gemma.encoders.base_encoder import ( + default_gemma4_i2v_system_prompt, + default_gemma4_t2v_system_prompt, + ) + + return default_gemma4_t2v_system_prompt() if t2v else default_gemma4_i2v_system_prompt() + + def build_image_generation_system_prompt() -> str: """System prompt for the image-generation (text-to-image) free-rewrite path, Z-Image-Turbo. diff --git a/backend/services/prompt_enhancer_pipeline/ltx_prompt_enhancer_pipeline.py b/backend/services/prompt_enhancer_pipeline/ltx_prompt_enhancer_pipeline.py index e608a70c5..dfcafe891 100644 --- a/backend/services/prompt_enhancer_pipeline/ltx_prompt_enhancer_pipeline.py +++ b/backend/services/prompt_enhancer_pipeline/ltx_prompt_enhancer_pipeline.py @@ -3,11 +3,17 @@ Builds the Gemma text encoder fresh for a single enhance call and frees it — matching every other local Gemma usage in this codebase (``ltx_pipelines.utils.blocks.PromptEncoder.__call__`` loads/frees it per text-encoding call too; none of them keep it resident). The full -``Gemma3ForConditionalGeneration`` (including ``lm_head``) is what gets built here, which is -why direct ``generate()``-based enhancement is available at no extra VRAM cost over plain -encoding. +generation model (including ``lm_head``) is what gets built here, which is why direct +``generate()``-based enhancement is available at no extra VRAM cost over plain encoding. -Generation is reimplemented here rather than calling the vendored +The root passed in is whichever checkpoint can actually generate for the active model, which is +not always the one that encodes it: 2.3's Gemma 3 does both, while 2.5 encodes with an +encode-only ``gemma4_unified`` and enhances with Gemma 4 E2B when present, else Gemma 3 if a +2.3 install already put it on disk. Everything below is model-type agnostic — ``get_gemma_ops`` +and the encoder's own defaults resolve the differences — so both roots load through the same +path. + +Generation is reimplemented here rather than calling upstream ``GemmaTextEncoder.enhance_t2v``/``enhance_i2v`` — those go through ``_pad_inputs_for_attention_alignment``, which right-pads the prompt to a multiple of 8 for Flash Attention *before* calling ``.generate()``. Right-padding a decoder-only model's prompt @@ -24,19 +30,24 @@ import torch if TYPE_CHECKING: - from ltx_core.text_encoders.gemma.encoders.base_encoder import GemmaTextEncoder + from ltx_core.text_encoders.gemma.encoders.base_encoder import LTXGemmaTextEncoder + + +def _generation_kwargs(text_encoder: "LTXGemmaTextEncoder") -> dict[str, Any]: + # Decoding settings differ per enhancer family — Gemma 3 samples at 0.7, Gemma 4 instruct is + # greedy with an n-gram block — and only the encoder knows which one it loaded. + return cast(dict[str, Any], cast(Any, text_encoder)._default_generation_kwargs()) def _generate( - text_encoder: "GemmaTextEncoder", + text_encoder: "LTXGemmaTextEncoder", messages: list[dict[str, object]], image: "torch.Tensor | None", seed: int, - max_new_tokens: int = 512, ) -> str: # transformers' Gemma3Processor/Gemma3ForConditionalGeneration stubs don't type these # dynamically-attached attributes precisely enough for strict mode; loosen locally rather - # than suppress line-by-line, matching this codebase's other vendored-internals call sites + # than suppress line-by-line, matching this codebase's other upstream-internals call sites # (e.g. services/text_encoder/ltx_text_encoder.py's PromptEncoder patches). encoder = cast(Any, text_encoder) assert encoder.processor is not None @@ -49,13 +60,19 @@ def _generate( fork_devices = [encoder.model.device] if encoder.model.device.type == "cuda" else [] with torch.inference_mode(), torch.random.fork_rng(devices=fork_devices): # type: ignore[reportUnknownMemberType] torch.manual_seed(seed) # type: ignore[reportUnknownMemberType] - outputs = encoder.model.generate( - **model_inputs, max_new_tokens=max_new_tokens, do_sample=True, temperature=0.7 - ) + outputs = encoder.model.generate(**model_inputs, **_generation_kwargs(text_encoder)) generated_ids = outputs[0][len(model_inputs.input_ids[0]) :] return cast(str, encoder.processor.tokenizer.decode(generated_ids, skip_special_tokens=True)) +def _default_system_prompt(text_encoder: "LTXGemmaTextEncoder", *, t2v: bool) -> str: + # LTXGemmaTextEncoder no longer exposes this as a public property (renamed from + # GemmaTextEncoder); its model-type-aware default lives on the private + # ``_default_system_prompt`` method, so reach into it like the other upstream-internals + # call sites in this module. + return cast(Any, text_encoder)._default_system_prompt(t2v=t2v) + + class LtxPromptEnhancerPipeline: @staticmethod def create(gemma_root: str, device: str) -> "LtxPromptEnhancerPipeline": @@ -65,27 +82,24 @@ def __init__(self, gemma_root: str, device: str) -> None: self._gemma_root = gemma_root self._device = device - def _build_text_encoder(self) -> "GemmaTextEncoder": + def _build_text_encoder(self) -> "LTXGemmaTextEncoder": # Mirrors ltx_pipelines.utils.blocks.PromptEncoder's non-streaming text-encoder build, # minus the embeddings-processor half — enhancement never needs video/audio embeddings. from ltx_core.loader.registry import DummyRegistry from ltx_core.loader.single_gpu_model_builder import SingleGPUModelBuilder from ltx_core.text_encoders.gemma import ( - GEMMA_LLM_KEY_OPS, - GEMMA_MODEL_OPS, GemmaTextEncoderConfigurator, - module_ops_from_gemma_root, + get_gemma_ops, + resolve_gemma_weight_paths, ) - from ltx_core.utils import find_matching_file - module_ops = module_ops_from_gemma_root(self._gemma_root) - model_folder = find_matching_file(self._gemma_root, "model*.safetensors").parent - weight_paths = [str(p) for p in model_folder.rglob("*.safetensors")] + sd_ops, module_ops = get_gemma_ops(self._gemma_root) + weight_paths = resolve_gemma_weight_paths(self._gemma_root) builder = SingleGPUModelBuilder( - model_path=tuple(weight_paths), - model_class_configurator=GemmaTextEncoderConfigurator, - model_sd_ops=GEMMA_LLM_KEY_OPS, - module_ops=(GEMMA_MODEL_OPS, *module_ops), + model_path=weight_paths, + model_class_configurator=GemmaTextEncoderConfigurator.with_gemma_model_path(self._gemma_root), + model_sd_ops=sd_ops, + module_ops=module_ops, registry=DummyRegistry(), ) return builder.build(device=torch.device(self._device), dtype=torch.bfloat16).eval() @@ -94,7 +108,7 @@ def enhance_t2v(self, prompt: str, system_prompt: str | None, seed: int) -> str: from ltx_pipelines.utils.gpu_model import gpu_model with gpu_model(self._build_text_encoder()) as text_encoder: - resolved_system_prompt = system_prompt or text_encoder.default_gemma_t2v_system_prompt + resolved_system_prompt = system_prompt or _default_system_prompt(text_encoder, t2v=True) messages: list[dict[str, object]] = [ {"role": "system", "content": resolved_system_prompt}, {"role": "user", "content": f"user prompt: {prompt}"}, @@ -109,7 +123,7 @@ def enhance_i2v(self, prompt: str, image_path: str, system_prompt: str | None, s image_tensor = resize_aspect_ratio_preserving(torch.tensor(image), 896).to(torch.uint8) with gpu_model(self._build_text_encoder()) as text_encoder: - resolved_system_prompt = system_prompt or text_encoder.default_gemma_i2v_system_prompt + resolved_system_prompt = system_prompt or _default_system_prompt(text_encoder, t2v=False) messages: list[dict[str, object]] = [ {"role": "system", "content": resolved_system_prompt}, { diff --git a/backend/services/retake_pipeline/ltx_retake_pipeline.py b/backend/services/retake_pipeline/ltx_retake_pipeline.py index 32f1e32fa..254dff103 100644 --- a/backend/services/retake_pipeline/ltx_retake_pipeline.py +++ b/backend/services/retake_pipeline/ltx_retake_pipeline.py @@ -19,13 +19,14 @@ from ltx_core.components.guiders import MultiModalGuiderParams from ltx_core.loader import LoraPathStrengthAndSDOps -from ltx_core.model.video_vae import TilingConfig, get_video_chunks_number +from ltx_core.model.video_vae import DimensionSizeConfig, TileSizeConfig, get_video_chunks_number from ltx_core.quantization import QuantizationPolicy from ltx_core.types import Audio from ltx_pipelines.utils.media_io import encode_video, get_videostream_metadata from api_types import ExtendMode -from services.ltx_pipeline_common import offload_mode_for_prefetch_count +from services.ltx_pipeline_common import build_model_paths, offload_mode_for_prefetch_count, resolve_tiling_config +from services.services_utils import TilingConfigType from services.retake_pipeline.retake_pipeline import RetakePipeline @@ -46,6 +47,9 @@ def create( *, loras: list[LoraPathStrengthAndSDOps] | None = None, quantization: QuantizationPolicy | None = None, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> RetakePipeline: return LTXRetakePipeline( checkpoint_path=checkpoint_path, @@ -54,6 +58,9 @@ def create( streaming_prefetch_count=streaming_prefetch_count, loras=loras or [], quantization=quantization, + video_vae_path=video_vae_path, + audio_vae_path=audio_vae_path, + duration_head_path=duration_head_path, ) def __init__( @@ -65,6 +72,9 @@ def __init__( *, loras: list[LoraPathStrengthAndSDOps], quantization: QuantizationPolicy | None, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> None: from ltx_pipelines.utils.blocks import ( AudioConditioner, @@ -78,41 +88,50 @@ def __init__( self.device = device self.dtype = torch.bfloat16 offload_mode = offload_mode_for_prefetch_count(streaming_prefetch_count, device) + model_paths = build_model_paths( + checkpoint_path, + gemma_root, + video_vae_path=video_vae_path, + audio_vae_path=audio_vae_path, + duration_head_path=duration_head_path, + ) + video_vae = model_paths.video_vae() + audio_vae = model_paths.audio_vae() + transformer = model_paths.transformer() self.prompt_encoder = PromptEncoder( - checkpoint_path=checkpoint_path, - gemma_root=gemma_root or "", - dtype=self.dtype, - device=device, + model_paths, + self.dtype, + device, offload_mode=offload_mode, ) self.image_conditioner = ImageConditioner( - checkpoint_path=checkpoint_path, - dtype=self.dtype, - device=device, + video_vae, + self.dtype, + device, ) self.audio_conditioner = AudioConditioner( - checkpoint_path=checkpoint_path, - dtype=self.dtype, - device=device, + audio_vae, + self.dtype, + device, ) self.stage = DiffusionStage.from_checkpoint( # type: ignore[reportUnknownMemberType] - checkpoint_path=checkpoint_path, - dtype=self.dtype, - device=device, + transformer, + self.dtype, + device, loras=tuple(loras), quantization=quantization, offload_mode=offload_mode, ) self.video_decoder = VideoDecoder( - checkpoint_path=checkpoint_path, - dtype=self.dtype, - device=device, + video_vae, + self.dtype, + device, ) self.audio_decoder = AudioDecoder( - checkpoint_path=checkpoint_path, - dtype=self.dtype, - device=device, + audio_vae, + self.dtype, + device, ) @torch.no_grad() @@ -137,7 +156,7 @@ def _run( # noqa: PLR0913, PLR0915 target_width: int | None = None, target_height: int | None = None, target_frames: int | None = None, - ) -> tuple[Iterator[torch.Tensor], Audio]: + ) -> tuple[Iterator[torch.Tensor], Audio, TilingConfigType]: from ltx_core.components.guiders import MultiModalGuider from ltx_core.components.noisers import GaussianNoiser from ltx_core.components.schedulers import LTX2Scheduler @@ -155,15 +174,14 @@ def _run( # noqa: PLR0913, PLR0915 effective_seed = int(torch.randint(0, 2**31, (1,)).item()) if seed < 0 else seed generator = torch.Generator(device=self.device).manual_seed(effective_seed) noiser = GaussianNoiser(generator=generator) - from ltx_core.model.video_vae import SpatialTilingConfig, TemporalTilingConfig dtype = self.dtype - tiling = TilingConfig.default() # Smaller tiles for source video encoding to reduce peak VRAM allocation # during the VAE encoder forward pass. - encoding_tiling = TilingConfig( - spatial_config=SpatialTilingConfig(tile_size_in_pixels=256, tile_overlap_in_pixels=64), - temporal_config=TemporalTilingConfig(tile_size_in_frames=24, tile_overlap_in_frames=16), + encoding_tiling = TileSizeConfig( + frames=DimensionSizeConfig(tile_size=24, overlap=16), + height=DimensionSizeConfig(tile_size=256, overlap=64), + width=DimensionSizeConfig(tile_size=256, overlap=64), ) # --- Encode source video (tiled) --- @@ -304,9 +322,16 @@ def _run( # noqa: PLR0913, PLR0915 # --- Decode video (lazy generator, tiled) --- assert video_state is not None + tiling = resolve_tiling_config( + self.video_decoder.checkpoint_path, + height=target_shape.height, + width=target_shape.width, + num_frames=target_shape.frames, + device=self.device, + ) decoded_video = self.video_decoder(video_state.latent, tiling, generator) - return decoded_video, decoded_audio + return decoded_video, decoded_audio, tiling @torch.no_grad() def generate( @@ -333,7 +358,7 @@ def generate( meta = get_videostream_metadata(video_path) fps = meta.fps num_frames = target_frames if target_frames is not None else meta.frames - video_iter, audio = self._run( + video_iter, audio, tiling_config = self._run( video_path=video_path, prompt=prompt, start_time=start_time, @@ -352,7 +377,6 @@ def generate( target_frames=target_frames, ) audio_out: Audio | None = audio - tiling_config = TilingConfig.default() video_chunks = get_video_chunks_number(num_frames, tiling_config) encode_video( video=video_iter, @@ -384,7 +408,7 @@ def extend( fps = meta.fps source_frames = target_frames if target_frames is not None else meta.frames total_frames = source_frames + extend_frames - video_iter, audio = self._run( + video_iter, audio, tiling_config = self._run( video_path=video_path, prompt=prompt, start_time=0.0, @@ -403,7 +427,6 @@ def extend( target_height=target_height, target_frames=target_frames, ) - tiling_config = TilingConfig.default() video_chunks = get_video_chunks_number(total_frames, tiling_config) encode_video( video=video_iter, diff --git a/backend/services/retake_pipeline/retake_pipeline.py b/backend/services/retake_pipeline/retake_pipeline.py index db0cce1e5..d97297b8c 100644 --- a/backend/services/retake_pipeline/retake_pipeline.py +++ b/backend/services/retake_pipeline/retake_pipeline.py @@ -23,6 +23,9 @@ def create( *, loras: list["LoraPathStrengthAndSDOps"] | None = None, quantization: "QuantizationPolicy | None" = None, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> "RetakePipeline": ... def generate( diff --git a/backend/services/services_utils.py b/backend/services/services_utils.py index 2b9792def..e98784da2 100644 --- a/backend/services/services_utils.py +++ b/backend/services/services_utils.py @@ -14,6 +14,7 @@ from numpy.typing import NDArray from ltx_core.model.video_vae import TilingConfig + from ltx_core.tiling import PipelineTiling JSONScalar: TypeAlias = str | int | float | bool | None @@ -30,9 +31,11 @@ FrameArray: TypeAlias = NDArray[np.uint8] TilingConfigType: TypeAlias = TilingConfig + PipelineTilingType: TypeAlias = PipelineTiling else: FrameArray: TypeAlias = object TilingConfigType: TypeAlias = object + PipelineTilingType: TypeAlias = object AudioType: TypeAlias = object TensorOrNone: TypeAlias = TensorType | None diff --git a/backend/services/text_encoder/ltx_text_encoder.py b/backend/services/text_encoder/ltx_text_encoder.py index 1eaac8e00..1c1452ed4 100644 --- a/backend/services/text_encoder/ltx_text_encoder.py +++ b/backend/services/text_encoder/ltx_text_encoder.py @@ -64,25 +64,25 @@ def install_patches(self, state_getter: Callable[[], AppState]) -> None: self._install_cleanup_memory_patch(state_getter) def _install_prompt_encoder_init_patch(self) -> None: - """Patch PromptEncoder.__init__ to accept None gemma_root (API encoding mode). + """Patch PromptEncoder.__init__ to accept a text-encoder-less ModelPaths (API encoding mode). - In API encoding mode, gemma_root is None since text encoding is done - remotely. The upstream PromptEncoder eagerly resolves file paths from - gemma_root in __init__, which crashes. This patch short-circuits init - when gemma_root is falsy, creating a stub that the __call__ patch will - intercept before any model loading. + In API encoding mode, text encoding is done remotely, so there is no local gemma root + -- ``model_paths.text_encoder_path`` is None. The upstream PromptEncoder eagerly resolves + file paths from it in __init__, which crashes. This patch short-circuits init when that + path is falsy, creating a stub that the __call__ patch will intercept before any model + loading. """ if self._prompt_encoder_init_patched: return try: from ltx_pipelines.utils.blocks import PromptEncoder + from ltx_pipelines.utils.model_paths import ModelPaths original_init = PromptEncoder.__init__ # type: ignore[reportUnknownVariableType, reportUnknownMemberType] def patched_init( self_encoder: PromptEncoder, - checkpoint_path: str, - gemma_root: str, + model_paths: ModelPaths, dtype: Any, device: Any, *args: Any, @@ -91,17 +91,17 @@ def patched_init( # Forward *args/**kwargs verbatim so this patch tracks the real # PromptEncoder.__init__ signature (registry, offload_mode, # text_encoder_builder, ...) instead of pinning a fixed arg list. - if not gemma_root: + if not model_paths.text_encoder_path: self_encoder._dtype = dtype # type: ignore[attr-defined] self_encoder._device = device # type: ignore[attr-defined] self_encoder._text_encoder_builder = None # type: ignore[attr-defined] self_encoder._embeddings_processor_builder = None # type: ignore[attr-defined] return - original_init(self_encoder, checkpoint_path, gemma_root, dtype, device, *args, **kwargs) + original_init(self_encoder, model_paths, dtype, device, *args, **kwargs) PromptEncoder.__init__ = patched_init # type: ignore[assignment] self._prompt_encoder_init_patched = True - logger.info("Installed PromptEncoder.__init__ patch for None gemma_root") + logger.info("Installed PromptEncoder.__init__ patch for text-encoder-less ModelPaths") except Exception as exc: logger.warning("Failed to patch PromptEncoder.__init__: %s", exc, exc_info=True) @@ -212,9 +212,19 @@ def get_model_id_from_checkpoint(self, checkpoint_path: str) -> str | None: logger.warning("Could not extract model_id from checkpoint: %s", exc, exc_info=True) return None - def encode_via_api(self, prompt: str, api_key: str, checkpoint_path: str, enhance_prompt: bool) -> TextEncodingResult | None: - model_id = self.get_model_id_from_checkpoint(checkpoint_path) + def encode_via_api( + self, + prompt: str, + api_key: str, + checkpoint_path: str, + enhance_prompt: bool, + api_model_id: str | None = None, + ) -> TextEncodingResult | None: + model_id = api_model_id or self.get_model_id_from_checkpoint(checkpoint_path) if not model_id: + logger.warning( + "Checkpoint %s carries no API model id; skipping LTX API text encoding", checkpoint_path + ) return None try: diff --git a/backend/services/text_encoder/text_encoder.py b/backend/services/text_encoder/text_encoder.py index 736326805..f51175083 100644 --- a/backend/services/text_encoder/text_encoder.py +++ b/backend/services/text_encoder/text_encoder.py @@ -13,5 +13,12 @@ class TextEncoder(Protocol): def install_patches(self, state_getter: Callable[[], AppState]) -> None: ... - def encode_via_api(self, prompt: str, api_key: str, checkpoint_path: str, enhance_prompt: bool) -> TextEncodingResult | None: + def encode_via_api( + self, + prompt: str, + api_key: str, + checkpoint_path: str, + enhance_prompt: bool, + api_model_id: str | None = None, + ) -> TextEncodingResult | None: ... diff --git a/backend/state/app_settings.py b/backend/state/app_settings.py index f3a3e987a..bda0a84f9 100644 --- a/backend/state/app_settings.py +++ b/backend/state/app_settings.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sys from typing import Any, Literal, TypeGuard, TypeVar, cast, get_args from pydantic import BaseModel, ConfigDict, Field, create_model, field_validator @@ -68,6 +69,8 @@ class AppSettings(SettingsBaseModel): locked_seed: int = 42 models_dir: str = "" active_ltx_model_id: LTXLocalModelId | None = None + # None = platform default (Mac on, CUDA/Linux off). An explicit bool is a user override. + use_conv_vae: bool | None = None @field_validator("prompt_cache_size", mode="before") @classmethod @@ -141,6 +144,14 @@ class SettingsResponse(SettingsBaseModel): locked_seed: int = 42 models_dir: str = "" active_ltx_model_id: LTXLocalModelId | None = None + use_conv_vae: bool = False + + +def resolved_use_conv_vae(settings: AppSettings) -> bool: + """Effective Fast decode setting: user override, else Mac on / CUDA off.""" + if settings.use_conv_vae is not None: + return settings.use_conv_vae + return sys.platform == "darwin" def to_settings_response(settings: AppSettings) -> SettingsResponse: @@ -151,7 +162,7 @@ def to_settings_response(settings: AppSettings) -> SettingsResponse: data["has_ltx_api_key"] = bool(ltx_key) data["has_fal_api_key"] = bool(fal_key) data["has_gemini_api_key"] = bool(gemini_key) - # models_dir passes through as-is (not secret) + data["use_conv_vae"] = resolved_use_conv_vae(settings) return SettingsResponse.model_validate(data) diff --git a/backend/state/app_state_types.py b/backend/state/app_state_types.py index e63c5f7f1..8cbe7917b 100644 --- a/backend/state/app_state_types.py +++ b/backend/state/app_state_types.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, NewType, Protocol -from api_types import ModelCheckpointID +from api_types import LTXLocalModelId, ModelCheckpointID from state.conditioning_cache import ConditioningCache if TYPE_CHECKING: @@ -122,11 +122,17 @@ class TextEncoderState: class VideoPipelineState: pipeline: FastVideoPipeline is_compiled: bool + # Cache key: API text-encode mode leaves gemma_root=None across versions, so without this a + # switch (2.5 <-> 2.3) wouldn't rebuild. + ltx_model_id: LTXLocalModelId loras: tuple[tuple[str, float], ...] = field(default_factory=tuple) # gemma_root the pipeline's text encoder was built with. Part of the cache key: switching # text-encoding mode (API<->local) changes it, and a cached pipeline built for the other # mode must be rebuilt (an API-mode pipeline has a stub encoder that can't encode locally). gemma_root: str | None = None + # Video VAE file the pipeline was built with. Fast decode swaps this path; a cached + # pipeline loaded for the other decoder must be rebuilt. + video_vae_path: str | None = None @dataclass @@ -142,17 +148,21 @@ class ICLoraState: lora_path: str depth_pipeline: DepthProcessorPipeline | None depth_model_path: str | None + ltx_model_id: LTXLocalModelId # cache key — see VideoPipelineState.ltx_model_id lora_strength: float = 1.0 pose_resources: PoseResources | None = None conditioning_cache: ConditioningCache = field(default_factory=ConditioningCache) gemma_root: str | None = None # cache key — see VideoPipelineState.gemma_root + video_vae_path: str | None = None # cache key — see VideoPipelineState.video_vae_path @dataclass class A2VPipelineState: pipeline: A2VPipeline + ltx_model_id: LTXLocalModelId # cache key — see VideoPipelineState.ltx_model_id loras: tuple[tuple[str, float], ...] = field(default_factory=tuple) gemma_root: str | None = None # cache key — see VideoPipelineState.gemma_root + video_vae_path: str | None = None # cache key — see VideoPipelineState.video_vae_path @dataclass @@ -160,7 +170,9 @@ class RetakePipelineState: pipeline: RetakePipeline distilled: bool quantized: bool + ltx_model_id: LTXLocalModelId # cache key — see VideoPipelineState.ltx_model_id gemma_root: str | None = None # cache key — see VideoPipelineState.gemma_root + video_vae_path: str | None = None # cache key — see VideoPipelineState.video_vae_path # ============================================================ diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index be517c11a..6258d25a4 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -121,18 +121,45 @@ def _test_model_path(test_state, cp_id: str) -> Path: @pytest.fixture def create_fake_model_files(test_state): - def _create(include_zit: bool = False): - ltx_spec = get_ltx_model_spec(get_latest_ltx_model_id()) - - for cp_id in (ltx_spec.model_cp, ltx_spec.upscale_cp): + def _create( + include_zit: bool = False, + model_id: str | None = None, + include_prompt_enhancer: bool = False, + ): + from runtime_config.model_download_specs import get_model_cp_spec + + ltx_spec = get_ltx_model_spec(model_id or get_latest_ltx_model_id()) + + for cp_id in ( + ltx_spec.model_cp, + ltx_spec.upscale_cp, + ltx_spec.video_vae_cp, + ltx_spec.video_vae_conv_cp, + ltx_spec.audio_vae_cp, + ltx_spec.duration_head_cp, + ): + if cp_id is None: + continue path = _test_model_path(test_state, cp_id) path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(b"\x00" * 1024) - te_dir = _test_model_path(test_state, ltx_spec.text_encoder_cp) - te_dir.mkdir(parents=True, exist_ok=True) - (te_dir / "model.safetensors").write_bytes(b"\x00" * 1024) - (te_dir / "tokenizer.model").write_bytes(b"\x00" * 1024) + def _write_cp(cp_id: str) -> None: + path = _test_model_path(test_state, cp_id) + if get_model_cp_spec(cp_id).is_folder: + path.mkdir(parents=True, exist_ok=True) + (path / "model.safetensors").write_bytes(b"\x00" * 1024) + (path / "tokenizer.model").write_bytes(b"\x00" * 1024) + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"\x00" * 1024) + + _write_cp(ltx_spec.text_encoder_cp) + + # Left out by default: it's an optional extra download, so tests opt in to the state + # where local Enhance is available for models that need it (2.5). + if include_prompt_enhancer and ltx_spec.prompt_enhancer_cp is not None: + _write_cp(ltx_spec.prompt_enhancer_cp) if include_zit: zit_dir = _test_model_path(test_state, IMG_GEN_MODEL_CP_ID) @@ -159,10 +186,14 @@ def _create(name: str = "style.safetensors") -> str: return _create +# Built-in depth/canny Union Control IC-LoRA ships with LTX 2.3 only. +_IC_LORA_MODEL_ID = "ltx-2.3-22b-distilled-1.1" + + @pytest.fixture def create_fake_ic_lora_files(test_state): def _create(include_depth: bool = True): - ltx_spec = get_ltx_model_spec(get_latest_ltx_model_id()) + ltx_spec = get_ltx_model_spec(_IC_LORA_MODEL_ID) for cp_id in get_ic_loras_cp_ids(ltx_spec.ic_loras_spec): path = _test_model_path(test_state, cp_id) path.parent.mkdir(parents=True, exist_ok=True) diff --git a/backend/tests/fakes/services.py b/backend/tests/fakes/services.py index 241e05aa7..7b85e01bf 100644 --- a/backend/tests/fakes/services.py +++ b/backend/tests/fakes/services.py @@ -8,6 +8,7 @@ from typing import Any, ClassVar from PIL import Image +from frame_math import AutoDurationSpec from api_types import ( ExtendMode, ImageConditioningInput, @@ -180,7 +181,7 @@ def generate_text_to_video( prompt: str, model: str, resolution: str, - duration: float, + duration: float | None, fps: float, generate_audio: bool, camera_motion: VideoCameraMotion = "none", @@ -209,7 +210,7 @@ def generate_image_to_video( image_uri: str, model: str, resolution: str, - duration: float, + duration: float | None, fps: float, generate_audio: bool, camera_motion: VideoCameraMotion = "none", @@ -264,6 +265,7 @@ def retake( duration: float, prompt: str, mode: RetakeMode, + model: str, ) -> LTXRetakeResult: self.retake_calls.append( { @@ -273,6 +275,7 @@ def retake( "duration": duration, "prompt": prompt, "mode": mode, + "model": model, } ) if self.raise_on_retake is not None: @@ -287,6 +290,7 @@ def extend( duration: float, prompt: str, mode: ExtendMode, + model: str, ) -> LTXRetakeResult: self.extend_calls.append( { @@ -295,6 +299,7 @@ def extend( "duration": duration, "prompt": prompt, "mode": mode, + "model": model, } ) if self.raise_on_extend is not None: @@ -676,8 +681,21 @@ def create( device: str | object, streaming_prefetch_count: int | None, loras: list[tuple[str, float]] | None = None, + *, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> "FakeFastVideoPipeline": - del checkpoint_path, gemma_root, upsampler_path, device, streaming_prefetch_count + del ( + checkpoint_path, + gemma_root, + upsampler_path, + device, + streaming_prefetch_count, + video_vae_path, + audio_vae_path, + duration_head_path, + ) pipeline = FakeFastVideoPipeline._singleton if pipeline is None: raise RuntimeError("FakeFastVideoPipeline singleton is not bound") @@ -690,7 +708,7 @@ def generate( seed: int, height: int, width: int, - num_frames: int, + num_frames: int | AutoDurationSpec, frame_rate: float, images: list[ImageConditioningInput], output_path: str, @@ -770,13 +788,14 @@ def bind_singleton(cls, pipeline: "FakePromptEnhancerPipeline") -> None: @staticmethod def create(gemma_root: str, device: str) -> "FakePromptEnhancerPipeline": - del gemma_root, device pipeline = FakePromptEnhancerPipeline._singleton if pipeline is None: raise RuntimeError("FakePromptEnhancerPipeline singleton is not bound") + pipeline.created_with.append({"gemma_root": gemma_root, "device": device}) return pipeline def __init__(self) -> None: + self.created_with: list[dict[str, Any]] = [] self.enhance_t2v_calls: list[dict[str, Any]] = [] self.enhance_i2v_calls: list[dict[str, Any]] = [] self.raise_on_enhance: Exception | None = None @@ -813,8 +832,23 @@ def create( device: str | object, streaming_prefetch_count: int | None, lora_strength: float = 1.0, + *, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> "FakeIcLoraPipeline": - del checkpoint_path, gemma_root, upsampler_path, lora_path, device, streaming_prefetch_count, lora_strength + del ( + checkpoint_path, + gemma_root, + upsampler_path, + lora_path, + device, + streaming_prefetch_count, + lora_strength, + video_vae_path, + audio_vae_path, + duration_head_path, + ) pipeline = FakeIcLoraPipeline._singleton if pipeline is None: raise RuntimeError("FakeIcLoraPipeline singleton is not bound") @@ -902,8 +936,21 @@ def create( device: str | object, streaming_prefetch_count: int | None, loras: list[tuple[str, float]] | None = None, + *, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> "FakeA2VPipeline": - del checkpoint_path, gemma_root, upsampler_path, device, streaming_prefetch_count + del ( + checkpoint_path, + gemma_root, + upsampler_path, + device, + streaming_prefetch_count, + video_vae_path, + audio_vae_path, + duration_head_path, + ) pipeline = FakeA2VPipeline._singleton if pipeline is None: raise RuntimeError("FakeA2VPipeline singleton is not bound") @@ -941,8 +988,21 @@ def create( *, loras: list[object] | None = None, quantization: object | None = None, + video_vae_path: str | None = None, + audio_vae_path: str | None = None, + duration_head_path: str | None = None, ) -> "FakeRetakePipeline": - del checkpoint_path, gemma_root, device, streaming_prefetch_count, loras, quantization + del ( + checkpoint_path, + gemma_root, + device, + streaming_prefetch_count, + loras, + quantization, + video_vae_path, + audio_vae_path, + duration_head_path, + ) pipeline = FakeRetakePipeline._singleton if pipeline is None: raise RuntimeError("FakeRetakePipeline singleton is not bound") @@ -982,13 +1042,21 @@ def __init__(self) -> None: def install_patches(self, state_getter) -> None: # noqa: ARG002 self.install_calls += 1 - def encode_via_api(self, prompt: str, api_key: str, checkpoint_path: str, enhance_prompt: bool) -> Any | None: + def encode_via_api( + self, + prompt: str, + api_key: str, + checkpoint_path: str, + enhance_prompt: bool, + api_model_id: str | None = None, + ) -> Any | None: self.encode_calls.append( { "prompt": prompt, "api_key": api_key, "checkpoint_path": checkpoint_path, "enhance_prompt": enhance_prompt, + "api_model_id": api_model_id, } ) if self.encode_responses: diff --git a/backend/tests/test_api_calls.py b/backend/tests/test_api_calls.py index 84d6bc46d..b2b23483f 100644 --- a/backend/tests/test_api_calls.py +++ b/backend/tests/test_api_calls.py @@ -9,6 +9,13 @@ from tests.http_error_assertions import assert_http_error from tests.fakes import FakeResponse +_LOCAL_2_3 = "ltx-2.3-22b-distilled-1.1" + + +def _install_local_2_3(test_state, create_fake_model_files, **kwargs) -> None: + create_fake_model_files(model_id=_LOCAL_2_3, **kwargs) + test_state.state.app_settings.active_ltx_model_id = _LOCAL_2_3 + def _gemini_ok(text: str = "Enhanced prompt text") -> FakeResponse: return FakeResponse( @@ -134,6 +141,29 @@ def test_happy_path_json_video_url(self, client, test_state): assert r.status_code == 200 assert r.json()["status"] == "complete" + def test_retake_defaults_to_ltx_2_3_pro(self, client, test_state): + self._force_api(test_state) + test_state.state.app_settings.ltx_api_key = "test-key" + video_path = self._make_video(test_state) + test_state.ltx_api_client.retake_result = LTXRetakeResult( + video_bytes=b"\x00\x00\x00\x1cftypisom" + b"\x00" * 500, + result_payload=None, + ) + + r = client.post("/api/retake", json=self._base_payload(video_path)) + assert r.status_code == 200 + assert len(test_state.ltx_api_client.retake_calls) == 1 + assert test_state.ltx_api_client.retake_calls[0]["model"] == "ltx-2-3-pro" + + def test_retake_rejects_2_5_model(self, client, test_state): + # ltxv-api retake only accepts ltx-2-pro / ltx-2-3-pro. + self._force_api(test_state) + test_state.state.app_settings.ltx_api_key = "test-key" + video_path = self._make_video(test_state) + + r = client.post("/api/retake", json={**self._base_payload(video_path), "model": "pro-2.5"}) + assert r.status_code == 422 + def test_api_retake_recoverable_via_progress(self, client, test_state): # After an API retake, /generation/progress must report complete + the result # path, so a page that unmounted mid-generation can recover the output. @@ -192,6 +222,15 @@ def test_no_api_key(self, client, test_state): r = client.post("/api/retake", json=self._base_payload(video_path)) assert r.status_code == 400 + def test_rejects_fast_tier_model(self, client, test_state): + # "fast" is not a RetakeExtendModel member, so pydantic rejects it (422) + # before the request handler runs. + self._force_api(test_state) + test_state.state.app_settings.ltx_api_key = "test-key" + video_path = self._make_video(test_state) + r = client.post("/api/retake", json={**self._base_payload(video_path), "model": "fast"}) + assert r.status_code == 422 + def test_upload_url_failure(self, client, test_state): self._force_api(test_state) test_state.state.app_settings.ltx_api_key = "test-key" @@ -244,7 +283,7 @@ def test_prompt_and_mode_forwarded(self, client, test_state): assert retake_call["mode"] == "replace_video" def test_local_retake_happy_path(self, client, test_state, create_fake_model_files): - create_fake_model_files(include_zit=False) + _install_local_2_3(test_state, create_fake_model_files, include_zit=False) test_state.state.app_settings.use_local_text_encoder = True test_state.config.local_generations_mode = "full_models_loading" @@ -255,11 +294,25 @@ def test_local_retake_happy_path(self, client, test_state, create_fake_model_fil assert data["status"] == "complete" assert data["video_path"] - def test_local_retake_mode_mapping(self, client, test_state, create_fake_model_files, fake_services): + def test_local_retake_rejected_on_2_5(self, client, test_state, create_fake_model_files): create_fake_model_files(include_zit=False) test_state.state.app_settings.use_local_text_encoder = True test_state.config.local_generations_mode = "full_models_loading" + video_path = self._make_valid_video(test_state) + r = client.post("/api/retake", json=self._base_payload(video_path)) + assert_http_error( + r, + status_code=409, + code="UNSUPPORTED_RETAKE", + message="Retake is not supported for the active LTX model.", + ) + + def test_local_retake_mode_mapping(self, client, test_state, create_fake_model_files, fake_services): + _install_local_2_3(test_state, create_fake_model_files, include_zit=False) + test_state.state.app_settings.use_local_text_encoder = True + test_state.config.local_generations_mode = "full_models_loading" + video_path = self._make_valid_video(test_state) client.post( "/api/retake", @@ -276,7 +329,7 @@ def test_local_retake_mode_mapping(self, client, test_state, create_fake_model_f assert retake_call["regenerate_audio"] is False def test_local_retake_forwards_selected_resolution(self, client, test_state, create_fake_model_files, fake_services): - create_fake_model_files(include_zit=False) + _install_local_2_3(test_state, create_fake_model_files, include_zit=False) test_state.state.app_settings.use_local_text_encoder = True test_state.config.local_generations_mode = "full_models_loading" @@ -312,7 +365,7 @@ def test_prefers_api_video_without_key_falls_back_to_local_retake( create_fake_model_files, fake_services, ): - create_fake_model_files(include_zit=False) + _install_local_2_3(test_state, create_fake_model_files, include_zit=False) test_state.config.local_generations_mode = "full_models_loading" test_state.state.app_settings.user_prefers_ltx_api_video_generations = True test_state.state.app_settings.ltx_api_key = "" @@ -365,6 +418,29 @@ def test_happy_path_binary_response(self, client, test_state): assert data["status"] == "complete" assert data["video_path"] + def test_extend_defaults_to_ltx_2_3_pro(self, client, test_state): + self._force_api(test_state) + test_state.state.app_settings.ltx_api_key = "test-key" + video_path = self._make_video(test_state) + test_state.ltx_api_client.extend_result = LTXRetakeResult( + video_bytes=b"\x00\x00\x00\x1cftypisom" + b"\x00" * 500, + result_payload=None, + ) + + r = client.post("/api/extend", json=self._base_payload(video_path)) + assert r.status_code == 200 + assert len(test_state.ltx_api_client.extend_calls) == 1 + assert test_state.ltx_api_client.extend_calls[0]["model"] == "ltx-2-3-pro" + + def test_extend_rejects_2_5_model(self, client, test_state): + # ltxv-api extend only accepts ltx-2-pro / ltx-2-3-pro. + self._force_api(test_state) + test_state.state.app_settings.ltx_api_key = "test-key" + video_path = self._make_video(test_state) + + r = client.post("/api/extend", json={**self._base_payload(video_path), "model": "pro-2.5"}) + assert r.status_code == 422 + def test_api_extend_recoverable_via_progress(self, client, test_state): self._force_api(test_state) test_state.state.app_settings.ltx_api_key = "test-key" @@ -457,7 +533,7 @@ def test_prompt_and_mode_forwarded(self, client, test_state): assert extend_call["duration"] == 6.0 def test_local_extend_happy_path(self, client, test_state, create_fake_model_files): - create_fake_model_files(include_zit=False) + _install_local_2_3(test_state, create_fake_model_files, include_zit=False) test_state.state.app_settings.use_local_text_encoder = True test_state.config.local_generations_mode = "full_models_loading" @@ -468,11 +544,25 @@ def test_local_extend_happy_path(self, client, test_state, create_fake_model_fil assert data["status"] == "complete" assert data["video_path"] - def test_local_extend_snaps_frames_and_forwards_mode(self, client, test_state, create_fake_model_files, fake_services): + def test_local_extend_rejected_on_2_5(self, client, test_state, create_fake_model_files): create_fake_model_files(include_zit=False) test_state.state.app_settings.use_local_text_encoder = True test_state.config.local_generations_mode = "full_models_loading" + video_path = self._make_valid_video(test_state) + r = client.post("/api/extend", json=self._base_payload(video_path)) + assert_http_error( + r, + status_code=409, + code="UNSUPPORTED_EXTEND", + message="Extend is not supported for the active LTX model.", + ) + + def test_local_extend_snaps_frames_and_forwards_mode(self, client, test_state, create_fake_model_files, fake_services): + _install_local_2_3(test_state, create_fake_model_files, include_zit=False) + test_state.state.app_settings.use_local_text_encoder = True + test_state.config.local_generations_mode = "full_models_loading" + video_path = self._make_valid_video(test_state, fps=24) client.post( "/api/extend", @@ -485,7 +575,7 @@ def test_local_extend_snaps_frames_and_forwards_mode(self, client, test_state, c assert extend_call["extend_frames"] % 8 == 0 def test_local_extend_corrects_source_resolution_to_div32(self, client, test_state, create_fake_model_files, fake_services): - create_fake_model_files(include_zit=False) + _install_local_2_3(test_state, create_fake_model_files, include_zit=False) test_state.state.app_settings.use_local_text_encoder = True test_state.config.local_generations_mode = "full_models_loading" @@ -497,7 +587,7 @@ def test_local_extend_corrects_source_resolution_to_div32(self, client, test_sta assert call["target_height"] == 64 def test_local_extend_corrects_frame_count(self, client, test_state, create_fake_model_files, fake_services): - create_fake_model_files(include_zit=False) + _install_local_2_3(test_state, create_fake_model_files, include_zit=False) test_state.state.app_settings.use_local_text_encoder = True test_state.config.local_generations_mode = "full_models_loading" @@ -509,7 +599,7 @@ def test_local_extend_corrects_frame_count(self, client, test_state, create_fake assert call["target_frames"] == 9 def test_local_extend_forwards_selected_resolution(self, client, test_state, create_fake_model_files, fake_services): - create_fake_model_files(include_zit=False) + _install_local_2_3(test_state, create_fake_model_files, include_zit=False) test_state.state.app_settings.use_local_text_encoder = True test_state.config.local_generations_mode = "full_models_loading" @@ -544,7 +634,7 @@ def test_prefers_api_video_without_key_falls_back_to_local_extend( create_fake_model_files, fake_services, ): - create_fake_model_files(include_zit=False) + _install_local_2_3(test_state, create_fake_model_files, include_zit=False) test_state.config.local_generations_mode = "full_models_loading" test_state.state.app_settings.user_prefers_ltx_api_video_generations = True test_state.state.app_settings.ltx_api_key = "" diff --git a/backend/tests/test_diffusion_stage_cache.py b/backend/tests/test_diffusion_stage_cache.py index ee7e287f4..9f587a82a 100644 --- a/backend/tests/test_diffusion_stage_cache.py +++ b/backend/tests/test_diffusion_stage_cache.py @@ -16,18 +16,22 @@ from ltx_core.block_streaming import StreamingModelBuilder from ltx_core.loader.single_gpu_model_builder import SingleGPUModelBuilder -from ltx_pipelines.utils.allocator_trim_strategy import AllocatorTrimStrategy +from ltx_core.allocator_trim_strategy import AllocatorTrimStrategy from services.patches import diffusion_stage_cache as dsc class _FakeModel: def __init__(self) -> None: self.freed_to: str | None = None + self.disposed = False def to(self, device: str) -> "_FakeModel": self.freed_to = device return self + def dispose(self) -> None: + self.disposed = True + class _FakeStage: """Duck-types the private DiffusionStage surface diffusion_stage_cache.py reads.""" diff --git a/backend/tests/test_diffvae_decode_vram.py b/backend/tests/test_diffvae_decode_vram.py new file mode 100644 index 000000000..34a657cf1 --- /dev/null +++ b/backend/tests/test_diffvae_decode_vram.py @@ -0,0 +1,159 @@ +"""DiffVAE decode must drop the resident transformer before building the VAE.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +from services.patches import diffusion_stage_cache as dsc +from services.patches import diffvae_decode_vram as patch + + +class _FakeModel: + def __init__(self) -> None: + self.freed_to: str | None = None + + def to(self, device: str) -> "_FakeModel": + self.freed_to = device + return self + + +class _FakeDecoder: + def __init__(self, device: torch.device) -> None: + self._device = device + self._checkpoint_path = "ltx-2.5-video-vae-bf16.safetensors" + self.diffvae_optimization = "chunked_eager" + + @property + def checkpoint_path(self) -> str: + return self._checkpoint_path + + +@pytest.fixture(autouse=True) +def _reset_cache_state() -> None: + dsc.set_enabled(True) + dsc.evict() + yield + dsc.set_enabled(True) + dsc.evict() + + +def test_patch_rebinds_video_decoder_call() -> None: + from ltx_pipelines.utils.blocks import VideoDecoder + + assert VideoDecoder.__call__ is patch._patched_video_decoder_call + + +def test_video_decoder_call_evicts_cached_transformer(monkeypatch) -> None: + model = _FakeModel() + dsc._cached_model = model + dsc._cached_key = ("planted",) + monkeypatch.setattr(patch, "_orig_video_decoder_call", lambda *args, **kwargs: "ok") + + assert patch._patched_video_decoder_call(object()) == "ok" + assert model.freed_to == "meta" + assert dsc._cached_model is None + + +def test_video_decoder_call_cleans_allocator_even_when_cache_empty(monkeypatch) -> None: + cleaned: list[bool] = [] + dsc.set_enabled(False) + dsc.evict() + monkeypatch.setattr(patch, "_orig_video_decoder_call", lambda *args, **kwargs: "ok") + monkeypatch.setattr(patch, "cleanup_memory", lambda: cleaned.append(True)) + + assert patch._patched_video_decoder_call(object()) == "ok" + assert cleaned == [True] + + +def test_pixel_shape_from_transformer_latent_is_540p() -> None: + # 20s 540p: (481-1)/8+1=61 latent frames, 576/32=18, 1024/32=32. + assert patch._pixel_shape_from_latent(torch.zeros(1, 128, 61, 18, 32)) == (576, 1024, 481) + + +def test_pixel_shape_from_5s_latent_is_not_stage4_scale() -> None: + # DiffVAE pixel_scale is 8 spatial and would report this as 144x256. + assert patch._pixel_shape_from_latent(torch.zeros(1, 128, 16, 18, 32)) == (576, 1024, 121) + + +def test_cuda_diffvae_reresolves_tiling_after_evict(monkeypatch) -> None: + captured: dict[str, object] = {} + sentinel = SimpleNamespace( + frames=SimpleNamespace(tile_size=80), + height=SimpleNamespace(tile_size=576), + width=SimpleNamespace(tile_size=608), + ) + latent = torch.zeros(1, 4, 61, 18, 32) + + def _recommend(checkpoint: str, **kwargs: object) -> object: + captured["checkpoint"] = checkpoint + captured["kwargs"] = kwargs + return sentinel + + monkeypatch.setattr(patch, "is_diffusion_video_vae", lambda _path: True) + monkeypatch.setattr(patch, "cuda_activation_budget_bytes", lambda _device: 6 * 1024**3) + monkeypatch.setattr(patch, "tiling_config_for_vae", _recommend) + monkeypatch.setattr(patch, "_orig_video_decoder_call", lambda _self, *args, **kwargs: (args, kwargs)) + + decoder = _FakeDecoder(torch.device("cuda")) + old_tiling = object() + args, kwargs = patch._patched_video_decoder_call(decoder, latent, old_tiling) + + assert args[0] is latent + assert args[1] is sentinel + assert kwargs == {} + assert captured["checkpoint"] == decoder.checkpoint_path + rec = captured["kwargs"] + assert rec["height"] == 576 + assert rec["width"] == 1024 + assert rec["num_frames"] == 481 + assert rec["free_bytes"] == 6 * 1024**3 + assert rec["device"] == decoder._device + + +def test_cuda_tile_budget_caps_optimistic_mem_get_info() -> None: + assert patch._cuda_tile_budget_bytes(int(29.5 * 1024**3)) == patch._CUDA_DIFFVAE_TILE_BUDGET_CAP_BYTES + assert patch._cuda_tile_budget_bytes(6 * 1024**3) == 6 * 1024**3 + + +def test_cuda_diffvae_caps_29gib_budget_before_recommend(monkeypatch) -> None: + captured: dict[str, object] = {} + monkeypatch.setattr(patch, "is_diffusion_video_vae", lambda _path: True) + monkeypatch.setattr(patch, "cuda_activation_budget_bytes", lambda _device: int(29.5 * 1024**3)) + monkeypatch.setattr( + patch, + "tiling_config_for_vae", + lambda _checkpoint, **kwargs: captured.update(kwargs) or object(), + ) + monkeypatch.setattr(patch, "_orig_video_decoder_call", lambda *_args, **_kwargs: "ok") + + patch._patched_video_decoder_call( + _FakeDecoder(torch.device("cuda")), + torch.zeros(1, 128, 61, 18, 32), + object(), + ) + assert captured["free_bytes"] == patch._CUDA_DIFFVAE_TILE_BUDGET_CAP_BYTES + assert captured["width"] == 1024 + assert captured["height"] == 576 + assert captured["num_frames"] == 481 + + +def test_non_cuda_decoder_keeps_pipeline_tiling(monkeypatch) -> None: + monkeypatch.setattr(patch, "tiling_config_for_vae", lambda *_args, **_kwargs: pytest.fail("should not re-resolve")) + monkeypatch.setattr(patch, "_orig_video_decoder_call", lambda _self, *args, **kwargs: (args, kwargs)) + latent = torch.zeros(1, 4, 61, 18, 32) + old_tiling = object() + args, _kwargs = patch._patched_video_decoder_call(_FakeDecoder(torch.device("cpu")), latent, old_tiling) + assert args[1] is old_tiling + + +def test_conv_vae_keeps_pipeline_tiling(monkeypatch) -> None: + monkeypatch.setattr(patch, "is_diffusion_video_vae", lambda _path: False) + monkeypatch.setattr(patch, "tiling_config_for_vae", lambda *_args, **_kwargs: pytest.fail("should not re-resolve")) + monkeypatch.setattr(patch, "_orig_video_decoder_call", lambda _self, *args, **kwargs: (args, kwargs)) + latent = torch.zeros(1, 4, 61, 18, 32) + old_tiling = object() + args, _kwargs = patch._patched_video_decoder_call(_FakeDecoder(torch.device("cuda")), latent, old_tiling) + assert args[1] is old_tiling diff --git a/backend/tests/test_diffvae_mps_tiling_budget.py b/backend/tests/test_diffvae_mps_tiling_budget.py new file mode 100644 index 000000000..1709cd2fd --- /dev/null +++ b/backend/tests/test_diffvae_mps_tiling_budget.py @@ -0,0 +1,104 @@ +"""DiffVAE MPS tiling budget: upstream treats non-CUDA free_bytes as 0.""" + +from __future__ import annotations + +import torch +from ltx_core.tiling import DimensionSizeConfig, TileSizeConfig, split_by_size + +from services.ltx_pipeline_common import ( + diffvae_activation_budget_bytes, + host_available_bytes, + resolve_diffvae_free_bytes, +) +from services.patches.diffvae_mps_tiling_budget import _ensure_mps_width_split + + +def _size_config(*, width_size: int, width_overlap: int = 160) -> TileSizeConfig: + return TileSizeConfig( + frames=DimensionSizeConfig(tile_size=128, overlap=40), + height=DimensionSizeConfig(tile_size=576, overlap=160), + width=DimensionSizeConfig(tile_size=width_size, overlap=width_overlap), + ) + + +def _width_tiles(config: TileSizeConfig, width: int) -> int: + return len(split_by_size(config.width.tile_size, config.width.overlap)(width).intervals) + + +def test_host_available_bytes_is_positive() -> None: + assert host_available_bytes() > 0 + + +def test_cpu_budget_is_positive() -> None: + assert diffvae_activation_budget_bytes(torch.device("cpu")) > 0 + + +def test_unset_non_cuda_free_bytes_uses_available_ram() -> None: + budget = resolve_diffvae_free_bytes(torch.device("cpu"), None) + assert budget is not None and budget > 0 + + +def test_explicit_free_bytes_are_kept() -> None: + assert resolve_diffvae_free_bytes(torch.device("cpu"), 3 * 1024**3) == 3 * 1024**3 + + +def test_zero_non_cuda_free_bytes_are_replaced() -> None: + # Upstream's MPS path passes 0, which makes usable_bytes=0 and raises before decode. + budget = resolve_diffvae_free_bytes(torch.device("mps"), 0) + assert budget is not None and budget > 0 + + +def test_cuda_unset_budget_stays_none() -> None: + # Let tiling_config_for_vae call cuda_activation_budget_bytes itself. + assert resolve_diffvae_free_bytes(torch.device("cuda"), None) is None + + +def test_patch_rebinds_tiling_config_for_vae() -> None: + import services.patches.diffvae_mps_tiling_budget as patch + from ltx_pipelines.utils import helpers + + assert helpers.tiling_config_for_vae is patch._patched_tiling_config_for_vae + + +def test_mps_full_width_540p_becomes_two_width_tiles() -> None: + original = _size_config(width_size=1024) + clamped = _ensure_mps_width_split(original, width=1024, device=torch.device("mps")) + assert clamped.width.tile_size == 608 + assert clamped.width.overlap == 160 + assert _width_tiles(clamped, 1024) == 2 + assert clamped.height == original.height + assert clamped.frames == original.frames + + +def test_mps_already_split_width_is_unchanged() -> None: + original = _size_config(width_size=608) + clamped = _ensure_mps_width_split(original, width=1024, device=torch.device("mps")) + assert clamped is original + + +def test_cuda_full_width_is_unchanged() -> None: + original = _size_config(width_size=1024) + clamped = _ensure_mps_width_split(original, width=1024, device=torch.device("cuda")) + assert clamped is original + + +def test_mps_too_narrow_to_split_is_unchanged() -> None: + original = _size_config(width_size=320) + clamped = _ensure_mps_width_split(original, width=320, device=torch.device("mps")) + assert clamped is original + + +def test_patched_tiling_config_applies_mps_width_split(monkeypatch) -> None: + import services.patches.diffvae_mps_tiling_budget as patch + + monkeypatch.setattr(patch, "_orig_tiling_config_for_vae", lambda *args, **kwargs: _size_config(width_size=1024)) + config = patch._patched_tiling_config_for_vae( + "unused.safetensors", + height=576, + width=1024, + num_frames=121, + device=torch.device("mps"), + free_bytes=17 * 1024**3, + ) + assert config.width.tile_size == 608 + assert _width_tiles(config, 1024) == 2 diff --git a/backend/tests/test_generation.py b/backend/tests/test_generation.py index 7304d5a02..c7c43573d 100644 --- a/backend/tests/test_generation.py +++ b/backend/tests/test_generation.py @@ -5,6 +5,8 @@ from dataclasses import dataclass from pathlib import Path +from frame_math import AutoDurationSpec +from runtime_config.model_download_specs import delete_cp_path, get_ltx_model_spec, resolve_model_path from services.ltx_api_client.ltx_api_client import LTXAPIClientError from state.app_state_types import GpuSlot, VideoPipelineState from tests.http_error_assertions import assert_http_error @@ -18,6 +20,9 @@ class _FakeEncodingResult: video_context: object = "fake_tensor" audio_context: object = None +_API_ENCODING_MODEL_ID = "ltx-2.3-22b-distilled-1.1" +_LOCAL_2_3 = "ltx-2.3-22b-distilled-1.1" + _T2V_JSON = { "prompt": "test", "resolution": "540p", @@ -27,6 +32,11 @@ class _FakeEncodingResult: } +def _install_local_2_3(test_state, create_fake_model_files, **kwargs) -> None: + create_fake_model_files(model_id=_LOCAL_2_3, **kwargs) + test_state.state.app_settings.active_ltx_model_id = _LOCAL_2_3 + + def _write_test_wav(path: Path, *, duration_seconds: float = 0.1, sample_rate: int = 8000) -> None: import wave @@ -48,6 +58,7 @@ def _fake_running_generation_state(test_state) -> None: active_pipeline=VideoPipelineState( pipeline=pipeline, is_compiled=False, + ltx_model_id="ltx-2.5-22b-distilled", ), ) test_state.generation.start_generation("running") @@ -58,6 +69,22 @@ def test_t2v_requires_downloaded_ltx_model(self, client): r = client.post("/api/generate", json=_T2V_JSON) assert_http_error(r, status_code=409, code="NO_DOWNLOADED_LTX_MODEL") + def test_t2v_on_2_5_uses_api_model_id_without_local_text_encoder( + self, client, test_state, fake_services, create_fake_model_files + ): + create_fake_model_files() + test_state.state.app_settings.ltx_api_key = "api-key" + test_state.state.app_settings.use_local_text_encoder = False + spec = get_ltx_model_spec("ltx-2.5-22b-distilled") + resolve_model_path(test_state.config.default_models_dir, spec.text_encoder_cp).unlink() + fake_services.text_encoder.encode_responses.append(_FakeEncodingResult()) + + r = client.post("/api/generate", json=_T2V_JSON) + + assert r.status_code == 200 + assert fake_services.text_encoder.encode_calls[0]["api_model_id"] == spec.api_text_encoder_model_id + assert fake_services.text_encoder.encode_calls[0]["enhance_prompt"] is True + def test_t2v_happy_path(self, client, test_state, fake_services, create_fake_model_files): create_fake_model_files() _enable_local_text_encoding(test_state) @@ -83,9 +110,79 @@ def test_t2v_happy_path(self, client, test_state, fake_services, create_fake_mod pipeline = fake_services.fast_video_pipeline assert len(pipeline.generate_calls) == 1 - def test_t2v_loras_forwarded_to_pipeline(self, client, test_state, fake_services, create_fake_model_files, create_fake_lora): + def test_t2v_auto_duration_on_2_5_forwards_envelope_range( + self, client, test_state, fake_services, create_fake_model_files + ): + create_fake_model_files() + _enable_local_text_encoding(test_state) + + r = client.post( + "/api/generate", + json={ + "prompt": "A lighthouse keeper climbs the stairs", + "resolution": "540p", + "model": "fast", + "duration": None, + "fps": 24, + }, + ) + + assert r.status_code == 200 + call = fake_services.fast_video_pipeline.generate_calls[0] + assert call["num_frames"] == AutoDurationSpec(min_seconds=5, max_seconds=20) + + def test_t2v_auto_duration_rejected_on_2_3( + self, client, test_state, create_fake_model_files + ): + _install_local_2_3(test_state, create_fake_model_files) + _enable_local_text_encoding(test_state) + + r = client.post( + "/api/generate", + json={ + "prompt": "A lighthouse keeper climbs the stairs", + "resolution": "540p", + "model": "fast", + "duration": None, + "fps": 24, + }, + ) + + assert_http_error( + r, + status_code=422, + code="INVALID_VIDEO_GENERATION_SPEC", + message="Automatic duration is not supported for local pipeline 'fast'", + ) + + def test_t2v_auto_duration_rejected_without_duration_head( + self, client, test_state, create_fake_model_files + ): create_fake_model_files() _enable_local_text_encoding(test_state) + delete_cp_path(test_state.config.default_models_dir, "ltx-2.5-duration-head") + + r = client.post( + "/api/generate", + json={ + "prompt": "A lighthouse keeper climbs the stairs", + "resolution": "540p", + "model": "fast", + "duration": None, + "fps": 24, + }, + ) + + assert_http_error( + r, + status_code=422, + code="INVALID_VIDEO_GENERATION_SPEC", + message="Automatic duration is not supported for local pipeline 'fast'", + ) + + def test_t2v_loras_forwarded_to_pipeline(self, client, test_state, fake_services, create_fake_model_files, create_fake_lora): + _install_local_2_3(test_state, create_fake_model_files) + _enable_local_text_encoding(test_state) lora_ref = create_fake_lora("style.safetensors") r = client.post( @@ -98,8 +195,9 @@ def test_t2v_loras_forwarded_to_pipeline(self, client, test_state, fake_services assert pipeline.create_loras[-1] == [(lora_ref, 0.8)] def test_same_loras_reuse_loaded_pipeline(self, client, test_state, fake_services, create_fake_model_files, create_fake_lora): - create_fake_model_files() + _install_local_2_3(test_state, create_fake_model_files) _enable_local_text_encoding(test_state) + test_state.state.app_settings.prompt_enhancer_enabled_t2v = False lora_ref = create_fake_lora("a.safetensors") body = {**_T2V_JSON, "loras": [{"ref": lora_ref, "scale": 1.0}]} @@ -110,7 +208,7 @@ def test_same_loras_reuse_loaded_pipeline(self, client, test_state, fake_service assert fake_services.fast_video_pipeline.create_loras == [[(lora_ref, 1.0)]] def test_changed_loras_reload_pipeline(self, client, test_state, fake_services, create_fake_model_files, create_fake_lora): - create_fake_model_files() + _install_local_2_3(test_state, create_fake_model_files) _enable_local_text_encoding(test_state) lora_ref = create_fake_lora("b.safetensors") @@ -126,9 +224,25 @@ def test_changed_loras_reload_pipeline(self, client, test_state, fake_services, [(lora_ref, 0.5)], ] - def test_t2v_loras_unknown_ref_rejected(self, client, test_state, create_fake_model_files): + def test_video_vae_path_change_reloads_pipeline( + self, client, test_state, fake_services, create_fake_model_files + ): create_fake_model_files() _enable_local_text_encoding(test_state) + test_state.state.app_settings.prompt_enhancer_enabled_t2v = False + test_state.state.app_settings.use_conv_vae = False + + assert client.post("/api/generate", json=_T2V_JSON).status_code == 200 + assert client.post("/api/generate", json=_T2V_JSON).status_code == 200 + assert fake_services.fast_video_pipeline.create_loras == [[]] + + test_state.state.app_settings.use_conv_vae = True + assert client.post("/api/generate", json=_T2V_JSON).status_code == 200 + assert fake_services.fast_video_pipeline.create_loras == [[], []] + + def test_t2v_loras_unknown_ref_rejected(self, client, test_state, create_fake_model_files): + _install_local_2_3(test_state, create_fake_model_files) + _enable_local_text_encoding(test_state) r = client.post( "/api/generate", @@ -137,6 +251,19 @@ def test_t2v_loras_unknown_ref_rejected(self, client, test_state, create_fake_mo assert r.status_code == 400 + def test_t2v_loras_forwarded_on_2_5(self, client, test_state, fake_services, create_fake_model_files, create_fake_lora): + create_fake_model_files() + _enable_local_text_encoding(test_state) + lora_ref = create_fake_lora("style.safetensors") + + r = client.post( + "/api/generate", + json={**_T2V_JSON, "loras": [{"ref": lora_ref, "scale": 0.8}]}, + ) + + assert r.status_code == 200 + assert fake_services.fast_video_pipeline.create_loras[-1] == [(lora_ref, 0.8)] + def test_already_running(self, client, test_state): _fake_running_generation_state(test_state) @@ -171,17 +298,53 @@ def test_i2v_rejects_invalid_image_content_400(self, client, test_state, create_ ) assert "Invalid image file" in data["message"] - def test_resolution_mapping_540p(self, client, test_state, fake_services, create_fake_model_files): + def test_resolution_mapping_540p_on_2_5(self, client, test_state, fake_services, create_fake_model_files): + # 2.5 540p is legal 16:9 on the /64 two-stage grid (1024×576). create_fake_model_files() _enable_local_text_encoding(test_state) r = client.post("/api/generate", json=_T2V_JSON) assert r.status_code == 200 + pipeline = fake_services.fast_video_pipeline + call = pipeline.generate_calls[0] + assert call["width"] == 1024 + assert call["height"] == 576 + + def test_resolution_mapping_540p_on_2_3(self, client, test_state, fake_services, create_fake_model_files): + # 2.3 540p is 960×544; snap_up_to_multiple(..., 64) maps height to 576 + # on the two-stage grid. + create_fake_model_files(model_id="ltx-2.3-22b-distilled-1.1") + test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled-1.1" + _enable_local_text_encoding(test_state) + + r = client.post("/api/generate", json=_T2V_JSON) + assert r.status_code == 200 + pipeline = fake_services.fast_video_pipeline call = pipeline.generate_calls[0] assert call["width"] == 960 - assert call["height"] == 512 + assert call["height"] == 576 + + def test_local_resolutions_are_all_on_the_two_stage_grid( + self, client, test_state, fake_services, create_fake_model_files + ): + # 2.5 Fast sizes are already /64. Two-stage halves each dimension onto a /32 latent grid, + # so anything not divisible by 64 gets silently snapped. + create_fake_model_files() + _enable_local_text_encoding(test_state) + + for resolution in ("540p", "720p", "1080p"): + for aspect_ratio in ("16:9", "9:16"): + fake_services.fast_video_pipeline.generate_calls.clear() + r = client.post( + "/api/generate", + json={**_T2V_JSON, "resolution": resolution, "aspectRatio": aspect_ratio, "duration": 5}, + ) + assert r.status_code == 200 + call = fake_services.fast_video_pipeline.generate_calls[0] + assert call["width"] % 64 == 0, f"{resolution} {aspect_ratio}: width {call['width']}" + assert call["height"] % 64 == 0, f"{resolution} {aspect_ratio}: height {call['height']}" def test_resolution_mapping_720p(self, client, test_state, fake_services, create_fake_model_files): create_fake_model_files() @@ -261,6 +424,29 @@ def test_a2v_generation_happy_path(self, client, test_state, fake_services, crea assert call["audio_max_duration"] is None def test_a2v_loras_forwarded_to_pipeline(self, client, test_state, fake_services, create_fake_model_files, create_fake_lora, tmp_path): + _install_local_2_3(test_state, create_fake_model_files) + _enable_local_text_encoding(test_state) + audio_file = tmp_path / "test_audio.wav" + _write_test_wav(audio_file) + lora_ref = create_fake_lora("groove.safetensors") + + r = client.post( + "/api/generate", + json={ + "prompt": "A music video", + "resolution": "540p", + "model": "fast", + "duration": 5, + "fps": 24, + "audioPath": str(audio_file), + "loras": [{"ref": lora_ref, "scale": 0.7}], + }, + ) + + assert r.status_code == 200 + assert fake_services.a2v_pipeline.create_loras[-1] == [(lora_ref, 0.7)] + + def test_a2v_loras_forwarded_on_2_5(self, client, test_state, fake_services, create_fake_model_files, create_fake_lora, tmp_path): create_fake_model_files() _enable_local_text_encoding(test_state) audio_file = tmp_path / "test_audio.wav" @@ -351,6 +537,53 @@ def test_a2v_forced_api_routes_to_ltx_api(self, client, test_state, fake_service assert call["model"] == "ltx-2-3-pro" assert call["resolution"] == "1920x1080" + def test_a2v_forced_api_routes_to_ltx_api_for_ltx_2_5_pro(self, client, test_state, fake_services, tmp_path): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + audio_file = tmp_path / "test_audio.wav" + _write_test_wav(audio_file) + + r = client.post( + "/api/generate", + json={ + "prompt": "A music video", + "resolution": "1080p", + "model": "pro-2.5", + "duration": 6, + "fps": 50, + "audioPath": str(audio_file), + }, + ) + assert r.status_code == 200 + assert r.json()["status"] == "complete" + assert len(fake_services.ltx_api_client.audio_to_video_calls) == 1 + call = fake_services.ltx_api_client.audio_to_video_calls[0] + assert call["model"] == "ltx-2-5-pro" + + def test_a2v_forced_api_routes_to_ltx_api_for_ltx_2_5_fast(self, client, test_state, fake_services, tmp_path): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + audio_file = tmp_path / "test_audio.wav" + _write_test_wav(audio_file) + + r = client.post( + "/api/generate", + json={ + "prompt": "A music video", + "resolution": "1080p", + "model": "fast-2.5", + "duration": 6, + "fps": 50, + "audioPath": str(audio_file), + }, + ) + assert r.status_code == 200 + assert r.json()["status"] == "complete" + assert len(fake_services.ltx_api_client.audio_to_video_calls) == 1 + call = fake_services.ltx_api_client.audio_to_video_calls[0] + assert call["model"] == "ltx-2-5-fast" + assert call["resolution"] == "1920x1080" + def test_a2v_prefers_api_routes_to_ltx_api(self, client, test_state, fake_services, tmp_path): test_state.config.local_generations_mode = "full_models_loading" test_state.state.app_settings.user_prefers_ltx_api_video_generations = True @@ -438,14 +671,14 @@ def test_a2v_forced_api_routes_to_ltx_api_with_audio_and_image( assert call["model"] == "ltx-2-3-pro" assert call["resolution"] == "1920x1080" - def test_a2v_uses_resolution_map(self, client, test_state, fake_services, create_fake_model_files, tmp_path): + def test_a2v_uses_resolution_map_on_2_5(self, client, test_state, fake_services, create_fake_model_files, tmp_path): create_fake_model_files() _enable_local_text_encoding(test_state) audio_file = tmp_path / "test_audio.wav" _write_test_wav(audio_file) for resolution, expected_w, expected_h in [ - ("540p", 960, 576), + ("540p", 1024, 576), ("720p", 1280, 704), ("1080p", 1920, 1088), ]: @@ -467,6 +700,30 @@ def test_a2v_uses_resolution_map(self, client, test_state, fake_services, create assert call["width"] == expected_w, f"{resolution}: expected width {expected_w}, got {call['width']}" assert call["height"] == expected_h, f"{resolution}: expected height {expected_h}, got {call['height']}" + def test_a2v_540p_on_2_3_uses_historical_pixels(self, client, test_state, fake_services, create_fake_model_files, tmp_path): + create_fake_model_files(model_id="ltx-2.3-22b-distilled-1.1") + test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled-1.1" + _enable_local_text_encoding(test_state) + audio_file = tmp_path / "test_audio.wav" + _write_test_wav(audio_file) + + r = client.post( + "/api/generate", + json={ + "prompt": "A music video", + "resolution": "540p", + "model": "fast", + "duration": 5, + "fps": 24, + "audioPath": str(audio_file), + }, + ) + + assert r.status_code == 200 + call = fake_services.a2v_pipeline.generate_calls[0] + assert call["width"] == 960 + assert call["height"] == 544 + def test_a2v_forced_api_rejects_missing_audio_file(self, client, test_state): test_state.config.local_generations_mode = "unsupported" test_state.state.app_settings.ltx_api_key = "api-key" @@ -600,6 +857,159 @@ def test_t2v_routes_to_ltx_api(self, client, test_state, fake_services): assert call["generate_audio"] is True assert call["camera_motion"] == "dolly_in" + def test_t2v_routes_to_ltx_api_for_ltx_2_5_fast(self, client, test_state, fake_services): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + + r = client.post( + "/api/generate", + json={ + "prompt": "A mountain lake", + "resolution": "1080p", + "model": "fast-2.5", + "duration": 6, + "fps": 50, + "audio": True, + "cameraMotion": "dolly_in", + }, + ) + + assert r.status_code == 200 + assert r.json()["status"] == "complete" + assert len(fake_services.ltx_api_client.text_to_video_calls) == 1 + call = fake_services.ltx_api_client.text_to_video_calls[0] + assert call["model"] == "ltx-2-5-fast" + assert call["resolution"] == "1920x1080" + assert call["duration"] == 6.0 + assert call["fps"] == 50.0 + assert call["generate_audio"] is True + assert call["camera_motion"] == "dolly_in" + + def test_t2v_auto_duration_sends_null_for_ltx_2_5_fast(self, client, test_state, fake_services): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + + r = client.post( + "/api/generate", + json={ + "prompt": "A lighthouse keeper climbs the stairs", + "resolution": "1080p", + "model": "fast-2.5", + "duration": None, + "fps": 24, + "audio": True, + }, + ) + + assert r.status_code == 200 + call = fake_services.ltx_api_client.text_to_video_calls[0] + assert call["model"] == "ltx-2-5-fast" + assert call["duration"] is None + + def test_api_auto_duration_does_not_require_local_duration_head( + self, client, test_state, fake_services, create_fake_model_files + ): + create_fake_model_files() + delete_cp_path(test_state.config.default_models_dir, "ltx-2.5-duration-head") + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + + r = client.post( + "/api/generate", + json={ + "prompt": "A lighthouse keeper climbs the stairs", + "resolution": "1080p", + "model": "fast-2.5", + "duration": None, + "fps": 24, + }, + ) + + assert r.status_code == 200 + call = fake_services.ltx_api_client.text_to_video_calls[0] + assert call["model"] == "ltx-2-5-fast" + assert call["duration"] is None + + def test_t2v_auto_duration_rejected_for_ltx_2_3_fast(self, client, test_state): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + + r = client.post( + "/api/generate", + json={ + "prompt": "A lighthouse keeper climbs the stairs", + "resolution": "1080p", + "model": "fast", + "duration": None, + "fps": 24, + }, + ) + + assert_http_error( + r, + status_code=422, + code="INVALID_VIDEO_GENERATION_SPEC", + message="Automatic duration is not supported for api pipeline 'fast'", + ) + + def test_a2v_rejects_auto_duration(self, client, test_state, tmp_path): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + audio_file = tmp_path / "test_audio.wav" + _write_test_wav(audio_file) + + r = client.post( + "/api/generate", + json={ + "prompt": "A music video", + "resolution": "1080p", + "model": "fast-2.5", + "duration": None, + "fps": 24, + "audioPath": str(audio_file), + }, + ) + + assert_http_error( + r, + status_code=422, + code="INVALID_VIDEO_GENERATION_SPEC", + message="Automatic duration cannot be combined with audio-to-video", + ) + + def test_i2v_routes_to_ltx_api_for_ltx_2_5_pro(self, client, test_state, fake_services, make_test_image, tmp_path): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + image_path = tmp_path / "input.png" + image_path.write_bytes(make_test_image().getvalue()) + + r = client.post( + "/api/generate", + json={ + "prompt": "Animate this frame", + "resolution": "2160p", + "model": "pro-2.5", + "duration": 8, + "fps": 25, + "audio": False, + "cameraMotion": "jib_up", + "imagePath": str(image_path), + }, + ) + + assert r.status_code == 200 + assert r.json()["status"] == "complete" + assert len(fake_services.ltx_api_client.upload_file_calls) == 1 + assert fake_services.ltx_api_client.upload_file_calls[0]["file_path"] == str(image_path) + assert len(fake_services.ltx_api_client.image_to_video_calls) == 1 + call = fake_services.ltx_api_client.image_to_video_calls[0] + assert call["image_uri"] == "storage://uploaded/input.png" + assert call["model"] == "ltx-2-5-pro" + assert call["resolution"] == "3840x2160" + assert call["duration"] == 8.0 + assert call["fps"] == 25.0 + assert call["camera_motion"] == "jib_up" + def test_i2v_routes_to_ltx_api(self, client, test_state, fake_services, make_test_image, tmp_path): test_state.config.local_generations_mode = "unsupported" test_state.state.app_settings.ltx_api_key = "api-key" @@ -796,6 +1206,33 @@ def test_invalid_forced_duration_rejected(self, client, test_state): message="Unsupported api text-to-video duration '5' for pipeline 'pro' at resolution '1080p' and fps '25'", ) + def test_forced_api_a2v_rejects_fast_tier_pipeline(self, client, test_state, tmp_path): + # ltxv-api audio-to-video does not accept ltx-2-3-fast. Reject pipeline "fast" + # here rather than a downstream 400. (ltx-2-5-fast does accept A2V.) + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + audio_file = tmp_path / "test_audio.wav" + _write_test_wav(audio_file) + + r = client.post( + "/api/generate", + json={ + "prompt": "A music video", + "resolution": "1080p", + "model": "fast", + "duration": 6, + "fps": 24, + "audioPath": str(audio_file), + }, + ) + + assert_http_error( + r, + status_code=422, + code="INVALID_VIDEO_GENERATION_SPEC", + message="Unsupported api audio-to-video resolution '1080p' for pipeline 'fast'", + ) + def test_invalid_forced_fps_rejected(self, client, test_state): test_state.config.local_generations_mode = "unsupported" test_state.state.app_settings.ltx_api_key = "api-key" @@ -1146,7 +1583,7 @@ def test_a2v_forced_api_passes_through_model_and_aspect(self, client, test_state json={ "prompt": "A portrait music video", "resolution": "1080p", - "model": "fast", + "model": "pro", "duration": 6, "fps": 25, "audioPath": str(audio_file), @@ -1158,7 +1595,7 @@ def test_a2v_forced_api_passes_through_model_and_aspect(self, client, test_state assert r.json()["status"] == "complete" call = fake_services.ltx_api_client.audio_to_video_calls[0] assert call["resolution"] == "1080x1920" - assert call["model"] == "ltx-2-3-fast" + assert call["model"] == "ltx-2-3-pro" class TestGenerateCancel: @@ -1183,14 +1620,56 @@ def test_models_specs_endpoint_returns_ordered_backend_specs(self, client): assert r.status_code == 200 data = r.json() assert [item["pipeline"] for item in data["local_models"]] == ["fast"] - assert data["local_models"][0]["spec"]["display_name"] == "LTX 2.3 Fast" + assert data["local_models"][0]["spec"]["display_name"] == "LTX 2.5 Fast" assert list(data["local_models"][0]["spec"]["supported_resolutions_durations"]["540p"]["fps_to_durations"].keys()) == ["24"] - assert [item["pipeline"] for item in data["api_models"]] == ["fast", "pro"] - assert list(data["api_models"][0]["spec"]["a2v_supported_resolutions_durations"].keys()) == ["1080p"] + assert [item["pipeline"] for item in data["api_models"]] == ["fast", "pro", "fast-2.5", "pro-2.5"] assert data["api_models"][0]["spec"]["supported_resolutions_durations"]["1080p"]["fps_to_durations"]["24"] == [ 6, 8, 10, 12, 14, 16, 18, 20, ] + api_models_by_pipeline = {item["pipeline"]: item for item in data["api_models"]} + # A2V envelope: none on fast; 1080p on pro, fast-2.5, and pro-2.5. + assert api_models_by_pipeline["fast"]["spec"]["a2v_supported_resolutions_durations"] is None + assert list(api_models_by_pipeline["pro"]["spec"]["a2v_supported_resolutions_durations"].keys()) == ["1080p"] + assert api_models_by_pipeline["fast-2.5"]["spec"]["display_name"] == "LTX-2.5 Fast (API)" + assert list(api_models_by_pipeline["fast-2.5"]["spec"]["a2v_supported_resolutions_durations"].keys()) == ["1080p"] + assert list(api_models_by_pipeline["pro-2.5"]["spec"]["a2v_supported_resolutions_durations"].keys()) == ["1080p"] + + local_caps = data["local_models"][0]["spec"]["capabilities"] + assert local_caps["a2v"] is True + assert local_caps["ic_lora"] is True + assert local_caps["user_loras"] is True + assert local_caps["retake"] is False + assert local_caps["extend"] is False + # No DurationHead on disk in this fixture — Auto stays off until that file is present. + assert local_caps["auto_duration"] is False + assert api_models_by_pipeline["fast"]["spec"]["capabilities"]["a2v"] is False + assert api_models_by_pipeline["fast"]["spec"]["capabilities"]["auto_duration"] is False + assert api_models_by_pipeline["fast-2.5"]["spec"]["capabilities"]["a2v"] is True + assert api_models_by_pipeline["fast-2.5"]["spec"]["capabilities"]["auto_duration"] is True + assert api_models_by_pipeline["pro"]["spec"]["capabilities"]["retake"] is True + assert api_models_by_pipeline["pro-2.5"]["spec"]["capabilities"]["retake"] is False + assert api_models_by_pipeline["pro-2.5"]["spec"]["capabilities"]["auto_duration"] is True + + def test_local_auto_duration_requires_duration_head_on_disk(self, client, create_fake_model_files): + create_fake_model_files() + r = client.get("/api/generate/models-specs") + assert r.status_code == 200 + assert r.json()["local_models"][0]["spec"]["capabilities"]["auto_duration"] is True + + def test_local_auto_duration_hidden_when_duration_head_missing( + self, client, test_state, create_fake_model_files + ): + create_fake_model_files() + delete_cp_path(test_state.config.default_models_dir, "ltx-2.5-duration-head") + r = client.get("/api/generate/models-specs") + assert r.status_code == 200 + data = r.json() + assert data["local_models"][0]["spec"]["capabilities"]["auto_duration"] is False + api_by_pipeline = {item["pipeline"]: item for item in data["api_models"]} + assert api_by_pipeline["fast-2.5"]["spec"]["capabilities"]["auto_duration"] is True + assert api_by_pipeline["pro-2.5"]["spec"]["capabilities"]["auto_duration"] is True + class TestGenerationProgress: def test_idle(self, client): @@ -1363,7 +1842,8 @@ class TestEnhancePromptFlag: """Verify enhance_prompt is passed correctly to the text encoder API.""" def _setup_api_encoding(self, test_state, fake_services, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_API_ENCODING_MODEL_ID) + test_state.state.app_settings.active_ltx_model_id = _API_ENCODING_MODEL_ID test_state.state.app_settings.ltx_api_key = "test-key" test_state.state.app_settings.use_local_text_encoder = False fake_services.text_encoder.encode_responses.append(_FakeEncodingResult()) @@ -1467,3 +1947,134 @@ def test_local_encoding_skips_api(self, client, test_state, fake_services, creat assert r.status_code == 200 assert len(fake_services.text_encoder.encode_calls) == 0 + + +class TestLocalEncodingEnhancement: + """The rewrite that API encoding gets server-side has to happen here for local encoding. + + Without it the enhancer setting silently does nothing whenever the local encoder is + selected, and the model sees the prompt exactly as typed. + """ + + def _setup_local(self, test_state, create_fake_model_files, *, with_enhancer: bool): + create_fake_model_files(include_prompt_enhancer=with_enhancer) + test_state.state.app_settings.use_local_text_encoder = True + test_state.state.app_settings.prompt_enhancer_enabled_t2v = True + test_state.state.app_settings.prompt_enhancer_enabled_i2v = True + + def test_t2v_prompt_is_enhanced_before_it_reaches_the_pipeline( + self, client, test_state, fake_services, create_fake_model_files + ): + self._setup_local(test_state, create_fake_model_files, with_enhancer=True) + fake_services.prompt_enhancer_pipeline.enhanced_prompt = "a long descriptive caption" + + r = client.post("/api/generate", json=_T2V_JSON) + assert r.status_code == 200 + + assert fake_services.prompt_enhancer_pipeline.enhance_t2v_calls[0]["prompt"] == "test" + assert fake_services.fast_video_pipeline.generate_calls[0]["prompt"] == "a long descriptive caption" + + def test_enhancer_runs_before_the_generation_is_marked_running( + self, client, test_state, fake_services, create_fake_model_files + ): + # The enhancer evicts whatever pipeline is resident to claim its VRAM, and eviction is + # refused once a generation is running — so getting a pipeline built at all is the + # assertion that the ordering held. + self._setup_local(test_state, create_fake_model_files, with_enhancer=True) + + r = client.post("/api/generate", json=_T2V_JSON) + assert r.status_code == 200 + assert len(fake_services.prompt_enhancer_pipeline.created_with) == 1 + + def test_disabled_setting_leaves_the_prompt_alone( + self, client, test_state, fake_services, create_fake_model_files + ): + self._setup_local(test_state, create_fake_model_files, with_enhancer=True) + test_state.state.app_settings.prompt_enhancer_enabled_t2v = False + + r = client.post("/api/generate", json=_T2V_JSON) + assert r.status_code == 200 + + assert fake_services.prompt_enhancer_pipeline.enhance_t2v_calls == [] + assert fake_services.fast_video_pipeline.generate_calls[0]["prompt"] == "test" + + def test_missing_enhancer_generates_with_the_prompt_as_typed( + self, client, test_state, fake_services, create_fake_model_files + ): + self._setup_local(test_state, create_fake_model_files, with_enhancer=False) + + r = client.post("/api/generate", json=_T2V_JSON) + assert r.status_code == 200 + + assert fake_services.prompt_enhancer_pipeline.enhance_t2v_calls == [] + assert fake_services.fast_video_pipeline.generate_calls[0]["prompt"] == "test" + + def test_2_5_uses_gemma3_fallback_to_enhance_before_generate( + self, client, test_state, fake_services, create_fake_model_files + ): + create_fake_model_files(include_prompt_enhancer=False) + create_fake_model_files(model_id="ltx-2.3-22b-distilled-1.1") + test_state.state.app_settings.active_ltx_model_id = "ltx-2.5-22b-distilled" + test_state.state.app_settings.use_local_text_encoder = True + test_state.state.app_settings.prompt_enhancer_enabled_t2v = True + fake_services.prompt_enhancer_pipeline.enhanced_prompt = "a long descriptive caption" + + r = client.post("/api/generate", json=_T2V_JSON) + assert r.status_code == 200 + assert fake_services.prompt_enhancer_pipeline.enhance_t2v_calls[0]["prompt"] == "test" + assert fake_services.fast_video_pipeline.generate_calls[0]["prompt"] == "a long descriptive caption" + + def test_enhancer_failure_does_not_fail_the_generation( + self, client, test_state, fake_services, create_fake_model_files + ): + self._setup_local(test_state, create_fake_model_files, with_enhancer=True) + fake_services.prompt_enhancer_pipeline.raise_on_enhance = RuntimeError("boom") + + r = client.post("/api/generate", json=_T2V_JSON) + assert r.status_code == 200 + assert fake_services.fast_video_pipeline.generate_calls[0]["prompt"] == "test" + + def test_camera_motion_is_appended_after_the_rewrite( + self, client, test_state, fake_services, create_fake_model_files + ): + self._setup_local(test_state, create_fake_model_files, with_enhancer=True) + fake_services.prompt_enhancer_pipeline.enhanced_prompt = "a long descriptive caption" + suffix = test_state.config.camera_motion_prompts["dolly_in"] + + r = client.post("/api/generate", json={**_T2V_JSON, "cameraMotion": "dolly_in"}) + assert r.status_code == 200 + + assert fake_services.prompt_enhancer_pipeline.enhance_t2v_calls[0]["prompt"] == "test" + assert ( + fake_services.fast_video_pipeline.generate_calls[0]["prompt"] + == "a long descriptive caption" + suffix + ) + + def test_i2v_routes_the_conditioning_image_to_the_enhancer( + self, client, test_state, fake_services, create_fake_model_files, make_test_image, tmp_path + ): + self._setup_local(test_state, create_fake_model_files, with_enhancer=True) + image_path = tmp_path / "input.png" + image_path.write_bytes(make_test_image().getvalue()) + + r = client.post("/api/generate", json={**_T2V_JSON, "imagePath": str(image_path)}) + assert r.status_code == 200 + + assert len(fake_services.prompt_enhancer_pipeline.enhance_i2v_calls) == 1 + assert fake_services.prompt_enhancer_pipeline.enhance_t2v_calls == [] + + def test_api_encoding_still_enhances_server_side( + self, client, test_state, fake_services, create_fake_model_files + ): + create_fake_model_files(model_id=_API_ENCODING_MODEL_ID, include_prompt_enhancer=True) + test_state.state.app_settings.active_ltx_model_id = _API_ENCODING_MODEL_ID + test_state.state.app_settings.ltx_api_key = "test-key" + test_state.state.app_settings.use_local_text_encoder = False + test_state.state.app_settings.prompt_enhancer_enabled_t2v = True + fake_services.text_encoder.encode_responses.append(_FakeEncodingResult()) + + r = client.post("/api/generate", json=_T2V_JSON) + assert r.status_code == 200 + + assert fake_services.prompt_enhancer_pipeline.enhance_t2v_calls == [] + assert fake_services.text_encoder.encode_calls[0]["enhance_prompt"] is True diff --git a/backend/tests/test_health.py b/backend/tests/test_health.py index 4608ae761..8270305db 100644 --- a/backend/tests/test_health.py +++ b/backend/tests/test_health.py @@ -9,6 +9,7 @@ def _set_video_pipeline(state): active_pipeline=VideoPipelineState( pipeline=FakeFastVideoPipeline(), is_compiled=False, + ltx_model_id="ltx-2.5-22b-distilled", ), ) diff --git a/backend/tests/test_ic_lora.py b/backend/tests/test_ic_lora.py index 72b150cd9..7f0cb8a5a 100644 --- a/backend/tests/test_ic_lora.py +++ b/backend/tests/test_ic_lora.py @@ -8,6 +8,7 @@ from tests.http_error_assertions import assert_http_error from tests.fakes import FakeCapture +from tests.conftest import _IC_LORA_MODEL_ID def _write_ic_lora_file(path: Path) -> None: @@ -19,6 +20,12 @@ def _write_ic_lora_file(path: Path) -> None: f.write(blob) +def _install_ic_lora_capable_model(create_fake_model_files, create_fake_ic_lora_files, *, include_depth: bool = True) -> None: + # Built-in control IC-LoRA is 2.3-only; 2.5 (latest) returns 409 for canny/depth. + create_fake_model_files(model_id=_IC_LORA_MODEL_ID) + create_fake_ic_lora_files(include_depth=include_depth) + + class TestIcLoraExtractConditioning: def test_canny_extraction(self, client, test_state): video_path = test_state.config.outputs_dir / "test_video.mp4" @@ -35,8 +42,7 @@ def test_canny_extraction(self, client, test_state): assert payload["conditioning"].startswith("data:image/jpeg;base64,") def test_depth_extraction(self, client, test_state, fake_services, create_fake_model_files, create_fake_ic_lora_files): - create_fake_model_files() - create_fake_ic_lora_files() + _install_ic_lora_capable_model(create_fake_model_files, create_fake_ic_lora_files) video_path = test_state.config.outputs_dir / "test_video.mp4" video_path.write_bytes(b"\x00" * 100) test_state.video_processor.register_video(str(video_path), FakeCapture(frames=["frame-a"])) @@ -63,8 +69,7 @@ def test_depth_extraction_requires_downloaded_ltx_model(self, client, test_state class TestIcLoraGenerate: def test_happy_path(self, client, test_state, create_fake_model_files, create_fake_ic_lora_files): - create_fake_model_files() - create_fake_ic_lora_files() + _install_ic_lora_capable_model(create_fake_model_files, create_fake_ic_lora_files) test_state.state.app_settings.use_local_text_encoder = True video_path = test_state.config.outputs_dir / "test_video.mp4" @@ -84,11 +89,37 @@ def test_happy_path(self, client, test_state, create_fake_model_files, create_fa assert response.json()["status"] == "complete" assert Path(response.json()["video_path"]).exists() + def test_builtin_control_rejected_on_2_5(self, client, test_state, create_fake_model_files): + create_fake_model_files() + test_state.state.app_settings.use_local_text_encoder = True + + video_path = test_state.config.outputs_dir / "test_video.mp4" + video_path.write_bytes(b"\x00" * 100) + test_state.video_processor.register_video(str(video_path), FakeCapture(frames=["frame-a", "frame-b"])) + + response = client.post( + "/api/ic-lora/generate", + json={ + "video_path": str(video_path), + "conditioning_type": "canny", + "prompt": "test prompt", + "images": [], + }, + ) + assert_http_error( + response, + status_code=409, + code="UNSUPPORTED_IC_LORA", + message=( + "Built-in control IC-LoRA is not available for the active LTX model. " + "Switch to an LTX 2.3 local model to use depth/canny control." + ), + ) + def test_canny_does_not_require_depth_cp(self, client, test_state, create_fake_model_files, create_fake_ic_lora_files): # canny preprocessing uses apply_canny, not the depth processor, so generation # must succeed even when the depth cp isn't installed (previously 500'd). - create_fake_model_files() - create_fake_ic_lora_files(include_depth=False) + _install_ic_lora_capable_model(create_fake_model_files, create_fake_ic_lora_files, include_depth=False) test_state.state.app_settings.use_local_text_encoder = True video_path = test_state.config.outputs_dir / "test_video.mp4" @@ -110,8 +141,7 @@ def test_canny_does_not_require_depth_cp(self, client, test_state, create_fake_m def test_local_ic_lora_recoverable_via_progress(self, client, test_state, create_fake_model_files, create_fake_ic_lora_files): # IC-LoRA is local-only and drives the generation state machine, so a page that # unmounted mid-generation can recover the output via /generation/progress. - create_fake_model_files() - create_fake_ic_lora_files() + _install_ic_lora_capable_model(create_fake_model_files, create_fake_ic_lora_files) test_state.state.app_settings.use_local_text_encoder = True video_path = test_state.config.outputs_dir / "test_video_recover.mp4" diff --git a/backend/tests/test_logging_policy.py b/backend/tests/test_logging_policy.py index bca1532cc..4a5911090 100644 --- a/backend/tests/test_logging_policy.py +++ b/backend/tests/test_logging_policy.py @@ -25,8 +25,9 @@ def test_http_500_logs_single_traceback(caplog, client, fake_services) -> None: assert records[0].exc_info is not None -def test_http_400_logs_without_traceback(caplog, client) -> None: +def test_http_400_logs_without_traceback(caplog, client, create_fake_model_files) -> None: caplog.set_level(logging.WARNING) + create_fake_model_files() response = client.post( "/api/generate", @@ -102,7 +103,7 @@ def test_logger_exception_usage_is_restricted_to_boundaries() -> None: } for path in backend_dir.rglob("*.py"): - if "tests" in path.parts or ".venv" in path.parts or "tmp" in path.parts: + if "tests" in path.parts or ".venv" in path.parts or "tmp" in path.parts or "vendor" in path.parts: continue content = path.read_text(encoding="utf-8") if "logger.exception(" in content: diff --git a/backend/tests/test_lora_catalog.py b/backend/tests/test_lora_catalog.py index 37d128525..ae049fb3d 100644 --- a/backend/tests/test_lora_catalog.py +++ b/backend/tests/test_lora_catalog.py @@ -29,6 +29,31 @@ def test_parse_valid_catalog(): assert r.default_settings.skip_stage_2 is True assert r.controls[0].id == "duration" assert r.controls[0].options == [5, 8] + assert r.supported_models == ["LTX-2.3", "LTX-2.5"] + assert r.supports_family("LTX-2.3") is True + assert r.supports_family("LTX-2.5") is True + + +def test_supported_models_rejects_empty_and_duplicates(): + empty = _VALID.replace( + '"requires_hf_login": true,', + '"requires_hf_login": true, "supported_models": [],', + ) + with pytest.raises(ValidationError, match="supported_models"): + parse_lora_catalog(empty) + dup = _VALID.replace( + '"requires_hf_login": true,', + '"requires_hf_login": true, "supported_models": ["LTX-2.3", "LTX-2.3"],', + ) + with pytest.raises(ValidationError, match="supported_models"): + parse_lora_catalog(dup) + + +def test_shipped_catalog_allows_2_3_and_2_5(): + catalog = Path(__file__).parent.parent / "runtime_config" / "lora_catalog.json" + cat = parse_lora_catalog(catalog.read_text(encoding="utf-8")) + for item in [*cat.loras, *cat.ic_loras]: + assert item.supported_models == ["LTX-2.3", "LTX-2.5"], item.id def test_controls_default_to_empty(): no_controls = _VALID.replace( diff --git a/backend/tests/test_ltx_api_client.py b/backend/tests/test_ltx_api_client.py index bd1f0fd9d..f6d73a77a 100644 --- a/backend/tests/test_ltx_api_client.py +++ b/backend/tests/test_ltx_api_client.py @@ -346,6 +346,7 @@ def test_retake_returns_direct_video_bytes(tmp_path) -> None: duration=3.0, prompt="make it dramatic", mode="replace_audio_and_video", + model="ltx-2-3-pro", ) assert result.video_bytes == b"retake-bytes" @@ -388,6 +389,7 @@ def test_retake_json_video_url_downloads_bytes(tmp_path) -> None: duration=4.0, prompt="test", mode="replace_video", + model="ltx-2-3-pro", ) assert result.video_bytes == b"downloaded-retake" @@ -424,6 +426,7 @@ def test_retake_json_without_video_url_returns_payload(tmp_path) -> None: duration=2.5, prompt="test", mode="replace_audio_and_video", + model="ltx-2-3-pro", ) assert result.video_bytes is None @@ -457,6 +460,7 @@ def test_retake_422_maps_to_safety_filter_error(tmp_path) -> None: duration=3.0, prompt="test", mode="replace_audio_and_video", + model="ltx-2-3-pro", ) assert exc.value.status_code == 422 @@ -486,6 +490,7 @@ def test_extend_async_submits_polls_and_downloads(tmp_path) -> None: duration=12.0, prompt="continue the motion", mode="end", + model="ltx-2-3-pro", ) assert result.video_bytes == b"extended-bytes" @@ -518,7 +523,7 @@ def test_extend_async_retries_transient_poll_blip(tmp_path) -> None: http.queue("get", FakeResponse(status_code=200, content=b"extended-bytes")) client = _async_client(http) - result = client.extend(api_key="k", video_path=input_path, duration=4.0, prompt="", mode="end") + result = client.extend(api_key="k", video_path=input_path, duration=4.0, prompt="", mode="end", model="ltx-2-3-pro") assert result.video_bytes == b"extended-bytes" @@ -533,7 +538,7 @@ def test_extend_async_unknown_terminal_status_surfaces(tmp_path) -> None: client = _async_client(http) with pytest.raises(LTXAPIClientError, match="rejected") as exc: - client.extend(api_key="k", video_path=input_path, duration=4.0, prompt="", mode="end") + client.extend(api_key="k", video_path=input_path, duration=4.0, prompt="", mode="end", model="ltx-2-3-pro") assert exc.value.status_code == 500 @@ -552,7 +557,7 @@ def test_extend_async_job_failed_raises(tmp_path) -> None: client = _async_client(http) with pytest.raises(LTXAPIClientError, match="model exploded"): - client.extend(api_key="k", video_path=input_path, duration=4.0, prompt="", mode="end") + client.extend(api_key="k", video_path=input_path, duration=4.0, prompt="", mode="end", model="ltx-2-3-pro") def test_extend_async_422_maps_to_safety_filter(tmp_path) -> None: @@ -563,7 +568,7 @@ def test_extend_async_422_maps_to_safety_filter(tmp_path) -> None: client = _async_client(http) with pytest.raises(LTXAPIClientError, match="Content rejected by safety filters") as exc: - client.extend(api_key="k", video_path=input_path, duration=4.0, prompt="", mode="end") + client.extend(api_key="k", video_path=input_path, duration=4.0, prompt="", mode="end", model="ltx-2-3-pro") assert exc.value.status_code == 422 @@ -576,7 +581,7 @@ def test_extend_async_connection_reset_maps_to_504(tmp_path) -> None: client = _async_client(http) with pytest.raises(LTXAPIClientError, match="please retry") as exc: - client.extend(api_key="k", video_path=input_path, duration=12.0, prompt="", mode="end") + client.extend(api_key="k", video_path=input_path, duration=12.0, prompt="", mode="end", model="ltx-2-3-pro") assert exc.value.status_code == 504 @@ -589,7 +594,7 @@ def test_extend_async_completed_without_url_raises(tmp_path) -> None: client = _async_client(http) with pytest.raises(LTXAPIClientError, match="without a video_url"): - client.extend(api_key="k", video_path=input_path, duration=4.0, prompt="", mode="end") + client.extend(api_key="k", video_path=input_path, duration=4.0, prompt="", mode="end", model="ltx-2-3-pro") def test_retake_upload_init_failure_maps_message() -> None: @@ -605,5 +610,6 @@ def test_retake_upload_init_failure_maps_message() -> None: duration=3.0, prompt="test", mode="replace_audio_and_video", + model="ltx-2-3-pro", ) assert exc.value.status_code == 401 diff --git a/backend/tests/test_ltx_capabilities.py b/backend/tests/test_ltx_capabilities.py new file mode 100644 index 000000000..f2155de33 --- /dev/null +++ b/backend/tests/test_ltx_capabilities.py @@ -0,0 +1,118 @@ +"""Unit tests for the Desktop LTX capabilities SSOT.""" + +from __future__ import annotations + +import pytest + +from runtime_config.ltx_capabilities import ( + api_caps, + effective_local_caps, + local_caps, + pixels_for, + supports, +) + + +def test_local_2_3_540p_is_historical_960x544(): + caps = local_caps("ltx-2.3-22b-distilled-1.1") + assert pixels_for(caps, "540p", "16:9") == (960, 544) + assert pixels_for(caps, "540p", "9:16") == (544, 960) + + +def test_local_2_3_v10_shares_2_3_pixel_map(): + assert pixels_for(local_caps("ltx-2.3-22b-distilled"), "540p", "16:9") == (960, 544) + + +def test_local_2_5_540p_is_legal_16_9(): + caps = local_caps("ltx-2.5-22b-distilled") + width, height = pixels_for(caps, "540p", "16:9") + assert (width, height) == (1024, 576) + assert width % 64 == 0 and height % 64 == 0 + + +def test_local_2_5_allows_ic_lora_and_user_loras(): + caps = local_caps("ltx-2.5-22b-distilled") + assert supports(caps, "ic_lora") is True + assert supports(caps, "user_loras") is True + assert supports(caps, "retake") is False + assert supports(caps, "extend") is False + + +def test_local_2_3_allows_ic_lora_user_loras_retake(): + caps = local_caps("ltx-2.3-22b-distilled-1.1") + assert supports(caps, "ic_lora") is True + assert supports(caps, "user_loras") is True + assert supports(caps, "retake") is True + assert supports(caps, "extend") is True + assert supports(caps, "auto_duration") is False + + +def test_local_2_5_keeps_t2v_i2v_a2v(): + caps = local_caps("ltx-2.5-22b-distilled") + assert supports(caps, "t2v") is True + assert supports(caps, "i2v") is True + assert supports(caps, "a2v") is True + assert supports(caps, "camera_motion") is True + assert supports(caps, "auto_duration") is True + + +def test_local_2_5_auto_duration_requires_duration_head_ready(): + model_id = "ltx-2.5-22b-distilled" + assert supports(effective_local_caps(model_id, duration_head_ready=True), "auto_duration") is True + assert supports(effective_local_caps(model_id, duration_head_ready=False), "auto_duration") is False + + +def test_local_2_3_auto_duration_stays_off_even_if_duration_head_ready(): + assert ( + supports( + effective_local_caps("ltx-2.3-22b-distilled-1.1", duration_head_ready=True), + "auto_duration", + ) + is False + ) + + +def test_api_fast_2_3_has_no_a2v_or_auto_duration(): + caps = api_caps("fast") + assert supports(caps, "a2v") is False + assert supports(caps, "retake") is False + assert supports(caps, "extend") is False + assert supports(caps, "auto_duration") is False + assert pixels_for(caps, "1080p", "16:9") == (1920, 1080) + + +def test_api_fast_2_5_has_a2v_and_auto_duration(): + caps = api_caps("fast-2.5") + assert supports(caps, "a2v") is True + assert supports(caps, "retake") is False + assert supports(caps, "extend") is False + assert supports(caps, "auto_duration") is True + assert pixels_for(caps, "1080p", "16:9") == (1920, 1080) + + +def test_api_pro_2_3_has_a2v_and_retake(): + caps = api_caps("pro") + assert supports(caps, "a2v") is True + assert supports(caps, "retake") is True + assert supports(caps, "extend") is True + assert supports(caps, "auto_duration") is False + + +def test_api_pro_2_5_has_a2v_and_auto_duration_not_retake(): + caps = api_caps("pro-2.5") + assert supports(caps, "a2v") is True + assert supports(caps, "retake") is False + assert supports(caps, "extend") is False + assert supports(caps, "auto_duration") is True + + +def test_pixels_for_unknown_resolution_raises(): + with pytest.raises(KeyError): + pixels_for(api_caps("fast"), "540p", "16:9") + + +def test_ic_lora_flag_is_on_for_every_local_model(): + from runtime_config.model_download_specs import ALL_LTX_LOCAL_MODEL_IDS + + for model_id in ALL_LTX_LOCAL_MODEL_IDS: + assert supports(local_caps(model_id), "ic_lora") is True diff --git a/backend/tests/test_ltx_runtime_paths.py b/backend/tests/test_ltx_runtime_paths.py new file mode 100644 index 000000000..eafaceae7 --- /dev/null +++ b/backend/tests/test_ltx_runtime_paths.py @@ -0,0 +1,122 @@ +"""Split 2.5 DurationHead path resolution.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from runtime_config.ltx_runtime_paths import resolve_ltx_runtime_paths +from runtime_config.model_download_specs import get_ltx_model_spec, resolve_model_path +from services.ltx_pipeline_common import build_model_paths + + +def _write_cp(models_dir: Path, cp_id: str) -> Path: + path = resolve_model_path(models_dir, cp_id) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"x") + return path + + +def _write_2_5_bundle( + models_dir: Path, + *, + include_diff_vae: bool = True, + include_conv_vae: bool = True, + include_duration_head: bool = True, +) -> None: + spec = get_ltx_model_spec("ltx-2.5-22b-distilled") + cps = [spec.model_cp, spec.upscale_cp, spec.audio_vae_cp] + if include_diff_vae: + cps.append(spec.video_vae_cp) + if include_conv_vae: + cps.append(spec.video_vae_conv_cp) + if include_duration_head: + cps.append(spec.duration_head_cp) + for cp_id in cps: + if cp_id is not None: + _write_cp(models_dir, cp_id) + + +def test_split_model_paths_pass_duration_head() -> None: + paths = build_model_paths( + "transformer.safetensors", + "gemma", + video_vae_path="video.safetensors", + audio_vae_path="audio.safetensors", + duration_head_path="duration.safetensors", + ) + assert paths.mode == "split" + assert paths.duration_head_path == "duration.safetensors" + + +def test_split_model_paths_omit_duration_head_when_missing() -> None: + paths = build_model_paths( + "transformer.safetensors", + "gemma", + video_vae_path="video.safetensors", + audio_vae_path="audio.safetensors", + ) + assert paths.duration_head_path is None + + +def test_monolith_duration_head_is_the_fat_checkpoint() -> None: + paths = build_model_paths("monolith.safetensors", "gemma") + assert paths.mode == "monolith" + assert paths.duration_head_path == "monolith.safetensors" + + +def test_resolve_runtime_paths_includes_downloaded_duration_head(tmp_path: Path) -> None: + spec = get_ltx_model_spec("ltx-2.5-22b-distilled") + _write_2_5_bundle(tmp_path) + paths = resolve_ltx_runtime_paths( + tmp_path, "ltx-2.5-22b-distilled", gemma_root=None, use_conv_vae=False + ) + assert paths.duration_head_path == str(resolve_model_path(tmp_path, spec.duration_head_cp)) + + +def test_resolve_runtime_paths_omits_missing_duration_head(tmp_path: Path) -> None: + spec = get_ltx_model_spec("ltx-2.5-22b-distilled") + _write_2_5_bundle(tmp_path, include_duration_head=False) + paths = resolve_ltx_runtime_paths( + tmp_path, "ltx-2.5-22b-distilled", gemma_root=None, use_conv_vae=False + ) + assert paths.duration_head_path is None + assert paths.video_vae_path == str(resolve_model_path(tmp_path, spec.video_vae_cp)) + + +def test_resolve_runtime_paths_picks_conv_vae(tmp_path: Path) -> None: + spec = get_ltx_model_spec("ltx-2.5-22b-distilled") + _write_2_5_bundle(tmp_path) + paths = resolve_ltx_runtime_paths( + tmp_path, "ltx-2.5-22b-distilled", gemma_root=None, use_conv_vae=True + ) + assert paths.video_vae_path == str(resolve_model_path(tmp_path, spec.video_vae_conv_cp)) + + +def test_resolve_runtime_paths_picks_diffvae(tmp_path: Path) -> None: + spec = get_ltx_model_spec("ltx-2.5-22b-distilled") + _write_2_5_bundle(tmp_path) + paths = resolve_ltx_runtime_paths( + tmp_path, "ltx-2.5-22b-distilled", gemma_root=None, use_conv_vae=False + ) + assert paths.video_vae_path == str(resolve_model_path(tmp_path, spec.video_vae_cp)) + + +def test_resolve_runtime_paths_2_3_ignores_conv_toggle(tmp_path: Path) -> None: + spec = get_ltx_model_spec("ltx-2.3-22b-distilled-1.1") + for cp_id in (spec.model_cp, spec.upscale_cp): + _write_cp(tmp_path, cp_id) + for use_conv_vae in (True, False): + paths = resolve_ltx_runtime_paths( + tmp_path, "ltx-2.3-22b-distilled-1.1", gemma_root=None, use_conv_vae=use_conv_vae + ) + assert paths.video_vae_path is None + + +def test_resolve_runtime_paths_does_not_fall_back_to_other_vae(tmp_path: Path) -> None: + _write_2_5_bundle(tmp_path, include_conv_vae=False) + with pytest.raises(FileNotFoundError, match="ltx-2.5-video-vae-conv"): + resolve_ltx_runtime_paths( + tmp_path, "ltx-2.5-22b-distilled", gemma_root=None, use_conv_vae=True + ) diff --git a/backend/tests/test_model_download_specs.py b/backend/tests/test_model_download_specs.py index 0f2e5eb9e..680003f93 100644 --- a/backend/tests/test_model_download_specs.py +++ b/backend/tests/test_model_download_specs.py @@ -10,19 +10,39 @@ from runtime_config.model_download_specs import ( ALL_MODEL_CP_IDS, ALL_LTX_LOCAL_MODEL_IDS, + LTX_2_5_FAMILY_DIR, ModelCheckpointSpec, + delete_cp_path, + get_existing_cp_path, get_ic_loras_cp_ids, get_latest_ltx_model_id, get_ltx_cps, get_ltx_model_cp_ids, get_ltx_model_spec, + get_local_prompt_enhancer_cp, get_model_cp_spec, + is_cp_downloaded, + selected_video_vae_cp, + unused_video_vae_cp, + is_duration_head_ready, + local_prompt_enhancer_candidates, + resolve_downloaded_prompt_enhancer_cp, resolve_downloading_dir, resolve_downloading_path, resolve_downloading_target_path, resolve_model_path, ) +_LTX_2_5_NATIVE_CPS: tuple[ModelCheckpointID, ...] = ( + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", + "gemma4-12b-with-proj-ltx-2.5", +) + def test_specs_cover_all_checkpoint_ids(): assert set(ALL_MODEL_CP_IDS) == {cp_id for cp_id in ALL_MODEL_CP_IDS} @@ -34,23 +54,90 @@ def test_primary_ltx_checkpoints_map_1_to_1_with_ltx_models(): def test_latest_ltx_model_is_relevant(): latest = get_latest_ltx_model_id() + assert latest == "ltx-2.5-22b-distilled" spec = get_ltx_model_spec(latest) assert spec.model_cp in get_ltx_cps() + assert spec.video_vae_cp is not None + assert spec.audio_vae_cp is not None + assert spec.ic_loras_spec is None -def test_ic_lora_cp_ids_are_deduped(): - spec = get_ltx_model_spec(get_latest_ltx_model_id()) +def test_ic_lora_cp_ids_are_deduped_for_2_3(): + spec = get_ltx_model_spec("ltx-2.3-22b-distilled-1.1") assert get_ic_loras_cp_ids(spec.ic_loras_spec) == ("ltx-2.3-22b-ic-lora-union-control-ref0.5",) -def test_ltx_model_cp_ids_include_deduped_ic_loras(): - spec = get_ltx_model_spec(get_latest_ltx_model_id()) - assert get_ltx_model_cp_ids(get_latest_ltx_model_id()) == ( +def test_ltx_2_5_model_cp_ids_include_split_vaes(): + spec = get_ltx_model_spec("ltx-2.5-22b-distilled") + assert get_ltx_model_cp_ids("ltx-2.5-22b-distilled") == ( spec.model_cp, spec.upscale_cp, spec.text_encoder_cp, - "ltx-2.3-22b-ic-lora-union-control-ref0.5", + spec.video_vae_cp, + spec.video_vae_conv_cp, + spec.audio_vae_cp, + spec.duration_head_cp, ) + assert spec.video_vae_conv_cp == "ltx-2.5-video-vae-conv" + assert spec.duration_head_cp == "ltx-2.5-duration-head" + + +def test_2_3_has_no_split_video_vaes(): + spec = get_ltx_model_spec("ltx-2.3-22b-distilled-1.1") + assert spec.video_vae_cp is None + assert spec.video_vae_conv_cp is None + assert selected_video_vae_cp(spec, use_conv_vae=True) is None + assert selected_video_vae_cp(spec, use_conv_vae=False) is None + + +def test_selected_video_vae_cp_follows_toggle(): + spec = get_ltx_model_spec("ltx-2.5-22b-distilled") + assert selected_video_vae_cp(spec, use_conv_vae=True) == spec.video_vae_conv_cp + assert selected_video_vae_cp(spec, use_conv_vae=False) == spec.video_vae_cp + assert unused_video_vae_cp(spec, use_conv_vae=True) == spec.video_vae_cp + assert unused_video_vae_cp(spec, use_conv_vae=False) == spec.video_vae_conv_cp + + +def test_2_3_has_no_duration_head_cp(): + assert get_ltx_model_spec("ltx-2.3-22b-distilled-1.1").duration_head_cp is None + + +def test_duration_head_ready_requires_the_split_file(tmp_path: Path): + model_id = "ltx-2.5-22b-distilled" + assert is_duration_head_ready(tmp_path, model_id) is False + spec = get_ltx_model_spec(model_id) + assert spec.duration_head_cp is not None + path = resolve_model_path(tmp_path, spec.duration_head_cp) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"\x00" * 1024) + assert is_duration_head_ready(tmp_path, model_id) is True + assert is_duration_head_ready(tmp_path, "ltx-2.3-22b-distilled-1.1") is False + + +def test_duration_head_download_filename_is_nested(): + spec = get_model_cp_spec("ltx-2.5-duration-head") + assert spec.download_filename == "model_patches/ltx-2.5-duration-head-bf16.safetensors" + assert spec.repo_id == "Lightricks/LTX-2.5" + + +def test_ltx_2_5_download_filenames_are_nested(): + transformer = get_model_cp_spec("ltx-2.5-22b-distilled") + assert transformer.download_filename.startswith("diffusion_models/") + assert transformer.repo_id == "Lightricks/LTX-2.5" + + +def test_ltx_2_5_native_weights_live_under_family_dir(): + for cp_id in _LTX_2_5_NATIVE_CPS: + relative = get_model_cp_spec(cp_id).relative_path + assert relative.parts[0] == LTX_2_5_FAMILY_DIR.name, cp_id + assert relative.parent == LTX_2_5_FAMILY_DIR + + +def test_shared_and_2_3_checkpoints_stay_at_models_root(): + for cp_id in ALL_MODEL_CP_IDS: + if cp_id in _LTX_2_5_NATIVE_CPS: + continue + assert get_model_cp_spec(cp_id).relative_path.parts[0] != LTX_2_5_FAMILY_DIR.name, cp_id def test_model_path_resolves_from_relative_path(tmp_path): @@ -69,6 +156,52 @@ def test_downloading_path_is_derived_from_spec(): == downloading_dir / "gemma-3-12b-it-qat-q4_0-unquantized" ) assert resolve_downloading_target_path(models_dir, "ltx-2.3-22b-distilled") == downloading_dir / "ltx-2.3-22b-distilled.safetensors" + assert resolve_downloading_target_path(models_dir, "ltx-2.5-22b-distilled") == ( + downloading_dir / LTX_2_5_FAMILY_DIR / "ltx-2.5-22b-distilled-transformer-bf16.safetensors" + ) + + +def test_2_5_write_path_is_family_dir(tmp_path): + path = resolve_model_path(tmp_path, "ltx-2.5-22b-distilled") + assert path.parent == tmp_path / LTX_2_5_FAMILY_DIR + assert path.name == "ltx-2.5-22b-distilled-transformer-bf16.safetensors" + + +def test_2_5_reads_legacy_flat_file_at_models_root(tmp_path): + spec = get_model_cp_spec("ltx-2.5-video-vae") + leftover = tmp_path / spec.relative_path.name + leftover.write_bytes(b"legacy") + assert is_cp_downloaded(tmp_path, "ltx-2.5-video-vae") is True + assert get_existing_cp_path(tmp_path, "ltx-2.5-video-vae") == leftover + + +def test_2_5_prefers_family_dir_over_legacy_root(tmp_path): + spec = get_model_cp_spec("ltx-2.5-video-vae") + leftover = tmp_path / spec.relative_path.name + leftover.write_bytes(b"legacy") + canonical = resolve_model_path(tmp_path, "ltx-2.5-video-vae") + canonical.parent.mkdir(parents=True) + canonical.write_bytes(b"canonical") + assert get_existing_cp_path(tmp_path, "ltx-2.5-video-vae") == canonical + + +def test_2_5_delete_removes_family_dir_and_legacy_root(tmp_path): + spec = get_model_cp_spec("ltx-2.5-audio-vae") + leftover = tmp_path / spec.relative_path.name + leftover.write_bytes(b"legacy") + canonical = resolve_model_path(tmp_path, "ltx-2.5-audio-vae") + canonical.parent.mkdir(parents=True) + canonical.write_bytes(b"canonical") + delete_cp_path(tmp_path, "ltx-2.5-audio-vae") + assert not leftover.exists() + assert not canonical.exists() + assert is_cp_downloaded(tmp_path, "ltx-2.5-audio-vae") is False + + +def test_2_3_has_no_root_fallback(tmp_path): + assert is_cp_downloaded(tmp_path, "ltx-2.3-22b-distilled") is False + with pytest.raises(FileNotFoundError): + get_existing_cp_path(tmp_path, "ltx-2.3-22b-distilled") def test_relative_paths_are_unique(): @@ -81,14 +214,47 @@ def test_model_path_rejects_parent_traversal(monkeypatch, tmp_path): relative_path=Path("../escape.safetensors"), expected_size_bytes=1, is_folder=False, - repo_id="test/repo", + repo_id="x/y", description="bad", ) - monkeypatch.setattr( "runtime_config.model_download_specs.get_model_cp_spec", - lambda cp_id: bad_spec, + lambda _cp_id: bad_spec, ) - - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="cannot traverse parents"): resolve_model_path(tmp_path, "ltx-2.3-22b-distilled") + + +def _write_folder_cp(models_dir: Path, cp_id: ModelCheckpointID) -> None: + path = resolve_model_path(models_dir, cp_id) + path.mkdir(parents=True, exist_ok=True) + (path / "model.safetensors").write_bytes(b"x") + + +def test_2_5_enhancer_candidates_prefer_e2b_then_gemma3(): + spec = get_ltx_model_spec("ltx-2.5-22b-distilled") + assert get_local_prompt_enhancer_cp(spec) == "gemma-4-e2b-it" + assert local_prompt_enhancer_candidates(spec) == ( + "gemma-4-e2b-it", + "gemma-3-12b-it-qat-q4_0-unquantized", + ) + + +def test_2_3_enhancer_is_the_encoder_only(): + spec = get_ltx_model_spec("ltx-2.3-22b-distilled-1.1") + assert local_prompt_enhancer_candidates(spec) == (spec.text_encoder_cp,) + assert "gemma-4-e2b-it" not in local_prompt_enhancer_candidates(spec) + + +def test_2_5_resolves_gemma3_when_e2b_is_missing(tmp_path: Path): + spec = get_ltx_model_spec("ltx-2.5-22b-distilled") + assert resolve_downloaded_prompt_enhancer_cp(tmp_path, spec) is None + _write_folder_cp(tmp_path, "gemma-3-12b-it-qat-q4_0-unquantized") + assert resolve_downloaded_prompt_enhancer_cp(tmp_path, spec) == "gemma-3-12b-it-qat-q4_0-unquantized" + + +def test_2_5_prefers_e2b_over_gemma3(tmp_path: Path): + spec = get_ltx_model_spec("ltx-2.5-22b-distilled") + _write_folder_cp(tmp_path, "gemma-3-12b-it-qat-q4_0-unquantized") + _write_folder_cp(tmp_path, "gemma-4-e2b-it") + assert resolve_downloaded_prompt_enhancer_cp(tmp_path, spec) == "gemma-4-e2b-it" diff --git a/backend/tests/test_models.py b/backend/tests/test_models.py index e278006b0..acc7e69d7 100644 --- a/backend/tests/test_models.py +++ b/backend/tests/test_models.py @@ -21,8 +21,17 @@ resolve_active_ltx_model_id, resolve_downloading_dir, resolve_model_path, + selected_video_vae_cp, + unused_video_vae_cp, +) +from state.app_settings import AppSettings, resolved_use_conv_vae +from state.app_state_types import ( + DownloadSessionComplete, + DownloadSessionError, + DownloadingSession, + FileDownloadRunning, + HfNotAuthenticated, ) -from state.app_state_types import DownloadSessionComplete, DownloadSessionError, DownloadingSession, FileDownloadRunning from tests.http_error_assertions import assert_http_error @@ -34,33 +43,140 @@ def _cp_path(test_state, cp_id: str) -> Path: return resolve_model_path(test_state.config.default_models_dir, cp_id) +def _use_conv_vae(test_state=None) -> bool: + settings = test_state.state.app_settings if test_state is not None else AppSettings() + return resolved_use_conv_vae(settings) + + +def _required_download_cps(*, include_text_encoder: bool, test_state=None) -> list[str]: + spec = _current_ltx_spec() + cps = [spec.model_cp, spec.upscale_cp] + selected = selected_video_vae_cp(spec, use_conv_vae=_use_conv_vae(test_state)) + if selected is not None: + cps.append(selected) + if spec.video_vae_conv_cp is not None and spec.video_vae_conv_cp not in cps: + cps.append(spec.video_vae_conv_cp) + if spec.audio_vae_cp is not None: + cps.append(spec.audio_vae_cp) + if spec.duration_head_cp is not None: + cps.append(spec.duration_head_cp) + if include_text_encoder: + cps.append(spec.text_encoder_cp) + return cps + + +def _optional_download_cps(*, include_text_encoder: bool, test_state=None) -> list[str]: + spec = _current_ltx_spec() + required = set(_required_download_cps(include_text_encoder=False, test_state=test_state)) + cps: list[str] = [] + unused = unused_video_vae_cp(spec, use_conv_vae=_use_conv_vae(test_state)) + if unused is not None and unused not in required: + cps.append(unused) + if include_text_encoder: + cps.append(spec.text_encoder_cp) + return cps + + +def _remove_text_encoder(test_state) -> None: + from runtime_config.model_download_specs import get_model_cp_spec + + text_encoder_path = _cp_path(test_state, _current_ltx_spec().text_encoder_cp) + te_spec = get_model_cp_spec(_current_ltx_spec().text_encoder_cp) + if te_spec.is_folder: + for child in text_encoder_path.iterdir(): + child.unlink() + text_encoder_path.rmdir() + else: + text_encoder_path.unlink(missing_ok=True) + + class TestRecommendations: def test_ltx_recommendation_requires_primary_local_bundle(self, client): - spec = _current_ltx_spec() response = client.get("/api/models/ltx-recommendation") assert response.status_code == 200 assert response.json() == { "status": "download", - "cps_to_download": [ - spec.model_cp, - spec.upscale_cp, - spec.text_encoder_cp, - ], + "cps_to_download": _required_download_cps(include_text_encoder=True), + "optional_cp_ids": _optional_download_cps(include_text_encoder=False), } - def test_ltx_recommendation_skips_text_encoder_when_api_key_exists(self, client, test_state): + def test_ltx_recommendation_skips_text_encoder_for_2_5_when_api_key_exists(self, client, test_state): test_state.state.app_settings.ltx_api_key = "test-key" - spec = _current_ltx_spec() response = client.get("/api/models/ltx-recommendation") assert response.status_code == 200 assert response.json() == { "status": "download", - "cps_to_download": [ - spec.model_cp, - spec.upscale_cp, - ], + "cps_to_download": _required_download_cps(include_text_encoder=False), + # Excused, not withheld: first-run still offers it so an offline setup stays possible. + "optional_cp_ids": _optional_download_cps(include_text_encoder=True), } + def test_required_video_vae_follows_fast_decode_toggle(self, client, test_state): + spec = _current_ltx_spec() + assert spec.video_vae_cp is not None + assert spec.video_vae_conv_cp is not None + + test_state.state.app_settings.use_conv_vae = True + payload = client.get("/api/models/ltx-recommendation").json() + assert spec.video_vae_conv_cp in payload["cps_to_download"] + assert spec.video_vae_cp not in payload["cps_to_download"] + assert spec.video_vae_cp in payload["optional_cp_ids"] + + test_state.state.app_settings.use_conv_vae = False + payload = client.get("/api/models/ltx-recommendation").json() + assert spec.video_vae_cp in payload["cps_to_download"] + assert spec.video_vae_conv_cp in payload["cps_to_download"] + assert spec.video_vae_conv_cp not in payload["optional_cp_ids"] + + def test_existing_2_5_prompts_missing_conv_vae_when_fast_decode_is_off( + self, client, test_state, create_fake_model_files + ): + # Windows default: Fast decode off, DiffVAE already on disk from the original 2.5 + # install. Conv must still surface as a required download (same LaunchGate as Mac). + create_fake_model_files() + test_state.state.app_settings.use_conv_vae = False + conv_path = _cp_path(test_state, _current_ltx_spec().video_vae_conv_cp) + conv_path.unlink() + + payload = client.get("/api/models/ltx-recommendation").json() + assert payload["status"] == "download" + assert payload["cps_to_download"] == [_current_ltx_spec().video_vae_conv_cp] + + by_id = {item["model_id"]: item for item in client.get("/api/models/ltx-versions").json()["versions"]} + assert by_id["ltx-2.5-22b-distilled"]["installed"] is False + assert _current_ltx_spec().video_vae_conv_cp in by_id["ltx-2.5-22b-distilled"]["cps_to_download"] + + def test_downloaded_text_encoder_is_not_offered_again(self, client, test_state, create_fake_model_files): + test_state.state.app_settings.ltx_api_key = "test-key" + create_fake_model_files() + _cp_path(test_state, _current_ltx_spec().upscale_cp).unlink() + + response = client.get("/api/models/ltx-recommendation") + assert response.status_code == 200 + assert response.json()["optional_cp_ids"] == [] + + def test_api_key_skips_text_encoder_for_supported_versions(self, client, test_state): + test_state.state.app_settings.ltx_api_key = "test-key" + response = client.get("/api/models/ltx-versions") + assert response.status_code == 200 + by_id = {item["model_id"]: item for item in response.json()["versions"]} + + assert _current_ltx_spec().text_encoder_cp not in by_id["ltx-2.5-22b-distilled"]["cps_to_download"] + spec_2_3 = get_ltx_model_spec("ltx-2.3-22b-distilled-1.1") + assert spec_2_3.text_encoder_cp not in by_id["ltx-2.3-22b-distilled-1.1"]["cps_to_download"] + + def test_prompt_enhancer_is_never_required(self, client, create_fake_model_files): + # 2.5's separate enhancer is an opt-in extra: missing it costs local Enhance only, so it + # must not hold back install/activation or show up as a pending download. + create_fake_model_files() + spec = _current_ltx_spec() + assert spec.prompt_enhancer_cp is not None + + by_id = {item["model_id"]: item for item in client.get("/api/models/ltx-versions").json()["versions"]} + assert by_id["ltx-2.5-22b-distilled"]["installed"] is True + assert spec.prompt_enhancer_cp not in by_id["ltx-2.5-22b-distilled"]["cps_to_download"] + assert client.post("/api/models/active-ltx-model", json={"model_id": "ltx-2.5-22b-distilled"}).status_code == 200 + def test_ltx_recommendation_ok_when_required_bundle_is_downloaded(self, client, create_fake_model_files): create_fake_model_files() response = client.get("/api/models/ltx-recommendation") @@ -83,22 +199,76 @@ def test_recommendation_surfaces_missing_shared_companion_for_current_base(self, assert response.json() == { "status": "download", "cps_to_download": [older_spec.upscale_cp], + "optional_cp_ids": [older_spec.text_encoder_cp], } def test_ltx_recommendation_reports_missing_text_encoder_for_current_model(self, client, test_state, create_fake_model_files): create_fake_model_files() - text_encoder_path = _cp_path(test_state, _current_ltx_spec().text_encoder_cp) - for child in text_encoder_path.iterdir(): - child.unlink() - text_encoder_path.rmdir() + _remove_text_encoder(test_state) response = client.get("/api/models/ltx-recommendation") assert response.status_code == 200 assert response.json() == { "status": "download", "cps_to_download": [_current_ltx_spec().text_encoder_cp], + "optional_cp_ids": [], } + def test_upgrade_from_2_3_downloads_split_companions( + self, client, test_state, create_fake_model_files, create_fake_ic_lora_files + ): + # Existing 2.3 install upgrading to 2.5 must pull the new transformer, upscaler, + # duration head, audio VAE, and both video VAEs (DiffVAE + conv). Fast decode can + # then toggle without another download. API key set so the TE stays optional. + create_fake_model_files(model_id="ltx-2.3-22b-distilled-1.1") + create_fake_ic_lora_files() + test_state.state.app_settings.ltx_api_key = "test-key" + test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled-1.1" + + response = client.get("/api/models/ltx-recommendation") + assert response.status_code == 200 + payload = response.json() + assert payload["status"] == "upgrade" + target = get_ltx_model_spec("ltx-2.5-22b-distilled") + assert set(payload["cps_to_download"]) == { + target.model_cp, + target.upscale_cp, + target.video_vae_cp, + target.video_vae_conv_cp, + target.audio_vae_cp, + target.duration_head_cp, + } + assert target.text_encoder_cp not in payload["cps_to_download"] + assert payload["loses_built_in_control"] is True + assert "ltx-2.3-22b-ic-lora-union-control-ref0.5" in payload["cps_to_delete"] + + def test_upgrade_from_2_3_includes_conv_vae_even_when_fast_decode_is_off( + self, client, test_state, create_fake_model_files, create_fake_ic_lora_files + ): + create_fake_model_files(model_id="ltx-2.3-22b-distilled-1.1") + create_fake_ic_lora_files() + test_state.state.app_settings.ltx_api_key = "test-key" + test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled-1.1" + test_state.state.app_settings.use_conv_vae = False + + payload = client.get("/api/models/ltx-recommendation").json() + assert payload["status"] == "upgrade" + assert "ltx-2.5-video-vae-conv" in payload["cps_to_download"] + assert "ltx-2.5-video-vae" in payload["cps_to_download"] + + def test_describe_checkpoints_labels_2_5_vaes(self, client): + response = client.post( + "/api/models/describe", + json={"cp_ids": ["ltx-2.5-video-vae", "ltx-2.5-video-vae-conv", "ltx-2.5-audio-vae", "ltx-2.5-duration-head", "ltx-2.5-22b-distilled"]}, + ) + assert response.status_code == 200 + by_id = {item["cp_id"]: item for item in response.json()["checkpoints"]} + assert by_id["ltx-2.5-22b-distilled"]["role"] == "base" + assert by_id["ltx-2.5-video-vae"]["role"] == "vae" + assert by_id["ltx-2.5-video-vae-conv"]["role"] == "vae" + assert by_id["ltx-2.5-audio-vae"]["role"] == "vae" + assert by_id["ltx-2.5-duration-head"]["role"] == "support" + def test_img_gen_recommendation(self, client, create_fake_model_files): response = client.get("/api/models/img-gen-recommendation") assert response.status_code == 200 @@ -111,15 +281,14 @@ def test_img_gen_recommendation(self, client, create_fake_model_files): def test_text_encoder_recommendation(self, client, create_fake_model_files, test_state): create_fake_model_files() - text_encoder_path = _cp_path(test_state, _current_ltx_spec().text_encoder_cp) - for child in text_encoder_path.iterdir(): - child.unlink() - text_encoder_path.rmdir() + _remove_text_encoder(test_state) response = client.get("/api/models/text-encoder-recommendation") assert response.status_code == 200 assert response.json()["cp_to_download"] == _current_ltx_spec().text_encoder_cp assert response.json()["expected_size_bytes"] > 0 + assert response.json()["api_encoding_supported"] is True + assert response.json()["ltx_version_label"] == "2.5" def test_describe_checkpoints(self, client, create_fake_model_files): spec = _current_ltx_spec() @@ -157,15 +326,22 @@ def test_ic_lora_recommendation(self, client, create_fake_model_files, create_fa create_fake_model_files() response = client.get("/api/models/ltx-ic-lora-recommendation") assert response.status_code == 200 - assert response.json()["cps_to_download"] == [ - *get_ic_loras_cp_ids(_current_ltx_spec().ic_loras_spec), - DEPTH_PROCESSOR_CP_ID, - ] - + # Latest (2.5) has no built-in Union Control IC-LoRA. + payload = response.json() + assert payload["cps_to_download"] == [] + assert payload["supported"] is False + + def test_ic_lora_recommendation_supported_on_active_2_3( + self, client, test_state, create_fake_model_files, create_fake_ic_lora_files + ): + create_fake_model_files(model_id="ltx-2.3-22b-distilled-1.1") create_fake_ic_lora_files() + test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled-1.1" response = client.get("/api/models/ltx-ic-lora-recommendation") assert response.status_code == 200 - assert response.json()["cps_to_download"] == [] + payload = response.json() + assert payload["supported"] is True + assert payload["cps_to_download"] == [] class TestDownloadProgress: @@ -345,12 +521,13 @@ def test_delete_removes_non_protected_checkpoint(self, client, test_state): class TestLtxVersions: - def test_two_base_versions_registered_newest_first(self): - assert ALL_LTX_LOCAL_MODEL_IDS[0] == "ltx-2.3-22b-distilled-1.1" + def test_base_versions_registered_newest_first(self): + assert ALL_LTX_LOCAL_MODEL_IDS[0] == "ltx-2.5-22b-distilled" + assert "ltx-2.3-22b-distilled-1.1" in ALL_LTX_LOCAL_MODEL_IDS assert "ltx-2.3-22b-distilled" in ALL_LTX_LOCAL_MODEL_IDS - assert get_latest_ltx_model_id() == "ltx-2.3-22b-distilled-1.1" + assert get_latest_ltx_model_id() == "ltx-2.5-22b-distilled" - def test_versions_share_companions(self): + def test_2_3_versions_share_companions(self): v11 = get_ltx_model_spec("ltx-2.3-22b-distilled-1.1") v10 = get_ltx_model_spec("ltx-2.3-22b-distilled") assert v11.upscale_cp == v10.upscale_cp @@ -359,42 +536,44 @@ def test_versions_share_companions(self): assert v11.model_cp != v10.model_cp def test_version_labels(self): - assert get_ltx_model_spec("ltx-2.3-22b-distilled-1.1").version_label == "1.1" - assert get_ltx_model_spec("ltx-2.3-22b-distilled").version_label == "1.0" + assert get_ltx_model_spec("ltx-2.5-22b-distilled").version_label == "2.5" + assert get_ltx_model_spec("ltx-2.3-22b-distilled-1.1").version_label == "2.3" + assert get_ltx_model_spec("ltx-2.3-22b-distilled").version_label == "2.3 (1.0)" - def test_1_1_checkpoint_spec(self): - spec = get_model_cp_spec("ltx-2.3-22b-distilled-1.1") - assert spec.repo_id == "Lightricks/LTX-2.3" - assert str(spec.relative_path) == "ltx-2.3-22b-distilled-1.1.safetensors" - assert spec.expected_size_bytes == 46_149_345_334 + def test_2_5_checkpoint_spec(self): + spec = get_model_cp_spec("ltx-2.5-22b-distilled") + assert spec.repo_id == "Lightricks/LTX-2.5" + assert spec.download_filename.startswith("diffusion_models/") - def test_distilled_1_1_has_upgrade_notes(self): - # The 1.1 spec carries authored "what's new" notes for the 1.0 -> 1.1 upgrade prompt. + def test_distilled_2_5_has_upgrade_notes(self): from runtime_config.model_download_specs import LTXLocalModelRelevant - relevance = get_ltx_model_spec("ltx-2.3-22b-distilled-1.1").relevance + relevance = get_ltx_model_spec("ltx-2.5-22b-distilled").relevance assert isinstance(relevance, LTXLocalModelRelevant) + assert relevance.upgrade_messages.get("ltx-2.3-22b-distilled-1.1") assert relevance.upgrade_messages.get("ltx-2.3-22b-distilled") class TestActiveModelResolution: - def _write_transformer(self, test_state, model_id: str) -> None: + def _write_generation_bundle(self, test_state, model_id: str) -> None: # resolve_active_ltx_model_id requires the full generation bundle (transformer + - # upscaler), so write both — otherwise the version isn't considered runnable. + # upscaler + split VAEs for 2.5). spec = get_ltx_model_spec(model_id) - for cp in (spec.model_cp, spec.upscale_cp): + for cp in (spec.model_cp, spec.upscale_cp, spec.video_vae_cp, spec.audio_vae_cp): + if cp is None: + continue path = resolve_model_path(test_state.config.default_models_dir, cp) path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(b"\x00" * 1024) def test_prefers_explicit_when_installed(self, test_state): models_dir = test_state.config.default_models_dir - self._write_transformer(test_state, "ltx-2.3-22b-distilled") - self._write_transformer(test_state, "ltx-2.3-22b-distilled-1.1") + self._write_generation_bundle(test_state, "ltx-2.3-22b-distilled") + self._write_generation_bundle(test_state, "ltx-2.3-22b-distilled-1.1") assert resolve_active_ltx_model_id(models_dir, "ltx-2.3-22b-distilled") == "ltx-2.3-22b-distilled" def test_falls_back_to_newest_installed_when_preferred_missing(self, test_state): models_dir = test_state.config.default_models_dir - self._write_transformer(test_state, "ltx-2.3-22b-distilled") + self._write_generation_bundle(test_state, "ltx-2.3-22b-distilled") # preferred 1.1 is NOT on disk -> fall back to the only installed (1.0) assert resolve_active_ltx_model_id(models_dir, "ltx-2.3-22b-distilled-1.1") == "ltx-2.3-22b-distilled" @@ -403,8 +582,8 @@ def test_none_when_nothing_installed(self, test_state): def test_generation_uses_active_setting(self, client, test_state): models_dir = test_state.config.default_models_dir - self._write_transformer(test_state, "ltx-2.3-22b-distilled") - self._write_transformer(test_state, "ltx-2.3-22b-distilled-1.1") + self._write_generation_bundle(test_state, "ltx-2.3-22b-distilled") + self._write_generation_bundle(test_state, "ltx-2.3-22b-distilled-1.1") test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled" resolved = resolve_active_ltx_model_id(models_dir, test_state.state.app_settings.active_ltx_model_id) assert resolved == "ltx-2.3-22b-distilled" @@ -418,20 +597,23 @@ def _write_transformer(self, test_state, model_id: str) -> None: path.write_bytes(b"\x00" * 1024) def test_versions_list_newest_first_with_flags(self, client, test_state, create_fake_model_files): - # create_fake_model_files installs the latest (1.1) bundle + # create_fake_model_files installs the latest (2.5) bundle create_fake_model_files() - test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled-1.1" + test_state.state.app_settings.active_ltx_model_id = "ltx-2.5-22b-distilled" response = client.get("/api/models/ltx-versions") assert response.status_code == 200 versions = response.json()["versions"] assert [v["model_id"] for v in versions] == [ + "ltx-2.5-22b-distilled", "ltx-2.3-22b-distilled-1.1", "ltx-2.3-22b-distilled", ] - newest, older = versions[0], versions[1] - assert newest["label"] == "1.1" + newest = versions[0] + older = versions[2] + assert newest["label"] == "2.5" assert newest["installed"] is True assert newest["active"] is True + assert versions[1]["installed"] is False assert older["installed"] is False assert older["is_newest"] is False assert newest["is_newest"] is True @@ -478,26 +660,29 @@ def test_set_active_succeeds_and_persists(self, client, test_state): class TestDeleteGuard: - def _write_transformer(self, test_state, model_id: str) -> None: - cp = get_ltx_model_spec(model_id).model_cp - path = resolve_model_path(test_state.config.default_models_dir, cp) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(b"\x00" * 1024) + def _write_generation_bundle(self, test_state, model_id: str) -> None: + spec = get_ltx_model_spec(model_id) + for cp in (spec.model_cp, spec.upscale_cp, spec.video_vae_cp, spec.audio_vae_cp): + if cp is None: + continue + path = resolve_model_path(test_state.config.default_models_dir, cp) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"\x00" * 1024) def test_cannot_delete_active_version_transformer(self, client, test_state, create_fake_model_files): - create_fake_model_files() # installs 1.1 bundle - self._write_transformer(test_state, "ltx-2.3-22b-distilled") # 1.0 also present - test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled-1.1" + create_fake_model_files() # installs latest (2.5) bundle + self._write_generation_bundle(test_state, "ltx-2.3-22b-distilled") # 1.0 also present + test_state.state.app_settings.active_ltx_model_id = "ltx-2.5-22b-distilled" response = client.request( - "DELETE", "/api/models/delete", json={"cp_ids": ["ltx-2.3-22b-distilled-1.1"]} + "DELETE", "/api/models/delete", json={"cp_ids": ["ltx-2.5-22b-distilled"]} ) assert response.status_code == 409 assert response.json()["code"] == "DELETE_PROTECTED_CHECKPOINT" def test_can_delete_non_active_version_transformer(self, client, test_state, create_fake_model_files): - create_fake_model_files() # 1.1 bundle, active by default resolution - self._write_transformer(test_state, "ltx-2.3-22b-distilled") # 1.0 present, not active - test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled-1.1" + create_fake_model_files() # 2.5 bundle, active by default resolution + self._write_generation_bundle(test_state, "ltx-2.3-22b-distilled") # 1.0 present, not active + test_state.state.app_settings.active_ltx_model_id = "ltx-2.5-22b-distilled" response = client.request( "DELETE", "/api/models/delete", json={"cp_ids": ["ltx-2.3-22b-distilled"]} ) @@ -505,10 +690,9 @@ def test_can_delete_non_active_version_transformer(self, client, test_state, cre def test_active_older_version_protected_newer_deletable(self, client, test_state, create_fake_model_files): # Both versions installed; active is the OLDER 1.0. - # Old impl (protected = newest installed = 1.1) would ALLOW deleting 1.0 and BLOCK deleting 1.1. - # New impl (protected = active = 1.0) must BLOCK deleting 1.0 and ALLOW deleting 1.1. - create_fake_model_files() # installs 1.1 bundle - self._write_transformer(test_state, "ltx-2.3-22b-distilled") # 1.0 also present + # Protected = active (1.0); newer non-active (2.5) must be deletable. + create_fake_model_files() # installs 2.5 bundle + self._write_generation_bundle(test_state, "ltx-2.3-22b-distilled") # 1.0 full bundle test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled" # active = 1.0 # Active (1.0) must be protected @@ -518,8 +702,116 @@ def test_active_older_version_protected_newer_deletable(self, client, test_state assert response_protected.status_code == 409 assert response_protected.json()["code"] == "DELETE_PROTECTED_CHECKPOINT" - # Non-active newer (1.1) must be deletable + # Non-active newer (2.5) must be deletable response_allowed = client.request( - "DELETE", "/api/models/delete", json={"cp_ids": ["ltx-2.3-22b-distilled-1.1"]} + "DELETE", "/api/models/delete", json={"cp_ids": ["ltx-2.5-22b-distilled"]} ) assert response_allowed.status_code == 200 + + +class TestActiveModelResolution: + def test_text_encoder_follows_active_not_newest_on_disk( + self, client, test_state, create_fake_model_files + ): + # Both bundles installed; newest-on-disk would pick 2.5/Gemma4, but active is 2.3/Gemma3. + create_fake_model_files() + create_fake_model_files(model_id="ltx-2.3-22b-distilled-1.1") + test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled-1.1" + + response = client.get("/api/models/text-encoder-recommendation") + assert response.status_code == 200 + assert response.json()["cp_to_download"] is None + assert ( + test_state.text.resolve_prompt_enhancer_root_if_downloaded() + == str( + resolve_model_path( + test_state.config.default_models_dir, + "gemma-3-12b-it-qat-q4_0-unquantized", + ) + ) + ) + + def test_ic_lora_follows_active_when_both_installed( + self, client, test_state, create_fake_model_files, create_fake_ic_lora_files + ): + create_fake_model_files() + create_fake_model_files(model_id="ltx-2.3-22b-distilled-1.1") + create_fake_ic_lora_files() + test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled-1.1" + + response = client.get("/api/models/ltx-ic-lora-recommendation") + assert response.status_code == 200 + assert response.json()["supported"] is True + + def test_models_specs_display_follows_active( + self, client, test_state, create_fake_model_files + ): + create_fake_model_files() + create_fake_model_files(model_id="ltx-2.3-22b-distilled-1.1") + test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled-1.1" + + response = client.get("/api/generate/models-specs") + assert response.status_code == 200 + assert response.json()["local_models"][0]["spec"]["display_name"] == "LTX 2.3 Fast" + + +class TestGatedCheckpointAccess: + def test_gated_download_rejected_when_signed_out(self, client, test_state): + test_state.state.hf_auth_state = HfNotAuthenticated() + response = client.post( + "/api/models/download", + json={"type": "download", "cp_ids": ["ltx-2.5-22b-distilled"]}, + ) + assert response.status_code == 403 + assert test_state.state.downloading_session is None + + def test_gated_download_starts_when_signed_in(self, client, test_state): + response = client.post( + "/api/models/download", + json={"type": "download", "cp_ids": ["ltx-2.5-22b-distilled"]}, + ) + assert response.status_code == 200 + assert _cp_path(test_state, "ltx-2.5-22b-distilled").exists() + + def test_nested_hf_path_flattens_to_local_basename(self, client, test_state): + # FakeModelDownloader writes to local_dir/; the staging step must + # flatten that to relative_path's basename before commit, or pipelines look for the + # wrong file. + spec = get_model_cp_spec("ltx-2.5-22b-distilled") + assert "/" in spec.download_filename + + response = client.post( + "/api/models/download", + json={"type": "download", "cp_ids": ["ltx-2.5-22b-distilled"]}, + ) + assert response.status_code == 200 + + committed = _cp_path(test_state, "ltx-2.5-22b-distilled") + assert committed.exists() + assert committed.parent.name == "ltx-2.5" + assert committed.name == spec.relative_path.name + nested_leftover = resolve_downloading_dir(test_state.config.default_models_dir) / Path( + spec.download_filename + ) + assert not nested_leftover.exists() + assert not (test_state.config.default_models_dir / Path(spec.download_filename)).exists() + + def test_public_download_still_allowed_when_signed_out(self, client, test_state): + test_state.state.hf_auth_state = HfNotAuthenticated() + response = client.post( + "/api/models/download", + json={"type": "download", "cp_ids": ["ltx-2.3-22b-distilled"]}, + ) + assert response.status_code == 200 + + def test_check_access_flags_gated_repo_when_signed_out(self, client, test_state): + test_state.state.hf_auth_state = HfNotAuthenticated() + response = client.post( + "/api/models/check-access", + json={"cp_ids": ["ltx-2.5-22b-distilled", "ltx-2.3-22b-distilled"]}, + ) + assert response.status_code == 200 + assert response.json()["access"] == { + "Lightricks/LTX-2.5": "not_authorized", + "Lightricks/LTX-2.3": "authorized", + } diff --git a/backend/tests/test_natten_libnatten_gate.py b/backend/tests/test_natten_libnatten_gate.py new file mode 100644 index 000000000..dbe4136fc --- /dev/null +++ b/backend/tests/test_natten_libnatten_gate.py @@ -0,0 +1,64 @@ +"""Flex-only natten must not count as available for cutlass-fna.""" + +from __future__ import annotations + +import sys +from types import SimpleNamespace + +from services.patches.natten_libnatten_gate import _gate_natten_available, _has_libnatten + + +def test_has_libnatten_is_false_when_import_fails(monkeypatch) -> None: + monkeypatch.setitem(sys.modules, "natten", None) + assert _has_libnatten() is False + + +def test_has_libnatten_is_false_for_flex_only_module(monkeypatch) -> None: + monkeypatch.setitem(sys.modules, "natten", SimpleNamespace()) + assert _has_libnatten() is False + + +def test_has_libnatten_is_false_when_flag_is_false(monkeypatch) -> None: + monkeypatch.setitem(sys.modules, "natten", SimpleNamespace(HAS_LIBNATTEN=False)) + assert _has_libnatten() is False + + +def test_has_libnatten_is_true_when_flag_set(monkeypatch) -> None: + monkeypatch.setitem(sys.modules, "natten", SimpleNamespace(HAS_LIBNATTEN=True)) + assert _has_libnatten() is True + + +def test_gate_clears_available_when_libnatten_missing() -> None: + mod = SimpleNamespace(_NATTEN_AVAILABLE=True) + _gate_natten_available(mod, has_libnatten=False) + assert mod._NATTEN_AVAILABLE is False + + +def test_gate_keeps_available_when_libnatten_present() -> None: + mod = SimpleNamespace(_NATTEN_AVAILABLE=True) + _gate_natten_available(mod, has_libnatten=True) + assert mod._NATTEN_AVAILABLE is True + + +def test_gate_leaves_missing_natten_alone() -> None: + mod = SimpleNamespace(_NATTEN_AVAILABLE=False) + _gate_natten_available(mod, has_libnatten=False) + assert mod._NATTEN_AVAILABLE is False + + +def test_gate_clears_available_when_libnatten_missing() -> None: + mod = SimpleNamespace(_NATTEN_AVAILABLE=True) + _gate_natten_available(mod, has_libnatten=False) + assert mod._NATTEN_AVAILABLE is False + + +def test_gate_keeps_available_when_libnatten_present() -> None: + mod = SimpleNamespace(_NATTEN_AVAILABLE=True) + _gate_natten_available(mod, has_libnatten=True) + assert mod._NATTEN_AVAILABLE is True + + +def test_gate_leaves_missing_natten_alone() -> None: + mod = SimpleNamespace(_NATTEN_AVAILABLE=False) + _gate_natten_available(mod, has_libnatten=False) + assert mod._NATTEN_AVAILABLE is False diff --git a/backend/tests/test_prompt_enhancement.py b/backend/tests/test_prompt_enhancement.py index baf991b25..93a330212 100644 --- a/backend/tests/test_prompt_enhancement.py +++ b/backend/tests/test_prompt_enhancement.py @@ -10,10 +10,18 @@ PromptTemplatePlaceholder, PromptTemplateSpec, ) +from runtime_config.model_download_specs import resolve_model_path from tests.fakes import FakeResponse from tests.http_error_assertions import assert_http_error +# 2.3's gemma3 encoder doubles as the local enhancer, so its plain bundle is enough to make the +# local provider usable (2.5 needs an extra opt-in download — see +# LTXLocalModelSpec.prompt_enhancer_cp). Everything below tests plumbing that's independent of +# the model generation, so pin it to that. +_LOCAL_ENHANCER_MODEL_ID = "ltx-2.3-22b-distilled-1.1" + + def _gemini_ok(text: str = "enhanced via gemini") -> FakeResponse: return FakeResponse( status_code=200, @@ -57,7 +65,7 @@ def _add_ic_lora(fake_services, **overrides: object) -> IcLoraCatalogItem: class TestNoSelection: def test_generic_fallback_uses_no_system_prompt(self, client, fake_services, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) r = client.post("/api/enhance-prompt", json={"prompt": "a cat"}) assert r.status_code == 200 assert r.json()["enhancedPrompt"] == fake_services.prompt_enhancer_pipeline.enhanced_prompt @@ -68,7 +76,7 @@ def test_generic_fallback_uses_no_system_prompt(self, client, fake_services, cre def test_image_path_routes_to_enhance_i2v( self, client, fake_services, create_fake_model_files, make_test_image, tmp_path ): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) image_path = tmp_path / "cat.png" image_path.write_bytes(make_test_image().getvalue()) @@ -79,7 +87,7 @@ def test_image_path_routes_to_enhance_i2v( assert len(fake_services.prompt_enhancer_pipeline.enhance_t2v_calls) == 0 def test_invalid_image_path_rejected(self, client, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) r = client.post("/api/enhance-prompt", json={"prompt": "a cat", "imagePath": "/tmp/does-not-exist.png"}) assert r.status_code == 400 @@ -87,7 +95,7 @@ def test_seed_is_independent_of_dev_mode_lock(self, client, test_state, fake_ser # Regression: enhance used StateHandlerBase._resolve_seed(), which returns a fixed # constant (1000) whenever dev mode is on — every call, including a "redo", would then # produce the exact same output. Two consecutive calls must not collapse to that. - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) test_state.config.dev_mode = True client.post("/api/enhance-prompt", json={"prompt": "a cat"}) client.post("/api/enhance-prompt", json={"prompt": "a cat"}) @@ -99,7 +107,7 @@ def test_seed_is_independent_of_dev_mode_lock(self, client, test_state, fake_ser class TestLoraSelection: def test_single_lora_system_prompt_and_trigger_enforced(self, client, fake_services, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) _add_lora( fake_services, id="cozy-felt", name="Cozy Felt", trigger="F3ltCut0u7", trigger_placement="anywhere", instructions=[InstructionSection(kind="summary", title="What it does", body="Felt look.")], @@ -115,7 +123,7 @@ def test_single_lora_system_prompt_and_trigger_enforced(self, client, fake_servi assert "Felt look." in call["system_prompt"] def test_multi_lora_prompt_includes_both(self, client, fake_services, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) _add_lora(fake_services, id="a", name="Alpha") _add_lora(fake_services, id="b", name="Beta") @@ -125,7 +133,7 @@ def test_multi_lora_prompt_includes_both(self, client, fake_services, create_fak assert "Alpha" in call["system_prompt"] and "Beta" in call["system_prompt"] def test_unknown_lora_id_rejected(self, client, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) r = client.post("/api/enhance-prompt", json={"prompt": "x", "loraCatalogIds": ["does-not-exist"]}) assert_http_error(r, status_code=404, code="LORA_CATALOG_ID_NOT_FOUND") @@ -134,7 +142,7 @@ class TestIcLoraSelection: def test_ic_lora_without_template_free_rewrite_and_trigger_enforced( self, client, fake_services, create_fake_model_files ): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) _add_ic_lora( fake_services, id="day-to-night", name="Day to Night", instructions=[InstructionSection(kind="summary", title="What it does", body="Relights to night.")], @@ -145,12 +153,12 @@ def test_ic_lora_without_template_free_rewrite_and_trigger_enforced( assert "Day to Night" in call["system_prompt"] def test_unknown_ic_lora_id_rejected(self, client, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) r = client.post("/api/enhance-prompt", json={"prompt": "x", "icLoraId": "does-not-exist"}) assert_http_error(r, status_code=404, code="LORA_CATALOG_ID_NOT_FOUND") def test_free_text_template_fill_stitches_deterministically(self, client, fake_services, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) _add_ic_lora( fake_services, id="colorization", name="Colorization", prompt_template=PromptTemplateSpec( @@ -167,7 +175,7 @@ def test_free_text_template_fill_stitches_deterministically(self, client, fake_s assert r.json()["enhancedPrompt"] == "Reference shows a grey rabbit. COLORIZE a brown rabbit." def test_enum_template_fill_stitches_deterministically(self, client, fake_services, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) _add_ic_lora( fake_services, id="crossview-prompt", name="CrossView", prompt_template=PromptTemplateSpec( @@ -188,7 +196,7 @@ def test_enum_template_fill_stitches_deterministically(self, client, fake_servic assert r.json()["enhancedPrompt"] == "crossview. new camera angle: to the right, lower, closer." def test_template_fill_rejects_invalid_enum_choice(self, client, fake_services, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) _add_ic_lora( fake_services, id="crossview-prompt", name="CrossView", prompt_template=PromptTemplateSpec( @@ -202,7 +210,7 @@ def test_template_fill_rejects_invalid_enum_choice(self, client, fake_services, assert r.status_code == 500 def test_template_fill_rejects_non_json_response(self, client, fake_services, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) _add_ic_lora( fake_services, id="upscale", name="Upscale", prompt_template=PromptTemplateSpec(template="upscale", placeholders={}), @@ -216,7 +224,7 @@ def test_template_fill_rejects_non_json_response(self, client, fake_services, cr class TestConditioningType: # canny/depth: the built-in "bring your own IC-LoRA" conditioning modes, no catalog entry. def test_depth_uses_dedicated_system_prompt(self, client, fake_services, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) r = client.post( "/api/enhance-prompt", json={"prompt": "a fox riding a skateboard", "conditioningType": "depth"}, @@ -228,7 +236,7 @@ def test_depth_uses_dedicated_system_prompt(self, client, fake_services, create_ assert "faithfully" in call["system_prompt"].lower() def test_canny_uses_dedicated_system_prompt(self, client, fake_services, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) r = client.post( "/api/enhance-prompt", json={"prompt": "a fox riding a skateboard", "conditioningType": "canny"}, @@ -241,7 +249,7 @@ def test_canny_uses_dedicated_system_prompt(self, client, fake_services, create_ class TestRequestValidation: def test_lora_ids_and_ic_lora_id_mutually_exclusive(self, client, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) r = client.post( "/api/enhance-prompt", json={"prompt": "x", "loraCatalogIds": ["a"], "icLoraId": "b"}, @@ -249,7 +257,7 @@ def test_lora_ids_and_ic_lora_id_mutually_exclusive(self, client, create_fake_mo assert r.status_code == 422 def test_conditioning_type_and_ic_lora_id_mutually_exclusive(self, client, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) r = client.post( "/api/enhance-prompt", json={"prompt": "x", "conditioningType": "depth", "icLoraId": "b"}, @@ -257,6 +265,107 @@ def test_conditioning_type_and_ic_lora_id_mutually_exclusive(self, client, creat assert r.status_code == 422 +class TestAudioVisualModels: + def test_generic_fallback_uses_the_audio_visual_caption_prompt( + self, client, test_state, create_fake_model_files + ): + # 2.5 was captioned as audio-visual: without its own caption instructions the enhancer + # writes a visual-only prompt, the soundscape is left unspecified, and the model fills it + # in — usually by having someone speak the prompt. + create_fake_model_files() + test_state.state.app_settings.gemini_api_key = "gemini-key" + test_state.http.queue("post", _gemini_ok()) + + r = client.post("/api/enhance-prompt", json={"prompt": "a puffin running", "provider": "api"}) + assert r.status_code == 200 + system_instruction = test_state.http.calls[-1].json_payload["systemInstruction"]["parts"][0]["text"] + assert "dialogue" in system_instruction.lower() + assert "soundscape" in system_instruction.lower() + + def test_2_3_keeps_the_provider_default(self, client, fake_services, create_fake_model_files): + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) + r = client.post("/api/enhance-prompt", json={"prompt": "a cat"}) + assert r.status_code == 200 + assert fake_services.prompt_enhancer_pipeline.enhance_t2v_calls[0]["system_prompt"] is None + + def test_local_provider_rejected_without_the_separate_enhancer(self, client, create_fake_model_files): + # 2.5's downloaded text encoder is encode-only, so a full generation bundle is not on its + # own enough to enhance locally. + create_fake_model_files() + assert_http_error( + client.post("/api/enhance-prompt", json={"prompt": "a cat", "provider": "local"}), + status_code=409, + code="LOCAL_TEXT_ENCODER_NOT_AVAILABLE", + ) + + def test_local_provider_runs_on_the_downloaded_enhancer( + self, client, test_state, fake_services, create_fake_model_files + ): + create_fake_model_files(include_prompt_enhancer=True) + + r = client.post("/api/enhance-prompt", json={"prompt": "a cat", "provider": "local"}) + assert r.status_code == 200 + # The enhancer root, not the encoder that generation uses. + assert fake_services.prompt_enhancer_pipeline.created_with[-1]["gemma_root"] == str( + resolve_model_path(test_state.config.default_models_dir, "gemma-4-e2b-it") + ) + + def test_recommendation_tracks_the_enhancer_download(self, client, create_fake_model_files): + create_fake_model_files() + before = client.get("/api/models/text-encoder-recommendation").json() + assert before["local_enhancement_supported"] is False + assert before["local_enhancer_cp"] == "gemma-4-e2b-it" + assert before["active_local_enhancer_cp"] is None + assert before["local_enhancer_expected_size_gb"] == 9.6 + + create_fake_model_files(include_prompt_enhancer=True) + after = client.get("/api/models/text-encoder-recommendation").json() + assert after["local_enhancement_supported"] is True + assert after["active_local_enhancer_cp"] == "gemma-4-e2b-it" + + def test_2_3_needs_no_separate_enhancer(self, client, create_fake_model_files): + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) + payload = client.get("/api/models/text-encoder-recommendation").json() + assert payload["local_enhancer_cp"] is None + assert payload["local_enhancement_supported"] is True + assert payload["active_local_enhancer_cp"] == "gemma-3-12b-it-qat-q4_0-unquantized" + + + def test_2_5_falls_back_to_gemma3_without_e2b( + self, client, test_state, fake_services, create_fake_model_files + ): + create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) + test_state.state.app_settings.active_ltx_model_id = "ltx-2.5-22b-distilled" + + rec = client.get("/api/models/text-encoder-recommendation").json() + assert rec["local_enhancement_supported"] is True + assert rec["local_enhancer_cp"] == "gemma-4-e2b-it" + assert rec["active_local_enhancer_cp"] == "gemma-3-12b-it-qat-q4_0-unquantized" + + r = client.post("/api/enhance-prompt", json={"prompt": "a cat", "provider": "local"}) + assert r.status_code == 200 + assert fake_services.prompt_enhancer_pipeline.created_with[-1]["gemma_root"] == str( + resolve_model_path(test_state.config.default_models_dir, "gemma-3-12b-it-qat-q4_0-unquantized") + ) + + def test_2_5_prefers_e2b_when_gemma3_is_also_present( + self, client, test_state, fake_services, create_fake_model_files + ): + create_fake_model_files(include_prompt_enhancer=True) + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) + test_state.state.app_settings.active_ltx_model_id = "ltx-2.5-22b-distilled" + + rec = client.get("/api/models/text-encoder-recommendation").json() + assert rec["active_local_enhancer_cp"] == "gemma-4-e2b-it" + + r = client.post("/api/enhance-prompt", json={"prompt": "a cat", "provider": "local"}) + assert r.status_code == 200 + assert fake_services.prompt_enhancer_pipeline.created_with[-1]["gemma_root"] == str( + resolve_model_path(test_state.config.default_models_dir, "gemma-4-e2b-it") + ) + + class TestGating: def test_local_enhance_works_even_when_generation_prefers_api_text_encoding( self, client, test_state, create_fake_model_files @@ -267,7 +376,9 @@ def test_local_enhance_works_even_when_generation_prefers_api_text_encoding( # API key and the checkpoint downloaded (the tiebreaker defaults to API) could previously # never use "Local" Enhance, even though the frontend's own checkpoint-presence check # offered it. - create_fake_model_files() + # 2.3, since the tiebreaker only exists for versions the LTX API can encode. + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) + test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled-1.1" test_state.state.app_settings.ltx_api_key = "ltx-key" assert test_state.text.should_use_local_encoding() is False # tiebreaker picks API @@ -280,7 +391,7 @@ def test_missing_local_gemma_rejected(self, client): assert_http_error(r, status_code=409, code="LOCAL_TEXT_ENCODER_NOT_AVAILABLE") def test_rejected_while_generation_running(self, client, test_state, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) test_state.pipelines.load_gpu_pipeline("fast") test_state.generation.start_generation("gen-1") @@ -288,7 +399,7 @@ def test_rejected_while_generation_running(self, client, test_state, create_fake assert r.status_code == 409 def test_enhance_failure_returns_500(self, client, fake_services, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) fake_services.prompt_enhancer_pipeline.raise_on_enhance = RuntimeError("boom") r = client.post("/api/enhance-prompt", json={"prompt": "x"}) assert r.status_code == 500 @@ -297,14 +408,14 @@ def test_enhance_marks_itself_busy_then_frees_the_slot(self, client, test_state, # Regression: enhance() must participate in the same generation-mutex bookkeeping every # other handler uses, so an orphaned enhance can't race a Generate click — and must # release it again on success so a later call isn't blocked forever. - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) assert test_state.generation.is_generation_running() is False r = client.post("/api/enhance-prompt", json={"prompt": "x"}) assert r.status_code == 200 assert test_state.generation.is_generation_running() is False def test_enhance_frees_the_slot_after_failure(self, client, fake_services, test_state, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) fake_services.prompt_enhancer_pipeline.raise_on_enhance = RuntimeError("boom") client.post("/api/enhance-prompt", json={"prompt": "x"}) assert test_state.generation.is_generation_running() is False @@ -315,7 +426,7 @@ def test_enhance_frees_the_slot_after_failure(self, client, fake_services, test_ class TestApiProvider: - # provider="api" never touches the local Gemma pipeline — no create_fake_model_files() call + # provider="api" never touches the local Gemma pipeline — no create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) call # anywhere in this class, proving the local-checkpoint requirement is fully bypassed. def test_free_rewrite_calls_gemini_without_local_gemma(self, client, test_state): test_state.state.app_settings.gemini_api_key = "key" @@ -452,7 +563,7 @@ class TestImageMediaType: def test_generation_uses_image_generation_system_prompt( self, client, fake_services, create_fake_model_files ): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) r = client.post("/api/enhance-prompt", json={"prompt": "a red car", "mediaType": "image"}) assert r.status_code == 200 call = fake_services.prompt_enhancer_pipeline.enhance_t2v_calls[0] @@ -462,7 +573,7 @@ def test_generation_uses_image_generation_system_prompt( def test_editing_uses_image_edit_system_prompt_and_routes_to_i2v( self, client, fake_services, create_fake_model_files, make_test_image, tmp_path ): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) image_path = tmp_path / "src.png" image_path.write_bytes(make_test_image().getvalue()) @@ -477,7 +588,7 @@ def test_editing_uses_image_edit_system_prompt_and_routes_to_i2v( assert "DESIRED RESULT" in call["system_prompt"] def test_image_media_type_rejects_lora_selection(self, client, create_fake_model_files): - create_fake_model_files() + create_fake_model_files(model_id=_LOCAL_ENHANCER_MODEL_ID) r = client.post( "/api/enhance-prompt", json={"prompt": "x", "mediaType": "image", "loraCatalogIds": ["a"]}, diff --git a/backend/tests/test_response_models.py b/backend/tests/test_response_models.py index 16230c3f3..f2250a8a2 100644 --- a/backend/tests/test_response_models.py +++ b/backend/tests/test_response_models.py @@ -13,6 +13,7 @@ def test_camelcase_keys(self, client, test_state): active_pipeline=VideoPipelineState( pipeline=pipeline, is_compiled=False, + ltx_model_id="ltx-2.5-22b-distilled", ), ) test_state.generation.start_generation("gen-1") diff --git a/backend/tests/test_settings.py b/backend/tests/test_settings.py index 6df0486fe..2d2d313a8 100644 --- a/backend/tests/test_settings.py +++ b/backend/tests/test_settings.py @@ -5,7 +5,7 @@ import json from pathlib import Path -from state.app_settings import AppSettings, UpdateSettingsRequest +from state.app_settings import AppSettings, UpdateSettingsRequest, resolved_use_conv_vae from state import build_initial_state from app_handler import ServiceBundle from tests.conftest import TEST_ADMIN_TOKEN @@ -28,6 +28,7 @@ def test_default_settings(self, client, default_app_settings, test_state): assert data["hasGeminiApiKey"] is False assert data["seedLocked"] is False assert data["lockedSeed"] == 42 + assert data["useConvVae"] is resolved_use_conv_vae(AppSettings()) # When no custom path is set, the response surfaces the runtime default # so the first-run UI can show the install location. assert data["modelsDir"] == str(test_state.config.default_models_dir) @@ -124,6 +125,44 @@ def test_unknown_field_rejected(self, client): r = client.post("/api/settings", json={"unknownSetting": True}) assert r.status_code == 422 + def test_use_conv_vae_round_trip(self, client, test_state): + r = client.post("/api/settings", json={"useConvVae": True}) + assert r.status_code == 200 + assert test_state.state.app_settings.use_conv_vae is True + assert client.get("/api/settings").json()["useConvVae"] is True + + r = client.post("/api/settings", json={"useConvVae": False}) + assert r.status_code == 200 + assert test_state.state.app_settings.use_conv_vae is False + assert client.get("/api/settings").json()["useConvVae"] is False + + def test_use_conv_vae_change_unloads_gpu_pipeline( + self, client, test_state, fake_services, create_fake_model_files + ): + create_fake_model_files() + test_state.state.app_settings.use_conv_vae = False + test_state.pipelines.load_gpu_pipeline("fast") + assert test_state.state.gpu_slot is not None + cleanup_before = fake_services.gpu_cleaner.cleanup_calls + + r = client.post("/api/settings", json={"useConvVae": True}) + assert r.status_code == 200 + assert test_state.state.gpu_slot is None + assert fake_services.gpu_cleaner.cleanup_calls > cleanup_before + + def test_use_conv_vae_unchanged_does_not_unload_gpu_pipeline( + self, client, test_state, fake_services, create_fake_model_files + ): + create_fake_model_files() + test_state.state.app_settings.use_conv_vae = True + test_state.pipelines.load_gpu_pipeline("fast") + cleanup_before = fake_services.gpu_cleaner.cleanup_calls + + r = client.post("/api/settings", json={"useConvVae": True}) + assert r.status_code == 200 + assert test_state.state.gpu_slot is not None + assert fake_services.gpu_cleaner.cleanup_calls == cleanup_before + class TestModelsDirAdminGuard: def test_models_dir_requires_admin_token(self, client, test_state): @@ -260,3 +299,25 @@ def test_user_prefers_api_video_generations_persists(self, client, test_state, d class TestSettingsSchemaDrift: def test_update_request_tracks_app_settings_fields(self): assert set(AppSettings.model_fields) == set(UpdateSettingsRequest.model_fields) + + +class TestResolvedUseConvVae: + def test_none_defaults_on_for_darwin(self, monkeypatch): + monkeypatch.setattr("state.app_settings.sys.platform", "darwin") + assert resolved_use_conv_vae(AppSettings()) is True + + def test_none_defaults_off_for_linux(self, monkeypatch): + monkeypatch.setattr("state.app_settings.sys.platform", "linux") + assert resolved_use_conv_vae(AppSettings()) is False + + def test_none_defaults_off_for_windows(self, monkeypatch): + monkeypatch.setattr("state.app_settings.sys.platform", "win32") + assert resolved_use_conv_vae(AppSettings()) is False + + def test_explicit_true_overrides_linux_default(self, monkeypatch): + monkeypatch.setattr("state.app_settings.sys.platform", "linux") + assert resolved_use_conv_vae(AppSettings(use_conv_vae=True)) is True + + def test_explicit_false_overrides_darwin_default(self, monkeypatch): + monkeypatch.setattr("state.app_settings.sys.platform", "darwin") + assert resolved_use_conv_vae(AppSettings(use_conv_vae=False)) is False diff --git a/backend/tests/test_state_actions.py b/backend/tests/test_state_actions.py index 39af9ce7a..37559456d 100644 --- a/backend/tests/test_state_actions.py +++ b/backend/tests/test_state_actions.py @@ -11,16 +11,16 @@ from handlers.generation_handler import _RESERVATION_TIMEOUT_S from runtime_config.model_download_specs import ( DEPTH_PROCESSOR_CP_ID, - get_latest_ltx_model_id, get_ltx_model_spec, resolve_model_path, ) from state.app_settings import UpdateSettingsRequest from state.app_state_types import CpuSlot, GpuSlot, ICLoraState, RetakePipelineState, VideoPipelineState +from tests.conftest import _IC_LORA_MODEL_ID -def _current_model_spec(): - return get_ltx_model_spec(get_latest_ltx_model_id()) +def _ic_lora_model_spec(): + return get_ltx_model_spec(_IC_LORA_MODEL_ID) def test_start_generation_requires_gpu(test_state): @@ -167,9 +167,10 @@ def test_retake_pipeline_eviction(test_state, create_fake_model_files): def test_ic_lora_load_includes_depth_resources(test_state, fake_services, create_fake_model_files, create_fake_ic_lora_files): - create_fake_model_files() + create_fake_model_files(model_id=_IC_LORA_MODEL_ID) create_fake_ic_lora_files() - model_spec = _current_model_spec() + model_spec = _ic_lora_model_spec() + assert model_spec.ic_loras_spec is not None lora_path = str(resolve_model_path(test_state.config.default_models_dir, model_spec.ic_loras_spec.canny_cp)) depth_path = str(resolve_model_path(test_state.config.default_models_dir, DEPTH_PROCESSOR_CP_ID)) @@ -183,9 +184,10 @@ def test_ic_lora_load_includes_depth_resources(test_state, fake_services, create def test_ic_lora_unload_clears_preprocessing_resources(test_state, create_fake_model_files, create_fake_ic_lora_files): - create_fake_model_files() + create_fake_model_files(model_id=_IC_LORA_MODEL_ID) create_fake_ic_lora_files() - model_spec = _current_model_spec() + model_spec = _ic_lora_model_spec() + assert model_spec.ic_loras_spec is not None lora_path = str(resolve_model_path(test_state.config.default_models_dir, model_spec.ic_loras_spec.canny_cp)) depth_path = str(resolve_model_path(test_state.config.default_models_dir, DEPTH_PROCESSOR_CP_ID)) test_state.pipelines.load_ic_lora(lora_path, depth_path) @@ -196,3 +198,23 @@ def test_ic_lora_unload_clears_preprocessing_resources(test_state, create_fake_m test_state.pipelines.unload_gpu_pipeline() assert test_state.state.gpu_slot is None + + +def test_pipeline_cache_rebuilds_when_active_model_changes(test_state, create_fake_model_files): + # Two API-encodable versions both resolve gemma_root=None, so without an ltx_model_id cache + # key the resident pipeline would keep serving the previous weights. + create_fake_model_files(model_id="ltx-2.3-22b-distilled") + create_fake_model_files(model_id="ltx-2.3-22b-distilled-1.1") + test_state.state.app_settings.ltx_api_key = "test-key" + test_state.state.app_settings.use_local_text_encoder = False + test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled-1.1" + + first = test_state.pipelines.load_gpu_pipeline("fast") + assert first.ltx_model_id == "ltx-2.3-22b-distilled-1.1" + assert first.gemma_root is None + + test_state.state.app_settings.active_ltx_model_id = "ltx-2.3-22b-distilled" + second = test_state.pipelines.load_gpu_pipeline("fast") + assert second.ltx_model_id == "ltx-2.3-22b-distilled" + assert second.gemma_root is None + assert second is not first diff --git a/backend/uv.lock b/backend/uv.lock index 850c08a80..a2e40c719 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -8,6 +8,18 @@ resolution-markers = [ "sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", ] +[manifest] + +[[manifest.dependency-metadata]] +name = "ltx-core" +version = "1.2.0" +requires-dist = ["torch~=2.7", "torchaudio", "einops", "numpy", "av", "transformers>=5.8.0,<5.15", "safetensors", "accelerate", "scipy>=1.14", "mps-sdpa>=0.2.0 ; platform_machine == 'arm64' and sys_platform == 'darwin'"] + +[[manifest.dependency-metadata]] +name = "ltx-pipelines" +version = "1.2.0" +requires-dist = ["ltx-core", "av", "tqdm", "pillow", "openimageio", "cloudpickle>=3.1"] + [[package]] name = "accelerate" version = "1.12.0" @@ -170,14 +182,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.1" +version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] [[package]] @@ -329,31 +341,26 @@ wheels = [ [[package]] name = "hf-xet" -version = "1.2.0" +version = "1.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5e/6e/0f11bacf08a67f7fb5ee09740f2ca54163863b07b70d579356e9222ce5d8/hf_xet-1.2.0.tar.gz", hash = "sha256:a8c27070ca547293b6890c4bf389f713f80e8c478631432962bb7f4bc0bd7d7f", size = 506020, upload-time = "2025-10-24T19:04:32.129Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/a5/85ef910a0aa034a2abcfadc360ab5ac6f6bc4e9112349bd40ca97551cff0/hf_xet-1.2.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:ceeefcd1b7aed4956ae8499e2199607765fbd1c60510752003b6cc0b8413b649", size = 2861870, upload-time = "2025-10-24T19:04:11.422Z" }, - { url = "https://files.pythonhosted.org/packages/ea/40/e2e0a7eb9a51fe8828ba2d47fe22a7e74914ea8a0db68a18c3aa7449c767/hf_xet-1.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b70218dd548e9840224df5638fdc94bd033552963cfa97f9170829381179c813", size = 2717584, upload-time = "2025-10-24T19:04:09.586Z" }, - { url = "https://files.pythonhosted.org/packages/a5/7d/daf7f8bc4594fdd59a8a596f9e3886133fdc68e675292218a5e4c1b7e834/hf_xet-1.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d40b18769bb9a8bc82a9ede575ce1a44c75eb80e7375a01d76259089529b5dc", size = 3315004, upload-time = "2025-10-24T19:04:00.314Z" }, - { url = "https://files.pythonhosted.org/packages/b1/ba/45ea2f605fbf6d81c8b21e4d970b168b18a53515923010c312c06cd83164/hf_xet-1.2.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd3a6027d59cfb60177c12d6424e31f4b5ff13d8e3a1247b3a584bf8977e6df5", size = 3222636, upload-time = "2025-10-24T19:03:58.111Z" }, - { url = "https://files.pythonhosted.org/packages/4a/1d/04513e3cab8f29ab8c109d309ddd21a2705afab9d52f2ba1151e0c14f086/hf_xet-1.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6de1fc44f58f6dd937956c8d304d8c2dea264c80680bcfa61ca4a15e7b76780f", size = 3408448, upload-time = "2025-10-24T19:04:20.951Z" }, - { url = "https://files.pythonhosted.org/packages/f0/7c/60a2756d7feec7387db3a1176c632357632fbe7849fce576c5559d4520c7/hf_xet-1.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f182f264ed2acd566c514e45da9f2119110e48a87a327ca271027904c70c5832", size = 3503401, upload-time = "2025-10-24T19:04:22.549Z" }, - { url = "https://files.pythonhosted.org/packages/4e/64/48fffbd67fb418ab07451e4ce641a70de1c40c10a13e25325e24858ebe5a/hf_xet-1.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:293a7a3787e5c95d7be1857358a9130694a9c6021de3f27fa233f37267174382", size = 2900866, upload-time = "2025-10-24T19:04:33.461Z" }, - { url = "https://files.pythonhosted.org/packages/e2/51/f7e2caae42f80af886db414d4e9885fac959330509089f97cccb339c6b87/hf_xet-1.2.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:10bfab528b968c70e062607f663e21e34e2bba349e8038db546646875495179e", size = 2861861, upload-time = "2025-10-24T19:04:19.01Z" }, - { url = "https://files.pythonhosted.org/packages/6e/1d/a641a88b69994f9371bd347f1dd35e5d1e2e2460a2e350c8d5165fc62005/hf_xet-1.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2a212e842647b02eb6a911187dc878e79c4aa0aa397e88dd3b26761676e8c1f8", size = 2717699, upload-time = "2025-10-24T19:04:17.306Z" }, - { url = "https://files.pythonhosted.org/packages/df/e0/e5e9bba7d15f0318955f7ec3f4af13f92e773fbb368c0b8008a5acbcb12f/hf_xet-1.2.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30e06daccb3a7d4c065f34fc26c14c74f4653069bb2b194e7f18f17cbe9939c0", size = 3314885, upload-time = "2025-10-24T19:04:07.642Z" }, - { url = "https://files.pythonhosted.org/packages/21/90/b7fe5ff6f2b7b8cbdf1bd56145f863c90a5807d9758a549bf3d916aa4dec/hf_xet-1.2.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:29c8fc913a529ec0a91867ce3d119ac1aac966e098cf49501800c870328cc090", size = 3221550, upload-time = "2025-10-24T19:04:05.55Z" }, - { url = "https://files.pythonhosted.org/packages/6f/cb/73f276f0a7ce46cc6a6ec7d6c7d61cbfe5f2e107123d9bbd0193c355f106/hf_xet-1.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e159cbfcfbb29f920db2c09ed8b660eb894640d284f102ada929b6e3dc410a", size = 3408010, upload-time = "2025-10-24T19:04:28.598Z" }, - { url = "https://files.pythonhosted.org/packages/b8/1e/d642a12caa78171f4be64f7cd9c40e3ca5279d055d0873188a58c0f5fbb9/hf_xet-1.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c91d5ae931510107f148874e9e2de8a16052b6f1b3ca3c1b12f15ccb491390f", size = 3503264, upload-time = "2025-10-24T19:04:30.397Z" }, - { url = "https://files.pythonhosted.org/packages/17/b5/33764714923fa1ff922770f7ed18c2daae034d21ae6e10dbf4347c854154/hf_xet-1.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:210d577732b519ac6ede149d2f2f34049d44e8622bf14eb3d63bbcd2d4b332dc", size = 2901071, upload-time = "2025-10-24T19:04:37.463Z" }, - { url = "https://files.pythonhosted.org/packages/96/2d/22338486473df5923a9ab7107d375dbef9173c338ebef5098ef593d2b560/hf_xet-1.2.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:46740d4ac024a7ca9b22bebf77460ff43332868b661186a8e46c227fdae01848", size = 2866099, upload-time = "2025-10-24T19:04:15.366Z" }, - { url = "https://files.pythonhosted.org/packages/7f/8c/c5becfa53234299bc2210ba314eaaae36c2875e0045809b82e40a9544f0c/hf_xet-1.2.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:27df617a076420d8845bea087f59303da8be17ed7ec0cd7ee3b9b9f579dff0e4", size = 2722178, upload-time = "2025-10-24T19:04:13.695Z" }, - { url = "https://files.pythonhosted.org/packages/9a/92/cf3ab0b652b082e66876d08da57fcc6fa2f0e6c70dfbbafbd470bb73eb47/hf_xet-1.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3651fd5bfe0281951b988c0facbe726aa5e347b103a675f49a3fa8144c7968fd", size = 3320214, upload-time = "2025-10-24T19:04:03.596Z" }, - { url = "https://files.pythonhosted.org/packages/46/92/3f7ec4a1b6a65bf45b059b6d4a5d38988f63e193056de2f420137e3c3244/hf_xet-1.2.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d06fa97c8562fb3ee7a378dd9b51e343bc5bc8190254202c9771029152f5e08c", size = 3229054, upload-time = "2025-10-24T19:04:01.949Z" }, - { url = "https://files.pythonhosted.org/packages/0b/dd/7ac658d54b9fb7999a0ccb07ad863b413cbaf5cf172f48ebcd9497ec7263/hf_xet-1.2.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4c1428c9ae73ec0939410ec73023c4f842927f39db09b063b9482dac5a3bb737", size = 3413812, upload-time = "2025-10-24T19:04:24.585Z" }, - { url = "https://files.pythonhosted.org/packages/92/68/89ac4e5b12a9ff6286a12174c8538a5930e2ed662091dd2572bbe0a18c8a/hf_xet-1.2.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a55558084c16b09b5ed32ab9ed38421e2d87cf3f1f89815764d1177081b99865", size = 3508920, upload-time = "2025-10-24T19:04:26.927Z" }, - { url = "https://files.pythonhosted.org/packages/cb/44/870d44b30e1dcfb6a65932e3e1506c103a8a5aea9103c337e7a53180322c/hf_xet-1.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:e6584a52253f72c9f52f9e549d5895ca7a471608495c4ecaa6cc73dba2b24d69", size = 2905735, upload-time = "2025-10-24T19:04:35.928Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/1b/ab/522a2ab67f27971a9d48ca666d4fca85ef7d5282d142e31fd087e27b1bbe/hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef", size = 920527, upload-time = "2026-08-03T22:33:13.243Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/62/3c062f593bd92ef4e77a0ef39541e3d82a0a1d3947c8a777a02a13a27828/hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d", size = 4074584, upload-time = "2026-08-03T22:32:47.364Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1e/c0ad437dd267a8e435bef594acf781bbc3874ff0b6435b4962d03ecf7cc4/hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675", size = 3867381, upload-time = "2026-08-03T22:32:49.049Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ee/7c0d7b6ab336167531b1c30af2af003f054af4c749becbd7209ae33a77c3/hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b", size = 4453982, upload-time = "2026-08-03T22:32:50.568Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/ad8eab1c9525246650cbaa821caa3cdbaca734ab1a5b8c91bea09cbd8d69/hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522", size = 4249445, upload-time = "2026-08-03T22:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/26/1eee8aedb0dafc1ab9717dc9ac602cde33361b232dc06803f1f6ed18b58c/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e", size = 4451099, upload-time = "2026-08-03T22:32:54.114Z" }, + { url = "https://files.pythonhosted.org/packages/67/57/0b88af1f194ab6c9c650547d9cc06bfeaab836ae4dcdb331676bfb8be95a/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9", size = 4664712, upload-time = "2026-08-03T22:32:55.547Z" }, + { url = "https://files.pythonhosted.org/packages/53/a0/26b717a9d1840e8abf48dcec64b5ed8fbe472671d38ad28d30e147132b33/hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338", size = 4025906, upload-time = "2026-08-03T22:32:57.391Z" }, + { url = "https://files.pythonhosted.org/packages/49/f6/4a9966633c6fef83af997e2cff68ec1963676d412bdfd096df2a93b8e185/hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765", size = 3849221, upload-time = "2026-08-03T22:32:59.123Z" }, + { url = "https://files.pythonhosted.org/packages/a2/50/7afa2c9c787405864fc47a0d1bbc02c62e9101947ed43c1f43899fc7d91d/hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d", size = 4071729, upload-time = "2026-08-03T22:33:00.721Z" }, + { url = "https://files.pythonhosted.org/packages/4b/69/55b8dcf636142ae660fec1869fcac14c4da2e8412e14d6eee1523be77e9f/hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a", size = 3876287, upload-time = "2026-08-03T22:33:02.251Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f", size = 4464663, upload-time = "2026-08-03T22:33:03.802Z" }, + { url = "https://files.pythonhosted.org/packages/9a/69/1f0cbc2fb22ae6082d094f743d1b8945a3f36f6089cb95f42b7ee348cda7/hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7", size = 4262538, upload-time = "2026-08-03T22:33:05.287Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/4f4f2301ade26e404462d3336fa11f7958d914cabbabdd6e03c3c5d5658c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb", size = 4460520, upload-time = "2026-08-03T22:33:06.81Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/311725e2a905534dfee2dcb5b08414f249147f1f12252bfc2bd24caa075c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c", size = 4675937, upload-time = "2026-08-03T22:33:08.616Z" }, + { url = "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b", size = 4033128, upload-time = "2026-08-03T22:33:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, ] [[package]] @@ -415,21 +422,22 @@ wheels = [ [[package]] name = "huggingface-hub" -version = "0.36.2" +version = "1.27.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "click" }, { name = "filelock" }, { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, { name = "packaging" }, { name = "pyyaml" }, - { name = "requests" }, { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7c/b7/8cb61d2eece5fb05a83271da168186721c450eb74e3c31f7ef3169fa475b/huggingface_hub-0.36.2.tar.gz", hash = "sha256:1934304d2fb224f8afa3b87007d58501acfda9215b334eed53072dd5e815ff7a", size = 649782, upload-time = "2026-02-06T09:24:13.098Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/9b/ddf3d02a8681f1b9ce52fda03d755dad6b74c4f8172304c4c8d2975450f9/huggingface_hub-1.27.0.tar.gz", hash = "sha256:c1fed40ea82a6b41b477f5243546549b792ae0a93abcea608cff66089bf8f8df", size = 942668, upload-time = "2026-08-07T12:48:05.161Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/af/48ac8483240de756d2438c380746e7130d1c6f75802ef22f3c6d49982787/huggingface_hub-0.36.2-py3-none-any.whl", hash = "sha256:48f0c8eac16145dfce371e9d2d7772854a4f591bcb56c9cf548accf531d54270", size = 566395, upload-time = "2026-02-06T09:24:11.133Z" }, + { url = "https://files.pythonhosted.org/packages/de/d8/95b735e183957c1f26d94c52977f09d466d55119cbbc1558ea4975e4c216/huggingface_hub-1.27.0-py3-none-any.whl", hash = "sha256:7df6827c2f956c60fbaa64646e979e566db76f619dd0a9729dfb8c5a3eb4f68d", size = 784926, upload-time = "2026-08-07T12:48:02.905Z" }, ] [[package]] @@ -503,10 +511,11 @@ wheels = [ [[package]] name = "ltx-core" -version = "1.1.7" -source = { git = "https://github.com/Lightricks/LTX-2.git?subdirectory=packages%2Fltx-core&rev=9377758131b1ffde4b7f766804590a6617bf2ab9#9377758131b1ffde4b7f766804590a6617bf2ab9" } +version = "1.2.0" +source = { git = "https://github.com/Lightricks/LTX-2.git?subdirectory=packages%2Fltx-core&tag=v1.2.0#d151147788a9284cca791edc6ce898007e727fe6" } dependencies = [ { name = "accelerate" }, + { name = "av" }, { name = "einops" }, { name = "mps-sdpa", marker = "platform_machine == 'arm64' and sys_platform == 'darwin'" }, { name = "numpy" }, @@ -533,6 +542,7 @@ dependencies = [ { name = "imageio-ffmpeg" }, { name = "ltx-core" }, { name = "ltx-pipelines" }, + { name = "natten", marker = "sys_platform == 'win32'" }, { name = "ninja", marker = "sys_platform == 'darwin'" }, { name = "opencv-python-headless" }, { name = "peft" }, @@ -548,6 +558,9 @@ dependencies = [ { name = "torch", version = "2.10.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, { name = "torch", version = "2.12.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'" }, + { name = "torchvision", version = "0.25.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "torchvision", version = "0.26.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin'" }, + { name = "torchvision", version = "0.27.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'" }, { name = "tqdm" }, { name = "transformers" }, { name = "triton", marker = "sys_platform == 'linux'" }, @@ -576,8 +589,9 @@ requires-dist = [ { name = "huggingface-hub", specifier = ">=0.23.0" }, { name = "imageio", specifier = ">=2.37.2" }, { name = "imageio-ffmpeg", specifier = ">=0.6.0" }, - { name = "ltx-core", git = "https://github.com/Lightricks/LTX-2.git?subdirectory=packages%2Fltx-core&rev=9377758131b1ffde4b7f766804590a6617bf2ab9" }, - { name = "ltx-pipelines", git = "https://github.com/Lightricks/LTX-2.git?subdirectory=packages%2Fltx-pipelines&rev=9377758131b1ffde4b7f766804590a6617bf2ab9" }, + { name = "ltx-core", git = "https://github.com/Lightricks/LTX-2.git?subdirectory=packages%2Fltx-core&tag=v1.2.0" }, + { name = "ltx-pipelines", git = "https://github.com/Lightricks/LTX-2.git?subdirectory=packages%2Fltx-pipelines&tag=v1.2.0" }, + { name = "natten", marker = "sys_platform == 'win32'", url = "https://storage.googleapis.com/ltx-desktop-artifacts/wheels/natten-0.21.6+torch2100cu128-cp313-cp313-win_amd64.whl" }, { name = "ninja", marker = "sys_platform == 'darwin'", specifier = ">=1.11" }, { name = "opencv-python-headless", specifier = ">=4.8.0" }, { name = "peft", specifier = ">=0.13.2" }, @@ -596,8 +610,10 @@ requires-dist = [ { name = "torch", marker = "sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", specifier = ">=2.3.0" }, { name = "torch", marker = "sys_platform == 'darwin'", specifier = ">=2.3.0,<2.12" }, { name = "torch", marker = "sys_platform == 'linux' or sys_platform == 'win32'", specifier = ">=2.3.0", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torchvision", marker = "sys_platform != 'linux' and sys_platform != 'win32'", specifier = ">=0.18.0" }, + { name = "torchvision", marker = "sys_platform == 'linux' or sys_platform == 'win32'", specifier = ">=0.18.0", index = "https://download.pytorch.org/whl/cu128" }, { name = "tqdm", specifier = ">=4.66.0" }, - { name = "transformers", specifier = ">=4.52,<5" }, + { name = "transformers", specifier = ">=5.8.0,<5.15" }, { name = "triton", marker = "sys_platform == 'linux'" }, { name = "triton-windows", marker = "sys_platform == 'win32'" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0" }, @@ -606,8 +622,8 @@ provides-extras = ["test", "dev"] [[package]] name = "ltx-pipelines" -version = "1.1.7" -source = { git = "https://github.com/Lightricks/LTX-2.git?subdirectory=packages%2Fltx-pipelines&rev=9377758131b1ffde4b7f766804590a6617bf2ab9#9377758131b1ffde4b7f766804590a6617bf2ab9" } +version = "1.2.0" +source = { git = "https://github.com/Lightricks/LTX-2.git?subdirectory=packages%2Fltx-pipelines&tag=v1.2.0#d151147788a9284cca791edc6ce898007e727fe6" } dependencies = [ { name = "av" }, { name = "cloudpickle" }, @@ -617,6 +633,18 @@ dependencies = [ { name = "tqdm" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -680,6 +708,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -707,6 +744,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b1/eb/629b9846b972471c4edacb244e1f59d72862fa57b7bc92a8c7fc5dabd9b0/mps_sdpa-0.2.0-py3-none-any.whl", hash = "sha256:2fa980c7028968164a6ae15e4ed380447c76411f4a258184f23d80cf5272f3fd", size = 58151, upload-time = "2026-04-30T16:12:52.089Z" }, ] +[[package]] +name = "natten" +version = "0.21.6+torch2100cu128" +source = { url = "https://storage.googleapis.com/ltx-desktop-artifacts/wheels/natten-0.21.6+torch2100cu128-cp313-cp313-win_amd64.whl" } +wheels = [ + { url = "https://storage.googleapis.com/ltx-desktop-artifacts/wheels/natten-0.21.6+torch2100cu128-cp313-cp313-win_amd64.whl", hash = "sha256:bc7dffe1d077322662b1da5e7cada58419d8b2acbcb79e63a64537ca1a6c55a6" }, +] + [[package]] name = "networkx" version = "3.6.1" @@ -1537,26 +1582,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + [[package]] name = "safetensors" -version = "0.7.0" +version = "0.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/29/9c/6e74567782559a63bd040a236edca26fd71bc7ba88de2ef35d75df3bca5e/safetensors-0.7.0.tar.gz", hash = "sha256:07663963b67e8bd9f0b8ad15bb9163606cd27cc5a1b96235a50d8369803b96b0", size = 200878, upload-time = "2025-11-19T15:18:43.199Z" } +sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/47/aef6c06649039accf914afef490268e1067ed82be62bcfa5b7e886ad15e8/safetensors-0.7.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c82f4d474cf725255d9e6acf17252991c3c8aac038d6ef363a4bf8be2f6db517", size = 467781, upload-time = "2025-11-19T15:18:35.84Z" }, - { url = "https://files.pythonhosted.org/packages/e8/00/374c0c068e30cd31f1e1b46b4b5738168ec79e7689ca82ee93ddfea05109/safetensors-0.7.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:94fd4858284736bb67a897a41608b5b0c2496c9bdb3bf2af1fa3409127f20d57", size = 447058, upload-time = "2025-11-19T15:18:34.416Z" }, - { url = "https://files.pythonhosted.org/packages/f1/06/578ffed52c2296f93d7fd2d844cabfa92be51a587c38c8afbb8ae449ca89/safetensors-0.7.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e07d91d0c92a31200f25351f4acb2bc6aff7f48094e13ebb1d0fb995b54b6542", size = 491748, upload-time = "2025-11-19T15:18:09.79Z" }, - { url = "https://files.pythonhosted.org/packages/ae/33/1debbbb70e4791dde185edb9413d1fe01619255abb64b300157d7f15dddd/safetensors-0.7.0-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8469155f4cb518bafb4acf4865e8bb9d6804110d2d9bdcaa78564b9fd841e104", size = 503881, upload-time = "2025-11-19T15:18:16.145Z" }, - { url = "https://files.pythonhosted.org/packages/8e/1c/40c2ca924d60792c3be509833df711b553c60effbd91da6f5284a83f7122/safetensors-0.7.0-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:54bef08bf00a2bff599982f6b08e8770e09cc012d7bba00783fc7ea38f1fb37d", size = 623463, upload-time = "2025-11-19T15:18:21.11Z" }, - { url = "https://files.pythonhosted.org/packages/9b/3a/13784a9364bd43b0d61eef4bea2845039bc2030458b16594a1bd787ae26e/safetensors-0.7.0-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:42cb091236206bb2016d245c377ed383aa7f78691748f3bb6ee1bfa51ae2ce6a", size = 532855, upload-time = "2025-11-19T15:18:25.719Z" }, - { url = "https://files.pythonhosted.org/packages/a0/60/429e9b1cb3fc651937727befe258ea24122d9663e4d5709a48c9cbfceecb/safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac7252938f0696ddea46f5e855dd3138444e82236e3be475f54929f0c510d48", size = 507152, upload-time = "2025-11-19T15:18:33.023Z" }, - { url = "https://files.pythonhosted.org/packages/3c/a8/4b45e4e059270d17af60359713ffd83f97900d45a6afa73aaa0d737d48b6/safetensors-0.7.0-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1d060c70284127fa805085d8f10fbd0962792aed71879d00864acda69dbab981", size = 541856, upload-time = "2025-11-19T15:18:31.075Z" }, - { url = "https://files.pythonhosted.org/packages/06/87/d26d8407c44175d8ae164a95b5a62707fcc445f3c0c56108e37d98070a3d/safetensors-0.7.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cdab83a366799fa730f90a4ebb563e494f28e9e92c4819e556152ad55e43591b", size = 674060, upload-time = "2025-11-19T15:18:37.211Z" }, - { url = "https://files.pythonhosted.org/packages/11/f5/57644a2ff08dc6325816ba7217e5095f17269dada2554b658442c66aed51/safetensors-0.7.0-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:672132907fcad9f2aedcb705b2d7b3b93354a2aec1b2f706c4db852abe338f85", size = 771715, upload-time = "2025-11-19T15:18:38.689Z" }, - { url = "https://files.pythonhosted.org/packages/86/31/17883e13a814bd278ae6e266b13282a01049b0c81341da7fd0e3e71a80a3/safetensors-0.7.0-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:5d72abdb8a4d56d4020713724ba81dac065fedb7f3667151c4a637f1d3fb26c0", size = 714377, upload-time = "2025-11-19T15:18:40.162Z" }, - { url = "https://files.pythonhosted.org/packages/4a/d8/0c8a7dc9b41dcac53c4cbf9df2b9c83e0e0097203de8b37a712b345c0be5/safetensors-0.7.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b0f6d66c1c538d5a94a73aa9ddca8ccc4227e6c9ff555322ea40bdd142391dd4", size = 677368, upload-time = "2025-11-19T15:18:41.627Z" }, - { url = "https://files.pythonhosted.org/packages/05/e5/cb4b713c8a93469e3c5be7c3f8d77d307e65fe89673e731f5c2bfd0a9237/safetensors-0.7.0-cp38-abi3-win32.whl", hash = "sha256:c74af94bf3ac15ac4d0f2a7c7b4663a15f8c2ab15ed0fc7531ca61d0835eccba", size = 326423, upload-time = "2025-11-19T15:18:45.74Z" }, - { url = "https://files.pythonhosted.org/packages/5d/e6/ec8471c8072382cb91233ba7267fd931219753bb43814cbc71757bfd4dab/safetensors-0.7.0-cp38-abi3-win_amd64.whl", hash = "sha256:d1239932053f56f3456f32eb9625590cc7582e905021f94636202a864d470755", size = 341380, upload-time = "2025-11-19T15:18:44.427Z" }, + { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, + { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, + { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, ] [[package]] @@ -1718,6 +1778,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/c6/76dc613121b793286a3f91621d7b75a2b493e0390ddca50f11993eadf192/setuptools-82.0.0-py3-none-any.whl", hash = "sha256:70b18734b607bd1da571d097d236cfcfacaf01de45717d59e6e04b96877532e0", size = 1003468, upload-time = "2026-02-08T15:08:38.723Z" }, ] +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + [[package]] name = "starlette" version = "0.52.1" @@ -1907,6 +1976,70 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/84/1c792b0b700eac9a96772cfd9f96c097b17bca3234a2fde3c64b8063660d/torchaudio-2.11.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:da2725e250866da42a12934c9a6552f65a18b7187fd7a6221387f0e605fb3b96", size = 679926, upload-time = "2026-03-23T18:13:24.452Z" }, ] +[[package]] +name = "torchvision" +version = "0.25.0+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "sys_platform == 'win32'", + "sys_platform == 'linux'", +] +dependencies = [ + { name = "numpy" }, + { name = "pillow" }, + { name = "torch", version = "2.10.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" } }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:8623e534ef6a815bd6407d4b52dd70c7154e2eda626ad4b9cb895d36c5a3305b", upload-time = "2026-01-21T22:32:23Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:1255a0ca2bf987acf9f103b96c5c4cfe3415fc4a1eef17fa08af527a04a4f573", upload-time = "2026-01-21T22:32:24Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:068e519838b4a8b32a09521244b170edd8c2ac9eeb6538b7bf492cd70e57ebf5", upload-time = "2026-01-21T22:32:25Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:12c253520a26483fe3c614f63ff16eca6d9b0b4ebe510699b7d15d88e6c0cd35", upload-time = "2026-01-21T22:32:26Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a9c0de893dce9c2913c9c7ae88a916910f92d02b99da149678806d18e8079f29", upload-time = "2026-01-21T22:32:27Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp313-cp313-win_amd64.whl", hash = "sha256:e2e0317e3861bba1b5aeba7c1cb4bcd50937cf0bffdbea478619d1f5f73e9050", upload-time = "2026-01-21T22:32:27Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:58b2971b55c761f1d2491bd80fcc4618ea97d363d387a9dd3aff23220cbee264", upload-time = "2026-01-21T22:32:28Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:1b6878b043513ea3dea1b90bfb5193455d9b248b8c4d5e66ea9f5d1643a43f13", upload-time = "2026-01-21T22:32:29Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp313-cp313t-win_amd64.whl", hash = "sha256:96cd2ba7b289117873b2a8f4c80605d38118d920b1045f3ce21a9f0ca68a701e", upload-time = "2026-01-21T22:32:30Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:e2dbf9ea9f4b2416822249e96ff3ad873d9a84e51285d6b9967732be3015c523", upload-time = "2026-01-21T22:32:31Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:5b7ad3fb6cf03ef2a2fd617cb4b4e41efa9bb0143c67f506c2a3e6765c7b12ad", upload-time = "2026-01-21T22:32:31Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp314-cp314-win_amd64.whl", hash = "sha256:a52ff3b072e89280f41499813e11c418d168ffc502b86cb17767bab29f432b3a", upload-time = "2026-01-21T22:32:32Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:687987fbcb074fd7f7a61cf2b407b1eac07588ace8351a3a36978546a00adc52", upload-time = "2026-01-21T22:32:33Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:84c5e2cb699235339b8a5c295e974a795244a45d1104ecee658d9d19600cdc75", upload-time = "2026-01-21T22:32:33Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp314-cp314t-win_amd64.whl", hash = "sha256:d1cf27bc2da13fd9e83694ae601b1bf4135c24d9c9e9ec249056896395a78a9e", upload-time = "2026-01-21T22:32:35Z" }, +] + +[[package]] +name = "torchvision" +version = "0.26.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform == 'darwin'", +] +dependencies = [ + { name = "numpy" }, + { name = "pillow" }, + { name = "torch", version = "2.11.0", source = { registry = "https://pypi.org/simple" } }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/e7/56b47cc3b132aea90ccce22bcb8975dec688b002150012acc842846039d0/torchvision-0.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c409e1c3fdebec7a3834465086dbda8bf7680eff79abf7fd2f10c6b59520a7a4", size = 1863502, upload-time = "2026-03-23T18:12:57.326Z" }, + { url = "https://files.pythonhosted.org/packages/da/80/0762f77f53605d10c9477be39bb47722cc8e383bbbc2531471ce0e396c07/torchvision-0.26.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5d63dd43162691258b1b3529b9041bac7d54caa37eae0925f997108268cbf7c4", size = 1860809, upload-time = "2026-03-23T18:12:47.629Z" }, + { url = "https://files.pythonhosted.org/packages/66/28/b4ad0a723ed95b003454caffcc41894b34bd8379df340848cae2c33871de/torchvision-0.26.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:358fc4726d0c08615b6d83b3149854f11efb2a564ed1acb6fce882e151412d23", size = 1951973, upload-time = "2026-03-23T18:12:48.781Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c8/9bffa9c7f7bdf95b2a0a2dc535c290b9f1cc580c3fb3033ab1246ffffdeb/torchvision-0.26.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:eb61804eb9dbe88c5a2a6c4da8dec1d80d2d0a6f18c999c524e32266cb1ebcd3", size = 1860813, upload-time = "2026-03-23T18:12:39.636Z" }, + { url = "https://files.pythonhosted.org/packages/45/8f/1f0402ac55c2ae15651ff831957d083fe70b2d12282e72612a30ba601512/torchvision-0.26.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:b7d3e295624a28b3b1769228ce1345d94cf4d390dd31136766f76f2d20f718da", size = 1860826, upload-time = "2026-03-23T18:12:34.1Z" }, +] + +[[package]] +name = "torchvision" +version = "0.27.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", +] +dependencies = [ + { name = "numpy" }, + { name = "pillow" }, + { name = "torch", version = "2.12.1", source = { registry = "https://pypi.org/simple" } }, +] + [[package]] name = "tqdm" version = "4.67.3" @@ -1921,23 +2054,22 @@ wheels = [ [[package]] name = "transformers" -version = "4.57.6" +version = "5.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock" }, { name = "huggingface-hub" }, { name = "numpy" }, { name = "packaging" }, { name = "pyyaml" }, { name = "regex" }, - { name = "requests" }, { name = "safetensors" }, { name = "tokenizers" }, { name = "tqdm" }, + { name = "typer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c4/35/67252acc1b929dc88b6602e8c4a982e64f31e733b804c14bc24b47da35e6/transformers-4.57.6.tar.gz", hash = "sha256:55e44126ece9dc0a291521b7e5492b572e6ef2766338a610b9ab5afbb70689d3", size = 10134912, upload-time = "2026-01-16T10:38:39.284Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/fb/2a2ba88f325e68a921d8b69ff63b477830b2e73ade9a3c8c8cab2f06d741/transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173", size = 9295927, upload-time = "2026-07-16T09:41:57.773Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/b8/e484ef633af3887baeeb4b6ad12743363af7cce68ae51e938e00aaa0529d/transformers-4.57.6-py3-none-any.whl", hash = "sha256:4c9e9de11333ddfe5114bc872c9f370509198acf0b87a832a0ab9458e2bd0550", size = 11993498, upload-time = "2026-01-16T10:38:31.289Z" }, + { url = "https://files.pythonhosted.org/packages/6f/67/8d85ca2323233ae3c0365a659c4e52ee1f587b440e4bc577e7d8e4416d0f/transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725", size = 11625234, upload-time = "2026-07-16T09:41:54.143Z" }, ] [[package]] @@ -1967,6 +2099,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/d3/58ad68518e04a97ce0549cad98eccbafac01ddba640379776a58b513020b/triton_windows-3.6.0.post25-cp314-cp314-win_amd64.whl", hash = "sha256:6f4c4775b22cfb18e9c60aead83deb7b9b970624ae3c13cd26b9be80b5cb8cd8", size = 48566374, upload-time = "2026-01-26T03:21:41.743Z" }, ] +[[package]] +name = "typer" +version = "0.27.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/40/4a3db7990d1f62a53182aa96eaef57aeb2886a27f90a195bc66713565d31/typer-0.27.1.tar.gz", hash = "sha256:a79bef8469a79c45498e7b814ecf8d603cc7644e9acbd9e19cac0334240b18df", size = 203994, upload-time = "2026-08-03T14:41:03.438Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/89/9518bc0c3929bee36b3a4a8e3daddd6e03f92f9961c66d4983b837160543/typer-0.27.1-py3-none-any.whl", hash = "sha256:53150287edd11baeb4e4722c8e394fcdf8181c0ae89485cba8d25c778d5edd56", size = 122874, upload-time = "2026-08-03T14:41:04.391Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" diff --git a/frontend/App.tsx b/frontend/App.tsx index 93751852f..437529111 100644 --- a/frontend/App.tsx +++ b/frontend/App.tsx @@ -30,7 +30,7 @@ type LtxUpgradeRecommendation = Extract { const ok = await window.electronAPI.acceptLicense() @@ -341,8 +342,9 @@ function AppContent() { const handleCompleteLtxUpgradePrompt = useCallback(async () => { setDismissedUpgradeTargetId(null) + notifyModelsChanged() await refreshLtxUpgradeRecommendation() - }, [refreshLtxUpgradeRecommendation]) + }, [notifyModelsChanged, refreshLtxUpgradeRecommendation]) const restartingOverlay = isBackendRestarting ? (
diff --git a/frontend/components/AssetPreviewModal.tsx b/frontend/components/AssetPreviewModal.tsx new file mode 100644 index 000000000..85120226a --- /dev/null +++ b/frontend/components/AssetPreviewModal.tsx @@ -0,0 +1,158 @@ +import { useEffect, useState } from 'react' +import { Check, ChevronLeft, ChevronRight, Copy, X } from 'lucide-react' +import type { Asset } from '../types/project-model' +import { pathToFileUrl } from '../lib/file-url' +import { formatPipelineDisplayName } from '../lib/video-generation-model-specs' + +// Duration can be a raw float (e.g. extend output: 12.041667s). Show a clean value: +// integers as-is, otherwise at most 2 decimals with trailing zeros trimmed. +function formatSeconds(seconds: number): string { + return Number.isInteger(seconds) ? String(seconds) : seconds.toFixed(2).replace(/\.?0+$/, '') +} + +function formatDurationLabel(asset: Asset): string | null { + if (asset.type === 'image') return 'Image' + // Requested Auto is persisted as generationParams.duration === null. Do not fall through + // to omitting the chip — that made Auto clips look like they had no duration at all. + if (asset.generationParams?.duration === null) return 'Auto duration' + if (asset.duration) return `${formatSeconds(asset.duration)}s` + return null +} + +export interface AssetPreviewModalProps { + asset: Asset + /** 0-based index into the current filtered gallery. */ + index: number + total: number + canGoPrev: boolean + canGoNext: boolean + onPrev: () => void + onNext: () => void + onClose: () => void +} + +export function AssetPreviewModal({ + asset, + index, + total, + canGoPrev, + canGoNext, + onPrev, + onNext, + onClose, +}: AssetPreviewModalProps) { + const [copiedPrompt, setCopiedPrompt] = useState(false) + + useEffect(() => { + const handleKey = (e: KeyboardEvent) => { + if (e.key === 'ArrowLeft') { e.preventDefault(); onPrev() } + else if (e.key === 'ArrowRight') { e.preventDefault(); onNext() } + else if (e.key === 'Escape') onClose() + } + window.addEventListener('keydown', handleKey) + return () => window.removeEventListener('keydown', handleKey) + }, [onPrev, onNext, onClose]) + + // Reset copy affordance when paging between assets. + useEffect(() => { + setCopiedPrompt(false) + }, [asset.id]) + + // Image assets also store a placeholder `model` (e.g. "fast" for Z-Image) that is not an + // LTX video pipeline — only label known video pipelines on video assets. Prefer the label + // captured at generation time: the local "fast" pipeline id is shared by every LTX version, + // so mapping it here would report the wrong version. + const modelLabel = asset.type === 'video' + ? asset.generationParams?.modelLabel ?? formatPipelineDisplayName(asset.generationParams?.model) + : null + const metaParts = [ + modelLabel, + asset.resolution || null, + formatDurationLabel(asset), + ].filter(Boolean) + + return ( +
+ + + + +
e.stopPropagation()}> +
+ + {index + 1} / {total} + + +
+ + {asset.type === 'video' ? ( +
+
+ ) +} diff --git a/frontend/components/FirstRunSetup.tsx b/frontend/components/FirstRunSetup.tsx index 037f12bac..50dce1ab9 100644 --- a/frontend/components/FirstRunSetup.tsx +++ b/frontend/components/FirstRunSetup.tsx @@ -1,9 +1,11 @@ -import { useState, useEffect, useRef, useCallback } from 'react' +import { useState, useEffect, useRef, useCallback, useMemo } from 'react' import { ApiClient, type ApiRequestBodyOf, type ApiSuccessOf } from '../lib/api-client' import { formatBytes } from '../lib/format' import { logger } from '../lib/logger' import { useHfAuth } from '../hooks/use-hf-auth' +import { useHfModelAccess } from '../hooks/use-hf-model-access' import { useAppSettings } from '../contexts/AppSettingsContext' +import { HfModelAccessGate } from './HfModelAccessGate' import './FirstRunSetup.css' interface LaunchGateProps { @@ -52,13 +54,24 @@ const CP_INFO_BY_ROLE: Record = { text_encoder: 'Reads your text prompt so the model understands it. You can skip this large download by entering an LTX ' + 'API key, which encodes prompts via the API instead.', + vae: + 'Decodes the model\'s latent frames into video (and audio, when present). Required for LTX versions that ' + + 'ship the transformer separately from their VAEs.', image: 'Generates still images from text prompts (used for image-to-video and image tools).', support: 'A supporting model used for guided generation (depth, edges, or pose control).', } // One line in the first-run "What will be downloaded" list: checkpoint name, an -// info-tooltip icon, and the size (or a "skipped" note when an API key covers it). -function DownloadItem({ item, skipped }: { item: CheckpointDescriptor; skipped: boolean }) { +// info-tooltip icon, and the size. Items an API key covers get a checkbox instead of being +// silently dropped, so the download stays available to anyone who wants to run offline. +function DownloadItem({ + item, + optIn, +}: { + item: CheckpointDescriptor + optIn?: { checked: boolean; onToggle: () => void } +}) { + const skipped = optIn !== undefined && !optIn.checked return (
+ {optIn && ( + + )} {item.name} - - {skipped ? 'Skipped (API key)' : formatBytes(item.size_bytes)} + + {item.downloaded ? 'Installed' : skipped ? 'Skipped (API key)' : formatBytes(item.size_bytes)}
) @@ -104,6 +125,7 @@ function DownloadItem({ item, skipped }: { item: CheckpointDescriptor; skipped: function buildDownloadSteps( ltxRecommendation: LtxRecommendation, imgGenRecommendation: ImgGenRecommendation, + extraCpIds: readonly ModelCheckpointID[] = [], ): DownloadStepSpec[] { const cpIds: ModelCheckpointID[] = [] if (ltxRecommendation.status === 'download') { @@ -112,6 +134,7 @@ function buildDownloadSteps( if (imgGenRecommendation.cp_to_download) { cpIds.push(imgGenRecommendation.cp_to_download) } + cpIds.push(...extraCpIds) const unique = uniqueCpIds(cpIds) return unique.length > 0 ? [{ type: 'download', cpIds: unique }] : [] } @@ -131,26 +154,56 @@ export function LaunchGate({ const [installMessage, setInstallMessage] = useState(INSTALL_MESSAGES[0]) const [availableSpace, setAvailableSpace] = useState('...') const [downloadItems, setDownloadItems] = useState([]) + const [optionalItems, setOptionalItems] = useState([]) + const [optedInCpIds, setOptedInCpIds] = useState([]) const [videoPath, setVideoPath] = useState('/splash/splash.mp4') const [ltxApiKey, setLtxApiKey] = useState('') + const [hasSavedLtxApiKey, setHasSavedLtxApiKey] = useState(false) const [licenseAccepted, setLicenseAccepted] = useState(false) const [licenseText, setLicenseText] = useState(null) const [licenseError, setLicenseError] = useState(null) const [actionError, setActionError] = useState(null) const [isActionPending, setIsActionPending] = useState(false) const { hfAuthStatus, hfAuthPolling, startHuggingFaceLogin } = useHfAuth(currentStep === 'location') + // A key typed here isn't saved yet, so the backend still lists the text encoder as required; + // move it to the opt-in list so the choice reads the same either way. + const keyEnteredNow = ltxApiKey.trim().length > 0 + const requiredItems = useMemo( + () => downloadItems.filter((item) => !(item.role === 'text_encoder' && keyEnteredNow)), + [downloadItems, keyEnteredNow], + ) + const allOptionalItems = useMemo( + () => [ + ...(keyEnteredNow ? downloadItems.filter((item) => item.role === 'text_encoder') : []), + ...optionalItems, + ], + [downloadItems, keyEnteredNow, optionalItems], + ) + const isOptedIn = (item: CheckpointDescriptor) => optedInCpIds.includes(item.cp_id as ModelCheckpointID) + const toggleOptIn = (item: CheckpointDescriptor) => { + const cpId = item.cp_id as ModelCheckpointID + setOptedInCpIds((ids) => (ids.includes(cpId) ? ids.filter((id) => id !== cpId) : [...ids, cpId])) + } + const pendingDownloadItems = useMemo( + () => + [...requiredItems, ...allOptionalItems.filter((item) => optedInCpIds.includes(item.cp_id as ModelCheckpointID))] + .filter((item) => !item.downloaded), + [requiredItems, allOptionalItems, optedInCpIds], + ) + const installCpIds = useMemo( + () => uniqueCpIds(pendingDownloadItems.map((item) => item.cp_id as ModelCheckpointID)), + [pendingDownloadItems], + ) + const { accessMap, allAuthorized, checking: checkingAccess, checkError, recheckAccess } = useHfModelAccess( + currentStep === 'location' ? installCpIds : [], + hfAuthStatus, + ) const { saveLtxApiKey } = useAppSettings() const downloadQueueRef = useRef([]) const runningDownloadProgress = downloadProgress?.status === 'downloading' ? downloadProgress : null const totalProgress = runningDownloadProgress?.total_progress ?? (downloadProgress?.status === 'complete' ? 100 : 0) - // The text encoder download is skipped when an API key is entered (the backend - // omits it once a key is saved); reflect that live in the preview + total. - const isItemSkipped = (item: CheckpointDescriptor): boolean => - item.role === 'text_encoder' && ltxApiKey.trim().length > 0 - const totalDownloadBytes = downloadItems - .filter((item) => !isItemSkipped(item)) - .reduce((sum, item) => sum + item.size_bytes, 0) + const totalDownloadBytes = pendingDownloadItems.reduce((sum, item) => sum + item.size_bytes, 0) // Format time remaining const formatTimeRemaining = (seconds: number): string => { @@ -203,21 +256,27 @@ export function LaunchGate({ } setInstallPath(settingsResult.data.modelsDir ?? '') + setHasSavedLtxApiKey(Boolean(settingsResult.data.hasLtxApiKey)) // Surface exactly what the install will download (base model, upscaler, text // encoder, image model) with per-checkpoint sizes and info. Same cp set the // installer actually downloads, so the preview can't drift from reality. const cpIds = buildDownloadSteps(ltxResult.data, imgGenResult.data).flatMap((step) => step.cpIds) - if (cpIds.length === 0) { + const optionalCpIds = ltxResult.data.status === 'download' ? ltxResult.data.optional_cp_ids : [] + if (cpIds.length === 0 && optionalCpIds.length === 0) { setDownloadItems([]) + setOptionalItems([]) return } - const describeResult = await ApiClient.describeCheckpoints({ cp_ids: cpIds }) + const describeResult = await ApiClient.describeCheckpoints({ cp_ids: [...cpIds, ...optionalCpIds] }) if (!describeResult.ok) { logger.error(`Failed to describe checkpoints: ${describeResult.error.message}`) return } - setDownloadItems(describeResult.data.checkpoints) + const described = describeResult.data.checkpoints + const isOptional = (item: CheckpointDescriptor) => optionalCpIds.includes(item.cp_id) + setDownloadItems(described.filter((item) => !isOptional(item))) + setOptionalItems(described.filter(isOptional)) }, [licenseOnly]) const startDownloadStep = useCallback(async (step: DownloadStepSpec) => { @@ -335,7 +394,13 @@ export function LaunchGate({ const nextLtxRecommendation = ltxResult.data const nextImgGenRecommendation = imgGenResult.data - const downloadSteps = buildDownloadSteps(nextLtxRecommendation, nextImgGenRecommendation) + // Saving the key above drops the encoder from the recommendation, so an explicit opt-in + // has to be added back here rather than read off the fresh response. + const downloadSteps = buildDownloadSteps( + nextLtxRecommendation, + nextImgGenRecommendation, + optedInCpIds, + ) if (downloadSteps.length === 0) { setCurrentStep('complete') return @@ -409,8 +474,9 @@ export function LaunchGate({ // Check if next button should be disabled const isNextDisabled = () => { if (currentStep === 'license') return !licenseAccepted || isActionPending - // HF sign-in is optional (base models are public) — don't block setup on it. - if (currentStep === 'location') return false + if (currentStep === 'location') { + return installCpIds.length > 0 && (!allAuthorized || checkingAccess) + } if (currentStep === 'complete') return isActionPending return false } @@ -662,7 +728,7 @@ export function LaunchGate({
{/* What will be downloaded */} - {downloadItems.length > 0 && ( + {(requiredItems.length > 0 || allOptionalItems.length > 0) && (
- {downloadItems.map((item) => ( - + {requiredItems.map((item) => ( + ))}
+ + {allOptionalItems.length > 0 && ( +
+ +

+ {allOptionalItems.every((item) => item.role === 'text_encoder') + ? "Your LTX API key covers this, so it isn't needed to generate. Download it to encode prompts on this computer instead — slower, but works offline and without a key." + : "Not required for the current setup. Tick any you want to keep on this computer."} +

+
+ {allOptionalItems.map((item) => ( + toggleOptIn(item) }} + /> + ))} +
+
+ )} )} @@ -699,7 +785,7 @@ export function LaunchGate({ marginLeft: 8, fontWeight: 400 }}> - Optional - Saves ~25 GB download + Optional — makes the text encoder optional @@ -707,7 +793,7 @@ export function LaunchGate({ type="password" value={ltxApiKey} onChange={(e) => setLtxApiKey(e.target.value)} - placeholder="Enter API key to skip text encoder download..." + placeholder="Enter API key to make the text encoder optional..." style={{ width: '100%', background: '#1a1a1a', @@ -720,13 +806,13 @@ export function LaunchGate({ }} />

- {ltxApiKey ? ( + {ltxApiKey || hasSavedLtxApiKey ? ( - ✓ Text encoder download will be skipped (using API instead) + ✓ Text encoder download is optional (using API instead). Tick it under Optional if you want it offline. ) : ( - 'If you have an LTX API key, entering it here skips the 25 GB text encoder download. ' + - 'The API provides faster text encoding (~1s vs 23s local).' + 'If you have an LTX API key, entering it here makes the text encoder optional. ' + + 'The API encodes prompts faster than running the local encoder.' )}

@@ -743,44 +829,60 @@ export function LaunchGate({ HuggingFace Account - {hfAuthStatus === 'authenticated' ? 'Signed in' : 'Optional'} + {hfAuthStatus === 'authenticated' + ? 'Signed in' + : allAuthorized + ? 'Optional' + : 'Required'} - {hfAuthStatus === 'authenticated' ? ( -

- ✓ Authenticated — gated models will download with your account. -

- ) : ( - <> -

- Optional. The base models download without an account — sign in only to - download gated models (some catalog LoRAs / IC-LoRAs require it). You can - also do this later in Settings. + {allAuthorized ? ( + hfAuthStatus === 'authenticated' ? ( +

+ ✓ Authenticated — gated models will download with your account.

- - + ) : ( + <> +

+ Optional for this install. Sign in if you later download gated models from Settings. +

+ + + ) + ) : ( + { + void startHuggingFaceLogin() + }} + checkError={checkError} + onRetryCheck={recheckAccess} + /> )} diff --git a/frontend/components/HfModelAccessGate.tsx b/frontend/components/HfModelAccessGate.tsx new file mode 100644 index 000000000..5cee9ec63 --- /dev/null +++ b/frontend/components/HfModelAccessGate.tsx @@ -0,0 +1,98 @@ +import { AlertCircle } from 'lucide-react' +import type { ApiSuccessOf } from '../lib/api-client' +import { Button } from './ui/button' + +type HfAuthStatus = ApiSuccessOf<'getHuggingFaceAuthStatus'>['status'] +type ModelAccessMap = ApiSuccessOf<'checkModelAccess'>['access'] + +interface HfModelAccessGateProps { + accessMap: ModelAccessMap + allAuthorized: boolean + hfAuthStatus: HfAuthStatus + hfAuthPolling: boolean + startHuggingFaceLogin: () => void + /** When the access check itself failed (network/backend), distinct from unauthorized. */ + checkError?: string | null + onRetryCheck?: () => void + className?: string +} + +export function HfModelAccessGate({ + accessMap, + allAuthorized, + hfAuthStatus, + hfAuthPolling, + startHuggingFaceLogin, + checkError = null, + onRetryCheck, + className, +}: HfModelAccessGateProps) { + if (allAuthorized) return null + + if (checkError) { + return ( +
+
+ + Couldn't verify Hugging Face access: {checkError} +
+ {onRetryCheck && ( + + )} +
+ ) + } + + const unauthorizedRepos = Object.entries(accessMap).filter(([, status]) => status === 'not_authorized') + if (unauthorizedRepos.length === 0) return null + + if (hfAuthStatus !== 'authenticated') { + return ( +
+
+ + + This model is gated on Hugging Face. Sign in, then accept the license to download. + +
+ +
+ ) + } + + return ( +
+

+ Accept the Hugging Face license for this model, then download. +

+ {unauthorizedRepos.map(([repoId]) => ( +
+ {repoId} + +
+ ))} +
+ ) +} diff --git a/frontend/components/ICLoraPanel.tsx b/frontend/components/ICLoraPanel.tsx index a514979df..69cabc333 100644 --- a/frontend/components/ICLoraPanel.tsx +++ b/frontend/components/ICLoraPanel.tsx @@ -111,6 +111,7 @@ export function ICLoraPanel({ const [isExtracting, setIsExtracting] = useState(false) const [requiredIcLoraCpIds, setRequiredIcLoraCpIds] = useState([]) + const [icLoraSupported, setIcLoraSupported] = useState(true) const [isCheckingIcLora, setIsCheckingIcLora] = useState(false) const [isDownloadingIcLora, setIsDownloadingIcLora] = useState(false) const [downloadProgress, setDownloadProgress] = useState(null) @@ -118,7 +119,7 @@ export function ICLoraPanel({ const [downloadSessionId, setDownloadSessionId] = useState(null) const [extractError, setExtractError] = useState(null) const [isDragOver, setIsDragOver] = useState(false) - const icLoraReady = requiredIcLoraCpIds.length === 0 + const icLoraReady = icLoraSupported && requiredIcLoraCpIds.length === 0 // Switching to an entry with a different input.kind (image vs video) invalidates the loaded // input — clear it so a stale video can't be submitted to an image-input entry (backend 400). @@ -174,7 +175,8 @@ export function ICLoraPanel({ const recommendationPayload = result.data setRequiredIcLoraCpIds(recommendationPayload.cps_to_download) - const isReady = recommendationPayload.cps_to_download.length === 0 + setIcLoraSupported(recommendationPayload.supported) + const isReady = recommendationPayload.supported && recommendationPayload.cps_to_download.length === 0 if (isReady) { setIsDownloadingIcLora(false) @@ -388,7 +390,10 @@ export function ICLoraPanel({ // Only the built-in canny/depth flow needs the bundled preprocessing cps. Recipes // (any input kind) and custom build their own control video, so never gate them. - const showDownloadGate = !isCustom && !isImage && !isCatalogIcLora && (isCheckingIcLora || !icLoraReady) + const showBuiltinGate = + !isCustom && !isImage && !isCatalogIcLora && (isCheckingIcLora || !icLoraReady) + const showUnsupportedGate = showBuiltinGate && !isCheckingIcLora && !icLoraSupported + const showDownloadGate = showBuiltinGate && !showUnsupportedGate const runningDownloadProgress = downloadProgress?.status === 'downloading' ? downloadProgress : null const gateItemIds = [...new Set([...(requiredIcLoraCpIds ?? []), ...(runningDownloadProgress?.all_files ?? [])])] @@ -439,7 +444,24 @@ export function ICLoraPanel({ )} - {showDownloadGate ? ( + {showUnsupportedGate ? ( +
+
+
+
+ +
+
+

Built-in control needs LTX 2.3

+

+ Depth and canny Union Control are not available on the active LTX 2.5 model. + Switch to an LTX 2.3 local model in Settings, or use a custom / catalog IC-LoRA. +

+
+
+
+
+ ) : showDownloadGate ? (
diff --git a/frontend/components/LtxUpgradePrompt.tsx b/frontend/components/LtxUpgradePrompt.tsx index df5beafd6..633665187 100644 --- a/frontend/components/LtxUpgradePrompt.tsx +++ b/frontend/components/LtxUpgradePrompt.tsx @@ -1,11 +1,17 @@ -import { useCallback, useEffect, useState } from 'react' +import { useCallback, useEffect, useMemo, useState } from 'react' import { AlertCircle, Download, Loader2, Sparkles, X } from 'lucide-react' -import { ApiClient, type ApiSuccessOf } from '../lib/api-client' +import { useHfAuth } from '../hooks/use-hf-auth' +import { useHfModelAccess } from '../hooks/use-hf-model-access' +import { ApiClient, type ApiRequestBodyOf, type ApiSuccessOf } from '../lib/api-client' import { logger } from '../lib/logger' +import { HfModelAccessGate } from './HfModelAccessGate' import { Button } from './ui/button' import './LtxUpgradePrompt.css' type UpgradeRecommendation = Extract, { status: 'upgrade' }> +type ModelCheckpointID = NonNullable< + NonNullable>['cp_ids'] +>[number] interface LtxUpgradePromptProps { recommendation: UpgradeRecommendation @@ -34,15 +40,23 @@ export function LtxUpgradePrompt({ null, ) const [errorMessage, setErrorMessage] = useState(null) - // Default to reclaiming disk (base models are tens of GB); unchecking keeps the old - // version installed so it stays selectable in the Settings version manager. - const [deleteOld, setDeleteOld] = useState(true) + // Default off when deleting the old bundle would also wipe built-in Union Control IC-LoRA + // (2.3 → 2.5). Otherwise default on to reclaim the tens of GB the old transformer uses. + const [deleteOld, setDeleteOld] = useState(!recommendation.loses_built_in_control) + + const cpsToDownload = useMemo( + () => recommendation.cps_to_download as ModelCheckpointID[], + [recommendation.cps_to_download], + ) + const { hfAuthStatus, hfAuthPolling, startHuggingFaceLogin } = useHfAuth(true) + const { accessMap, allAuthorized, checking: checkingAccess, checkError, recheckAccess } = useHfModelAccess( + cpsToDownload, + hfAuthStatus, + ) const hasOldToDelete = recommendation.cps_to_delete.length > 0 const canClose = phase === 'idle' - // LTX base-model checkpoints are public on Hugging Face, so the upgrade download needs no - // sign-in or model-access acceptance — once the user opts in, it can start immediately. - const canStartUpgrade = wantsUpgrade && phase === 'idle' + const canStartUpgrade = wantsUpgrade && phase === 'idle' && allAuthorized && !checkingAccess const runningProgress = downloadProgress?.status === 'downloading' ? downloadProgress : null const totalProgress = runningProgress?.total_progress ?? (phase === 'finishing' ? 100 : 0) @@ -222,7 +236,9 @@ export function LtxUpgradePrompt({ {hasOldToDelete && (

{deleteOld - ? 'Your previous checkpoint will be removed from disk once the download completes.' + ? recommendation.loses_built_in_control + ? 'Your previous checkpoint and its built-in depth/canny/pose control models will be removed from disk.' + : 'Your previous checkpoint will be removed from disk once the download completes.' : 'Your previous checkpoint will be kept — switch between versions anytime in Settings → Models.'}

)} @@ -247,13 +263,32 @@ export function LtxUpgradePrompt({ disabled={!canClose} className="h-4 w-4 rounded border-blue-300/40 bg-slate-950 text-blue-500 focus:ring-blue-400" /> - Delete the previous checkpoint to free up disk space + + {recommendation.loses_built_in_control + ? 'Delete the previous checkpoint (also removes built-in control models)' + : 'Delete the previous checkpoint to free up disk space'} + )}
{wantsUpgrade && (
+ {!allAuthorized && ( +
+ { + void startHuggingFaceLogin() + }} + checkError={checkError} + onRetryCheck={recheckAccess} + /> +
+ )} {canStartUpgrade && (
diff --git a/frontend/components/SettingsModal.tsx b/frontend/components/SettingsModal.tsx index 39de26c5b..403473383 100644 --- a/frontend/components/SettingsModal.tsx +++ b/frontend/components/SettingsModal.tsx @@ -6,6 +6,7 @@ import { useAppSettings, type AppSettings } from '../contexts/AppSettingsContext import { ApiClient, type ApiSuccessOf } from '../lib/api-client' import { logger } from '../lib/logger' import { ApiKeyHelperRow, LtxApiKeyInput, LtxApiKeyHelperRow } from './LtxApiKeyInput' +import { HfModelAccessGate } from './HfModelAccessGate' import { useHfAuth } from '../hooks/use-hf-auth' import { useHfModelAccess } from '../hooks/use-hf-model-access' @@ -17,6 +18,9 @@ interface SettingsModalProps { type TabId = 'general' | 'models' | 'apiKeys' | 'promptEnhancer' | 'about' +/** A checkpoint this modal can download: the text encoder or the optional prompt enhancer. */ +type TextEncodingCp = NonNullable['cp_to_download']> + /** Focuses an API Keys tab input once the modal has switched to that tab. * Shared by the LTX and FAL key inputs — each call gets its own ref/pending state. */ function useApiKeyFocus(isOpen: boolean, activeTab: TabId, setActiveTab: (tab: TabId) => void) { @@ -93,7 +97,7 @@ function SettingToggle({ title, description, enabled, onToggle, statusOn, status } export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProps) { - const { settings, updateSettings, saveLtxApiKey, saveFalApiKey, saveGeminiApiKey, forceApiGenerations, cudaAvailable } = useAppSettings() + const { settings, updateSettings, saveLtxApiKey, saveFalApiKey, saveGeminiApiKey, forceApiGenerations, cudaAvailable, notifyModelsChanged } = useAppSettings() const onSettingsChange = (next: AppSettings) => updateSettings(next) const [activeTab, setActiveTab] = useState('general') const ltxApiKey = useApiKeyFocus(isOpen, activeTab, setActiveTab) @@ -103,7 +107,9 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp const [geminiApiKeyInput, setGeminiApiKeyInput] = useState('') const geminiApiKeyInputRef = useRef(null) const [textEncoderRecommendation, setTextEncoderRecommendation] = useState | null>(null) - const [isDownloading, setIsDownloading] = useState(false) + // Which checkpoint is downloading, not just whether one is — the encoder and the optional + // prompt enhancer each have their own card and must show progress only on their own. + const [downloadingCp, setDownloadingCp] = useState(null) const [downloadError, setDownloadError] = useState(null) const [downloadSessionId, setDownloadSessionId] = useState(null) const [downloadProgress, setDownloadProgress] = useState | null>(null) @@ -114,7 +120,22 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp : [textEncoderRecommendation.cp_to_download]), [forceApiGenerations, textEncoderRecommendation?.cp_to_download], ) - const { accessMap: teAccessMap, allAuthorized: teAllAuthorized } = useHfModelAccess(textEncoderModelTypes, hfAuthStatus) + const { accessMap: teAccessMap, allAuthorized: teAllAuthorized, checkError: teCheckError, recheckAccess: recheckTeAccess } = useHfModelAccess(textEncoderModelTypes, hfAuthStatus) + const preferredEnhancerDownloaded = textEncoderRecommendation !== null + && textEncoderRecommendation.local_enhancer_cp !== null + && textEncoderRecommendation.active_local_enhancer_cp === textEncoderRecommendation.local_enhancer_cp + const enhancerCpToDownload = textEncoderRecommendation !== null + && textEncoderRecommendation.local_enhancer_cp !== null + && !preferredEnhancerDownloaded + ? textEncoderRecommendation.local_enhancer_cp + : null + const enhancerModelTypes = useMemo( + () => (enhancerCpToDownload === null ? [] : [enhancerCpToDownload]), + [enhancerCpToDownload], + ) + const { accessMap: enhancerAccessMap, allAuthorized: enhancerAllAuthorized, checkError: enhancerCheckError, recheckAccess: recheckEnhancerAccess } = useHfModelAccess(enhancerModelTypes, hfAuthStatus) + const apiEncodingSupported = textEncoderRecommendation?.api_encoding_supported ?? true + const localEncoderSelected = settings.useLocalTextEncoder || !apiEncodingSupported const [appVersion, setAppVersion] = useState('') const [noticesText, setNoticesText] = useState(null) const [noticesLoading, setNoticesLoading] = useState(false) @@ -170,9 +191,16 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp const data = result.data setTextEncoderRecommendation(data) - if (data.cp_to_download === null) { - setIsDownloading(false) - } + // A download that finished elsewhere (another surface, or before this modal opened) would + // otherwise leave its card stuck showing progress. + const stillPending = [ + data.cp_to_download, + data.local_enhancer_cp !== null + && data.active_local_enhancer_cp !== data.local_enhancer_cp + ? data.local_enhancer_cp + : null, + ] + setDownloadingCp((cp) => (cp !== null && !stillPending.includes(cp) ? null : cp)) } void fetchRecommendation() @@ -180,20 +208,23 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp // Poll download progress via session ID useEffect(() => { - if (!isDownloading || !downloadSessionId) return + if (downloadingCp === null || !downloadSessionId) return const poll = async () => { const result = await ApiClient.getModelDownloadProgress({ sessionId: downloadSessionId }) if (!result.ok) return setDownloadProgress(result.data) if (result.data.status === 'complete') { - setIsDownloading(false) + setDownloadingCp(null) setDownloadSessionId(null) const rec = await ApiClient.getTextEncoderRecommendation() if (rec.ok) setTextEncoderRecommendation(rec.data) + // Enhance reads local availability outside this modal, so it has to be told the set of + // installed checkpoints changed. + notifyModelsChanged() } else if (result.data.status === 'error') { setDownloadError(result.data.error ?? 'Download failed') - setIsDownloading(false) + setDownloadingCp(null) setDownloadSessionId(null) } } @@ -201,21 +232,16 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp void poll() const interval = setInterval(() => { void poll() }, 1000) return () => clearInterval(interval) - }, [isDownloading, downloadSessionId]) + }, [downloadingCp, downloadSessionId, notifyModelsChanged]) - // Handle text encoder download - const handleDownloadTextEncoder = async () => { - if (!textEncoderRecommendation?.cp_to_download) return - setIsDownloading(true) + const handleDownloadCheckpoint = async (cpId: TextEncodingCp) => { + setDownloadingCp(cpId) setDownloadError(null) setDownloadProgress(null) - const result = await ApiClient.startModelDownload({ - type: 'download', - cp_ids: [textEncoderRecommendation.cp_to_download], - }) + const result = await ApiClient.startModelDownload({ type: 'download', cp_ids: [cpId] }) if (!result.ok) { setDownloadError(result.error.message) - setIsDownloading(false) + setDownloadingCp(null) return } if (result.data.status === 'started') { @@ -239,6 +265,13 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp }) } + const handleToggleFastDecode = () => { + onSettingsChange({ + ...settings, + useConvVae: !settings.useConvVae, + }) + } + const handleToggleLocalEncoder = () => { onSettingsChange({ ...settings, @@ -522,10 +555,13 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp {/* LTX API Option (Default) */}
{ + if (!apiEncodingSupported) return if (!settings.useLocalTextEncoder) return if (!settings.hasLtxApiKey) { ltxApiKey.openAndFocus() @@ -539,21 +575,35 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp
LTX API - Recommended + {apiEncodingSupported ? ( + Recommended + ) : ( + Unavailable + )}

Fast cloud-based text encoding (~1 second). Requires an LTX API key configured in the API Keys tab.

- {!settings.useLocalTextEncoder && } + {!localEncoderSelected && }
+ {!apiEncodingSupported && ( +
+ + + Not available for LTX {textEncoderRecommendation?.ltx_version_label ?? ''} — prompts for this + version can only be encoded by the local encoder. + +
+ )} + {/* Warning when selected but no key */} - {!settings.useLocalTextEncoder && !settings.hasLtxApiKey && ( + {apiEncodingSupported && !settings.useLocalTextEncoder && !settings.hasLtxApiKey && (
API key required — configure it in the API Keys tab. @@ -561,7 +611,7 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp )} {/* Prompt Cache Size — only relevant for API text encoding */} - {!settings.useLocalTextEncoder && settings.hasLtxApiKey && ( + {!localEncoderSelected && settings.hasLtxApiKey && (
@@ -583,7 +633,7 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp {/* Local Encoder Option */}
!settings.useLocalTextEncoder && handleToggleLocalEncoder()} > @@ -595,27 +645,31 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp Local Encoder + {!apiEncodingSupported && ( + Required + )}

- Run on your computer (~23 seconds). Requires 25 GB download. + Run on your computer (slower than the API). Requires{' '} + {textEncoderRecommendation?.expected_size_gb ?? '~25'} GB download.

- {settings.useLocalTextEncoder && } + {localEncoderSelected && }
{/* Download Status - show when this option is selected */} - {settings.useLocalTextEncoder && ( + {localEncoderSelected && (
{textEncoderRecommendation?.cp_to_download === null ? (
Downloaded ({textEncoderRecommendation?.expected_size_gb ?? 0} GB)
- ) : isDownloading ? ( + ) : downloadingCp === textEncoderRecommendation?.cp_to_download ? (
Downloading text encoder... @@ -631,28 +685,24 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp Not downloaded ({textEncoderRecommendation?.expected_size_gb || 0} GB required)
- {hfAuthStatus === 'authenticated' && !teAllAuthorized && Object.keys(teAccessMap).length > 0 && ( -
- {Object.entries(teAccessMap) - .filter(([, status]) => status === 'not_authorized') - .map(([repoId]) => ( -
- {repoId} - -
- ))} -
- )} + { + void startHuggingFaceLogin() + }} + checkError={teCheckError} + onRetryCheck={recheckTeAccess} + className="space-y-1.5 mb-2" + />
)}
+ + {/* Optional local prompt enhancer — only for models whose encoder can't generate */} + {textEncoderRecommendation?.local_enhancer_cp && ( +
+
+ + Local Prompt Enhancer + Optional +
+

+ LTX {textEncoderRecommendation.ltx_version_label}'s text encoder can only encode + prompts, so enhancing them on your computer needs a separate instruct model. + Gemma 3 already downloaded for 2.3 works; Gemma 4 E2B is the smaller optional + upgrade. Without either, the Enhance button can still use Gemini if you have a + key, and Generate uses the prompt as typed. +

+ +
+ {preferredEnhancerDownloaded ? ( +
+ + Downloaded ({textEncoderRecommendation.local_enhancer_expected_size_gb ?? 0} GB) +
+ ) : downloadingCp === textEncoderRecommendation.local_enhancer_cp ? ( +
+
+ Downloading prompt enhancer... + {downloadProgress?.status === 'downloading' ? Math.round(downloadProgress.current_file_progress) : 0}% +
+
+
+
+
+ ) : ( +
+ {textEncoderRecommendation.local_enhancement_supported && ( +
+ + Using Gemma 3 already on disk +
+ )} + { + void startHuggingFaceLogin() + }} + checkError={enhancerCheckError} + onRetryCheck={recheckEnhancerAccess} + className="space-y-1.5 mb-2" + /> + + {downloadError && ( +

{downloadError}

+ )} +
+ )} +
+
+ )}
)} + {/* Fast decode — all platforms. Swaps the 2.5 video VAE; takes effect on next load. */} + + {/* Torch Compile + Diffusion Stage Cache -- CUDA only, no-op on MPS/CPU */} {cudaAvailable && (

- Sign in to HuggingFace to download model files. + Sign in to download gated models (such as LTX 2.5) and accept Hugging Face licenses.

@@ -1061,80 +1196,66 @@ export function SettingsModal({ isOpen, onClose, initialTab }: SettingsModalProp

- Automatically enhances your prompts via the LTX API with rich visual details, sound descriptions, - and motion cues to help generate higher quality videos. Control independently for each generation type. + When enabled, Generate rewrites your prompt with visual detail, sound, and camera + motion before the model sees it. Local generations use the on-device enhancer; + LTX API text encoding enhances on the server. The Enhance button in Gen Space is + separate — it rewrites the prompt box so you can edit it first. Control + independently for each generation type.

- {!settings.hasLtxApiKey ? ( -
-
-
- -
-

LTX API key required

-

- Prompt enhancement runs server-side on the LTX API. To use this feature, you need to configure - an API key in the API Keys tab. -

-
-
- + {!settings.hasLtxApiKey && ( +

+ An LTX API key is only needed when text encoding goes through the LTX API. + Local generations use the local enhancer instead (download it under Models + if this version ships one separately). +

+ )} + + {/* T2V Toggle */} +
handleTogglePromptEnhancer('t2v')} + > +
+ T2V +
+ Text-to-Video +

+ {settings.promptEnhancerEnabledT2V ? 'Prompts will be enhanced before T2V generation' : 'T2V prompts used as-is'} +

- ) : ( - <> - {/* T2V Toggle */} -
handleTogglePromptEnhancer('t2v')} - > -
- T2V -
- Text-to-Video -

- {settings.promptEnhancerEnabledT2V ? 'Prompts will be enhanced before T2V generation' : 'T2V prompts used as-is'} -

-
-
-
-
-
-
+
+
+
+
- {/* I2V Toggle */} -
handleTogglePromptEnhancer('i2v')} - > -
- I2V -
- Image-to-Video -

- {settings.promptEnhancerEnabledI2V ? 'Prompts will be enhanced before I2V generation' : 'I2V prompts used as-is'} -

-
-
-
-
-
+ {/* I2V Toggle */} +
handleTogglePromptEnhancer('i2v')} + > +
+ I2V +
+ Image-to-Video +

+ {settings.promptEnhancerEnabledI2V ? 'Prompts will be enhanced before I2V generation' : 'I2V prompts used as-is'} +

- - )} +
+
+
+
+
)} diff --git a/frontend/components/SettingsPanel.tsx b/frontend/components/SettingsPanel.tsx index c82bace60..a446615b9 100644 --- a/frontend/components/SettingsPanel.tsx +++ b/frontend/components/SettingsPanel.tsx @@ -5,6 +5,7 @@ import { resolveVideoGenerationOptions, sanitizeVideoGenerationSettings, type VideoGenerationModelSpecItem, + type VideoGenerationPipeline, } from '../lib/video-generation-model-specs' export type GenerationMode = 'text-to-video' | 'image-to-video' | 'text-to-image' @@ -20,8 +21,8 @@ export interface LoraSelection { } export interface GenerationSettings { - model: 'fast' | 'pro' - duration: number + model: VideoGenerationPipeline + duration: number | null videoResolution: string fps: number audio: boolean @@ -84,7 +85,7 @@ export function SettingsPanel({ } }, [hasAudio, hideDuration, isImageMode, minimumDuration, onSettingsChange, settings, videoModelSpecs]) - const handleChange = (key: keyof GenerationSettings, value: string | number | boolean) => { + const handleChange = (key: keyof GenerationSettings, value: string | number | boolean | null) => { if (isImageMode) { onSettingsChange({ ...settings, [key]: value } as GenerationSettings) return @@ -171,10 +172,17 @@ export function SettingsPanel({ {!hideDuration && ( void handleDownload()} - disabled={busy} + disabled={busy || !canDownload} className="bg-blue-600 hover:bg-blue-500 text-white text-xs" > @@ -181,8 +195,20 @@ function VersionRow({
+ {!version.installed && ( + + )} + {error && ( -
+
{error}
@@ -194,9 +220,10 @@ function VersionRow({ export function BaseModelSection() { const [versions, setVersions] = useState([]) const [modelsDir, setModelsDir] = useState('') - // A download keeps running in the backend even if this modal is closed. Track the active - // session so a row can reattach to its progress on remount instead of resetting to "Download". const [activeDownload, setActiveDownload] = useState<{ sessionId: string; cpIds: string[] } | null>(null) + const { hfAuthStatus, hfAuthPolling, startHuggingFaceLogin } = useHfAuth(true) + const { notifyModelsChanged } = useAppSettings() + const knownActiveRef = useRef(null) const refreshVersions = useCallback(async () => { const [versionsResult, activeResult] = await Promise.all([ @@ -208,6 +235,13 @@ export function BaseModelSection() { return } setVersions(versionsResult.data.versions) + // Signal only on a real change so mounting the panel doesn't refetch generation specs. + const nextActive = versionsResult.data.versions.find((item) => item.active)?.model_id ?? null + const nextKey = `${nextActive}|${versionsResult.data.versions.filter((item) => item.installed).map((item) => item.model_id).join(',')}` + if (knownActiveRef.current !== null && knownActiveRef.current !== nextKey) { + notifyModelsChanged() + } + knownActiveRef.current = nextKey if (activeResult.ok) { setActiveDownload( activeResult.data.session_id @@ -215,7 +249,7 @@ export function BaseModelSection() { : null, ) } - }, []) + }, [notifyModelsChanged]) useEffect(() => { void refreshVersions() @@ -231,7 +265,6 @@ export function BaseModelSection() { return ( <> - {/* Models Folder */}
@@ -269,7 +302,6 @@ export function BaseModelSection() {
- {/* Base Model Versions */}
@@ -277,7 +309,7 @@ export function BaseModelSection() {

The active version is used for new generations. Download a version to make it available, - then set it active. + then set it active. Newer versions may require a Hugging Face sign-in.

{versions.length === 0 ? ( @@ -293,6 +325,11 @@ export function BaseModelSection() { ? activeDownload.sessionId : null } + hfAuthStatus={hfAuthStatus} + hfAuthPolling={hfAuthPolling} + startHuggingFaceLogin={() => { + void startHuggingFaceLogin() + }} /> )) )} diff --git a/frontend/contexts/AppSettingsContext.tsx b/frontend/contexts/AppSettingsContext.tsx index 6833b62ac..b0bf24df0 100644 --- a/frontend/contexts/AppSettingsContext.tsx +++ b/frontend/contexts/AppSettingsContext.tsx @@ -22,6 +22,7 @@ export interface AppSettings { seedLocked: boolean lockedSeed: number modelsDir: string + useConvVae: boolean } export const DEFAULT_APP_SETTINGS: AppSettings = { @@ -40,6 +41,7 @@ export const DEFAULT_APP_SETTINGS: AppSettings = { seedLocked: false, lockedSeed: 42, modelsDir: '', + useConvVae: false, } type BackendProcessStatus = 'alive' | 'restarting' | 'dead' @@ -57,6 +59,11 @@ interface AppSettingsContextValue { shouldVideoGenerateWithLtxApi: boolean shouldImageGenerateWithFalApi: boolean cudaAvailable: boolean + // Bumped whenever installed models change (download / delete / activate a version). Generation + // model specs are derived from the *active* local model, so anything reading them must refetch; + // without this they stay pinned to whatever was installed at app start. + modelsVersion: number + notifyModelsChanged: () => void } const AppSettingsContext = createContext(null) @@ -90,6 +97,7 @@ function normalizeAppSettings(data: Partial): AppSettings { seedLocked: data.seedLocked ?? DEFAULT_APP_SETTINGS.seedLocked, lockedSeed: data.lockedSeed ?? DEFAULT_APP_SETTINGS.lockedSeed, modelsDir: data.modelsDir ?? DEFAULT_APP_SETTINGS.modelsDir, + useConvVae: data.useConvVae ?? DEFAULT_APP_SETTINGS.useConvVae, } } @@ -103,6 +111,11 @@ export function AppSettingsProvider({ children }: { children: ReactNode }) { const [forceApiGenerations, setForceApiGenerations] = useState(true) const [cudaAvailable, setCudaAvailable] = useState(false) const [backendProcessStatus, setBackendProcessStatus] = useState(null) + const [modelsVersion, setModelsVersion] = useState(0) + + const notifyModelsChanged = useCallback(() => { + setModelsVersion((current) => current + 1) + }, []) useEffect(() => { if (backendProcessStatus !== 'alive') return @@ -160,7 +173,7 @@ export function AppSettingsProvider({ children }: { children: ReactNode }) { return () => { cancelled = true } - }, [backendProcessStatus]) + }, [backendProcessStatus, modelsVersion]) useEffect(() => { let cancelled = false @@ -291,8 +304,10 @@ export function AppSettingsProvider({ children }: { children: ReactNode }) { shouldVideoGenerateWithLtxApi, shouldImageGenerateWithFalApi, cudaAvailable, + modelsVersion, + notifyModelsChanged, }), - [cudaAvailable, forceApiGenerations, isLoaded, refreshSettings, runtimePolicyLoaded, saveFalApiKey, saveGeminiApiKey, saveLtxApiKey, settings, shouldVideoGenerateWithLtxApi, shouldImageGenerateWithFalApi, updateSettings], + [cudaAvailable, forceApiGenerations, isLoaded, modelsVersion, notifyModelsChanged, refreshSettings, runtimePolicyLoaded, saveFalApiKey, saveGeminiApiKey, saveLtxApiKey, settings, shouldVideoGenerateWithLtxApi, shouldImageGenerateWithFalApi, updateSettings], ) return {children} diff --git a/frontend/generated/backend-openapi.json b/frontend/generated/backend-openapi.json index f439beab7..d16b14378 100644 --- a/frontend/generated/backend-openapi.json +++ b/frontend/generated/backend-openapi.json @@ -11,10 +11,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -48,6 +56,7 @@ "anyOf": [ { "enum": [ + "ltx-2.5-22b-distilled", "ltx-2.3-22b-distilled-1.1", "ltx-2.3-22b-distilled" ], @@ -184,6 +193,17 @@ ], "title": "Seedlocked" }, + "useConvVae": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Useconvvae" + }, "useLocalTextEncoder": { "anyOf": [ { @@ -326,10 +346,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -371,10 +399,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "title": "Cp Id", @@ -393,6 +429,7 @@ "base", "upscaler", "text_encoder", + "vae", "image", "support" ], @@ -424,10 +461,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -501,10 +546,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -521,10 +574,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -542,10 +603,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -757,6 +826,12 @@ "title": "Mode", "type": "string" }, + "model": { + "const": "pro", + "default": "pro", + "title": "Model", + "type": "string" + }, "prompt": { "default": "", "title": "Prompt", @@ -977,20 +1052,27 @@ "type": "string" }, "duration": { - "default": 5, - "enum": [ - 5, - 6, - 8, - 10, - 12, - 14, - 16, - 18, - 20 + "anyOf": [ + { + "enum": [ + 5, + 6, + 8, + 10, + 12, + 14, + 16, + 18, + 20 + ], + "type": "integer" + }, + { + "type": "null" + } ], - "title": "Duration", - "type": "integer" + "default": 5, + "title": "Duration" }, "fps": { "default": 24, @@ -1025,7 +1107,9 @@ "default": "fast", "enum": [ "fast", - "pro" + "pro", + "fast-2.5", + "pro-2.5" ], "title": "Model", "type": "string" @@ -1508,6 +1592,17 @@ "title": "Requires Hf Login", "type": "boolean" }, + "supported_models": { + "items": { + "enum": [ + "LTX-2.3", + "LTX-2.5" + ], + "type": "string" + }, + "title": "Supported Models", + "type": "array" + }, "tags": { "items": { "type": "string" @@ -2165,10 +2260,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -2309,12 +2412,68 @@ "type": "object" }, "JsonValue": {}, + "LTXOfferingCapabilitiesSpec": { + "description": "Feature flags for one local model or API pipeline. Pixel maps stay backend-only.", + "properties": { + "a2v": { + "title": "A2V", + "type": "boolean" + }, + "auto_duration": { + "title": "Auto Duration", + "type": "boolean" + }, + "camera_motion": { + "title": "Camera Motion", + "type": "boolean" + }, + "extend": { + "title": "Extend", + "type": "boolean" + }, + "i2v": { + "title": "I2V", + "type": "boolean" + }, + "ic_lora": { + "title": "Ic Lora", + "type": "boolean" + }, + "retake": { + "title": "Retake", + "type": "boolean" + }, + "t2v": { + "title": "T2V", + "type": "boolean" + }, + "user_loras": { + "title": "User Loras", + "type": "boolean" + } + }, + "required": [ + "t2v", + "i2v", + "a2v", + "ic_lora", + "retake", + "extend", + "user_loras", + "camera_motion", + "auto_duration" + ], + "title": "LTXOfferingCapabilitiesSpec", + "type": "object" + }, "LTXVideoGenerationModelSpecItem": { "properties": { "pipeline": { "enum": [ "fast", - "pro" + "pro", + "fast-2.5", + "pro-2.5" ], "title": "Pipeline", "type": "string" @@ -2385,6 +2544,16 @@ ], "title": "A2V Supported Resolutions Durations" }, + "capabilities": { + "anyOf": [ + { + "$ref": "#/components/schemas/LTXOfferingCapabilitiesSpec" + }, + { + "type": "null" + } + ] + }, "display_name": { "title": "Display Name", "type": "string" @@ -2561,6 +2730,17 @@ "title": "Requires Hf Login", "type": "boolean" }, + "supported_models": { + "items": { + "enum": [ + "LTX-2.3", + "LTX-2.5" + ], + "type": "string" + }, + "title": "Supported Models", + "type": "array" + }, "tags": { "items": { "type": "string" @@ -2764,10 +2944,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -2775,6 +2963,34 @@ "title": "Cps To Download", "type": "array" }, + "optional_cp_ids": { + "default": [], + "items": { + "enum": [ + "ltx-2.3-22b-distilled", + "ltx-2.3-22b-distilled-1.1", + "ltx-2.3-spatial-upscaler-x2-1.0", + "ltx-2.3-spatial-upscaler-x2-1.1", + "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", + "dpt-hybrid-midas", + "yolox-l-torchscript", + "dw-ll-ucoco-384-bs5", + "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", + "z-image-turbo" + ], + "type": "string" + }, + "title": "Optional Cp Ids", + "type": "array" + }, "status": { "const": "download", "title": "Status", @@ -2798,16 +3014,29 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" }, "title": "Cps To Download", "type": "array" + }, + "supported": { + "default": true, + "title": "Supported", + "type": "boolean" } }, "required": [ @@ -2849,10 +3078,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -2879,10 +3116,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "title": "Model Cp", @@ -2890,6 +3135,7 @@ }, "model_id": { "enum": [ + "ltx-2.5-22b-distilled", "ltx-2.3-22b-distilled-1.1", "ltx-2.3-22b-distilled" ], @@ -2954,10 +3200,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -2973,10 +3227,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -2984,8 +3246,14 @@ "title": "Cps To Download", "type": "array" }, + "loses_built_in_control": { + "default": false, + "title": "Loses Built In Control", + "type": "boolean" + }, "ltx_model_id": { "enum": [ + "ltx-2.5-22b-distilled", "ltx-2.3-22b-distilled-1.1", "ltx-2.3-22b-distilled" ], @@ -3056,10 +3324,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -3082,10 +3358,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -3350,6 +3634,12 @@ "title": "Mode", "type": "string" }, + "model": { + "const": "pro", + "default": "pro", + "title": "Model", + "type": "string" + }, "prompt": { "default": "", "title": "Prompt", @@ -3418,6 +3708,7 @@ "properties": { "model_id": { "enum": [ + "ltx-2.5-22b-distilled", "ltx-2.3-22b-distilled-1.1", "ltx-2.3-22b-distilled" ], @@ -3437,6 +3728,7 @@ "anyOf": [ { "enum": [ + "ltx-2.5-22b-distilled", "ltx-2.3-22b-distilled-1.1", "ltx-2.3-22b-distilled" ], @@ -3513,6 +3805,11 @@ "title": "Seedlocked", "type": "boolean" }, + "useConvVae": { + "default": false, + "title": "Useconvvae", + "type": "boolean" + }, "useLocalTextEncoder": { "default": false, "title": "Uselocaltextencoder", @@ -3656,6 +3953,41 @@ }, "TextEncoderRecommendationResponse": { "properties": { + "active_local_enhancer_cp": { + "anyOf": [ + { + "enum": [ + "ltx-2.3-22b-distilled", + "ltx-2.3-22b-distilled-1.1", + "ltx-2.3-spatial-upscaler-x2-1.0", + "ltx-2.3-spatial-upscaler-x2-1.1", + "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", + "dpt-hybrid-midas", + "yolox-l-torchscript", + "dw-ll-ucoco-384-bs5", + "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", + "z-image-turbo" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Active Local Enhancer Cp" + }, + "api_encoding_supported": { + "title": "Api Encoding Supported", + "type": "boolean" + }, "cp_to_download": { "anyOf": [ { @@ -3665,10 +3997,18 @@ "ltx-2.3-spatial-upscaler-x2-1.0", "ltx-2.3-spatial-upscaler-x2-1.1", "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", "dpt-hybrid-midas", "yolox-l-torchscript", "dw-ll-ucoco-384-bs5", "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", "z-image-turbo" ], "type": "string" @@ -3686,12 +4026,68 @@ "expected_size_gb": { "title": "Expected Size Gb", "type": "number" + }, + "local_enhancement_supported": { + "title": "Local Enhancement Supported", + "type": "boolean" + }, + "local_enhancer_cp": { + "anyOf": [ + { + "enum": [ + "ltx-2.3-22b-distilled", + "ltx-2.3-22b-distilled-1.1", + "ltx-2.3-spatial-upscaler-x2-1.0", + "ltx-2.3-spatial-upscaler-x2-1.1", + "ltx-2.3-22b-ic-lora-union-control-ref0.5", + "ltx-2.5-22b-distilled", + "ltx-2.5-spatial-upscaler-x2-1.0", + "ltx-2.5-video-vae", + "ltx-2.5-video-vae-conv", + "ltx-2.5-audio-vae", + "ltx-2.5-duration-head", + "dpt-hybrid-midas", + "yolox-l-torchscript", + "dw-ll-ucoco-384-bs5", + "gemma-3-12b-it-qat-q4_0-unquantized", + "gemma4-12b-with-proj-ltx-2.5", + "gemma-4-e2b-it", + "z-image-turbo" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Local Enhancer Cp" + }, + "local_enhancer_expected_size_gb": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Local Enhancer Expected Size Gb" + }, + "ltx_version_label": { + "title": "Ltx Version Label", + "type": "string" } }, "required": [ "cp_to_download", "expected_size_bytes", - "expected_size_gb" + "expected_size_gb", + "api_encoding_supported", + "ltx_version_label", + "local_enhancement_supported", + "local_enhancer_cp", + "local_enhancer_expected_size_gb", + "active_local_enhancer_cp" ], "title": "TextEncoderRecommendationResponse", "type": "object" diff --git a/frontend/generated/backend-openapi.ts b/frontend/generated/backend-openapi.ts index 1c29d9fbb..84eec01ab 100644 --- a/frontend/generated/backend-openapi.ts +++ b/frontend/generated/backend-openapi.ts @@ -714,14 +714,14 @@ export interface components { /** ActiveDownloadResponse */ ActiveDownloadResponse: { /** Cp Ids */ - cp_ids: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo")[]; + cp_ids: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo")[]; /** Session Id */ session_id: string | null; }; /** AppSettingsPatch */ AppSettingsPatch: { /** Activeltxmodelid */ - activeLtxModelId?: ("ltx-2.3-22b-distilled-1.1" | "ltx-2.3-22b-distilled") | null; + activeLtxModelId?: ("ltx-2.5-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-22b-distilled") | null; /** Diffusionstagecacheenabled */ diffusionStageCacheEnabled?: boolean | null; /** Falapikey */ @@ -744,6 +744,8 @@ export interface components { promptEnhancerProviderPreference?: ("local" | "api") | null; /** Seedlocked */ seedLocked?: boolean | null; + /** Useconvvae */ + useConvVae?: boolean | null; /** Uselocaltextencoder */ useLocalTextEncoder?: boolean | null; /** Usetorchcompile */ @@ -794,7 +796,7 @@ export interface components { /** CheckModelAccessRequest */ CheckModelAccessRequest: { /** Cp Ids */ - cp_ids?: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo")[]; + cp_ids?: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo")[]; }; /** CheckModelAccessResponse */ CheckModelAccessResponse: { @@ -809,7 +811,7 @@ export interface components { * Cp Id * @enum {string} */ - cp_id: "ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo"; + cp_id: "ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo"; /** Downloaded */ downloaded: boolean; /** Name */ @@ -818,14 +820,14 @@ export interface components { * Role * @enum {string} */ - role: "base" | "upscaler" | "text_encoder" | "image" | "support"; + role: "base" | "upscaler" | "text_encoder" | "vae" | "image" | "support"; /** Size Bytes */ size_bytes: number; }; /** DescribeCheckpointsRequest */ DescribeCheckpointsRequest: { /** Cp Ids */ - cp_ids: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo")[]; + cp_ids: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo")[]; }; /** DescribeCheckpointsResponse */ DescribeCheckpointsResponse: { @@ -853,11 +855,11 @@ export interface components { /** DownloadProgressRunningResponse */ DownloadProgressRunningResponse: { /** All Files */ - all_files: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo")[]; + all_files: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo")[]; /** Completed Files */ - completed_files: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo")[]; + completed_files: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo")[]; /** Current Downloading File */ - current_downloading_file: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo") | null; + current_downloading_file: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo") | null; /** Current File Progress */ current_file_progress: number; /** Error */ @@ -948,6 +950,12 @@ export interface components { * @enum {string} */ mode: "start" | "end"; + /** + * Model + * @default pro + * @constant + */ + model: "pro"; /** * Prompt * @default @@ -1056,9 +1064,8 @@ export interface components { /** * Duration * @default 5 - * @enum {integer} */ - duration: 5 | 6 | 8 | 10 | 12 | 14 | 16 | 18 | 20; + duration: (5 | 6 | 8 | 10 | 12 | 14 | 16 | 18 | 20) | null; /** * Fps * @default 24 @@ -1074,7 +1081,7 @@ export interface components { * @default fast * @enum {string} */ - model: "fast" | "pro"; + model: "fast" | "pro" | "fast-2.5" | "pro-2.5"; /** * Negativeprompt * @default @@ -1243,6 +1250,8 @@ export interface components { recommended_strength?: number | null; /** Requires Hf Login */ requires_hf_login: boolean; + /** Supported Models */ + supported_models?: ("LTX-2.3" | "LTX-2.5")[]; /** Tags */ tags?: string[]; /** Trigger */ @@ -1492,7 +1501,7 @@ export interface components { /** ImageGenRecommendationResponse */ ImageGenRecommendationResponse: { /** Cp To Download */ - cp_to_download: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo") | null; + cp_to_download: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo") | null; }; /** InputSpec */ InputSpec: { @@ -1543,13 +1552,37 @@ export interface components { title: "What it does" | "Input" | "Prompt" | "Tips" | "Notes"; }; JsonValue: unknown; + /** + * LTXOfferingCapabilitiesSpec + * @description Feature flags for one local model or API pipeline. Pixel maps stay backend-only. + */ + LTXOfferingCapabilitiesSpec: { + /** A2V */ + a2v: boolean; + /** Auto Duration */ + auto_duration: boolean; + /** Camera Motion */ + camera_motion: boolean; + /** Extend */ + extend: boolean; + /** I2V */ + i2v: boolean; + /** Ic Lora */ + ic_lora: boolean; + /** Retake */ + retake: boolean; + /** T2V */ + t2v: boolean; + /** User Loras */ + user_loras: boolean; + }; /** LTXVideoGenerationModelSpecItem */ LTXVideoGenerationModelSpecItem: { /** * Pipeline * @enum {string} */ - pipeline: "fast" | "pro"; + pipeline: "fast" | "pro" | "fast-2.5" | "pro-2.5"; spec: components["schemas"]["LTXVideoGenerationSpec"]; }; /** LTXVideoGenerationResolutionSpec */ @@ -1565,6 +1598,7 @@ export interface components { a2v_supported_resolutions_durations?: { [key: string]: components["schemas"]["LTXVideoGenerationResolutionSpec"]; } | null; + capabilities?: components["schemas"]["LTXOfferingCapabilitiesSpec"] | null; /** Display Name */ display_name: string; /** Supported Resolutions Durations */ @@ -1613,6 +1647,8 @@ export interface components { recommended_strength?: number | null; /** Requires Hf Login */ requires_hf_login: boolean; + /** Supported Models */ + supported_models?: ("LTX-2.3" | "LTX-2.5")[]; /** Tags */ tags?: string[]; /** Trigger */ @@ -1690,7 +1726,12 @@ export interface components { /** LtxDownloadRecommendationResponse */ LtxDownloadRecommendationResponse: { /** Cps To Download */ - cps_to_download: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo")[]; + cps_to_download: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo")[]; + /** + * Optional Cp Ids + * @default [] + */ + optional_cp_ids: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo")[]; /** * Status * @constant @@ -1700,7 +1741,12 @@ export interface components { /** LtxIcLoraRecommendationResponse */ LtxIcLoraRecommendationResponse: { /** Cps To Download */ - cps_to_download: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo")[]; + cps_to_download: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo")[]; + /** + * Supported + * @default true + */ + supported: boolean; }; /** LtxInsufficientFundsErrorResponse */ LtxInsufficientFundsErrorResponse: { @@ -1717,7 +1763,7 @@ export interface components { /** Active */ active: boolean; /** Cps To Download */ - cps_to_download: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo")[]; + cps_to_download: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo")[]; /** Installed */ installed: boolean; /** Is Newest */ @@ -1728,12 +1774,12 @@ export interface components { * Model Cp * @enum {string} */ - model_cp: "ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo"; + model_cp: "ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo"; /** * Model Id * @enum {string} */ - model_id: "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-22b-distilled"; + model_id: "ltx-2.5-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-22b-distilled"; /** Size Bytes */ size_bytes: number; }; @@ -1753,14 +1799,19 @@ export interface components { /** LtxUpgradeRecommendationResponse */ LtxUpgradeRecommendationResponse: { /** Cps To Delete */ - cps_to_delete: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo")[]; + cps_to_delete: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo")[]; /** Cps To Download */ - cps_to_download: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo")[]; + cps_to_download: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo")[]; + /** + * Loses Built In Control + * @default false + */ + loses_built_in_control: boolean; /** * Ltx Model Id * @enum {string} */ - ltx_model_id: "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-22b-distilled"; + ltx_model_id: "ltx-2.5-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-22b-distilled"; /** * Status * @constant @@ -1779,12 +1830,12 @@ export interface components { /** ModelDeleteRequest */ ModelDeleteRequest: { /** Cp Ids */ - cp_ids?: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo")[]; + cp_ids?: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo")[]; }; /** ModelDownloadRequest */ ModelDownloadRequest: { /** Cp Ids */ - cp_ids?: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo")[]; + cp_ids?: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo")[]; /** * Type * @default download @@ -1906,6 +1957,12 @@ export interface components { * @enum {string} */ mode: "replace_audio_and_video" | "replace_video" | "replace_audio"; + /** + * Model + * @default pro + * @constant + */ + model: "pro"; /** * Prompt * @default @@ -1938,12 +1995,12 @@ export interface components { * Model Id * @enum {string} */ - model_id: "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-22b-distilled"; + model_id: "ltx-2.5-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-22b-distilled"; }; /** SettingsResponse */ SettingsResponse: { /** Activeltxmodelid */ - activeLtxModelId?: ("ltx-2.3-22b-distilled-1.1" | "ltx-2.3-22b-distilled") | null; + activeLtxModelId?: ("ltx-2.5-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-22b-distilled") | null; /** * Diffusionstagecacheenabled * @default false @@ -1996,6 +2053,11 @@ export interface components { * @default false */ seedLocked: boolean; + /** + * Useconvvae + * @default false + */ + useConvVae: boolean; /** * Uselocaltextencoder * @default false @@ -2076,12 +2138,24 @@ export interface components { }; /** TextEncoderRecommendationResponse */ TextEncoderRecommendationResponse: { + /** Active Local Enhancer Cp */ + active_local_enhancer_cp: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo") | null; + /** Api Encoding Supported */ + api_encoding_supported: boolean; /** Cp To Download */ - cp_to_download: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "z-image-turbo") | null; + cp_to_download: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo") | null; /** Expected Size Bytes */ expected_size_bytes: number; /** Expected Size Gb */ expected_size_gb: number; + /** Local Enhancement Supported */ + local_enhancement_supported: boolean; + /** Local Enhancer Cp */ + local_enhancer_cp: ("ltx-2.3-22b-distilled" | "ltx-2.3-22b-distilled-1.1" | "ltx-2.3-spatial-upscaler-x2-1.0" | "ltx-2.3-spatial-upscaler-x2-1.1" | "ltx-2.3-22b-ic-lora-union-control-ref0.5" | "ltx-2.5-22b-distilled" | "ltx-2.5-spatial-upscaler-x2-1.0" | "ltx-2.5-video-vae" | "ltx-2.5-video-vae-conv" | "ltx-2.5-audio-vae" | "ltx-2.5-duration-head" | "dpt-hybrid-midas" | "yolox-l-torchscript" | "dw-ll-ucoco-384-bs5" | "gemma-3-12b-it-qat-q4_0-unquantized" | "gemma4-12b-with-proj-ltx-2.5" | "gemma-4-e2b-it" | "z-image-turbo") | null; + /** Local Enhancer Expected Size Gb */ + local_enhancer_expected_size_gb: number | null; + /** Ltx Version Label */ + ltx_version_label: string; }; }; responses: never; diff --git a/frontend/hooks/use-extend.ts b/frontend/hooks/use-extend.ts index 6a252d243..967a3b70d 100644 --- a/frontend/hooks/use-extend.ts +++ b/frontend/hooks/use-extend.ts @@ -2,6 +2,7 @@ import { useCallback, useState } from 'react' import { ApiClient } from '../lib/api-client' import { withGenerationActive } from '../lib/generation-active' import { logger } from '../lib/logger' +import type { RetakeExtendModel } from './use-retake' export type ExtendDirection = 'start' | 'end' @@ -15,6 +16,7 @@ export interface ExtendSubmitParams { prompt: string mode: ExtendDirection resolution?: { width: number; height: number } + model: RetakeExtendModel } export interface ExtendResult { @@ -48,6 +50,7 @@ export function useExtend() { prompt: params.prompt, mode: params.mode, resolution: params.resolution, + model: params.model, }) if (!result.ok) { diff --git a/frontend/hooks/use-generation.ts b/frontend/hooks/use-generation.ts index 22d9b00c8..ae96159a2 100644 --- a/frontend/hooks/use-generation.ts +++ b/frontend/hooks/use-generation.ts @@ -15,6 +15,9 @@ export interface GenerationRecoveryContext { // Absent for ic-lora/retake: those recover as standalone video assets (Phase 1), // so there are no video/image settings to restore. settings?: GenerationSettings + // Retake/extend write this instead of a full `settings` blob — the recovery importer + // prefers it over `settings.model` (which defaults to 'fast' when absent). + model?: string inputImageUrl?: string inputAudioUrl?: string genType?: 'image' | 'enhance' @@ -184,7 +187,7 @@ export function useGeneration(): UseGenerationReturn { settings: GenerationSettings, audioPath?: string | null, ) => { - const statusMsg = settings.model === 'pro' + const statusMsg = settings.model.startsWith('pro') ? 'Loading Pro model & generating...' : 'Generating video...' @@ -231,7 +234,7 @@ export function useGeneration(): UseGenerationReturn { let lastPhase = '' let inferenceStartTime = 0 // Estimated inference time in seconds based on model - const estimatedInferenceTime = settings.model === 'pro' ? 120 : 45 + const estimatedInferenceTime = settings.model.startsWith('pro') ? 120 : 45 const pollProgress = async () => { if (!shouldApplyPollingUpdates) return diff --git a/frontend/hooks/use-hf-auth.ts b/frontend/hooks/use-hf-auth.ts index 76e8fe294..69361e451 100644 --- a/frontend/hooks/use-hf-auth.ts +++ b/frontend/hooks/use-hf-auth.ts @@ -14,7 +14,7 @@ interface UseHfAuthResult { const NOOP = async () => {} export function useHfAuth(enabled: boolean): UseHfAuthResult { - // HF sign-in is useful for gated catalog LoRA/IC-LoRA downloads; catalogs are always on. + // Used for gated downloads (LTX 2.5 base models, catalog LoRAs / IC-LoRAs). const [hfAuthStatus, setHfAuthStatus] = useState('not_authenticated') const [hfAuthPolling, setHfAuthPolling] = useState(false) diff --git a/frontend/hooks/use-hf-model-access.ts b/frontend/hooks/use-hf-model-access.ts index 65d4b799c..681047387 100644 --- a/frontend/hooks/use-hf-model-access.ts +++ b/frontend/hooks/use-hf-model-access.ts @@ -11,21 +11,23 @@ interface UseHfModelAccessResult { accessMap: ModelAccessMap allAuthorized: boolean checking: boolean + /** Set when the access check itself failed — distinct from "not authorized". */ + checkError: string | null recheckAccess: () => void } -const NOOP = () => {} - export function useHfModelAccess(modelTypes: readonly ModelCheckpointID[], hfAuthStatus: HfAuthStatus): UseHfModelAccessResult { - // Bundled checkpoints are public — when signed out there's nothing to verify (treated as - // authorized below). The per-repo access check only runs when signed in, covering any future - // gated checkpoint without forcing sign-in. const [accessMap, setAccessMap] = useState({}) const [checking, setChecking] = useState(false) const [polling, setPolling] = useState(false) + const [checkError, setCheckError] = useState(null) - const allAuthorized = modelTypes.length > 0 - && modelTypes.every((modelType) => accessMap[modelType] === 'authorized') + // Nothing to authorize, or every returned repo is authorized. Empty map with pending + // checkpoints is NOT authorized — that covers both "still checking" and a failed check + // (which leaves the map empty and sets checkError for the gate to show). + const allAuthorized = modelTypes.length === 0 + || (Object.keys(accessMap).length > 0 + && Object.values(accessMap).every((status) => status === 'authorized')) const doCheck = useCallback(async () => { if (modelTypes.length === 0) return @@ -33,21 +35,25 @@ export function useHfModelAccess(modelTypes: readonly ModelCheckpointID[], hfAut const result = await ApiClient.checkModelAccess({ cp_ids: [...modelTypes] }) if (!result.ok) { logger.error(`Model access check failed: ${result.error.message}`) + setAccessMap({}) + setCheckError(result.error.message) setChecking(false) return } const { access } = result.data setAccessMap(access) + setCheckError(null) const allOk = Object.values(access).every((s) => s === 'authorized') if (allOk) setPolling(false) setChecking(false) }, [modelTypes]) - // Initial check when authenticated + // Signed out still needs a check: gated repos (LTX 2.5) can't be downloaded without a token. useEffect(() => { - if (hfAuthStatus !== 'authenticated' || modelTypes.length === 0) { + if (modelTypes.length === 0) { setAccessMap((current) => (Object.keys(current).length === 0 ? current : {})) + setCheckError(null) setPolling(false) return } @@ -66,10 +72,5 @@ export function useHfModelAccess(modelTypes: readonly ModelCheckpointID[], hfAut void doCheck() }, [doCheck]) - // Signed out: bundled checkpoints are public, so nothing gates the download. - if (hfAuthStatus !== 'authenticated') { - return { accessMap: {}, allAuthorized: true, checking: false, recheckAccess: NOOP } - } - - return { accessMap, allAuthorized, checking, recheckAccess } + return { accessMap, allAuthorized, checking, checkError, recheckAccess } } diff --git a/frontend/hooks/use-prompt-enhancer-provider.ts b/frontend/hooks/use-prompt-enhancer-provider.ts index 2b2a1a2e5..8da6c8a9a 100644 --- a/frontend/hooks/use-prompt-enhancer-provider.ts +++ b/frontend/hooks/use-prompt-enhancer-provider.ts @@ -33,23 +33,31 @@ export function usePromptEnhancerProvider(enabled: boolean): UsePromptEnhancerPr settings: { hasGeminiApiKey, promptEnhancerProviderPreference }, updateSettings, forceApiGenerations, + modelsVersion, } = useAppSettings() - const [isTextEncoderDownloaded, setIsTextEncoderDownloaded] = useState(false) + const [isLocalEncoderUsable, setIsLocalEncoderUsable] = useState(false) useEffect(() => { if (!enabled) return let cancelled = false void ApiClient.getTextEncoderRecommendation().then((result) => { - if (!cancelled) setIsTextEncoderDownloaded(result.ok && result.data.cp_to_download === null) + // Deliberately not cp_to_download: the encoder that runs generations isn't always the one + // that can enhance (LTX 2.5's encodes only, and enhances from a separate checkpoint), so + // the backend reports enhancer availability on its own. + if (!cancelled) { + setIsLocalEncoderUsable(result.ok && result.data.local_enhancement_supported) + } }) return () => { cancelled = true } - }, [enabled]) + // modelsVersion: the enhancer is a download the user can make mid-session, and Enhance should + // become available without a restart. + }, [enabled, modelsVersion]) // Downloaded isn't enough on its own — forceApiGenerations is the pure "insufficient memory // for local models this run" signal (deliberately NOT shouldVideoGenerateWithLtxApi, which // also folds in the user's own preference to use the LTX API for VIDEO specifically — that's // unrelated to whether the much smaller Gemma text encoder can run locally right now). - const hasLocalTextEncoder = isTextEncoderDownloaded && !forceApiGenerations + const hasLocalTextEncoder = isLocalEncoderUsable && !forceApiGenerations const canToggleProvider = hasLocalTextEncoder && hasGeminiApiKey // Default to local (the first available option) when the user hasn't made an explicit choice, diff --git a/frontend/hooks/use-retake.ts b/frontend/hooks/use-retake.ts index 13ba2fa22..43121d179 100644 --- a/frontend/hooks/use-retake.ts +++ b/frontend/hooks/use-retake.ts @@ -1,10 +1,26 @@ import { useCallback, useState } from 'react' +import type { components } from '../generated/backend-openapi' import { ApiClient } from '../lib/api-client' import { withGenerationActive } from '../lib/generation-active' import { logger } from '../lib/logger' export type RetakeMode = 'replace_audio_and_video' | 'replace_video' | 'replace_audio' +// ltxv-api /v1/retake and /v2/extend accept ltx-2-pro / ltx-2-3-pro. +// Desktop maps those to pipeline "pro". +export type RetakeExtendModel = components['schemas']['RetakeRequest']['model'] + +// Runtime options for the retake/extend MODEL dropdown. Checked against the OpenAPI union +// so a schema change that adds/removes a value fails typecheck until this list is updated. +export const RETAKE_EXTEND_MODELS = ['pro'] as const satisfies ReadonlyArray + +/** Map a persisted video pipeline id onto the nearest retake/extend model. */ +export function retakeExtendModelFromPipeline( + _model: string | undefined | null, +): RetakeExtendModel { + return 'pro' +} + export interface RetakeSubmitParams { videoPath: string startTime: number @@ -12,6 +28,7 @@ export interface RetakeSubmitParams { prompt: string mode: RetakeMode resolution?: { width: number; height: number } + model: RetakeExtendModel } export interface RetakeResult { @@ -51,6 +68,7 @@ export function useRetake() { prompt: params.prompt, mode: params.mode, resolution: params.resolution, + model: params.model, }) if (!result.ok) { diff --git a/frontend/hooks/use-video-generation-model-specs.ts b/frontend/hooks/use-video-generation-model-specs.ts index 2954b640c..0b367418d 100644 --- a/frontend/hooks/use-video-generation-model-specs.ts +++ b/frontend/hooks/use-video-generation-model-specs.ts @@ -1,4 +1,5 @@ import { useEffect, useState } from 'react' +import { useAppSettings } from '../contexts/AppSettingsContext' import { ApiClient } from '../lib/api-client' import type { VideoGenerationModelSpecsResponse } from '../lib/video-generation-model-specs' @@ -9,6 +10,9 @@ interface VideoGenerationModelSpecsState { } export function useVideoGenerationModelSpecs(): VideoGenerationModelSpecsState { + // Local specs describe the *active* LTX version, so they go stale as soon as the user + // downloads, deletes, or activates a version — refetch on the models-changed signal. + const { modelsVersion } = useAppSettings() const [state, setState] = useState({ modelSpecs: null, isLoading: true, @@ -45,7 +49,7 @@ export function useVideoGenerationModelSpecs(): VideoGenerationModelSpecsState { isActive = false abortController.abort() } - }, []) + }, [modelsVersion]) return state } diff --git a/frontend/lib/generation-recovery-importers.ts b/frontend/lib/generation-recovery-importers.ts index 12207f425..3d5bbf06e 100644 --- a/frontend/lib/generation-recovery-importers.ts +++ b/frontend/lib/generation-recovery-importers.ts @@ -6,8 +6,9 @@ import type { RecoveryGenType, RecoveryImporter } from './generation-recovery' // Mirrors GenSpace's own live-generation completion effect (see the `videoPath` effect in // GenSpace.tsx) but operates on the recovery marker's captured context instead of live component // state, so it can run in the background while that project's GenSpace isn't mounted. ic-lora/ -// retake/extend all write a marker with no `settings` (see GenerationRecoveryContext) and recover -// as a standalone video asset here too, same as GenSpace's own mount-recovery effect does. +// retake/extend write a marker with no full `settings` (see GenerationRecoveryContext) and +// recover as a standalone video asset here too; retake/extend additionally set `ctx.model` so +// the pipeline label isn't forced to 'fast'. const importVideo: RecoveryImporter = async (ctx, result, { addAsset, modelsDir }) => { const videoPath = typeof result === 'string' ? result : result[0] if (!videoPath) return @@ -29,12 +30,12 @@ const importVideo: RecoveryImporter = async (ctx, result, { addAsset, modelsDir height: copied.height, prompt: ctx.prompt, resolution: s?.videoResolution ?? '', - duration: s?.duration, + duration: s?.duration ?? undefined, generationParams: { mode: genMode, prompt: ctx.prompt, - model: s?.model ?? 'fast', - duration: s?.duration ?? 0, + model: ctx.model ?? s?.model ?? 'fast', + duration: s?.duration ?? null, resolution: s?.videoResolution ?? '', fps: s?.fps ?? 24, audio: s?.audio ?? false, diff --git a/frontend/lib/video-generation-model-specs.ts b/frontend/lib/video-generation-model-specs.ts index 112b5e398..b92b935b7 100644 --- a/frontend/lib/video-generation-model-specs.ts +++ b/frontend/lib/video-generation-model-specs.ts @@ -3,15 +3,21 @@ import type { components } from '../generated/backend-openapi' export type VideoGenerationModelSpecsResponse = components['schemas']['GenerateVideoModelsSpecsResponse'] export type VideoGenerationModelSpecItem = components['schemas']['LTXVideoGenerationModelSpecItem'] export type VideoGenerationResolutionSpec = components['schemas']['LTXVideoGenerationResolutionSpec'] +export type VideoGenerationOfferingCapabilities = NonNullable< + VideoGenerationModelSpecItem['spec']['capabilities'] +> export type VideoGenerationPipeline = components['schemas']['GenerateVideoRequest']['model'] export type VideoGenerationResolution = components['schemas']['GenerateVideoRequest']['resolution'] -export type VideoGenerationDuration = components['schemas']['GenerateVideoRequest']['duration'] +export type VideoGenerationDuration = Exclude< + components['schemas']['GenerateVideoRequest']['duration'], + null +> export type VideoGenerationFps = components['schemas']['GenerateVideoRequest']['fps'] export type VideoGenerationAspectRatio = components['schemas']['GenerateVideoRequest']['aspectRatio'] export interface VideoGenerationSettingsShape { model: string - duration: number + duration: number | null videoResolution: string fps: number aspectRatio?: string @@ -27,6 +33,7 @@ export interface ResolvedVideoGenerationOptions { selectedResolution: VideoGenerationResolution | null selectedFps: VideoGenerationFps | null selectedDuration: VideoGenerationDuration | null + autoDurationAvailable: boolean hasCompatibleOptions: boolean } @@ -45,10 +52,12 @@ function getResolutionMap( options: { hasAudio: boolean }, ): Record { const { hasAudio } = options - if (hasAudio && item.spec.a2v_supported_resolutions_durations) { - return item.spec.a2v_supported_resolutions_durations + if (!hasAudio) { + return item.spec.supported_resolutions_durations } - return item.spec.supported_resolutions_durations + // A model with no a2v spec doesn't support audio-conditioned generation at all — + // must not fall back to the plain (non-a2v) matrix, or it looks compatible when it isn't. + return item.spec.a2v_supported_resolutions_durations ?? {} } function getResolutionEntries( @@ -100,13 +109,35 @@ function getCompatibleModelOptions( options: { hasAudio: boolean; minimumDuration: number | undefined }, ): VideoGenerationModelSpecItem[] { const { hasAudio, minimumDuration } = options - if (minimumDuration === undefined) return modelSpecs + // Always filter by resolution compatibility — hasAudio alone (independent of any + // minimumDuration constraint) can exclude a model, e.g. a fast-tier pipeline with no + // a2v spec. Skipping this whenever minimumDuration is unset used to let incompatible + // (audio-unsupported) models stay selectable and get stuck with no valid resolution. return modelSpecs.filter((item) => ( getCompatibleResolutionEntries(item, { hasAudio, minimumDuration }).length > 0 )) } -function chooseOption(current: string | number, options: T[]): T | null { +function emptyResolvedOptions( + modelOptions: VideoGenerationModelSpecItem[], + extras: Partial = {}, +): ResolvedVideoGenerationOptions { + return { + modelOptions, + resolutionOptions: [], + fpsOptions: [], + durationOptions: [], + selectedModel: null, + selectedResolution: null, + selectedFps: null, + selectedDuration: null, + autoDurationAvailable: false, + hasCompatibleOptions: false, + ...extras, + } +} + +function chooseOption(current: string | number | null, options: T[]): T | null { return options.find((option) => option === current) ?? options[0] ?? null } @@ -119,6 +150,12 @@ export function getVideoGenerationModelSpecs( return useApiSpecs ? specs.api_models : specs.local_models } +export function getLocalOfferingCapabilities( + specs: VideoGenerationModelSpecsResponse | null | undefined, +): VideoGenerationOfferingCapabilities | null { + return specs?.local_models[0]?.spec.capabilities ?? null +} + export function resolveVideoGenerationOptions({ settings, modelSpecs, @@ -129,74 +166,46 @@ export function resolveVideoGenerationOptions item.pipeline === settings.model) ?? modelOptions[0] ?? null if (!selectedModelItem) { - return { - modelOptions, - resolutionOptions: [], - fpsOptions: [], - durationOptions: [], - selectedModel: null, - selectedResolution: null, - selectedFps: null, - selectedDuration: null, - hasCompatibleOptions: false, - } + return emptyResolvedOptions(modelOptions) } const resolutionEntries = getCompatibleResolutionEntries(selectedModelItem, { hasAudio, minimumDuration }) const resolutionOptions = resolutionEntries.map(([resolution]) => resolution) const selectedResolution = chooseOption(settings.videoResolution, resolutionOptions) if (!selectedResolution) { - return { - modelOptions, - resolutionOptions, - fpsOptions: [], - durationOptions: [], - selectedModel: selectedModelItem.pipeline, - selectedResolution: null, - selectedFps: null, - selectedDuration: null, - hasCompatibleOptions: false, - } + return emptyResolvedOptions(modelOptions, { selectedModel: selectedModelItem.pipeline, resolutionOptions }) } const selectedResolutionSpec = resolutionEntries.find(([resolution]) => resolution === selectedResolution)?.[1] ?? null if (!selectedResolutionSpec) { - return { - modelOptions, - resolutionOptions, - fpsOptions: [], - durationOptions: [], + return emptyResolvedOptions(modelOptions, { selectedModel: selectedModelItem.pipeline, + resolutionOptions, selectedResolution, - selectedFps: null, - selectedDuration: null, - hasCompatibleOptions: false, - } + }) } const fpsOptions = getCompatibleFps(selectedResolutionSpec, { minimumDuration }) const selectedFps = chooseOption(settings.fps, fpsOptions) if (!selectedFps) { - return { - modelOptions, - resolutionOptions, - fpsOptions, - durationOptions: [], + return emptyResolvedOptions(modelOptions, { selectedModel: selectedModelItem.pipeline, + resolutionOptions, selectedResolution, - selectedFps: null, - selectedDuration: null, - hasCompatibleOptions: false, - } + fpsOptions, + }) } const durationOptions = filterDurationsByMinimum( getDurationsForFps(selectedResolutionSpec, selectedFps), minimumDuration, ) + const autoDurationAvailable = !hasAudio && Boolean(selectedModelItem.spec.capabilities?.auto_duration) const selectedDuration = durationSelection === 'smallest_valid' ? durationOptions[0] ?? null - : chooseOption(settings.duration, durationOptions) + : autoDurationAvailable && settings.duration === null + ? null + : chooseOption(settings.duration, durationOptions) return { modelOptions, @@ -207,7 +216,8 @@ export function resolveVideoGenerationOptions = { + fast: 'LTX Fast', + pro: 'LTX Pro', + 'fast-2.5': 'LTX-2.5 Fast', + 'pro-2.5': 'LTX-2.5 Pro', +} + +/** Returns a display label for a known video pipeline, or null if unknown/absent. */ +export function formatPipelineDisplayName(model: string | undefined | null): string | null { + if (!model) return null + return PIPELINE_DISPLAY_NAMES[model] ?? null +} + +/** + * Version-correct label for `pipeline` taken from the backend specs currently in effect. + * Returns null when the pipeline isn't in `modelSpecs`, so callers can fall back. + */ +export function resolvePipelineDisplayName( + modelSpecs: VideoGenerationModelSpecItem[], + pipeline: string | undefined | null, +): string | null { + if (!pipeline) return null + return modelSpecs.find((item) => item.pipeline === pipeline)?.spec.display_name ?? null +} diff --git a/frontend/lib/video-resolution.ts b/frontend/lib/video-resolution.ts index 5765d8e78..3fe1b8ea9 100644 --- a/frontend/lib/video-resolution.ts +++ b/frontend/lib/video-resolution.ts @@ -1,7 +1,7 @@ // Resolution tier options for local retake/extend. Offers the source resolution plus -// standard lower tiers (named by short edge: 1080p / 720p / 540p); the backend snaps the -// chosen size to a valid (÷32, not-upscaled) resolution. Local only — the cloud preserves -// source resolution. +// standard lower tiers (named by short edge: 1080p / 720p / 540p). Grid sizes such as +// 576/704/1088 map to the nearest named tier. The backend snaps the chosen size to a +// valid (÷32, not-upscaled) resolution. Local only — the cloud preserves source resolution. export interface ResolutionOption { key: string @@ -12,9 +12,18 @@ export interface ResolutionOption { } const STANDARD_TIERS = [1080, 720, 540] -// A source whose short edge is within this of a tier is labelled as that tier (so a -// 1088px source still reads "1080p"), and that tier is dropped from the lower list. -const MATCH_TOLERANCE = 0.03 +const NAMED_TIERS = [2160, 1440, 1080, 720, 540] as const + +/** Map a pixel short-edge (incl. /64 grid sizes like 576, 704, 1088) to the picker tier name. */ +export function namedResolutionTier(shortEdge: number): (typeof NAMED_TIERS)[number] { + return NAMED_TIERS.reduce((best, tier) => + Math.abs(tier - shortEdge) < Math.abs(best - shortEdge) ? tier : best, + ) +} + +export function namedResolutionDisplayName(tier: number): string { + return tier >= 2160 ? '4K' : `${tier}p` +} export function resolutionOptions(width: number, height: number): ResolutionOption[] { if (!width || !height) return [] @@ -22,9 +31,7 @@ export function resolutionOptions(width: number, height: number): ResolutionOpti const longEdge = Math.max(width, height) const portrait = height > width - // Name "Original" by the standard tier it matches (within tolerance), else its own height. - const matchedTier = STANDARD_TIERS.find((t) => Math.abs(t - shortEdge) <= shortEdge * MATCH_TOLERANCE) - const originalTier = matchedTier ?? shortEdge + const originalTier = namedResolutionTier(shortEdge) const options: ResolutionOption[] = [ { key: 'original', label: `${originalTier}p (Original)`, width: null, height: null }, diff --git a/frontend/types/project-model.ts b/frontend/types/project-model.ts index 3743b8d9a..facdc62dc 100644 --- a/frontend/types/project-model.ts +++ b/frontend/types/project-model.ts @@ -56,7 +56,11 @@ export const generationParamsSchema = z.object({ mode: z.enum(generationModeValues), prompt: z.string(), model: z.string(), - duration: z.number(), + // Local pipeline ids ("fast") are shared across LTX versions, so `model` alone can't say + // which version produced the asset. Captured at generation time from the backend spec's + // display name; absent on assets written before this existed. + modelLabel: z.string().optional(), + duration: z.number().nullable(), resolution: z.string(), fps: z.number(), audio: z.boolean(), diff --git a/frontend/views/GenSpace.tsx b/frontend/views/GenSpace.tsx index 2f8de5750..274b30f34 100644 --- a/frontend/views/GenSpace.tsx +++ b/frontend/views/GenSpace.tsx @@ -3,7 +3,7 @@ import { Trash2, Download, Image, Video, X, Heart, Film, Volume2, VolumeX, Sparkles, Sparkle, Clock, Monitor, ChevronUp, Scissors, Music, Undo2, Redo2, Loader2, - ChevronLeft, ChevronRight, Copy, Check, MoveHorizontal, Wand2 + MoveHorizontal, Wand2 } from 'lucide-react' import { useProjects } from '../contexts/ProjectContext' import type { GenSpaceRetakeSource } from '../contexts/ProjectContext' @@ -13,7 +13,12 @@ import { setActiveGenerationOwner, hasValidBaselineId } from '../lib/generation- import { withGenerationActive } from '../lib/generation-active' import { useVideoGenerationModelSpecs } from '../hooks/use-video-generation-model-specs' import { createLocalGenerationError, type GenerationError } from '../lib/generation-errors' -import { useRetake } from '../hooks/use-retake' +import { + useRetake, + RETAKE_EXTEND_MODELS, + retakeExtendModelFromPipeline, + type RetakeExtendModel, +} from '../hooks/use-retake' import { useExtend, type ExtendDirection, EXTEND_SECONDS, DEFAULT_EXTEND_SECONDS } from '../hooks/use-extend' import { resolutionOptions, type ResolutionOption } from '../lib/video-resolution' import { useIcLora, type IcLoraAudioMode } from '../hooks/use-ic-lora' @@ -26,15 +31,20 @@ import { usePromptEnhancerProvider } from '../hooks/use-prompt-enhancer-provider import { useGlobalGenerationLock } from '../hooks/use-global-generation-lock' import type { ICLoraConditioningType } from '../components/ICLoraPanel' import type { Asset } from '../types/project-model' +import { AssetPreviewModal } from '../components/AssetPreviewModal' import { GenerationErrorDialog } from '../components/GenerationErrorDialog' import { addVisualAssetToProject } from '../lib/asset-copy' import { pathToFileUrl } from '../lib/file-url' import { areVideoGenerationSettingsEquivalent, + formatPipelineDisplayName, getVideoGenerationModelSpecs, + getLocalOfferingCapabilities, + resolvePipelineDisplayName, resolveVideoGenerationOptions, sanitizeVideoGenerationSettings, type VideoGenerationModelSpecItem, + type VideoGenerationPipeline, } from '../lib/video-generation-model-specs' import { logger } from '../lib/logger' import { ApiClient, type ApiSuccessOf } from '../lib/api-client' @@ -202,13 +212,15 @@ function AssetCard({ )} {asset.type === 'video' && ( <> - + {onRetake && ( + + )} {onExtend && ( - - {/* Next button */} - - - {/* Content area */} -
e.stopPropagation()}> - {/* Top bar: counter + close */} -
- - {selectedIndex + 1} / {filteredAssets.length} - - -
- - {selectedAsset.type === 'video' ? ( -
-
+ setSelectedAsset(null)} + /> )} {(error || localError) && ( diff --git a/frontend/views/editor/ClipContextMenu.tsx b/frontend/views/editor/ClipContextMenu.tsx index 63850896c..78e0ba5f7 100644 --- a/frontend/views/editor/ClipContextMenu.tsx +++ b/frontend/views/editor/ClipContextMenu.tsx @@ -48,6 +48,7 @@ export interface ClipContextMenuProps { onRetakeClip: (clip: TimelineClip) => void onICLoraClip: (clip: TimelineClip) => void canUseIcLora: boolean + canUseRetake: boolean onCaptureFrameForVideo: (clip: TimelineClip) => void onCreateVideoFromAudio: (clip: TimelineClip) => void } @@ -119,6 +120,7 @@ export function ClipContextMenu({ onRetakeClip, onICLoraClip, canUseIcLora, + canUseRetake, onCaptureFrameForVideo, onCreateVideoFromAudio, }: ClipContextMenuProps) { @@ -230,6 +232,7 @@ export function ClipContextMenu({ onRetakeClip={onRetakeClip} onICLoraClip={onICLoraClip} canUseIcLora={canUseIcLora} + canUseRetake={canUseRetake} onCaptureFrameForVideo={onCaptureFrameForVideo} onCreateVideoFromAudio={onCreateVideoFromAudio} close={close} @@ -261,7 +264,7 @@ function SingleClipMenu({ duplicateClip, splitClipAtPlayhead, removeClip, updateClip, getLiveAsset, getMaxClipDuration, onRevealAsset, - onCreateVideoFromImage, onRetakeClip, onICLoraClip, canUseIcLora, + onCreateVideoFromImage, onRetakeClip, onICLoraClip, canUseIcLora, canUseRetake, onCaptureFrameForVideo, onCreateVideoFromAudio, close, @@ -289,6 +292,7 @@ function SingleClipMenu({ onRetakeClip: (clip: TimelineClip) => void onICLoraClip: (clip: TimelineClip) => void canUseIcLora: boolean + canUseRetake: boolean onCaptureFrameForVideo: (clip: TimelineClip) => void onCreateVideoFromAudio: (clip: TimelineClip) => void close: () => void @@ -492,8 +496,10 @@ function SingleClipMenu({ )} {isVideo && contextClip.assetId && ( <> - { onRetakeClip(contextClip); close() }} /> + {canUseRetake && ( + { onRetakeClip(contextClip); close() }} /> + )} {canUseIcLora && ( { onICLoraClip(contextClip); close() }} /> diff --git a/frontend/views/editor/ClipPropertiesPanel.tsx b/frontend/views/editor/ClipPropertiesPanel.tsx index d789fefdd..ce63a8eec 100644 --- a/frontend/views/editor/ClipPropertiesPanel.tsx +++ b/frontend/views/editor/ClipPropertiesPanel.tsx @@ -10,6 +10,7 @@ import { import type { Asset, TimelineClip, LetterboxSettings, TextOverlayStyle, TransitionType } from '../../types/project-model' // EFFECTS HIDDEN: removed EffectMask import { DEFAULT_COLOR_CORRECTION, DEFAULT_LETTERBOX } from '../../types/project-model' // EFFECTS HIDDEN: removed EFFECT_DEFINITIONS, DEFAULT_EFFECT_MASK import { TEXT_PRESETS } from '../../types/project' +import { namedResolutionTier } from '../../lib/video-resolution' import { formatTime } from './video-editor-utils' import { Tooltip } from '../../components/ui/tooltip' import { @@ -139,7 +140,8 @@ export function ClipPropertiesPanel(props: ClipPropertiesPanelProps) { // Determine if this is an upscaled take (take index > 0 and resolution is higher than original) const originalRes = liveAsset?.generationParams?.resolution - const isUpscaled = dims && originalRes ? dims.height > parseInt(originalRes, 10) : false + const qualityTier = dims ? namedResolutionTier(Math.min(dims.width, dims.height)) : 0 + const isUpscaled = dims && originalRes ? qualityTier > parseInt(originalRes, 10) : false return (
@@ -172,7 +174,7 @@ export function ClipPropertiesPanel(props: ClipPropertiesPanelProps) {
Quality - {dims.height >= 2160 ? 'Ultra HD' : dims.height >= 1080 ? 'Full HD' : dims.height >= 720 ? 'HD' : 'SD'} + {qualityTier >= 2160 ? 'Ultra HD' : qualityTier >= 1080 ? 'Full HD' : qualityTier >= 720 ? 'HD' : 'SD'} {isUpscaled && (Upscaled)}
diff --git a/frontend/views/editor/VideoEditorTimelineEditingPanel.tsx b/frontend/views/editor/VideoEditorTimelineEditingPanel.tsx index ea690c6e6..8eec8a05c 100644 --- a/frontend/views/editor/VideoEditorTimelineEditingPanel.tsx +++ b/frontend/views/editor/VideoEditorTimelineEditingPanel.tsx @@ -29,6 +29,7 @@ import { ApiClient } from '../../lib/api-client' import { pathToFileUrl } from '../../lib/file-url' import { areVideoGenerationSettingsEquivalent, + getLocalOfferingCapabilities, getVideoGenerationModelSpecs, resolveVideoGenerationOptions, sanitizeVideoGenerationSettings, @@ -186,6 +187,12 @@ export function VideoEditorTimelineEditingPanel(props: VideoEditorTimelineEditin isLoading: isLoadingVideoGenerationModelSpecs, errorMessage: videoGenerationModelSpecsErrorMessage, } = useVideoGenerationModelSpecs() + const localCaps = getLocalOfferingCapabilities(videoGenerationModelSpecsResponse) + const canUseRetake = shouldVideoGenerateWithLtxApi || Boolean(localCaps?.retake) + const requestRetakeClip = (clip: TimelineClip) => { + if (!canUseRetake) return + handleRetakeClip(clip) + } const assets = useEditorStore(selectAssets) const timelines = useEditorStore(selectTimelines) @@ -946,7 +953,7 @@ export function VideoEditorTimelineEditingPanel(props: VideoEditorTimelineEditin height: copied.height, prompt: generatingGap.prompt, resolution: isImageResult ? generatingGap.settings.imageResolution : generatingGap.settings.videoResolution, - duration: assetType === 'video' ? generatingGap.settings.duration : undefined, + duration: assetType === 'video' ? generatingGap.settings.duration ?? undefined : undefined, generationParams: { mode: generatingGap.mode, prompt: generatingGap.prompt, @@ -2523,7 +2530,7 @@ export function VideoEditorTimelineEditingPanel(props: VideoEditorTimelineEditin {(() => { const resInfo = getClipResolution(clip) if (!resInfo) return null - return {resInfo.height >= 2160 ? '4K' : `${resInfo.height}p`} + return {resInfo.displayName} })()} {clip.speed !== 1 && {clip.speed}x} {clip.reversed && REV} @@ -2590,10 +2597,10 @@ export function VideoEditorTimelineEditingPanel(props: VideoEditorTimelineEditin - {clip.type === 'video' && ( + {canUseRetake && clip.type === 'video' && (