diff --git a/backend/api_model_specs.py b/backend/api_model_specs.py index 8e93f35d3..7550123e5 100644 --- a/backend/api_model_specs.py +++ b/backend/api_model_specs.py @@ -29,10 +29,37 @@ } -def _resolution_spec( - *, - fps_to_durations: dict[LTXVideoGenFps, tuple[LTXVideoGenDuration, ...]], -) -> LTXVideoGenerationResolutionSpec: +_ApiDurationEnvelope = tuple[LTXVideoGenDuration, ...] +_ApiFpsDurationMap = dict[LTXVideoGenFps, _ApiDurationEnvelope] +_ApiResolutionMap = dict[LTXVideoGenResolution, LTXVideoGenerationResolutionSpec] + +# API duration envelopes (seconds). ltxv-api accepts 2–20s. GenSpace floors the +# picker at 6s; gap fill uses smallest_valid from this full list. +# 20s is Fast 720p/1080p at 24/25 and the A2V standard-tier audio cap. +_API_DURATIONS_TO_10S: _ApiDurationEnvelope = (2, 3, 4, 5, 6, 8, 10) +_API_DURATIONS_TO_20S: _ApiDurationEnvelope = (2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 18, 20) + +_API_FPS_STANDARD: _ApiFpsDurationMap = { + 24: _API_DURATIONS_TO_10S, + 25: _API_DURATIONS_TO_10S, + 48: _API_DURATIONS_TO_10S, + 50: _API_DURATIONS_TO_10S, +} +_API_FPS_EXTENDED: _ApiFpsDurationMap = { + 24: _API_DURATIONS_TO_20S, + 25: _API_DURATIONS_TO_20S, + 48: _API_DURATIONS_TO_10S, + 50: _API_DURATIONS_TO_10S, +} +# ltx-2-5-pro has no 48 fps in MODEL_CAPABILITY_MATRIX. +_API_FPS_PRO_2_5: _ApiFpsDurationMap = { + 24: _API_DURATIONS_TO_10S, + 25: _API_DURATIONS_TO_10S, + 50: _API_DURATIONS_TO_10S, +} + + +def _resolution_spec(fps_to_durations: _ApiFpsDurationMap) -> LTXVideoGenerationResolutionSpec: return LTXVideoGenerationResolutionSpec( fps_to_durations={ fps: list(durations) @@ -41,37 +68,44 @@ def _resolution_spec( ) +_API_STANDARD_RESOLUTION = _resolution_spec(_API_FPS_STANDARD) +_API_EXTENDED_RESOLUTION = _resolution_spec(_API_FPS_EXTENDED) +_API_PRO_2_5_RESOLUTION = _resolution_spec(_API_FPS_PRO_2_5) + +# Fast t2v/i2v: 720p/1080p get 20s at 24/25; 1440p/4K stay at 10s. +_API_FAST_RESOLUTIONS: _ApiResolutionMap = { + "720p": _API_EXTENDED_RESOLUTION, + "1080p": _API_EXTENDED_RESOLUTION, + "1440p": _API_STANDARD_RESOLUTION, + "2160p": _API_STANDARD_RESOLUTION, +} +# Pro 2.3 t2v/i2v: 10s at every fps and resolution, including 720p. +_API_PRO_RESOLUTIONS: _ApiResolutionMap = { + "720p": _API_STANDARD_RESOLUTION, + "1080p": _API_STANDARD_RESOLUTION, + "1440p": _API_STANDARD_RESOLUTION, + "2160p": _API_STANDARD_RESOLUTION, +} +# Pro 2.5 t2v/i2v/a2v: 720p+1080p, 24/25/50, 10s. No 48 fps, no 1440p/4K. +_API_PRO_2_5_RESOLUTIONS: _ApiResolutionMap = { + "720p": _API_PRO_2_5_RESOLUTION, + "1080p": _API_PRO_2_5_RESOLUTION, +} +# A2V for Pro 2.3 / Fast 2.5: 20s audio at 720p/1080p, 10s at 1440p/4K. +_API_A2V_RESOLUTIONS: _ApiResolutionMap = { + "720p": _API_EXTENDED_RESOLUTION, + "1080p": _API_EXTENDED_RESOLUTION, + "1440p": _API_STANDARD_RESOLUTION, + "2160p": _API_STANDARD_RESOLUTION, +} + + ltx_api_model_specs: tuple[tuple[LTXVideoGenPipeline, LTXVideoGenerationSpec], ...] = ( ( "fast", LTXVideoGenerationSpec( display_name="LTX-2.3 Fast (API)", - supported_resolutions_durations={ - "1080p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10, 12, 14, 16, 18, 20), - 25: (6, 8, 10, 12, 14, 16, 18, 20), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - "1440p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10), - 25: (6, 8, 10), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - "2160p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10), - 25: (6, 8, 10), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - }, + supported_resolutions_durations=_API_FAST_RESOLUTIONS, # No A2V envelope: ltxv-api audio-to-video does not accept ltx-2-3-fast. ), ), @@ -79,128 +113,24 @@ def _resolution_spec( "pro", LTXVideoGenerationSpec( display_name="LTX-2.3 Pro (API)", - supported_resolutions_durations={ - "1080p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10), - 25: (6, 8, 10), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - "1440p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10), - 25: (6, 8, 10), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - "2160p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10), - 25: (6, 8, 10), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - }, - a2v_supported_resolutions_durations={ - "1080p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10), - 25: (6, 8, 10), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - }, + supported_resolutions_durations=_API_PRO_RESOLUTIONS, + a2v_supported_resolutions_durations=_API_A2V_RESOLUTIONS, ), ), - # ltx-2-5-fast: t2v/i2v duration envelope matches API Fast. A2V is 1080p - # (ltxv-api MAX_AUDIO_SECONDS for this model). ( "fast-2.5", LTXVideoGenerationSpec( display_name="LTX-2.5 Fast (API)", - supported_resolutions_durations={ - "1080p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10, 12, 14, 16, 18, 20), - 25: (6, 8, 10, 12, 14, 16, 18, 20), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - "1440p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10), - 25: (6, 8, 10), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - "2160p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10), - 25: (6, 8, 10), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - }, - a2v_supported_resolutions_durations={ - "1080p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10), - 25: (6, 8, 10), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - }, + supported_resolutions_durations=_API_FAST_RESOLUTIONS, + a2v_supported_resolutions_durations=_API_A2V_RESOLUTIONS, ), ), ( "pro-2.5", LTXVideoGenerationSpec( display_name="LTX-2.5 Pro (API)", - supported_resolutions_durations={ - "1080p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10), - 25: (6, 8, 10), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - "1440p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10), - 25: (6, 8, 10), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - "2160p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10), - 25: (6, 8, 10), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - }, - a2v_supported_resolutions_durations={ - "1080p": _resolution_spec( - fps_to_durations={ - 24: (6, 8, 10), - 25: (6, 8, 10), - 48: (6, 8, 10), - 50: (6, 8, 10), - }, - ), - }, + supported_resolutions_durations=_API_PRO_2_5_RESOLUTIONS, + a2v_supported_resolutions_durations=_API_PRO_2_5_RESOLUTIONS, ), ), ) diff --git a/backend/api_types.py b/backend/api_types.py index 7880ac565..02b13ddd6 100644 --- a/backend/api_types.py +++ b/backend/api_types.py @@ -381,7 +381,7 @@ class LtxInsufficientFundsErrorResponse(BaseModel): LTXVideoGenResolution: TypeAlias = Literal["540p", "720p", "1080p", "1440p", "2160p"] -LTXVideoGenDuration: TypeAlias = Literal[5, 6, 8, 10, 12, 14, 16, 18, 20] +LTXVideoGenDuration: TypeAlias = Literal[2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 18, 20] LTXVideoGenFps: TypeAlias = Literal[24, 25, 48, 50] LTXVideoGenPipeline: TypeAlias = Literal["fast", "pro", "fast-2.5", "pro-2.5"] diff --git a/backend/handlers/text_handler.py b/backend/handlers/text_handler.py index 1e5eaa172..b18deee85 100644 --- a/backend/handlers/text_handler.py +++ b/backend/handlers/text_handler.py @@ -185,7 +185,7 @@ def _prepare_api_embeddings(self, prompt: str, enhance_prompt: bool) -> TextEnco api_key=settings.ltx_api_key, checkpoint_path=str(get_existing_cp_path(self.models_dir, model_spec.model_cp)), enhance_prompt=enhance_prompt, - api_model_id=model_spec.api_text_encoder_model_id, + api_model=model_spec.api_prompt_embedding_model, ) if encoded is not None: self._cache_prompt(prompt, enhance_prompt, encoded) diff --git a/backend/ltx2_server.py b/backend/ltx2_server.py index 65d532357..30efaab9f 100644 --- a/backend/ltx2_server.py +++ b/backend/ltx2_server.py @@ -51,7 +51,7 @@ del _safetensors_loader_fix import services.patches.safetensors_metadata_fix as _safetensors_metadata_fix # pyright: ignore[reportUnusedImport] # Remove once safetensors supports read-only mmap del _safetensors_metadata_fix -import services.patches.pinned_pool_fix as _pinned_pool_fix # pyright: ignore[reportUnusedImport] # Remove once ltx-core restores bounded pinned pool +import services.patches.pinned_pool_fix as _pinned_pool_fix # pyright: ignore[reportUnusedImport] # Remove once ltx-core alloc_buffer does not report pinned-host failure as CUDA VRAM OOM (LTX-Desktop#141) del _pinned_pool_fix import services.patches.ic_lora_stage2_lora as _ic_lora_stage2_lora # pyright: ignore[reportUnusedImport] # EXPERIMENTAL: remove once upstream ships PR #494 (use_lora_in_stage_2) del _ic_lora_stage2_lora diff --git a/backend/runtime_config/ltx_api_text_encoder_ids.py b/backend/runtime_config/ltx_api_text_encoder_ids.py index ac335a31a..33b8791c1 100644 --- a/backend/runtime_config/ltx_api_text_encoder_ids.py +++ b/backend/runtime_config/ltx_api_text_encoder_ids.py @@ -1,10 +1,28 @@ -"""API text-encoder model ids used by /v1/prompt-embedding. +"""Selectors for `/v1/prompt-embedding`. -These values are the worker `encrypted_wandb_properties` payload (Fernet), not a -user secret. Split 2.5 checkpoints omit this header, so generation falls back -to this override. Rotate the blob when serving rotates the key. +LTX 2.5 OS checkpoints identify the text encoder via `gemma_source_checkpoint` +metadata (`ltx_version` + `gemma_version`), not a usable `model_id`. The public +endpoint accepts exactly one of: + +- `model_id` — legacy Comfy / LTX 2.3 (`encrypted_wandb_properties` on the checkpoint) +- `model` — `{ ltx_version, gemma_version }` for LTX 2.5+ + +Split 2.5 checkpoints omit `encrypted_wandb_properties`, so generation uses this +`model` override. Keep it in lockstep with the gateway's +`LTX_2_5_GEMMA_SOURCE_CHECKPOINT`. """ -# Worker key for LTX 2.5 Fast prompt-embedding. Do not replace with a plaintext -# id unless the worker is verified to accept one. -LTX_2_5_API_TEXT_ENCODER_MODEL_ID = "gAAAAABqcenRC9mecBlpR24xY8xWrCgERvgD5LK80sJw1J9WuSbi_J_SYXw4L3UCrCBciI5T6KJ7o5ZOld2hM6rYbs6R7_XftiN6MghrdbNt1bMZ_boxWcZ-HFvWQYgkAhJGjqLUPLyiCzDf4Idi1UQ8JebKcdK2qTxMCAEDIqLEAg_IwfPHd93YPHzT11gURnQ9vWEkJU385nRu8SRnI1ubKyGx78FmPSa8CRdh3I0zrJawmi0On2BnehplBzKU-Ub2IZaGKOP5pN9yLPW6fjgHi28d20hlNu5SoQl2fD1QnJBYNPbd6YIWYgmL7dYZzZFNLLpwYsXaY48Kh_uPY8p_wALCu_Vl2ONcEvy9spFRW5tRn0YaCs8SuIrv_8MaEO8xG0s5vLmsRfhA29yvhUIqOAuTo13vuqCKlVXv9gm17wjzkK2zcPzE_e26KUGyQepqLVHDWWxxZyPtXiWIkdOwFyzdTYlEjegEMmpMoT2dk-A-AOLdmx6QSpNd7oHcgoS78OsMO-hKG9pdjpmA6ryn3YwvAYumiFuTFM9ELKph2LEdFCyMUFp4mBaLTIxpROEc9b6xSRTQOYxSvzMkPsuJYYsdJXHB4MiDYGmzNoghIpFNABiv_i2Q8H4b4M5wi8PuYLQICiKbYCEOvlF6U_9TlIZpWf3Pt7DU5ufsCHAhYmUmSYQDerlNISYojFqOBRzyik3tqExlDdrIr51bn83YyJSnvC_q88K06dtwC4CrmVyGI0QDMcGowxLqDvdrsfYwxBLBUY_No1D6qA8_SyxlMchRF0rOzUJ_gI3MJJGeQ9loWIb-oG_RveHXQsWTP82wy7TmlOxi1BkplatMPR07qAINEYGBMApZ7ZuMWAV70c80tevyZkSCBu4OY3wMZmVHDQHIO4NM4_261A8e7ISg7fP6mJIkBehc9yGvZuE3eGQK1PD6USYjbabs-TVsLo1ZUn5_5kCn8cMCnyT2djYi3zG2CaFTfBmUSsF9ubXzk4BLDY3b1B9V4RtQf0cOUMSOvBDqi2Rvv0YXMxZcS4OdQDdUxzGC5_XekkM_HWITsyazl8jx713Dk_5fV-Z3n0_pjSVLVZUVr1DRnPkUsSDUpzq9zZ6G-i2c2AoUA2T1LRvZCKs8inqz5Y-6WJJP2R9IMpG22pjWc_Rpg2Fdd1SDnd9umy_AR-VNu2ZP0IC13MpvheyoQnPfIln7yZVupdvitfIeFKaibatgmWgVqtEevP_zavu-MnmDBpKd2mTBAPHD7X9QJXkZUtgebPjJQ3j9SQrJTIS899O4QZxC0r4h9bzpM7bEsKGO6KrnCsMISE5KgNGccPATxnOTuhQznjCfT763DpQz7MJxI8PRHEo2aR4b1uSjILrNMLczMIOx6GZPAxvPHM7HjBQZQSZThiRhhqh1kJk47w13vN2VNy0XSIwPjVPj84Af_yxB6K-2BiLQtn6ndhjDlTJ-nPBhLXltfnfKKksUt_LzYl2pd76i15oUCJrs52SNAyeN_D2uEcQ0FiSUB4u-hiYilpg_dtV2N2Jx9DmeewTcJEY9VCQr7Ccvp__EgJaDLFgFCPMPMAp_BsgUbJJIQEDGhjYJLypOIpBqJgwfIprvn7P7dVTseNL9MrZEgYI4IbABj6hVHsPGDBtTXiTU9PtTrLBiMAlJNE4Kdkd99YJuTWqLdps-rIfyagNoeJ5ItPzmDvy0oeDyyl93dLRpHtwyRtvBUB3IWGknVfX7Gj8GzfnvmIGw85Rh000AhtIIp8zH7WXIxNc8SF5zQZM06_LVePt77Ag58ihsADiRJZEniUlTGSGynZNr3Rrv4tAZUIkhIIUVGVNejtLQhLLIwY_TsblO_6H2cBb4Ep-2T4vnNk1zD0pwMDQXoHy_U3aG00FOgsSje4sTbqGOvUeO7PCEhZ8a9OoDXoNV_Myi" +from typing import TypedDict + + +class LtxApiPromptEmbeddingModel(TypedDict): + ltx_version: str + gemma_version: str + + +# Same pair the LTX API video path uses for LTX-2.5 prompt-encode. +# Source: https://github.com/LightricksResearch/ltxv-api/pull/1604 +LTX_2_5_API_PROMPT_EMBEDDING_MODEL: LtxApiPromptEmbeddingModel = { + "ltx_version": "2.5.0", + "gemma_version": "gemma4-12b-ltx-v1", +} diff --git a/backend/runtime_config/ltx_capabilities.py b/backend/runtime_config/ltx_capabilities.py index a33a58a91..c718ed406 100644 --- a/backend/runtime_config/ltx_capabilities.py +++ b/backend/runtime_config/ltx_capabilities.py @@ -42,6 +42,18 @@ class LtxOfferingCapabilities: resolution_pixels_16_9: dict[LTXVideoGenResolution, tuple[int, int]] +@dataclass(frozen=True) +class LocalOfferingCapabilities(LtxOfferingCapabilities): + """Same shape as the base class — this exists purely so local_caps()'s return + type can't be confused with api_caps()'s at the type-checker level.""" + + +@dataclass(frozen=True) +class ApiOfferingCapabilities(LtxOfferingCapabilities): + """Same shape as the base class — this exists purely so api_caps()'s return + type can't be confused with local_caps()'s at the type-checker level.""" + + # Local Fast sizes: one /64 two-stage grid for 2.3 and 2.5. Splitting 540p # (2.3 960×544 vs 2.5 1024×576) made a model switch fail assert_resolution. _LOCAL_PIXELS_16_9: dict[LTXVideoGenResolution, tuple[int, int]] = { @@ -51,12 +63,13 @@ class LtxOfferingCapabilities: } _API_PIXELS_16_9: dict[LTXVideoGenResolution, tuple[int, int]] = { + "720p": (1280, 720), "1080p": (1920, 1080), "1440p": (2560, 1440), "2160p": (3840, 2160), } -_LOCAL_2_3 = LtxOfferingCapabilities( +_LOCAL_2_3 = LocalOfferingCapabilities( t2v=True, i2v=True, a2v=True, @@ -72,13 +85,13 @@ class LtxOfferingCapabilities: # DistilledA2V is wired for local 2.5. Auto duration is DurationHead on the # distilled checkpoint (t2v/i2v; A2V length comes from the audio). Advertised # only when those weights are on disk — see effective_local_caps(). -_LOCAL_2_5 = LtxOfferingCapabilities( +_LOCAL_2_5 = LocalOfferingCapabilities( t2v=True, i2v=True, a2v=True, ic_lora=True, - retake=False, - extend=False, + retake=True, + extend=True, user_loras=True, camera_motion=True, auto_duration=True, @@ -87,7 +100,7 @@ class LtxOfferingCapabilities: # API rows follow ltxv-api handlers. camera_motion is a named LoRA on the tia2v # stack, not a Desktop capability on Fast. -_API_FAST = LtxOfferingCapabilities( +_API_FAST = ApiOfferingCapabilities( t2v=True, i2v=True, a2v=False, @@ -100,7 +113,7 @@ class LtxOfferingCapabilities: resolution_pixels_16_9=_API_PIXELS_16_9, ) -_API_FAST_2_5 = LtxOfferingCapabilities( +_API_FAST_2_5 = ApiOfferingCapabilities( t2v=True, i2v=True, a2v=True, @@ -113,7 +126,7 @@ class LtxOfferingCapabilities: resolution_pixels_16_9=_API_PIXELS_16_9, ) -_API_PRO = LtxOfferingCapabilities( +_API_PRO = ApiOfferingCapabilities( t2v=True, i2v=True, a2v=True, @@ -128,7 +141,7 @@ class LtxOfferingCapabilities: # ltxv-api retake/extend accept ltx-2-pro / ltx-2-3-pro. Auto duration is on both # 2.5 API variants (t2v/i2v duration=null). -_API_PRO_2_5 = LtxOfferingCapabilities( +_API_PRO_2_5 = ApiOfferingCapabilities( t2v=True, i2v=True, a2v=True, @@ -142,7 +155,7 @@ class LtxOfferingCapabilities: ) -def local_caps(model_id: LTXLocalModelId) -> LtxOfferingCapabilities: +def local_caps(model_id: LTXLocalModelId) -> LocalOfferingCapabilities: match model_id: case "ltx-2.5-22b-distilled": return _LOCAL_2_5 @@ -156,7 +169,7 @@ def effective_local_caps( model_id: LTXLocalModelId, *, duration_head_ready: bool, -) -> LtxOfferingCapabilities: +) -> LocalOfferingCapabilities: """Static offering flags, with Auto duration on only when DurationHead weights are on disk. Local-only. API 2.5 Auto duration is independent of this file. @@ -167,7 +180,7 @@ def effective_local_caps( return caps -def api_caps(pipeline: LTXVideoGenPipeline) -> LtxOfferingCapabilities: +def api_caps(pipeline: LTXVideoGenPipeline) -> ApiOfferingCapabilities: match pipeline: case "fast": return _API_FAST diff --git a/backend/runtime_config/model_download_specs.py b/backend/runtime_config/model_download_specs.py index d6b268f43..9bafad251 100644 --- a/backend/runtime_config/model_download_specs.py +++ b/backend/runtime_config/model_download_specs.py @@ -18,7 +18,10 @@ LTXVideoGenerationSpec, ModelCheckpointID, ) -from runtime_config.ltx_api_text_encoder_ids import LTX_2_5_API_TEXT_ENCODER_MODEL_ID +from runtime_config.ltx_api_text_encoder_ids import ( + LTX_2_5_API_PROMPT_EMBEDDING_MODEL, + LtxApiPromptEmbeddingModel, +) logger = logging.getLogger(__name__) @@ -92,8 +95,9 @@ class LTXLocalModelSpec: supported_pipelines: tuple[tuple[LTXVideoGenPipeline, LTXVideoGenerationSpec], ...] version_label: str supports_api_text_encoding: bool = True - # Overrides encrypted_wandb_properties for published checkpoints that omit it. - api_text_encoder_model_id: str | None = None + # `/v1/prompt-embedding` `model` selector for published checkpoints that omit + # `encrypted_wandb_properties` (LTX 2.5 OS split). XOR with checkpoint `model_id`. + api_prompt_embedding_model: LtxApiPromptEmbeddingModel | None = None # True when the model was captioned as audio-visual: enhancement must describe the soundscape # (and quote dialogue) as well as the visuals, or the prompt lands outside the training # distribution and the model improvises the missing audio — typically as someone speaking. @@ -380,7 +384,7 @@ def get_ltx_model_spec(model_id: LTXLocalModelId) -> LTXLocalModelSpec: ), supported_pipelines=_DISTILLED_PIPELINES_2_5, version_label="2.5", - api_text_encoder_model_id=LTX_2_5_API_TEXT_ENCODER_MODEL_ID, + api_prompt_embedding_model=LTX_2_5_API_PROMPT_EMBEDDING_MODEL, wants_audio_visual_captions=True, prompt_enhancer_cp="gemma-4-e2b-it", is_latest=True, diff --git a/backend/services/patches/pinned_pool_fix.py b/backend/services/patches/pinned_pool_fix.py index 83bf3edec..4d0f42e7c 100644 --- a/backend/services/patches/pinned_pool_fix.py +++ b/backend/services/patches/pinned_pool_fix.py @@ -1,28 +1,28 @@ -"""Monkey-patch: replace pin-all-at-init with bounded pinned pool. +"""Monkey-patch: do not report pinned-host allocation failure as CUDA VRAM OOM. -The upstream _LayerStore.__init__ pins ALL layer tensors upfront via -pin_memory(), allocating ~24 GB of page-locked host memory for a 22B model. -This wastes USS (private memory) and CUDA reserved memory since most layers -are idle at any given time. +``OffloadMode.CPU`` streaming pre-allocates one pinned host buffer per +transformer block via ``ltx_core.block_streaming.utils.alloc_buffer``. For the +22B distilled checkpoint that is ~23 GiB of page-locked RAM (more if PyTorch's +CachingHostAllocator rounds each block up to the next power of two). -This patch replaces _LayerStore with an on-demand pinning strategy: only the -layers about to be transferred to GPU are pinned (prefetch_count + 1 layers). -After a layer is evicted from GPU, its pinned copy is freed and the original -source data is restored. +``alloc_buffer`` first tries ``cudaHostRegister`` on a regular CPU tensor. On +Windows that call typically fails (pointer/size are not page-aligned for WDDM). +The failure is left as CUDA's sticky last error. The fallback +``torch.empty(..., pin_memory=True)`` then raises ``CUDA error: out of memory`` +even with ~14 GiB VRAM free, and every later CUDA call (including +``cudaMemGetInfo``) fails until the process is restarted. -Benefits (measured on RTX 5090, 22B distilled, spc=2, FP8): - - Peak USS: -11 GB (33.9 -> 22.4 GB) - - torch_peak_reserved: -4.6 GB (12.7 -> 8.1 GB) - - Duration: -12% (278 -> 245s) +See https://github.com/Lightricks/LTX-Desktop/issues/141 -Remove this patch once the upstream ltx-core package includes the fix. +This patch: +- On Windows, skips both ``cudaHostRegister`` and the CachingHostAllocator + fallback and allocates pageable host memory. H2D copies stay correct; they + just will not overlap compute. +- On other platforms, clears the sticky CUDA error after a failed register and + falls back to pageable memory if ``pin_memory=True`` itself OOMs. -NOTE: the upstream MPS-support work rewrote the weight-streaming subsystem -(`ltx_core.layer_streaming` -> `ltx_core.block_streaming`). When that module is -absent this patch cleanly no-ops — it only ever applied to the old _LayerStore, -and it is irrelevant on MPS (DISK streaming). Whether the new block_streaming -pins all blocks upfront (the waste this patch fixed) must be re-evaluated on CUDA -before shipping there. +Remove once ltx-core ``alloc_buffer`` does not poison the CUDA context or +mis-report pinned-host failure as VRAM OOM. Usage: import services.patches.pinned_pool_fix # noqa: F401 @@ -30,91 +30,87 @@ from __future__ import annotations -import itertools import logging +import sys import torch -from torch import nn - -try: - from ltx_core.layer_streaming import _LayerStore - - _AVAILABLE = True -except ModuleNotFoundError: - _AVAILABLE = False - logging.getLogger(__name__).info( - "pinned_pool_fix: ltx_core.layer_streaming absent (streaming subsystem rewritten " - "upstream to block_streaming) — skipping bounded-pinned-pool patch." - ) - - -def _patched_init(self: _LayerStore, layers: nn.ModuleList, target_device: torch.device) -> None: - self.target_device = target_device - self.num_layers = len(layers) - self._on_gpu: set[int] = set() - - # Keep a reference to the source data for each layer so we can pin it - # on demand and restore it after eviction. - self._source_data: list[dict[str, torch.Tensor]] = [] - for layer in layers: - source: dict[str, torch.Tensor] = {} - for name, tensor in itertools.chain(layer.named_parameters(), layer.named_buffers()): - source[name] = tensor.data - self._source_data.append(source) - - # Hold pinned tensors alive until the H2D transfer completes. - # Without this, the CachingHostAllocator can reclaim a pinned tensor - # as soon as its Python reference is dropped, even if an async H2D - # transfer is still reading from it. - self._pinned_in_flight: dict[int, list[torch.Tensor]] = {} - - -def _patched_move_to_gpu(self: _LayerStore, idx: int, layer: nn.Module, *, non_blocking: bool = False) -> None: - """Pin layer *idx* on demand, then transfer to GPU.""" - self._check_idx(idx) - if idx in self._on_gpu: - return - source = self._source_data[idx] - pinned_refs: list[torch.Tensor] = [] - for name, param in itertools.chain(layer.named_parameters(), layer.named_buffers()): - pinned = source[name].pin_memory() - param.data = pinned.to(self.target_device, non_blocking=non_blocking) - pinned_refs.append(pinned) - # Keep pinned tensors alive until eviction — the async H2D transfer - # may still be reading from them. - self._pinned_in_flight[idx] = pinned_refs - self._on_gpu.add(idx) - - -def _patched_evict_to_cpu(self: _LayerStore, idx: int, layer: nn.Module) -> None: - """Restore source data, freeing the GPU and pinned copies.""" - self._check_idx(idx) - if idx not in self._on_gpu: +from ltx_core.block_streaming import utils as bs_utils + +logger = logging.getLogger(__name__) + + +def _require_attr(name: str) -> None: + # Explicit raise so this still fires under `python -O` (asserts are stripped). + if not hasattr(bs_utils, name): + raise RuntimeError( + f"ltx_core.block_streaming.utils.{name} not found — patch needs updating." + ) + + +_require_attr("alloc_buffer") +_require_attr("_alloc_pinned_exact") + +_orig_alloc_pinned_exact = bs_utils._alloc_pinned_exact +_windows_pageable_logged = False + + +def _clear_cuda_sticky_error() -> None: + """Drop a leftover CUDA runtime error so later API calls are not poisoned.""" + if not torch.cuda.is_available(): return - source = self._source_data[idx] - for name, param in itertools.chain(layer.named_parameters(), layer.named_buffers()): - param.data = source[name] - # Release pinned tensors — the H2D transfer is complete by now. - self._pinned_in_flight.pop(idx, None) - self._on_gpu.discard(idx) - - -def _patched_cleanup(self: _LayerStore) -> None: - """Release all source data and in-flight pinned references.""" - for source_dict in self._source_data: - source_dict.clear() - self._source_data.clear() - self._pinned_in_flight.clear() - - -# Apply patches -if _AVAILABLE: - assert hasattr(_LayerStore, "__init__"), "_LayerStore.__init__ not found — patch needs updating." - assert hasattr(_LayerStore, "move_to_gpu"), "_LayerStore.move_to_gpu not found — patch needs updating." - assert hasattr(_LayerStore, "evict_to_cpu"), "_LayerStore.evict_to_cpu not found — patch needs updating." - assert hasattr(_LayerStore, "cleanup"), "_LayerStore.cleanup not found — patch needs updating." - - _LayerStore.__init__ = _patched_init # type: ignore[assignment] - _LayerStore.move_to_gpu = _patched_move_to_gpu # type: ignore[assignment] - _LayerStore.evict_to_cpu = _patched_evict_to_cpu # type: ignore[assignment] - _LayerStore.cleanup = _patched_cleanup # type: ignore[assignment] + try: + torch.cuda.cudart().cudaGetLastError() + except Exception: + logger.debug("pinned_pool_fix: failed to clear CUDA last error", exc_info=True) + + +def _is_cuda_oom(exc: BaseException) -> bool: + msg = str(exc).lower() + return "out of memory" in msg or "cudaerrormemoryallocation" in msg + + +def _alloc_pinned_exact_cleared(nbytes: int) -> torch.Tensor | None: + buf = _orig_alloc_pinned_exact(nbytes) + if buf is None: + # cudaHostRegister returns the error to Python but also sets the thread's + # sticky last error. Clear it before any later CUDA call. + _clear_cuda_sticky_error() + return buf + + +def _patched_alloc_buffer(nbytes: int, device: torch.device | None, pin_memory: bool) -> torch.Tensor: + """Like upstream ``alloc_buffer``, but never raises a misleading CUDA VRAM OOM.""" + if pin_memory and not torch.cuda.is_available(): + pin_memory = False + cpu_pin = pin_memory and (device is None or torch.device(device).type == "cpu") + if cpu_pin and sys.platform == "win32": + global _windows_pageable_logged + if not _windows_pageable_logged: + logger.warning( + "Using pageable host memory for streaming weight buffers on Windows. " + "ltx-core would pin via cudaHostRegister / CachingHostAllocator, which " + "fails as a misleading CUDA OOM while VRAM is free (LTX-Desktop#141)." + ) + _windows_pageable_logged = True + return torch.empty(nbytes, dtype=torch.uint8, device=device, pin_memory=False) + if cpu_pin: + buf = _alloc_pinned_exact_cleared(nbytes) + if buf is not None: + return buf + try: + return torch.empty(nbytes, dtype=torch.uint8, device=device, pin_memory=True) + except Exception as exc: + if not _is_cuda_oom(exc): + raise + _clear_cuda_sticky_error() + logger.warning( + "Pinned host-memory allocation of %s bytes failed (%s). " + "Falling back to pageable CPU memory. This is not a GPU VRAM shortage.", + nbytes, + exc, + ) + pin_memory = False + return torch.empty(nbytes, dtype=torch.uint8, device=device, pin_memory=pin_memory) + + +bs_utils.alloc_buffer = _patched_alloc_buffer diff --git a/backend/services/retake_pipeline/ltx_retake_pipeline.py b/backend/services/retake_pipeline/ltx_retake_pipeline.py index 254dff103..5a2ba69bc 100644 --- a/backend/services/retake_pipeline/ltx_retake_pipeline.py +++ b/backend/services/retake_pipeline/ltx_retake_pipeline.py @@ -15,14 +15,24 @@ from __future__ import annotations from collections.abc import Iterator +from functools import partial +from typing import Any import torch +from ltx_core.components.diffusion_steps import EulerAncestralDiffusionStep from ltx_core.components.guiders import MultiModalGuiderParams from ltx_core.loader import LoraPathStrengthAndSDOps from ltx_core.model.video_vae import DimensionSizeConfig, TileSizeConfig, get_video_chunks_number from ltx_core.quantization import QuantizationPolicy from ltx_core.types import Audio +from ltx_pipelines.distilled import ( + ANCESTRAL_ETA, + ANCESTRAL_NOISE_SEED_OFFSET, + ANCESTRAL_S_NOISE, + should_use_ancestral_sampler, +) from ltx_pipelines.utils.media_io import encode_video, get_videostream_metadata +from ltx_pipelines.utils.samplers import euler_ancestral_denoising_loop from api_types import ExtendMode from services.ltx_pipeline_common import build_model_paths, offload_mode_for_prefetch_count, resolve_tiling_config @@ -36,6 +46,30 @@ _EXTEND_MASK_DELTA_SECONDS = 0.5 +def distilled_stage_sampler_kwargs( + *, + distilled: bool, + use_ancestral: bool, + seed: int, + dtype: torch.dtype, +) -> dict[str, Any]: + """Optional ``stepper``/``loop`` overrides for :class:`DiffusionStage`. + + Distilled 2.5+ checkpoints need the ancestral sampler; 2.3 and the guided + (non-distilled) path keep DiffusionStage's deterministic defaults. + ``use_ancestral`` is resolved once at pipeline init from checkpoint metadata. + """ + if not distilled or not use_ancestral: + return {} + return { + "stepper": EulerAncestralDiffusionStep(eta=ANCESTRAL_ETA, s_noise=ANCESTRAL_S_NOISE), + "loop": partial( + euler_ancestral_denoising_loop, + noise_seed=seed + ANCESTRAL_NOISE_SEED_OFFSET, + model_dtype=dtype, + ), + } + class LTXRetakePipeline: @staticmethod @@ -98,6 +132,7 @@ def __init__( video_vae = model_paths.video_vae() audio_vae = model_paths.audio_vae() transformer = model_paths.transformer() + self.use_ancestral_sampler = should_use_ancestral_sampler(transformer) self.prompt_encoder = PromptEncoder( model_paths, @@ -134,6 +169,24 @@ def __init__( device, ) + def _invoke_diffusion_stage( + self, + *, + distilled: bool, + seed: int, + **stage_kwargs: Any, + ) -> Any: + """Call ``DiffusionStage`` with distilled 2.5 ancestral overrides when needed.""" + return self.stage( + **stage_kwargs, + **distilled_stage_sampler_kwargs( + distilled=distilled, + use_ancestral=self.use_ancestral_sampler, + seed=seed, + dtype=self.dtype, + ), + ) + @torch.no_grad() def _run( # noqa: PLR0913, PLR0915 self, @@ -302,8 +355,9 @@ def _run( # noqa: PLR0913, PLR0915 ) # --- Run diffusion stage --- - - video_state, audio_state = self.stage( + video_state, audio_state = self._invoke_diffusion_stage( + distilled=distilled, + seed=effective_seed, denoiser=denoiser, sigmas=sigmas, noiser=noiser, @@ -315,7 +369,6 @@ def _run( # noqa: PLR0913, PLR0915 audio=audio_modality_spec, ) - # --- Decode audio first (eager, small) --- assert audio_state is not None decoded_audio = self.audio_decoder(audio_state.latent) diff --git a/backend/services/text_encoder/ltx_text_encoder.py b/backend/services/text_encoder/ltx_text_encoder.py index eb4f24a74..805009a8b 100644 --- a/backend/services/text_encoder/ltx_text_encoder.py +++ b/backend/services/text_encoder/ltx_text_encoder.py @@ -11,7 +11,9 @@ import torch +from runtime_config.ltx_api_text_encoder_ids import LtxApiPromptEmbeddingModel from services.http_client.http_client import HTTPClient +from services.services_utils import JSONValue from state.app_state_types import TextEncodingResult if TYPE_CHECKING: @@ -243,14 +245,27 @@ def encode_via_api( api_key: str, checkpoint_path: str, enhance_prompt: bool, - api_model_id: str | None = None, + api_model: LtxApiPromptEmbeddingModel | None = None, ) -> TextEncodingResult | None: - model_id = api_model_id or self.get_model_id_from_checkpoint(checkpoint_path) - if not model_id: - logger.warning( - "Checkpoint %s carries no API model id; skipping LTX API text encoding", checkpoint_path - ) - return None + # Gateway XOR: exactly one of `model` (2.5+) or `model_id` (legacy 2.3 / Comfy). + json_payload: dict[str, JSONValue] = { + "prompt": prompt, + "enhance_prompt": enhance_prompt, + } + if api_model is not None: + json_payload["model"] = { + "ltx_version": api_model["ltx_version"], + "gemma_version": api_model["gemma_version"], + } + else: + model_id = self.get_model_id_from_checkpoint(checkpoint_path) + if not model_id: + logger.warning( + "Checkpoint %s carries no API model selector; skipping LTX API text encoding", + checkpoint_path, + ) + return None + json_payload["model_id"] = model_id try: start = time.time() @@ -260,11 +275,7 @@ def encode_via_api( "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", }, - json_payload={ - "prompt": prompt, - "model_id": model_id, - "enhance_prompt": enhance_prompt, - }, + json_payload=json_payload, timeout=60, ) diff --git a/backend/services/text_encoder/text_encoder.py b/backend/services/text_encoder/text_encoder.py index f51175083..618fd014c 100644 --- a/backend/services/text_encoder/text_encoder.py +++ b/backend/services/text_encoder/text_encoder.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Protocol if TYPE_CHECKING: + from runtime_config.ltx_api_text_encoder_ids import LtxApiPromptEmbeddingModel from state.app_state_types import AppState, TextEncodingResult @@ -19,6 +20,6 @@ def encode_via_api( api_key: str, checkpoint_path: str, enhance_prompt: bool, - api_model_id: str | None = None, + api_model: LtxApiPromptEmbeddingModel | None = None, ) -> TextEncodingResult | None: ... diff --git a/backend/tests/fakes/services.py b/backend/tests/fakes/services.py index 6e82592d1..6976ac112 100644 --- a/backend/tests/fakes/services.py +++ b/backend/tests/fakes/services.py @@ -1023,23 +1023,21 @@ def create( audio_vae_path: str | None = None, duration_head_path: str | None = None, ) -> "FakeRetakePipeline": - del ( - checkpoint_path, - gemma_root, - device, - streaming_prefetch_count, - loras, - quantization, - video_vae_path, - audio_vae_path, - duration_head_path, - ) + del device, streaming_prefetch_count, loras, quantization pipeline = FakeRetakePipeline._singleton if pipeline is None: raise RuntimeError("FakeRetakePipeline singleton is not bound") + pipeline.create_calls.append({ + "checkpoint_path": checkpoint_path, + "gemma_root": gemma_root, + "video_vae_path": video_vae_path, + "audio_vae_path": audio_vae_path, + "duration_head_path": duration_head_path, + }) return pipeline def __init__(self) -> None: + self.create_calls: list[dict[str, Any]] = [] self.generate_calls: list[dict[str, Any]] = [] self.extend_calls: list[dict[str, Any]] = [] self.raise_on_generate: Exception | None = None @@ -1079,7 +1077,7 @@ def encode_via_api( api_key: str, checkpoint_path: str, enhance_prompt: bool, - api_model_id: str | None = None, + api_model: dict[str, str] | None = None, ) -> Any | None: self.encode_calls.append( { @@ -1087,7 +1085,7 @@ def encode_via_api( "api_key": api_key, "checkpoint_path": checkpoint_path, "enhance_prompt": enhance_prompt, - "api_model_id": api_model_id, + "api_model": api_model, } ) if self.encode_responses: diff --git a/backend/tests/test_api_calls.py b/backend/tests/test_api_calls.py index 154351072..1859ad636 100644 --- a/backend/tests/test_api_calls.py +++ b/backend/tests/test_api_calls.py @@ -331,19 +331,15 @@ def test_local_retake_happy_path(self, client, test_state, create_fake_model_fil assert data["status"] == "complete" assert data["video_path"] - def test_local_retake_rejected_on_2_5(self, client, test_state, create_fake_model_files): + def test_local_retake_happy_path_on_2_5(self, client, test_state, create_fake_model_files): create_fake_model_files(include_zit=False) test_state.state.app_settings.use_local_text_encoder = True test_state.config.local_generations_mode = "full_models_loading" video_path = self._make_valid_video(test_state) r = client.post("/api/retake", json=self._base_payload(video_path)) - assert_http_error( - r, - status_code=409, - code="UNSUPPORTED_RETAKE", - message="Retake is not supported for the active LTX model.", - ) + assert r.status_code == 200 + assert r.json()["status"] == "complete" def test_local_retake_mode_mapping(self, client, test_state, create_fake_model_files, fake_services): _install_local_2_3(test_state, create_fake_model_files, include_zit=False) @@ -581,19 +577,15 @@ def test_local_extend_happy_path(self, client, test_state, create_fake_model_fil assert data["status"] == "complete" assert data["video_path"] - def test_local_extend_rejected_on_2_5(self, client, test_state, create_fake_model_files): + def test_local_extend_happy_path_on_2_5(self, client, test_state, create_fake_model_files): create_fake_model_files(include_zit=False) test_state.state.app_settings.use_local_text_encoder = True test_state.config.local_generations_mode = "full_models_loading" video_path = self._make_valid_video(test_state) r = client.post("/api/extend", json=self._base_payload(video_path)) - assert_http_error( - r, - status_code=409, - code="UNSUPPORTED_EXTEND", - message="Extend is not supported for the active LTX model.", - ) + assert r.status_code == 200 + assert r.json()["status"] == "complete" def test_local_extend_snaps_frames_and_forwards_mode(self, client, test_state, create_fake_model_files, fake_services): _install_local_2_3(test_state, create_fake_model_files, include_zit=False) diff --git a/backend/tests/test_generation.py b/backend/tests/test_generation.py index ce28ec02f..d787cdd8f 100644 --- a/backend/tests/test_generation.py +++ b/backend/tests/test_generation.py @@ -103,7 +103,7 @@ def test_t2v_requires_downloaded_ltx_model(self, client): r = client.post("/api/generate", json=_T2V_JSON) assert_http_error(r, status_code=409, code="NO_DOWNLOADED_LTX_MODEL") - def test_t2v_on_2_5_uses_api_model_id_without_local_text_encoder( + def test_t2v_on_2_5_uses_api_model_selector_without_local_text_encoder( self, client, test_state, fake_services, create_fake_model_files ): create_fake_model_files() @@ -116,7 +116,7 @@ def test_t2v_on_2_5_uses_api_model_id_without_local_text_encoder( r = client.post("/api/generate", json=_T2V_JSON) assert r.status_code == 200 - assert fake_services.text_encoder.encode_calls[0]["api_model_id"] == spec.api_text_encoder_model_id + assert fake_services.text_encoder.encode_calls[0]["api_model"] == spec.api_prompt_embedding_model assert fake_services.text_encoder.encode_calls[0]["enhance_prompt"] is True def test_t2v_happy_path(self, client, test_state, fake_services, create_fake_model_files): @@ -1023,7 +1023,7 @@ def test_i2v_routes_to_ltx_api_for_ltx_2_5_pro(self, client, test_state, fake_se "/api/generate", json={ "prompt": "Animate this frame", - "resolution": "2160p", + "resolution": "1080p", "model": "pro-2.5", "duration": 8, "fps": 25, @@ -1041,7 +1041,7 @@ def test_i2v_routes_to_ltx_api_for_ltx_2_5_pro(self, client, test_state, fake_se call = fake_services.ltx_api_client.image_to_video_calls[0] assert call["image_uri"] == "storage://uploaded/input.png" assert call["model"] == "ltx-2-5-pro" - assert call["resolution"] == "3840x2160" + assert call["resolution"] == "1920x1080" assert call["duration"] == 8.0 assert call["fps"] == 25.0 assert call["camera_motion"] == "jib_up" @@ -1204,7 +1204,7 @@ def test_invalid_forced_resolution_rejected(self, client, test_state): "/api/generate", json={ "prompt": "A city skyline", - "resolution": "720p", + "resolution": "540p", "model": "pro", "duration": 6, "fps": 25, @@ -1216,7 +1216,53 @@ def test_invalid_forced_resolution_rejected(self, client, test_state): r, status_code=422, code="INVALID_VIDEO_GENERATION_SPEC", - message="Unsupported api text-to-video resolution '720p' for pipeline 'pro'", + message="Unsupported api text-to-video resolution '540p' for pipeline 'pro'", + ) + + def test_pro_2_5_rejects_4k_resolution(self, client, test_state): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + + r = client.post( + "/api/generate", + json={ + "prompt": "A city skyline", + "resolution": "2160p", + "model": "pro-2.5", + "duration": 6, + "fps": 25, + "audio": False, + }, + ) + + assert_http_error( + r, + status_code=422, + code="INVALID_VIDEO_GENERATION_SPEC", + message="Unsupported api text-to-video resolution '2160p' for pipeline 'pro-2.5'", + ) + + def test_pro_2_5_rejects_48_fps(self, client, test_state): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + + r = client.post( + "/api/generate", + json={ + "prompt": "A city skyline", + "resolution": "1080p", + "model": "pro-2.5", + "duration": 6, + "fps": 48, + "audio": False, + }, + ) + + assert_http_error( + r, + status_code=422, + code="INVALID_VIDEO_GENERATION_SPEC", + message="Unsupported api text-to-video fps '48' for pipeline 'pro-2.5' at resolution '1080p'", ) def test_invalid_forced_duration_rejected(self, client, test_state): @@ -1229,7 +1275,7 @@ def test_invalid_forced_duration_rejected(self, client, test_state): "prompt": "A city skyline", "resolution": "1080p", "model": "pro", - "duration": 5, + "duration": 12, "fps": 25, "audio": False, }, @@ -1239,7 +1285,7 @@ def test_invalid_forced_duration_rejected(self, client, test_state): r, status_code=422, code="INVALID_VIDEO_GENERATION_SPEC", - message="Unsupported api text-to-video duration '5' for pipeline 'pro' at resolution '1080p' and fps '25'", + message="Unsupported api text-to-video duration '12' for pipeline 'pro' at resolution '1080p' and fps '25'", ) def test_forced_api_a2v_rejects_fast_tier_pipeline(self, client, test_state, tmp_path): @@ -1381,6 +1427,46 @@ def test_portrait_resolution_1080p(self, client, test_state, fake_services): call = fake_services.ltx_api_client.text_to_video_calls[0] assert call["resolution"] == "1080x1920" + def test_portrait_resolution_720p(self, client, test_state, fake_services): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + + r = client.post( + "/api/generate", + json={ + "prompt": "A portrait video", + "resolution": "720p", + "model": "fast", + "duration": 6, + "fps": 25, + "aspectRatio": "9:16", + }, + ) + + assert r.status_code == 200 + call = fake_services.ltx_api_client.text_to_video_calls[0] + assert call["resolution"] == "720x1280" + + def test_landscape_resolution_720p(self, client, test_state, fake_services): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + + r = client.post( + "/api/generate", + json={ + "prompt": "A landscape video", + "resolution": "720p", + "model": "pro-2.5", + "duration": 6, + "fps": 25, + }, + ) + + assert r.status_code == 200 + call = fake_services.ltx_api_client.text_to_video_calls[0] + assert call["resolution"] == "1280x720" + assert call["model"] == "ltx-2-5-pro" + def test_portrait_resolution_1440p(self, client, test_state, fake_services): test_state.config.local_generations_mode = "unsupported" test_state.state.app_settings.ltx_api_key = "api-key" @@ -1582,7 +1668,101 @@ def test_a2v_portrait_resolution(self, client, test_state, fake_services, tmp_pa call = fake_services.ltx_api_client.audio_to_video_calls[0] assert call["resolution"] == "1080x1920" - def test_a2v_forced_api_rejects_non_1080p(self, client, test_state, fake_services, tmp_path): + def test_a2v_720p_routes_to_ltx_api(self, client, test_state, fake_services, tmp_path): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + audio_file = tmp_path / "test_audio.wav" + _write_test_wav(audio_file) + + r = client.post( + "/api/generate", + json={ + "prompt": "A music video", + "resolution": "720p", + "model": "pro", + "duration": 6, + "fps": 25, + "audioPath": str(audio_file), + }, + ) + + assert r.status_code == 200 + call = fake_services.ltx_api_client.audio_to_video_calls[0] + assert call["resolution"] == "1280x720" + + def test_a2v_1440p_capped_at_10s(self, client, test_state, tmp_path): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + audio_file = tmp_path / "test_audio.wav" + _write_test_wav(audio_file) + + r = client.post( + "/api/generate", + json={ + "prompt": "A music video", + "resolution": "1440p", + "model": "pro", + "duration": 20, + "fps": 24, + "audioPath": str(audio_file), + }, + ) + + assert_http_error( + r, + status_code=422, + code="INVALID_VIDEO_GENERATION_SPEC", + message="Unsupported api audio-to-video duration '20' for pipeline 'pro' at resolution '1440p' and fps '24'", + ) + + def test_a2v_1440p_10s_routes_to_ltx_api(self, client, test_state, fake_services, tmp_path): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + audio_file = tmp_path / "test_audio.wav" + _write_test_wav(audio_file) + + r = client.post( + "/api/generate", + json={ + "prompt": "A music video", + "resolution": "1440p", + "model": "pro", + "duration": 10, + "fps": 24, + "audioPath": str(audio_file), + }, + ) + + assert r.status_code == 200 + call = fake_services.ltx_api_client.audio_to_video_calls[0] + assert call["resolution"] == "2560x1440" + + def test_a2v_pro_2_5_rejects_20s(self, client, test_state, tmp_path): + test_state.config.local_generations_mode = "unsupported" + test_state.state.app_settings.ltx_api_key = "api-key" + audio_file = tmp_path / "test_audio.wav" + _write_test_wav(audio_file) + + r = client.post( + "/api/generate", + json={ + "prompt": "A music video", + "resolution": "1080p", + "model": "pro-2.5", + "duration": 20, + "fps": 24, + "audioPath": str(audio_file), + }, + ) + + assert_http_error( + r, + status_code=422, + code="INVALID_VIDEO_GENERATION_SPEC", + message="Unsupported api audio-to-video duration '20' for pipeline 'pro-2.5' at resolution '1080p' and fps '24'", + ) + + def test_a2v_forced_api_rejects_fast(self, client, test_state, fake_services, tmp_path): test_state.config.local_generations_mode = "unsupported" test_state.state.app_settings.ltx_api_key = "test_key" audio_file = tmp_path / "test_audio.wav" @@ -1746,23 +1926,42 @@ def test_models_specs_endpoint_returns_ordered_backend_specs(self, client): assert list(data["local_models"][0]["spec"]["supported_resolutions_durations"]["540p"]["fps_to_durations"].keys()) == ["24"] assert [item["pipeline"] for item in data["api_models"]] == ["fast", "pro", "fast-2.5", "pro-2.5"] assert data["api_models"][0]["spec"]["supported_resolutions_durations"]["1080p"]["fps_to_durations"]["24"] == [ - 6, 8, 10, 12, 14, 16, 18, 20, + 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 18, 20, ] api_models_by_pipeline = {item["pipeline"]: item for item in data["api_models"]} - # A2V envelope: none on fast; 1080p on pro, fast-2.5, and pro-2.5. + # A2V: none on fast; 720p–4K on pro/fast-2.5; 720p+1080p on pro-2.5 (no 48 fps). assert api_models_by_pipeline["fast"]["spec"]["a2v_supported_resolutions_durations"] is None - assert list(api_models_by_pipeline["pro"]["spec"]["a2v_supported_resolutions_durations"].keys()) == ["1080p"] + assert list(api_models_by_pipeline["pro"]["spec"]["a2v_supported_resolutions_durations"].keys()) == [ + "720p", "1080p", "1440p", "2160p", + ] assert api_models_by_pipeline["fast-2.5"]["spec"]["display_name"] == "LTX-2.5 Fast (API)" - assert list(api_models_by_pipeline["fast-2.5"]["spec"]["a2v_supported_resolutions_durations"].keys()) == ["1080p"] - assert list(api_models_by_pipeline["pro-2.5"]["spec"]["a2v_supported_resolutions_durations"].keys()) == ["1080p"] + assert list(api_models_by_pipeline["fast-2.5"]["spec"]["a2v_supported_resolutions_durations"].keys()) == [ + "720p", "1080p", "1440p", "2160p", + ] + assert list(api_models_by_pipeline["pro-2.5"]["spec"]["a2v_supported_resolutions_durations"].keys()) == [ + "720p", "1080p", + ] + assert list(api_models_by_pipeline["pro-2.5"]["spec"]["supported_resolutions_durations"].keys()) == [ + "720p", "1080p", + ] + assert list(api_models_by_pipeline["fast"]["spec"]["supported_resolutions_durations"].keys()) == [ + "720p", "1080p", "1440p", "2160p", + ] + assert "48" not in api_models_by_pipeline["pro-2.5"]["spec"]["supported_resolutions_durations"]["1080p"]["fps_to_durations"] + assert api_models_by_pipeline["pro"]["spec"]["a2v_supported_resolutions_durations"]["1440p"]["fps_to_durations"]["24"] == [ + 2, 3, 4, 5, 6, 8, 10, + ] + assert api_models_by_pipeline["pro"]["spec"]["a2v_supported_resolutions_durations"]["720p"]["fps_to_durations"]["24"] == [ + 2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 18, 20, + ] local_caps = data["local_models"][0]["spec"]["capabilities"] assert local_caps["a2v"] is True assert local_caps["ic_lora"] is True assert local_caps["user_loras"] is True - assert local_caps["retake"] is False - assert local_caps["extend"] is False + assert local_caps["retake"] is True + assert local_caps["extend"] is True # No DurationHead on disk in this fixture — Auto stays off until that file is present. assert local_caps["auto_duration"] is False assert api_models_by_pipeline["fast"]["spec"]["capabilities"]["a2v"] is False diff --git a/backend/tests/test_ltx_capabilities.py b/backend/tests/test_ltx_capabilities.py index 2f2ae7072..a2c6e5637 100644 --- a/backend/tests/test_ltx_capabilities.py +++ b/backend/tests/test_ltx_capabilities.py @@ -5,6 +5,7 @@ import pytest from runtime_config.ltx_capabilities import ( + LocalOfferingCapabilities, api_caps, effective_local_caps, local_caps, @@ -51,8 +52,8 @@ def test_local_2_5_allows_ic_lora_and_user_loras(): caps = local_caps("ltx-2.5-22b-distilled") assert supports(caps, "ic_lora") is True assert supports(caps, "user_loras") is True - assert supports(caps, "retake") is False - assert supports(caps, "extend") is False + assert supports(caps, "retake") is True + assert supports(caps, "extend") is True def test_local_2_3_allows_ic_lora_user_loras_retake(): @@ -79,6 +80,11 @@ def test_local_2_5_auto_duration_requires_duration_head_ready(): assert supports(effective_local_caps(model_id, duration_head_ready=False), "auto_duration") is False +def test_effective_local_caps_preserves_local_subclass(): + caps = effective_local_caps("ltx-2.5-22b-distilled", duration_head_ready=False) + assert isinstance(caps, LocalOfferingCapabilities) + + def test_local_2_3_auto_duration_stays_off_even_if_duration_head_ready(): assert ( supports( @@ -96,6 +102,8 @@ def test_api_fast_2_3_has_no_a2v_or_auto_duration(): assert supports(caps, "extend") is False assert supports(caps, "auto_duration") is False assert pixels_for(caps, "1080p", "16:9") == (1920, 1080) + assert pixels_for(caps, "720p", "16:9") == (1280, 720) + assert pixels_for(caps, "720p", "9:16") == (720, 1280) def test_api_fast_2_5_has_a2v_and_auto_duration(): @@ -105,6 +113,7 @@ def test_api_fast_2_5_has_a2v_and_auto_duration(): assert supports(caps, "extend") is False assert supports(caps, "auto_duration") is True assert pixels_for(caps, "1080p", "16:9") == (1920, 1080) + assert pixels_for(caps, "720p", "16:9") == (1280, 720) def test_api_pro_2_3_has_a2v_and_retake(): diff --git a/backend/tests/test_ltx_text_encoder.py b/backend/tests/test_ltx_text_encoder.py index bbcba6309..98040aa53 100644 --- a/backend/tests/test_ltx_text_encoder.py +++ b/backend/tests/test_ltx_text_encoder.py @@ -1,19 +1,26 @@ -"""Unpickler hardening for the LTX prompt-embedding response.""" +"""Unpickler hardening and `/v1/prompt-embedding` request shape.""" from __future__ import annotations import collections import io +import json import pickle +import struct +from pathlib import Path import pytest import torch +from runtime_config.ltx_api_text_encoder_ids import LTX_2_5_API_PROMPT_EMBEDDING_MODEL +from runtime_config.model_download_specs import get_ltx_model_spec from services.text_encoder.ltx_text_encoder import ( # noqa: SLF001 + LTXTextEncoder, _ALLOWED_PICKLE_GLOBALS, _CpuUnpickler, _first_embedding_tensor, ) +from tests.fakes.services import FakeHTTPClient, FakeResponse def _unpickler() -> _CpuUnpickler: @@ -70,3 +77,126 @@ def test_first_embedding_tensor_rejects_non_tensor_nests() -> None: _first_embedding_tensor([torch.zeros(1)]) with pytest.raises(pickle.UnpicklingError, match="not a tensor"): _first_embedding_tensor([["nope"]]) + + +def _embedding_response_bytes() -> bytes: + embeddings = torch.randn(1, 8, 4096 + 384, dtype=torch.bfloat16) + return pickle.dumps([[embeddings]]) + + +def _write_safetensors_with_metadata(path: Path, metadata: dict[str, str]) -> None: + header = { + "__metadata__": metadata, + "x": {"dtype": "F32", "shape": [1], "data_offsets": [0, 4]}, + } + encoded = json.dumps(header, separators=(",", ":")).encode("utf-8") + path.write_bytes(struct.pack(" LTXTextEncoder: + http.queue("post", FakeResponse(status_code=200, content=_embedding_response_bytes())) + return LTXTextEncoder( + device=torch.device("cpu"), + http=http, + ltx_api_base_url="https://api.ltx.video", + ) + + +def test_ltx_2_5_spec_uses_prompt_embedding_model_not_fernet_model_id() -> None: + spec = get_ltx_model_spec("ltx-2.5-22b-distilled") + assert spec.api_prompt_embedding_model == LTX_2_5_API_PROMPT_EMBEDDING_MODEL + assert spec.api_prompt_embedding_model == { + "ltx_version": "2.5.0", + "gemma_version": "gemma4-12b-ltx-v1", + } + assert get_ltx_model_spec("ltx-2.3-22b-distilled-1.1").api_prompt_embedding_model is None + + +def test_encode_via_api_sends_model_selector_without_model_id(tmp_path: Path) -> None: + http = FakeHTTPClient() + encoder = _encoder_with_queued_embedding(http) + checkpoint = tmp_path / "ltx-2.5.safetensors" + checkpoint.write_bytes(b"not-a-checkpoint") + + result = encoder.encode_via_api( + prompt="A beautiful sunset", + api_key="key", + checkpoint_path=str(checkpoint), + enhance_prompt=True, + api_model=LTX_2_5_API_PROMPT_EMBEDDING_MODEL, + ) + + assert result is not None + assert http.calls[0].url == "https://api.ltx.video/v1/prompt-embedding" + assert http.calls[0].json_payload == { + "prompt": "A beautiful sunset", + "enhance_prompt": True, + "model": { + "ltx_version": "2.5.0", + "gemma_version": "gemma4-12b-ltx-v1", + }, + } + assert "model_id" not in http.calls[0].json_payload + + +def test_encode_via_api_sends_checkpoint_model_id_for_legacy_2_3(tmp_path: Path) -> None: + http = FakeHTTPClient() + encoder = _encoder_with_queued_embedding(http) + checkpoint = tmp_path / "ltx-2.3.safetensors" + _write_safetensors_with_metadata(checkpoint, {"encrypted_wandb_properties": "legacy-model-id"}) + + result = encoder.encode_via_api( + prompt="A beautiful sunset", + api_key="key", + checkpoint_path=str(checkpoint), + enhance_prompt=False, + ) + + assert result is not None + assert http.calls[0].json_payload == { + "prompt": "A beautiful sunset", + "enhance_prompt": False, + "model_id": "legacy-model-id", + } + assert "model" not in http.calls[0].json_payload + + +def test_encode_via_api_model_selector_wins_over_checkpoint_model_id(tmp_path: Path) -> None: + http = FakeHTTPClient() + encoder = _encoder_with_queued_embedding(http) + checkpoint = tmp_path / "ltx-2.5.safetensors" + _write_safetensors_with_metadata(checkpoint, {"encrypted_wandb_properties": "retired-fernet-blob"}) + + encoder.encode_via_api( + prompt="prompt", + api_key="key", + checkpoint_path=str(checkpoint), + enhance_prompt=False, + api_model=LTX_2_5_API_PROMPT_EMBEDDING_MODEL, + ) + + payload = http.calls[0].json_payload + assert payload is not None + assert "model" in payload + assert "model_id" not in payload + + +def test_encode_via_api_skips_request_when_no_selector(tmp_path: Path) -> None: + http = FakeHTTPClient() + encoder = LTXTextEncoder( + device=torch.device("cpu"), + http=http, + ltx_api_base_url="https://api.ltx.video", + ) + checkpoint = tmp_path / "empty.safetensors" + checkpoint.write_bytes(b"not-a-checkpoint") + + result = encoder.encode_via_api( + prompt="prompt", + api_key="key", + checkpoint_path=str(checkpoint), + enhance_prompt=False, + ) + + assert result is None + assert http.calls == [] diff --git a/backend/tests/test_pinned_pool_fix.py b/backend/tests/test_pinned_pool_fix.py new file mode 100644 index 000000000..29c801d76 --- /dev/null +++ b/backend/tests/test_pinned_pool_fix.py @@ -0,0 +1,114 @@ +"""Streaming pinned-host alloc must not surface as a CUDA VRAM OOM.""" + +from __future__ import annotations + +import pytest +import torch +from ltx_core.block_streaming import utils as bs_utils + +import services.patches.pinned_pool_fix as patch + + +@pytest.fixture(autouse=True) +def _reset_windows_log_flag() -> None: + patch._windows_pageable_logged = False + yield + patch._windows_pageable_logged = False + + +def test_patch_rebinds_alloc_buffer() -> None: + assert bs_utils.alloc_buffer is patch._patched_alloc_buffer + + +def test_require_attr_fails_loudly_when_symbol_missing() -> None: + with pytest.raises(RuntimeError, match="definitely_missing_symbol not found"): + patch._require_attr("definitely_missing_symbol") + + +def test_unpinned_allocation_unchanged() -> None: + buf = patch._patched_alloc_buffer(64, torch.device("cpu"), pin_memory=False) + assert buf.shape == (64,) + assert buf.dtype == torch.uint8 + assert not buf.is_pinned() + + +def test_windows_pin_request_uses_pageable_memory(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(patch.sys, "platform", "win32") + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + + def register_must_not_run(_nbytes: int) -> torch.Tensor | None: + raise AssertionError("cudaHostRegister path must be skipped on Windows") + + monkeypatch.setattr(patch, "_alloc_pinned_exact_cleared", register_must_not_run) + + buf = patch._patched_alloc_buffer(1024, torch.device("cpu"), pin_memory=True) + assert buf.numel() == 1024 + assert buf.dtype == torch.uint8 + assert not buf.is_pinned() + + +def test_windows_allocate_layout_views_stays_pageable(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(patch.sys, "platform", "win32") + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + views = bs_utils.allocate_layout_views({"k": (torch.Size([4, 4]), torch.bfloat16)}, pin_memory=True) + assert not views["k"].is_pinned() + assert views["k"].shape == torch.Size([4, 4]) + + +def test_linux_clears_sticky_error_after_register_failure(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(patch.sys, "platform", "linux") + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(patch, "_orig_alloc_pinned_exact", lambda _nbytes: None) + cleared: list[bool] = [] + monkeypatch.setattr(patch, "_clear_cuda_sticky_error", lambda: cleared.append(True)) + + real_empty = torch.empty + + def empty_unpinned(*args: object, pin_memory: bool = False, **kwargs: object) -> torch.Tensor: + return real_empty(*args, pin_memory=False, **kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(torch, "empty", empty_unpinned) + + buf = patch._patched_alloc_buffer(32, torch.device("cpu"), pin_memory=True) + assert buf.numel() == 32 + assert cleared == [True] + + +def test_linux_pin_oom_falls_back_to_pageable(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(patch.sys, "platform", "linux") + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(patch, "_alloc_pinned_exact_cleared", lambda _nbytes: None) + cleared: list[bool] = [] + monkeypatch.setattr(patch, "_clear_cuda_sticky_error", lambda: cleared.append(True)) + + real_empty = torch.empty + + def empty_maybe_pin(*args: object, pin_memory: bool = False, **kwargs: object) -> torch.Tensor: + if pin_memory: + raise RuntimeError("CUDA error: out of memory") + return real_empty(*args, pin_memory=False, **kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(torch, "empty", empty_maybe_pin) + + buf = patch._patched_alloc_buffer(128, torch.device("cpu"), pin_memory=True) + assert buf.numel() == 128 + assert not buf.is_pinned() + assert cleared == [True] + + +def test_linux_pin_non_oom_is_not_swallowed(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(patch.sys, "platform", "linux") + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(patch, "_alloc_pinned_exact_cleared", lambda _nbytes: None) + + real_empty = torch.empty + + def empty_maybe_pin(*args: object, pin_memory: bool = False, **kwargs: object) -> torch.Tensor: + if pin_memory: + raise RuntimeError("CUDA error: invalid argument") + return real_empty(*args, pin_memory=False, **kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(torch, "empty", empty_maybe_pin) + + with pytest.raises(RuntimeError, match="invalid argument"): + patch._patched_alloc_buffer(8, torch.device("cpu"), pin_memory=True) diff --git a/backend/tests/test_retake_extend_generation.py b/backend/tests/test_retake_extend_generation.py new file mode 100644 index 000000000..e12e19bd0 --- /dev/null +++ b/backend/tests/test_retake_extend_generation.py @@ -0,0 +1,144 @@ +"""Local Retake/Extend routing for LTX 2.3 and 2.5. + +The fake pipeline never runs GPU code. These tests pin that each model id +loads the matching checkpoint/VAE paths and that a model switch rebuilds +the cached Retake pipeline. +""" + +from __future__ import annotations + +import uuid + +import pytest + +from api_types import LTXLocalModelId +from runtime_config.ltx_runtime_paths import resolve_ltx_runtime_paths +from runtime_config.model_download_specs import get_existing_cp_path, get_ltx_model_spec +from state.app_settings import resolved_use_conv_vae +from state.app_state_types import GpuSlot, RetakePipelineState + +_LOCAL_2_3: LTXLocalModelId = "ltx-2.3-22b-distilled" +_LOCAL_2_5: LTXLocalModelId = "ltx-2.5-22b-distilled" + + +def _make_valid_video( + test_state, + *, + frames: int = 73, + width: int = 64, + height: int = 64, + fps: int = 24, +) -> str: + import imageio.v2 as imageio + import numpy as np + + video_file = test_state.config.outputs_dir / f"retake_extend_{uuid.uuid4().hex[:6]}.mp4" + writer = imageio.get_writer(str(video_file), fps=fps, codec="libx264", macro_block_size=None) + frame = np.zeros((height, width, 3), dtype=np.uint8) + for _ in range(frames): + writer.append_data(frame) + writer.close() + return str(video_file) + + +def _install_local(test_state, create_fake_model_files, model_id: LTXLocalModelId) -> None: + create_fake_model_files(model_id=model_id) + test_state.state.app_settings.active_ltx_model_id = model_id + test_state.state.app_settings.use_local_text_encoder = True + + +def _expected_create_paths(test_state, model_id: LTXLocalModelId) -> dict[str, str | None]: + spec = get_ltx_model_spec(model_id) + gemma_root = str(get_existing_cp_path(test_state.config.default_models_dir, spec.text_encoder_cp)) + paths = resolve_ltx_runtime_paths( + test_state.config.default_models_dir, + model_id, + gemma_root=gemma_root, + use_conv_vae=resolved_use_conv_vae(test_state.state.app_settings), + ) + return { + "checkpoint_path": paths.checkpoint_path, + "gemma_root": paths.gemma_root, + "video_vae_path": paths.video_vae_path, + "audio_vae_path": paths.audio_vae_path, + "duration_head_path": paths.duration_head_path, + } + + +def _assert_create_paths(fake_services, test_state, model_id: LTXLocalModelId) -> None: + assert fake_services.retake_pipeline.create_calls[-1] == _expected_create_paths(test_state, model_id) + + +def _assert_retake_pipeline_model(test_state, model_id: LTXLocalModelId) -> None: + slot = test_state.state.gpu_slot + assert isinstance(slot, GpuSlot) + assert isinstance(slot.active_pipeline, RetakePipelineState) + assert slot.active_pipeline.ltx_model_id == model_id + + +@pytest.mark.parametrize("model_id", [_LOCAL_2_3, _LOCAL_2_5]) +def test_local_retake_routes_to_model_paths( + client, test_state, create_fake_model_files, fake_services, model_id: LTXLocalModelId +) -> None: + _install_local(test_state, create_fake_model_files, model_id) + video_path = _make_valid_video(test_state) + + r = client.post( + "/api/retake", + json={"video_path": video_path, "start_time": 1.0, "duration": 3.0, "prompt": "make it dramatic"}, + ) + assert r.status_code == 200 + data = r.json() + assert data["status"] == "complete" + assert data["video_path"] + _assert_create_paths(fake_services, test_state, model_id) + _assert_retake_pipeline_model(test_state, model_id) + + +@pytest.mark.parametrize("model_id", [_LOCAL_2_3, _LOCAL_2_5]) +@pytest.mark.parametrize("mode", ["start", "end"]) +def test_local_extend_routes_to_model_paths( + client, test_state, create_fake_model_files, fake_services, model_id: LTXLocalModelId, mode: str +) -> None: + _install_local(test_state, create_fake_model_files, model_id) + video_path = _make_valid_video(test_state, frames=9) + + r = client.post( + "/api/extend", + json={"video_path": video_path, "duration": 4.0, "prompt": "continue the motion", "mode": mode}, + ) + assert r.status_code == 200 + data = r.json() + assert data["status"] == "complete" + assert data["video_path"] + _assert_create_paths(fake_services, test_state, model_id) + _assert_retake_pipeline_model(test_state, model_id) + assert fake_services.retake_pipeline.extend_calls[-1]["mode"] == mode + + +def test_retake_pipeline_rebuilds_when_switching_2_3_to_2_5( + client, test_state, create_fake_model_files, fake_services +) -> None: + create_fake_model_files(model_id=_LOCAL_2_3) + create_fake_model_files(model_id=_LOCAL_2_5) + test_state.state.app_settings.use_local_text_encoder = True + test_state.state.app_settings.active_ltx_model_id = _LOCAL_2_3 + + video_path = _make_valid_video(test_state) + payload = {"video_path": video_path, "start_time": 1.0, "duration": 3.0, "prompt": "make it dramatic"} + + first = client.post("/api/retake", json=payload) + assert first.status_code == 200 + _assert_create_paths(fake_services, test_state, _LOCAL_2_3) + _assert_retake_pipeline_model(test_state, _LOCAL_2_3) + + test_state.state.app_settings.active_ltx_model_id = _LOCAL_2_5 + second = client.post("/api/retake", json=payload) + assert second.status_code == 200 + _assert_create_paths(fake_services, test_state, _LOCAL_2_5) + _assert_retake_pipeline_model(test_state, _LOCAL_2_5) + assert len(fake_services.retake_pipeline.create_calls) == 2 + assert ( + fake_services.retake_pipeline.create_calls[0]["checkpoint_path"] + != fake_services.retake_pipeline.create_calls[1]["checkpoint_path"] + ) diff --git a/backend/tests/test_retake_sampler_selection.py b/backend/tests/test_retake_sampler_selection.py new file mode 100644 index 000000000..90b878f0e --- /dev/null +++ b/backend/tests/test_retake_sampler_selection.py @@ -0,0 +1,117 @@ +"""GPU-free checks for Retake/Extend distilled sampler selection. + +``_run()`` cannot be exercised here without loading real checkpoints. The helper +it calls, plus ``_invoke_diffusion_stage`` (the ``self.stage`` call site), are +the whole decision. +""" + +from __future__ import annotations + +from functools import partial +from pathlib import Path +from typing import Any, cast + +import torch +from ltx_core.components.diffusion_steps import EulerAncestralDiffusionStep +from ltx_pipelines.distilled import ( + ANCESTRAL_ETA, + ANCESTRAL_NOISE_SEED_OFFSET, + ANCESTRAL_S_NOISE, + should_use_ancestral_sampler, +) +from ltx_pipelines.utils.samplers import euler_ancestral_denoising_loop +from safetensors.torch import save_file + +from services.retake_pipeline.ltx_retake_pipeline import ( + LTXRetakePipeline, + distilled_stage_sampler_kwargs, +) + + +def _checkpoint_with_version(tmp_path: Path, version: str) -> str: + path = tmp_path / f"transformer-{version}.safetensors" + save_file({"dummy": torch.zeros(1)}, str(path), metadata={"model_version": version}) + return str(path) + + +def _assert_ancestral_kwargs(kwargs: dict[str, Any], *, seed: int, dtype: torch.dtype) -> None: + stepper = kwargs["stepper"] + assert isinstance(stepper, EulerAncestralDiffusionStep) + assert stepper.eta == ANCESTRAL_ETA + assert stepper.s_noise == ANCESTRAL_S_NOISE + loop = kwargs["loop"] + assert isinstance(loop, partial) + assert loop.func is euler_ancestral_denoising_loop + assert loop.keywords["noise_seed"] == seed + ANCESTRAL_NOISE_SEED_OFFSET + assert loop.keywords["model_dtype"] is dtype + + +def test_distilled_2_5_selects_ancestral_sampler(tmp_path: Path) -> None: + seed = 7 + dtype = torch.bfloat16 + path = _checkpoint_with_version(tmp_path, "2.5") + assert should_use_ancestral_sampler(path) + kwargs = distilled_stage_sampler_kwargs( + distilled=True, + use_ancestral=should_use_ancestral_sampler(path), + seed=seed, + dtype=dtype, + ) + _assert_ancestral_kwargs(kwargs, seed=seed, dtype=dtype) + + +def test_distilled_2_3_keeps_deterministic_defaults(tmp_path: Path) -> None: + path = _checkpoint_with_version(tmp_path, "2.3") + assert not should_use_ancestral_sampler(path) + kwargs = distilled_stage_sampler_kwargs( + distilled=True, + use_ancestral=should_use_ancestral_sampler(path), + seed=1, + dtype=torch.bfloat16, + ) + assert kwargs == {} + + +def test_guided_path_never_selects_ancestral_sampler() -> None: + kwargs = distilled_stage_sampler_kwargs( + distilled=False, + use_ancestral=True, + seed=1, + dtype=torch.bfloat16, + ) + assert kwargs == {} + + +def _pipeline_with_recording_stage(*, use_ancestral: bool) -> tuple[LTXRetakePipeline, list[dict[str, Any]]]: + calls: list[dict[str, Any]] = [] + + def fake_stage(**kwargs: Any) -> tuple[object, object]: + calls.append(kwargs) + return object(), object() + + pipeline = cast(LTXRetakePipeline, object.__new__(LTXRetakePipeline)) + pipeline.dtype = torch.bfloat16 + pipeline.use_ancestral_sampler = use_ancestral + pipeline.stage = fake_stage # type: ignore[method-assign] + return pipeline, calls + + +def test_invoke_diffusion_stage_forwards_ancestral_kwargs() -> None: + seed = 7 + pipeline, calls = _pipeline_with_recording_stage(use_ancestral=True) + denoiser = object() + pipeline._invoke_diffusion_stage(distilled=True, seed=seed, denoiser=denoiser) + + assert len(calls) == 1 + kwargs = calls[0] + assert kwargs["denoiser"] is denoiser + _assert_ancestral_kwargs(kwargs, seed=seed, dtype=pipeline.dtype) + + +def test_invoke_diffusion_stage_omits_sampler_overrides_when_not_ancestral() -> None: + pipeline, calls = _pipeline_with_recording_stage(use_ancestral=False) + pipeline._invoke_diffusion_stage(distilled=True, seed=7, denoiser=object()) + + assert len(calls) == 1 + assert "stepper" not in calls[0] + assert "loop" not in calls[0] diff --git a/frontend/components/SettingsPanel.tsx b/frontend/components/SettingsPanel.tsx index a446615b9..4986a8f81 100644 --- a/frontend/components/SettingsPanel.tsx +++ b/frontend/components/SettingsPanel.tsx @@ -1,5 +1,6 @@ import { useEffect, useMemo } from 'react' import { Select } from './ui/select' +import { videoGenerationResolutionLabel } from '../lib/video-resolution' import { areVideoGenerationSettingsEquivalent, resolveVideoGenerationOptions, @@ -199,7 +200,7 @@ export function SettingsPanel({ > {resolvedVideoOptions.resolutionOptions.map((resolution) => ( ))} diff --git a/frontend/generated/backend-openapi.json b/frontend/generated/backend-openapi.json index 7d8da34f2..f7ee1fcab 100644 --- a/frontend/generated/backend-openapi.json +++ b/frontend/generated/backend-openapi.json @@ -1110,6 +1110,9 @@ "anyOf": [ { "enum": [ + 2, + 3, + 4, 5, 6, 8, @@ -2555,6 +2558,9 @@ "additionalProperties": { "items": { "enum": [ + 2, + 3, + 4, 5, 6, 8, diff --git a/frontend/generated/backend-openapi.ts b/frontend/generated/backend-openapi.ts index 8cd1ef63b..71ee10df5 100644 --- a/frontend/generated/backend-openapi.ts +++ b/frontend/generated/backend-openapi.ts @@ -1103,7 +1103,7 @@ export interface components { * Duration * @default 5 */ - duration: (5 | 6 | 8 | 10 | 12 | 14 | 16 | 18 | 20) | null; + duration: (2 | 3 | 4 | 5 | 6 | 8 | 10 | 12 | 14 | 16 | 18 | 20) | null; /** * Fps * @default 24 @@ -1629,7 +1629,7 @@ export interface components { LTXVideoGenerationResolutionSpec: { /** Fps To Durations */ fps_to_durations: { - [key: string]: (5 | 6 | 8 | 10 | 12 | 14 | 16 | 18 | 20)[]; + [key: string]: (2 | 3 | 4 | 5 | 6 | 8 | 10 | 12 | 14 | 16 | 18 | 20)[]; }; }; /** LTXVideoGenerationSpec */ diff --git a/frontend/hooks/use-generation.ts b/frontend/hooks/use-generation.ts index 389db9ace..55fb3081a 100644 --- a/frontend/hooks/use-generation.ts +++ b/frontend/hooks/use-generation.ts @@ -18,6 +18,9 @@ export interface GenerationRecoveryContext { // Retake/extend write this instead of a full `settings` blob — the recovery importer // prefers it over `settings.model` (which defaults to 'fast' when absent). model?: string + // Frozen at click. Local `fast` and API `fast` share an id; display_name must not be + // re-resolved from whichever offering is selected when the job later finishes. + modelLabel?: string inputImageUrl?: string inputAudioUrl?: string genType?: 'image' | 'enhance' diff --git a/frontend/lib/generation-recovery-importers.ts b/frontend/lib/generation-recovery-importers.ts index 3d5bbf06e..213f40227 100644 --- a/frontend/lib/generation-recovery-importers.ts +++ b/frontend/lib/generation-recovery-importers.ts @@ -35,6 +35,7 @@ const importVideo: RecoveryImporter = async (ctx, result, { addAsset, modelsDir mode: genMode, prompt: ctx.prompt, model: ctx.model ?? s?.model ?? 'fast', + modelLabel: ctx.modelLabel, duration: s?.duration ?? null, resolution: s?.videoResolution ?? '', fps: s?.fps ?? 24, diff --git a/frontend/lib/video-generation-model-specs.ts b/frontend/lib/video-generation-model-specs.ts index b92b935b7..0f1b97ea7 100644 --- a/frontend/lib/video-generation-model-specs.ts +++ b/frontend/lib/video-generation-model-specs.ts @@ -39,6 +39,9 @@ export interface ResolvedVideoGenerationOptions { type DurationSelectionMode = 'preserve' | 'smallest_valid' +/** GenSpace picker floor. The API envelope includes 2–5s so gap fill can request shorts. */ +export const GENSPACE_MIN_SELECTABLE_DURATION_S = 6 + interface ResolveVideoGenerationOptionsParams { settings: T modelSpecs: VideoGenerationModelSpecItem[] @@ -156,6 +159,14 @@ export function getLocalOfferingCapabilities( return specs?.local_models[0]?.spec.capabilities ?? null } +export function getApiOfferingCapabilities( + specs: VideoGenerationModelSpecsResponse | null | undefined, + pipeline: string | null | undefined, +): VideoGenerationOfferingCapabilities | null { + if (!specs || !pipeline) return null + return specs.api_models.find((item) => item.pipeline === pipeline)?.spec.capabilities ?? null +} + export function resolveVideoGenerationOptions({ settings, modelSpecs, diff --git a/frontend/lib/video-resolution.ts b/frontend/lib/video-resolution.ts index 3fe1b8ea9..3452df9c0 100644 --- a/frontend/lib/video-resolution.ts +++ b/frontend/lib/video-resolution.ts @@ -25,6 +25,11 @@ export function namedResolutionDisplayName(tier: number): string { return tier >= 2160 ? '4K' : `${tier}p` } +/** Display label for generation resolution ids (`2160p` → `4K`). */ +export function videoGenerationResolutionLabel(resolution: string): string { + return resolution === '2160p' ? '4K' : resolution +} + export function resolutionOptions(width: number, height: number): ResolutionOption[] { if (!width || !height) return [] const shortEdge = Math.min(width, height) @@ -34,7 +39,7 @@ export function resolutionOptions(width: number, height: number): ResolutionOpti const originalTier = namedResolutionTier(shortEdge) const options: ResolutionOption[] = [ - { key: 'original', label: `${originalTier}p (Original)`, width: null, height: null }, + { key: 'original', label: `${namedResolutionDisplayName(originalTier)} (Original)`, width: null, height: null }, ] for (const tier of STANDARD_TIERS) { // Only smaller tiers, and drop the one that already maps to Original. diff --git a/frontend/views/GenSpace.tsx b/frontend/views/GenSpace.tsx index 63b5e70a6..fde19a6a3 100644 --- a/frontend/views/GenSpace.tsx +++ b/frontend/views/GenSpace.tsx @@ -20,7 +20,7 @@ import { type RetakeExtendModel, } from '../hooks/use-retake' import { useExtend, type ExtendDirection, EXTEND_SECONDS, DEFAULT_EXTEND_SECONDS } from '../hooks/use-extend' -import { resolutionOptions, type ResolutionOption } from '../lib/video-resolution' +import { resolutionOptions, videoGenerationResolutionLabel, type ResolutionOption } from '../lib/video-resolution' import { useIcLora, type IcLoraAudioMode } from '../hooks/use-ic-lora' import { useCustomIcLoraEnabled } from '../hooks/use-custom-ic-lora-enabled' import { useDevFlags } from '../contexts/DevFlagsContext' @@ -38,6 +38,8 @@ import { pathToFileUrl } from '../lib/file-url' import { areVideoGenerationSettingsEquivalent, formatPipelineDisplayName, + GENSPACE_MIN_SELECTABLE_DURATION_S, + getApiOfferingCapabilities, getVideoGenerationModelSpecs, getLocalOfferingCapabilities, resolvePipelineDisplayName, @@ -48,7 +50,7 @@ import { } from '../lib/video-generation-model-specs' import { logger } from '../lib/logger' import { ApiClient, type ApiSuccessOf } from '../lib/api-client' -import type { LoraSelection } from '../components/SettingsPanel' +import type { GenerationSettings, LoraSelection } from '../components/SettingsPanel' import { RetakePanel } from '../components/RetakePanel' import { ExtendPanel } from '../components/ExtendPanel' import { ICLoraPanel, CONDITIONING_TYPES } from '../components/ICLoraPanel' @@ -642,6 +644,7 @@ function PromptBar({ settings, modelSpecs: videoModelSpecs, hasAudio: Boolean(inputAudio), + minimumDuration: isLocalMode ? undefined : GENSPACE_MIN_SELECTABLE_DURATION_S, }) : null const showVideoFpsControl = Boolean( @@ -1036,11 +1039,18 @@ function PromptBar({ title="RESOLUTION" value={resolvedVideoOptions.selectedResolution ?? settings.videoResolution} onChange={(v) => onSettingsChange({ ...settings, videoResolution: v })} - options={resolvedVideoOptions.resolutionOptions.map((value) => ({ value, label: value }))} + options={resolvedVideoOptions.resolutionOptions.map((value) => ({ + value, + label: videoGenerationResolutionLabel(value), + }))} trigger={ <> - {(resolvedVideoOptions.selectedResolution ?? settings.videoResolution).replace('p', '')} + + {videoGenerationResolutionLabel( + resolvedVideoOptions.selectedResolution ?? settings.videoResolution, + ).replace(/p$/, '')} + } /> @@ -1268,6 +1278,17 @@ export function GenSpace() { // Provenance for the completion effect below: imagePaths/isGenerating alone can't tell // it whether the request that just finished was an edit or a plain generation. const lastImageEditRef = useRef<{ source: string; strength: number } | null>(null) + // Click-time t2v/i2v/a2v/image snapshot. The live picker can change while the job + // runs; the completion effects must tag the asset with what was actually submitted. + // Survives a refresh via the recovery marker restore below — the ref itself does not. + const generateSubmissionRef = useRef<{ + kind: 'video' | 'image' + prompt: string + settings: GenerationSettings + modelLabel?: string + inputImageUrl: string | null + inputAudioUrl: string | null + } | null>(null) const [settings, setSettings] = useState(() => ({ ...DEFAULT_VIDEO_SETTINGS })) const videoModelSpecs = getVideoGenerationModelSpecs(videoGenerationModelSpecsResponse, { useApiSpecs: shouldVideoGenerateWithLtxApi, @@ -1282,9 +1303,10 @@ export function GenSpace() { if (mode !== 'video' || videoModelSpecs.length === 0) return next return sanitizeVideoGenerationSettings(next, videoModelSpecs, { hasAudio: Boolean(inputAudio), + minimumDuration: shouldVideoGenerateWithLtxApi ? GENSPACE_MIN_SELECTABLE_DURATION_S : undefined, }) ?? next }, - [inputAudio, mode, videoModelSpecs], + [inputAudio, mode, shouldVideoGenerateWithLtxApi, videoModelSpecs], ) const { @@ -1305,10 +1327,16 @@ export function GenSpace() { // Locally installed LoRAs are only usable in local generation mode. const isLocalMode = !shouldVideoGenerateWithLtxApi const localCaps = getLocalOfferingCapabilities(videoGenerationModelSpecsResponse) + // Retake/Extend always request RETAKE_EXTEND_MODELS (currently "pro" / 2.3 Pro), + // not the t2v/i2v picker in settings.model. + const apiCaps = getApiOfferingCapabilities( + videoGenerationModelSpecsResponse, + RETAKE_EXTEND_MODELS[0], + ) const canUseUserLoras = isLocalMode && Boolean(localCaps?.user_loras) const canUseIcLora = !forceApiGenerations && Boolean(localCaps?.ic_lora) - const canUseRetake = !isLocalMode || Boolean(localCaps?.retake) - const canUseExtend = !isLocalMode || Boolean(localCaps?.extend) + const canUseRetake = isLocalMode ? Boolean(localCaps?.retake) : Boolean(apiCaps?.retake) + const canUseExtend = isLocalMode ? Boolean(localCaps?.extend) : Boolean(apiCaps?.extend) // Enhance itself is independent of the video-generation backend — the backend enhance // endpoint only cares about the enhancer provider (local Gemma vs. Gemini), not whether video // generation runs locally or via the LTX API. If no catalog LoRA is selected (e.g. because the @@ -1602,6 +1630,9 @@ export function GenSpace() { useEffect(() => { if (!genSpaceRetakeSource) return + // Specs start null on remount (Project unmounts GenSpace off-tab). Don't drop an + // incoming timeline retake until we know whether the offering actually supports it. + if (isLoadingVideoGenerationModelSpecs) return if (!canUseRetake) { setGenSpaceRetakeSource(null) return @@ -1619,7 +1650,13 @@ export function GenSpace() { }) setRetakePanelKey((prev) => prev + 1) setGenSpaceRetakeSource(null) - }, [genSpaceRetakeSource, setGenSpaceRetakeSource, activeProject?.assets, canUseRetake]) + }, [ + genSpaceRetakeSource, + setGenSpaceRetakeSource, + activeProject?.assets, + canUseRetake, + isLoadingVideoGenerationModelSpecs, + ]) useEffect(() => { if (!genSpaceIcLoraSource) return @@ -1642,10 +1679,11 @@ export function GenSpace() { }, [genSpaceIcLoraSource, canUseIcLora, setGenSpaceIcLoraSource]) useEffect(() => { + if (isLoadingVideoGenerationModelSpecs) return if (mode === 'ic-lora' && !canUseIcLora) setMode('video') if (mode === 'retake' && !canUseRetake) setMode('video') if (mode === 'extend' && !canUseExtend) setMode('video') - }, [canUseIcLora, canUseRetake, canUseExtend, mode]) + }, [canUseIcLora, canUseRetake, canUseExtend, mode, isLoadingVideoGenerationModelSpecs]) useEffect(() => { if (!canUseUserLoras && selectedLoras.length > 0) setSelectedLoras([]) @@ -1770,6 +1808,14 @@ export function GenSpace() { variations: s.variations ?? prev.variations, imageEditStrength: s.imageEditStrength ?? prev.imageEditStrength, })) + generateSubmissionRef.current = { + kind: ctx.genType === 'image' ? 'image' : 'video', + prompt: ctx.prompt, + settings: s, + modelLabel: ctx.modelLabel, + inputImageUrl: ctx.inputImageUrl ?? null, + inputAudioUrl: ctx.inputAudioUrl ?? null, + } // The completion effect reads this ref (not settings/inputImage) to tag a // recovered image asset as an edit — restore it so recovery matches the // live handleGenerate() path. @@ -1793,10 +1839,32 @@ export function GenSpace() { if (persistedVideoKeyRef.current === generationKey) return persistedVideoKeyRef.current = generationKey - const genMode = inputAudio + const submission = generateSubmissionRef.current + if (submission?.kind !== 'video') { + logger.error('Video completed without a click-time submission; tagging from live picker state') + } + const usedPrompt = submission?.kind === 'video' ? submission.prompt : lastPrompt + const usedSettings: GenerationSettings = submission?.kind === 'video' + ? submission.settings + : { + model: settings.model as VideoGenerationPipeline, + duration: settings.duration, + videoResolution: settings.videoResolution, + fps: settings.fps, + audio: settings.audio, + cameraMotion: 'none', + aspectRatio: settings.aspectRatio, + imageResolution: settings.imageResolution, + imageAspectRatio: settings.aspectRatio ?? '16:9', + imageSteps: 4, + variations: settings.variations, + imageEditStrength: settings.imageEditStrength, + } + const usedImage = submission?.kind === 'video' ? submission.inputImageUrl : inputImage + const usedAudio = submission?.kind === 'video' ? submission.inputAudioUrl : inputAudio + const genMode = usedAudio ? 'audio-to-video' - : inputImage ? 'image-to-video' : 'text-to-video' - const savedVideoSettings = sanitizeVideoSettings(settings) + : usedImage ? 'image-to-video' : 'text-to-video' ;(async () => { try { @@ -1809,25 +1877,27 @@ export function GenSpace() { smallThumbnailPath: copied.smallThumbnailPath, width: copied.width, height: copied.height, - prompt: lastPrompt, - resolution: savedVideoSettings.videoResolution, - duration: savedVideoSettings.duration ?? undefined, + prompt: usedPrompt, + resolution: usedSettings.videoResolution, + duration: usedSettings.duration ?? undefined, generationParams: { mode: genMode as 'text-to-video' | 'image-to-video' | 'audio-to-video', - prompt: lastPrompt, - model: savedVideoSettings.model, - modelLabel: resolvePipelineDisplayName(videoModelSpecs, savedVideoSettings.model) ?? undefined, - duration: savedVideoSettings.duration, - resolution: savedVideoSettings.videoResolution, - fps: savedVideoSettings.fps, - audio: savedVideoSettings.audio || false, + prompt: usedPrompt, + model: usedSettings.model, + modelLabel: (submission?.kind === 'video' ? submission.modelLabel : undefined) + ?? resolvePipelineDisplayName(videoModelSpecs, usedSettings.model) + ?? undefined, + duration: usedSettings.duration, + resolution: usedSettings.videoResolution, + fps: usedSettings.fps, + audio: usedSettings.audio || false, cameraMotion: 'none', - imageAspectRatio: savedVideoSettings.aspectRatio, + imageAspectRatio: usedSettings.aspectRatio, imageSteps: 4, - inputImageUrl: inputImage || undefined, - inputAudioUrl: inputAudio || undefined, - loras: canUseUserLoras && selectedLoras.length > 0 - ? selectedLoras.map(l => ({ ...l, ref: toModelsDirRelativeRef(l.ref, appSettings.modelsDir) })) + inputImageUrl: usedImage || undefined, + inputAudioUrl: usedAudio || undefined, + loras: usedSettings.loras && usedSettings.loras.length > 0 + ? usedSettings.loras.map(l => ({ ...l, ref: toModelsDirRelativeRef(l.ref, appSettings.modelsDir) })) : undefined, }, takes: [{ @@ -1840,13 +1910,14 @@ export function GenSpace() { }], activeTakeIndex: 0, }) + generateSubmissionRef.current = null reset() } catch (err) { persistedVideoKeyRef.current = null logger.error(`Failed to persist generated video asset: ${err}`) } })() - }, [videoPath, currentProjectId, isGenerating, sanitizeVideoSettings, settings, inputImage, inputAudio, lastPrompt, addAsset, reset, selectedLoras, canUseUserLoras, appSettings.modelsDir]) + }, [videoPath, currentProjectId, isGenerating, settings, inputImage, inputAudio, lastPrompt, addAsset, reset, appSettings.modelsDir, videoModelSpecs]) // When retake completes, add as take or new asset useEffect(() => { @@ -2095,6 +2166,24 @@ export function GenSpace() { if (imagePaths.length === 0 || !currentProjectId || isGenerating) return if (addingImagesRef.current) return addingImagesRef.current = true + const submission = generateSubmissionRef.current + if (submission?.kind !== 'image') { + logger.error('Image completed without a click-time submission; tagging from live picker state') + } + const usedPrompt = submission?.kind === 'image' ? submission.prompt : lastPrompt + const usedSettings: GenerationSettings = submission?.kind === 'image' + ? submission.settings + : { + model: 'fast', + duration: 5, + videoResolution: settings.videoResolution, + fps: 24, + audio: false, + cameraMotion: 'none', + imageResolution: settings.imageResolution, + imageAspectRatio: settings.aspectRatio ?? '16:9', + imageSteps: 4, + } const editContext = lastImageEditRef.current const genMode = editContext ? 'image-edit' : 'text-to-image' @@ -2118,18 +2207,18 @@ export function GenSpace() { smallThumbnailPath: copied.smallThumbnailPath, width: copied.width, height: copied.height, - prompt: lastPrompt, - resolution: settings.imageResolution, + prompt: usedPrompt, + resolution: usedSettings.imageResolution, generationParams: { mode: genMode, - prompt: lastPrompt, + prompt: usedPrompt, model: 'fast', duration: 5, - resolution: settings.imageResolution, + resolution: usedSettings.imageResolution, fps: 24, audio: false, cameraMotion: 'none', - imageAspectRatio: settings.aspectRatio, + imageAspectRatio: usedSettings.imageAspectRatio || usedSettings.aspectRatio, imageSteps: editContext ? IMAGE_STEPS_EDIT : IMAGE_STEPS_GENERATE, ...(editContext ? { inputImageUrl: editContext.source, imageEditStrength: editContext.strength } : {}), }, @@ -2144,6 +2233,7 @@ export function GenSpace() { activeTakeIndex: 0, }) } + generateSubmissionRef.current = null reset() } catch (err) { logger.error(`Failed to persist generated image asset(s): ${err}`) @@ -2525,43 +2615,64 @@ export function GenSpace() { lastImageEditRef.current = editSource ? { source: editSource, strength: settings.imageEditStrength ?? 0.6 } : null - const imageSettings = { - model: 'fast' as VideoGenerationPipeline, + const imageSettings: GenerationSettings = { + model: 'fast', duration: 5, videoResolution: settings.videoResolution, fps: 24, audio: false, cameraMotion: 'none', imageResolution: settings.imageResolution, - imageAspectRatio: settings.aspectRatio, + imageAspectRatio: settings.aspectRatio ?? '16:9', imageSteps: editSource ? IMAGE_STEPS_EDIT : IMAGE_STEPS_GENERATE, variations: settings.variations, imageEditStrength: settings.imageEditStrength, } - await writeRecoveryContext({ prompt, settings: imageSettings, genType: 'image', inputImageUrl: editSource ?? undefined }) + const modelLabel = resolvePipelineDisplayName(videoModelSpecs, imageSettings.model) ?? undefined + generateSubmissionRef.current = { + kind: 'image', + prompt, + settings: imageSettings, + modelLabel, + inputImageUrl: editSource, + inputAudioUrl: null, + } + await writeRecoveryContext({ + prompt, + settings: imageSettings, + modelLabel, + genType: 'image', + inputImageUrl: editSource ?? undefined, + }) generateImage(prompt, imageSettings, editSource) } else { // Generate video (t2v if no image/audio, i2v if image, a2v if audio) const imagePath = inputImage || null const audioPath = inputAudio || null const videoSettings = sanitizeVideoSettings(settings) - const genSettings = { + if (!videoSettings) return + const genSettings: GenerationSettings = { + ...videoSettings, model: videoSettings.model as VideoGenerationPipeline, - duration: videoSettings.duration, - videoResolution: videoSettings.videoResolution, - fps: videoSettings.fps, - audio: videoSettings.audio || false, cameraMotion: 'none', - aspectRatio: videoSettings.aspectRatio, - imageResolution: videoSettings.imageResolution, - imageAspectRatio: videoSettings.aspectRatio, + imageAspectRatio: videoSettings.aspectRatio ?? '16:9', imageSteps: 4, // Local LoRA refs are filesystem paths the cloud API can't resolve. loras: canUseUserLoras && selectedLoras.length > 0 ? selectedLoras : undefined, } + const modelLabel = resolvePipelineDisplayName(videoModelSpecs, genSettings.model) ?? undefined + generateSubmissionRef.current = { + kind: 'video', + prompt, + settings: genSettings, + modelLabel, + inputImageUrl: imagePath, + inputAudioUrl: audioPath, + } await writeRecoveryContext({ prompt, settings: genSettings, + modelLabel, inputImageUrl: imagePath ?? undefined, inputAudioUrl: audioPath ?? undefined, }) @@ -2641,6 +2752,7 @@ export function GenSpace() { settings, modelSpecs: videoModelSpecs, hasAudio: Boolean(inputAudio), + minimumDuration: shouldVideoGenerateWithLtxApi ? GENSPACE_MIN_SELECTABLE_DURATION_S : undefined, }).hasCompatibleOptions ) // One global backend slot: Stop / Generate-disable must follow the in-flight job, not the diff --git a/frontend/views/editor/VideoEditorTimelineEditingPanel.tsx b/frontend/views/editor/VideoEditorTimelineEditingPanel.tsx index 9eecea549..3597ecdbd 100644 --- a/frontend/views/editor/VideoEditorTimelineEditingPanel.tsx +++ b/frontend/views/editor/VideoEditorTimelineEditingPanel.tsx @@ -20,6 +20,7 @@ import { ClipWaveform } from '../../components/AudioWaveform' import type { GenerationSettings } from '../../components/SettingsPanel' import { useAppSettings } from '../../contexts/AppSettingsContext' import { useVideoGenerationModelSpecs } from '../../hooks/use-video-generation-model-specs' +import { RETAKE_EXTEND_MODELS } from '../../hooks/use-retake' import type { GenerationError } from '../../lib/generation-errors' import { addVisualAssetToProject } from '../../lib/asset-copy' import { GapGenerationModal } from './GapGenerationModal' @@ -29,6 +30,7 @@ import { ApiClient } from '../../lib/api-client' import { pathToFileUrl } from '../../lib/file-url' import { areVideoGenerationSettingsEquivalent, + getApiOfferingCapabilities, getLocalOfferingCapabilities, getVideoGenerationModelSpecs, resolveVideoGenerationOptions, @@ -191,7 +193,15 @@ export function VideoEditorTimelineEditingPanel(props: VideoEditorTimelineEditin errorMessage: videoGenerationModelSpecsErrorMessage, } = useVideoGenerationModelSpecs() const localCaps = getLocalOfferingCapabilities(videoGenerationModelSpecsResponse) - const canUseRetake = shouldVideoGenerateWithLtxApi || Boolean(localCaps?.retake) + // Retake/Extend always request RETAKE_EXTEND_MODELS (currently "pro" / 2.3 Pro), + // not the timeline clip's t2v/i2v generation pipeline. + const apiCaps = getApiOfferingCapabilities( + videoGenerationModelSpecsResponse, + RETAKE_EXTEND_MODELS[0], + ) + const canUseRetake = shouldVideoGenerateWithLtxApi + ? Boolean(apiCaps?.retake) + : Boolean(localCaps?.retake) const requestRetakeClip = (clip: TimelineClip) => { if (!canUseRetake) return handleRetakeClip(clip) diff --git a/package.json b/package.json index 6365720a0..6e747ff31 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ltx-desktop", - "version": "1.2.4", + "version": "1.2.5", "description": "LTX-2 Video Generation - Desktop App", "type": "module", "main": "dist-electron/main.js",