diff --git a/backend/ltx2_server.py b/backend/ltx2_server.py index f250fb7c..63fa9ea1 100644 --- a/backend/ltx2_server.py +++ b/backend/ltx2_server.py @@ -235,11 +235,14 @@ def _resolve_app_data_dir() -> Path: def _resolve_local_generations_mode() -> LocalGenerationMode: + from runtime_config.accelerator import accelerator_backend + gpu_info = GpuInfoImpl() system = platform.system() cuda_available = gpu_info.get_cuda_available() mps_available = gpu_info.get_mps_available() vram_gb = gpu_info.get_vram_total_gb() + fp8_capable = accelerator_backend() == "cuda" # On Darwin there's no discrete VRAM (unified memory), so gate on *available* RAM, # not total — total overstates real headroom once the OS/Electron/app are running. # See GpuInfoImpl.get_available_ram_gb. @@ -252,16 +255,18 @@ def _resolve_local_generations_mode() -> LocalGenerationMode: vram_gb=vram_gb, mps_available=mps_available, ram_gb=available_ram_gb, + fp8_capable=fp8_capable, ) logger.info( "Runtime policy local_generations_mode=%s (system=%s cuda_available=%s mps_available=%s " - "vram_gb=%s available_ram_gb=%s)", + "vram_gb=%s available_ram_gb=%s fp8_capable=%s)", mode, system, cuda_available, mps_available, vram_gb, available_ram_gb, + fp8_capable, ) return mode @@ -344,6 +349,8 @@ def log_hardware_info() -> None: "LTX 2.5 decode uses eager SDPA on Mac (no Triton; slower than Linux/Windows)." ) logger.info(gpu_line) + from runtime_config.accelerator import accelerator_backend + logger.info(f"Accelerator: {accelerator_backend()} | HIP: {getattr(torch.version, 'hip', None)}") logger.info(f"SageAttention: {'enabled' if use_sage_attention else 'disabled'}") logger.info(f"Python: {sys.version.split()[0]} | Torch: {torch.__version__}") diff --git a/backend/runtime_config/accelerator.py b/backend/runtime_config/accelerator.py new file mode 100644 index 00000000..07688b8c --- /dev/null +++ b/backend/runtime_config/accelerator.py @@ -0,0 +1,27 @@ +"""Accelerator backend detection. + +ROCm PyTorch reports itself through the same `torch.cuda` interface as NVIDIA +CUDA builds (`torch.cuda.is_available()`, `device.type == "cuda"`, etc.), so +code that checks `device.type == "cuda"` to mean "this is an NVIDIA GPU" is +wrong under ROCm. `torch.version.hip` is the actual discriminator: it is set +on ROCm builds and `None` on CUDA builds. +""" + +from __future__ import annotations + +import torch + +AcceleratorBackend = str # "rocm" | "cuda" | "mps" | "cpu" + + +def accelerator_backend() -> AcceleratorBackend: + if getattr(torch.version, "hip", None): + return "rocm" + + if torch.cuda.is_available(): + return "cuda" + + if hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + return "mps" + + return "cpu" diff --git a/backend/runtime_config/runtime_policy.py b/backend/runtime_config/runtime_policy.py index 91bbcce4..04882a12 100644 --- a/backend/runtime_config/runtime_policy.py +++ b/backend/runtime_config/runtime_policy.py @@ -48,6 +48,7 @@ def decide_local_generation_mode( vram_gb: int | None, mps_available: bool = False, ram_gb: int | None = None, + fp8_capable: bool = True, ) -> LocalGenerationMode: """Pick the local-generation mode for this runtime. @@ -92,6 +93,14 @@ def decide_local_generation_mode( return "unsupported" if vram_gb < 15: return "unsupported" + # full_models_loading's >=31 GB floor assumes the fp8-halved (~23 GB) transformer + # (see module docstring). Without fp8 (ROCm today — see + # runtime_config.accelerator.accelerator_backend), holding the full bf16 + # (~42-46 GB) transformer resident instead would OOM at this floor, so stay on + # the streaming path regardless of VRAM until a real bf16-full-resident floor is + # established on non-CUDA hardware. + if not fp8_capable: + return "streaming_models_loading" if vram_gb < 31: return "streaming_models_loading" return "full_models_loading" diff --git a/backend/services/services_utils.py b/backend/services/services_utils.py index e98784da..9c2c9854 100644 --- a/backend/services/services_utils.py +++ b/backend/services/services_utils.py @@ -90,7 +90,15 @@ def effective_edit_steps(num_inference_steps: int, strength: float) -> int: def device_supports_fp8(device: str | torch.device | object | None) -> bool: - return get_device_type(device) == "cuda" + # ROCm PyTorch also reports device.type == "cuda" (see runtime_config.accelerator), + # so this must not be a plain device-type check: ROCm has no fp8_cast kernel support + # here yet, and would otherwise be silently misidentified as CUDA/NVIDIA. + if get_device_type(device) != "cuda": + return False + + from runtime_config.accelerator import accelerator_backend + + return accelerator_backend() == "cuda" def sync_device(device: str | torch.device | object | None) -> None: