From cee0b978073916c9b6f2e77afd67144163276ca0 Mon Sep 17 00:00:00 2001
From: Jeffrey Wang <jeffreywangcf@gmail.com>
Date: Tue, 1 Sep 2026 18:56:44 -0500
Subject: [PATCH] fallback to ffmpeg

---
 .gitignore                               |  1 +
 src/engine/ov_genai/whisper.py           | 36 ++++++++++++++++++++++--
 tests/unit/test_ov_genai_whisper_unit.py | 22 ++++++++++++++-
 3 files changed, 56 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index c4a6107..6ca98a9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,6 +19,7 @@ docker-compose.override.yaml
 .mypy_cache/
 .venv/
 .vscode/
+.idea/
 .builds/
 .cursor/
 openarc_bench.db
diff --git a/src/engine/ov_genai/whisper.py b/src/engine/ov_genai/whisper.py
index db5814c..e0cab87 100644
--- a/src/engine/ov_genai/whisper.py
+++ b/src/engine/ov_genai/whisper.py
@@ -3,11 +3,15 @@
 import gc
 import io
 import logging
+import os
+import subprocess
+import tempfile
 from typing import Any, AsyncIterator, Dict, Union
 
 import librosa
 import numpy as np
 from openvino_genai import WhisperPipeline
+from soundfile import LibsndfileError
 
 from src.server.model_registry import ModelRegistry
 from src.server.schemas.registration import ModelLoadConfig
@@ -22,6 +26,30 @@ def __init__(self, load_config: ModelLoadConfig):
         self.load_config = load_config
         pass
 
+    def decode_with_ffmpeg(self, audio_bytes: bytes) -> np.ndarray:
+        src = tempfile.NamedTemporaryFile(delete=False)
+        try:
+            src.write(audio_bytes)
+            src.flush()
+            src.close()
+            proc = subprocess.run(
+                [
+                    "ffmpeg",
+                    "-loglevel", "error",
+                    "-i", src.name,         # Read the uploaded audio from the temporary file.
+                    "-ac", "1",             # Convert the audio to mono.
+                    "-ar", "16000",         # Resample to 16 kHz.
+                    "-f", "f32le",          # Emit raw 32-bit little-endian float PCM.
+                    "pipe:1",               # Write the decoded audio to std out.
+                ],
+                capture_output=True,
+            )
+            if proc.returncode != 0:
+                raise ValueError(f"ffmpeg could not decode audio: {proc.stderr.decode(errors='replace')[:500]}")
+            return np.frombuffer(proc.stdout, dtype=np.float32)
+        finally:
+            os.unlink(src.name)  # always delete the temp file
+
     def prepare_audio(self, gen_config: OVGenAI_WhisperGenConfig) -> list[float]:
         """
         Prepare audio inputs from base64 string for the Whisper pipeline.
@@ -30,8 +58,12 @@ def prepare_audio(self, gen_config: OVGenAI_WhisperGenConfig) -> list[float]:
         audio_bytes = base64.b64decode(gen_config.audio_base64)
         
         audio_buffer = io.BytesIO(audio_bytes)
-        
-        audio, sr = librosa.load(audio_buffer, sr=16000, mono=True)
+
+        try:
+            audio, sr = librosa.load(audio_buffer, sr=16000, mono=True)
+        except LibsndfileError as exc:
+            logger.info(f"librosa decode failed ({exc}), retrying with ffmpeg")
+            audio = self.decode_with_ffmpeg(audio_bytes)
 
         return audio.astype(np.float32).tolist()
 
diff --git a/tests/unit/test_ov_genai_whisper_unit.py b/tests/unit/test_ov_genai_whisper_unit.py
index ff2d2f4..e67fb3b 100644
--- a/tests/unit/test_ov_genai_whisper_unit.py
+++ b/tests/unit/test_ov_genai_whisper_unit.py
@@ -71,6 +71,27 @@ def test_prepare_audio_calls_librosa(monkeypatch: pytest.MonkeyPatch, load_confi
     assert audio_list == audio_array.tolist()
 
 
+def test_prepare_audio_falls_back_to_ffmpeg(monkeypatch: pytest.MonkeyPatch, load_config: ModelLoadConfig) -> None:
+    whisper = OVGenAI_Whisper(load_config)
+    audio_array = np.array([0.1, -0.2], dtype=np.float32)
+
+    load_mock = MagicMock(side_effect=whisper_module.LibsndfileError(1))
+    monkeypatch.setattr(whisper_module.librosa, "load", load_mock)
+    proc_mock = MagicMock(returncode=0, stdout=audio_array.tobytes())
+    run_mock = MagicMock(return_value=proc_mock)
+    monkeypatch.setattr(whisper_module.subprocess, "run", run_mock)
+
+    audio_list = whisper.prepare_audio(OVGenAI_WhisperGenConfig(audio_base64=_sample_audio_base64()))
+
+    load_mock.assert_called_once()
+    run_mock.assert_called_once()
+    command = run_mock.call_args.args[0]
+    assert command[:4] == ["ffmpeg", "-loglevel", "error", "-i"]
+    assert command[5:] == ["-ac", "1", "-ar", "16000", "-f", "f32le", "pipe:1"]
+    assert run_mock.call_args.kwargs["capture_output"] is True
+    assert audio_list == audio_array.tolist()
+
+
 def test_collect_metrics_formats_values(load_config: ModelLoadConfig) -> None:
     whisper = OVGenAI_Whisper(load_config)
     metrics = whisper.collect_metrics(DummyPerfMetrics())
@@ -151,4 +172,3 @@ def test_unload_model_resets_state(monkeypatch: pytest.MonkeyPatch, load_config:
     assert whisper.whisper_model is None
     registry.register_unload.assert_called_once_with("model-name")
     gc_mock.assert_called_once()
-