diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 8a4eedd63..5af2e8f0e 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -243,6 +243,7 @@ interface Window { error?: string; userNotified?: boolean; microphoneFallbackRequired?: boolean; + captureStartedAtMs?: number; }>; stopNativeScreenRecording: () => Promise<{ success: boolean; diff --git a/electron/ipc/captions/generate.test.ts b/electron/ipc/captions/generate.test.ts new file mode 100644 index 000000000..a794a1c08 --- /dev/null +++ b/electron/ipc/captions/generate.test.ts @@ -0,0 +1,70 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { getCompanionAudioFallbackInfoMock, resolveRecordingSessionMock } = vi.hoisted(() => ({ + getCompanionAudioFallbackInfoMock: vi.fn(), + resolveRecordingSessionMock: vi.fn(), +})); + +vi.mock("electron", () => ({ app: { getPath: () => "/tmp" } })); +vi.mock("../recording/diagnostics", () => ({ + getCompanionAudioFallbackInfo: getCompanionAudioFallbackInfoMock, +})); +vi.mock("../project/session", () => ({ + resolveRecordingSession: resolveRecordingSessionMock, +})); + +import { parseMaxVolumeDb, resolveCaptionAudioCandidates } from "./generate"; + +describe("resolveCaptionAudioCandidates", () => { + beforeEach(() => { + getCompanionAudioFallbackInfoMock.mockReset(); + resolveRecordingSessionMock.mockReset(); + resolveRecordingSessionMock.mockResolvedValue(null); + }); + + it("falls back from the recording to system and microphone sidecars", async () => { + const videoPath = "/recordings/recording.mp4"; + getCompanionAudioFallbackInfoMock.mockResolvedValue({ + paths: [videoPath, "/recordings/recording.system.m4a", "/recordings/recording.mic.wav"], + candidatePaths: ["/recordings/recording.system.m4a", "/recordings/recording.mic.wav"], + startDelayMsByPath: { "/recordings/recording.mic.wav": 125 }, + }); + + await expect(resolveCaptionAudioCandidates(videoPath)).resolves.toEqual([ + { + path: "/recordings/recording.system.m4a", + label: "system and microphone recording", + secondaryPath: "/recordings/recording.mic.wav", + secondaryStartDelayMs: 125, + }, + { path: videoPath, label: "recording" }, + { path: "/recordings/recording.system.m4a", label: "system audio recording" }, + { + path: "/recordings/recording.mic.wav", + label: "microphone recording", + startDelayMs: 125, + }, + ]); + }); + + it("falls back to the recording and linked webcam without sidecars", async () => { + getCompanionAudioFallbackInfoMock.mockResolvedValue({ + paths: [], + startDelayMsByPath: {}, + }); + resolveRecordingSessionMock.mockResolvedValue({ webcamPath: "/recordings/webcam.mp4" }); + + await expect(resolveCaptionAudioCandidates("/recordings/recording.mp4")).resolves.toEqual([ + { path: "/recordings/recording.mp4", label: "recording" }, + { path: "/recordings/webcam.mp4", label: "linked webcam recording" }, + ]); + }); + + it("parses finite and silent FFmpeg volume measurements", () => { + expect(parseMaxVolumeDb("[Parsed_volumedetect] max_volume: -18.4 dB")).toBe(-18.4); + expect(parseMaxVolumeDb("[Parsed_volumedetect] max_volume: -inf dB")).toBe( + Number.NEGATIVE_INFINITY, + ); + expect(parseMaxVolumeDb("no volume measurement")).toBeNull(); + }); +}); diff --git a/electron/ipc/captions/generate.ts b/electron/ipc/captions/generate.ts index 2a3f49cd7..095429e1b 100644 --- a/electron/ipc/captions/generate.ts +++ b/electron/ipc/captions/generate.ts @@ -7,6 +7,7 @@ import { app } from "electron"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; import { getBundledWhisperExecutableCandidates } from "../paths/binaries"; import { resolveRecordingSession } from "../project/session"; +import { getCompanionAudioFallbackInfo } from "../recording/diagnostics"; import { normalizeVideoSourcePath } from "../utils"; import { parseSrtCues, parseWhisperJsonCues, shouldRetryWhisperWithoutJson } from "./parser"; import { segmentCuesIntoPhrases } from "./segment"; @@ -83,27 +84,114 @@ export async function resolveWhisperExecutablePath(preferredPath?: string | null } export async function resolveCaptionAudioCandidates(videoPath: string) { - const candidates: Array<{ path: string; label: string }> = []; + const candidates: Array<{ + path: string; + label: string; + startDelayMs?: number; + secondaryPath?: string; + secondaryStartDelayMs?: number; + }> = []; const seenPaths = new Set(); - const pushCandidate = (candidatePath: string | null | undefined, label: string) => { + const pushCandidate = ( + candidatePath: string | null | undefined, + label: string, + startDelayMs?: number, + secondaryPath?: string, + secondaryStartDelayMs?: number, + ) => { const normalizedCandidatePath = normalizeVideoSourcePath(candidatePath); - if (!normalizedCandidatePath || seenPaths.has(normalizedCandidatePath)) { + const normalizedSecondaryPath = normalizeVideoSourcePath(secondaryPath); + if (!normalizedCandidatePath) { + return; + } + const candidateKey = normalizedSecondaryPath + ? `${normalizedCandidatePath}\0${normalizedSecondaryPath}` + : normalizedCandidatePath; + if (seenPaths.has(candidateKey)) { return; } - seenPaths.add(normalizedCandidatePath); - candidates.push({ path: normalizedCandidatePath, label }); + seenPaths.add(candidateKey); + candidates.push({ + path: normalizedCandidatePath, + label, + ...(Number.isFinite(startDelayMs) && (startDelayMs ?? 0) > 0 ? { startDelayMs } : {}), + ...(normalizedSecondaryPath ? { secondaryPath: normalizedSecondaryPath } : {}), + ...(Number.isFinite(secondaryStartDelayMs) && (secondaryStartDelayMs ?? 0) > 0 + ? { secondaryStartDelayMs } + : {}), + }); }; + const companionAudio = await getCompanionAudioFallbackInfo(videoPath); + const companionPaths = companionAudio.candidatePaths ?? companionAudio.paths; + const systemPath = companionPaths.find((path) => path.toLowerCase().includes(".system.")); + const microphonePath = companionPaths.find((path) => path.toLowerCase().includes(".mic.")); + if (systemPath && microphonePath) { + pushCandidate( + systemPath, + "system and microphone recording", + companionAudio.startDelayMsByPath[systemPath], + microphonePath, + companionAudio.startDelayMsByPath[microphonePath], + ); + } else if (microphonePath) { + pushCandidate( + microphonePath, + "microphone recording", + companionAudio.startDelayMsByPath[microphonePath], + ); + } + pushCandidate(videoPath, "recording"); + for (const companionPath of companionPaths) { + if (companionPath.toLowerCase().includes(".system.")) { + pushCandidate( + companionPath, + "system audio recording", + companionAudio.startDelayMsByPath[companionPath], + ); + } + } + + for (const companionPath of companionPaths) { + if (companionPath.toLowerCase().includes(".mic.")) { + pushCandidate( + companionPath, + "microphone recording", + companionAudio.startDelayMsByPath[companionPath], + ); + } + } + const requestedRecordingSession = await resolveRecordingSession(videoPath); pushCandidate(requestedRecordingSession?.webcamPath, "linked webcam recording"); return candidates; } +export function parseMaxVolumeDb(output: string) { + const match = output.match(/max_volume:\s*(-inf|[-+]?\d+(?:\.\d+)?)\s*dB/i); + if (!match || match[1].toLowerCase() === "-inf") { + return match ? Number.NEGATIVE_INFINITY : null; + } + + const value = Number(match[1]); + return Number.isFinite(value) ? value : null; +} + +async function hasAudibleCaptionAudio(ffmpegPath: string, wavPath: string) { + const { stderr } = await execFileAsync( + ffmpegPath, + ["-hide_banner", "-i", wavPath, "-af", "volumedetect", "-f", "null", "-"], + { timeout: 5 * 60 * 1000, maxBuffer: 20 * 1024 * 1024 }, + ); + const maxVolumeDb = parseMaxVolumeDb(stderr); + return maxVolumeDb === null || maxVolumeDb > -60; +} + export async function extractCaptionAudioSource(options: { videoPath: string; ffmpegPath: string; @@ -121,25 +209,43 @@ export async function extractCaptionAudioSource(options: { for (const candidate of candidates) { try { await ensureReadableFile(candidate.path); + if (candidate.secondaryPath) { + await ensureReadableFile(candidate.secondaryPath); + } + const mixedAudioFilter = candidate.secondaryPath + ? [ + `[0:a]adelay=${Math.round(candidate.startDelayMs ?? 0)}:all=1[primary]`, + `[1:a]adelay=${Math.round(candidate.secondaryStartDelayMs ?? 0)}:all=1[secondary]`, + "[primary][secondary]amix=inputs=2:duration=longest:normalize=0[mixed]", + ].join(";") + : null; await execFileAsync( options.ffmpegPath, [ "-y", "-i", candidate.path, - "-map", - "0:a:0", + ...(candidate.secondaryPath ? ["-i", candidate.secondaryPath] : []), + ...(mixedAudioFilter + ? ["-filter_complex", mixedAudioFilter, "-map", "[mixed]"] + : ["-map", "0:a:0"]), "-vn", "-ac", "1", "-ar", "16000", + ...(!mixedAudioFilter && candidate.startDelayMs + ? ["-af", `adelay=${Math.round(candidate.startDelayMs)}:all=1`] + : []), "-c:a", "pcm_s16le", options.wavPath, ], { timeout: 5 * 60 * 1000, maxBuffer: 20 * 1024 * 1024 }, ); + if (!(await hasAudibleCaptionAudio(options.ffmpegPath, options.wavPath))) { + throw new Error("Extracted audio is silent"); + } attemptedCandidates.push({ ...candidate, readable: true, extractedAudio: true }); return candidate; } catch (error) { diff --git a/electron/ipc/captions/whisper.ts b/electron/ipc/captions/whisper.ts index c8e774c62..82a798ca7 100644 --- a/electron/ipc/captions/whisper.ts +++ b/electron/ipc/captions/whisper.ts @@ -29,7 +29,7 @@ export async function getWhisperSmallModelStatus() { return { success: true, exists: false, - path: null, + path: WHISPER_SMALL_MODEL_PATH, }; } } diff --git a/electron/ipc/constants.ts b/electron/ipc/constants.ts index 2c5cf8f17..f2e239a21 100644 --- a/electron/ipc/constants.ts +++ b/electron/ipc/constants.ts @@ -23,6 +23,9 @@ export const WHISPER_SMALL_MODEL_PATH = path.join(WHISPER_MODEL_DIR, "ggml-small export const COMPANION_AUDIO_LAYOUTS = [ { platform: "mac" as const, systemSuffix: ".system.m4a", micSuffix: ".mic.m4a" }, { platform: "win" as const, systemSuffix: ".system.wav", micSuffix: ".mic.wav" }, + // Browser microphone capture on macOS is finalized as PCM WAV while + // ScreenCaptureKit system audio remains M4A. + { platform: "mac" as const, systemSuffix: ".system.m4a", micSuffix: ".mic.wav" }, { platform: "mac" as const, systemSuffix: ".system.webm", micSuffix: ".mic.webm" }, ]; diff --git a/electron/ipc/recording/audioFilters.test.ts b/electron/ipc/recording/audioFilters.test.ts index 1e80e2665..2deb85376 100644 --- a/electron/ipc/recording/audioFilters.test.ts +++ b/electron/ipc/recording/audioFilters.test.ts @@ -57,6 +57,7 @@ describe("browser microphone sidecar post-processing", () => { }); it("adds bounded speech gain only for the no-AGC browser mic profile", () => { + expect(getBrowserMicSidecarFilters("raw")).toEqual([]); expect(getBrowserMicSidecarFilters("processed")).toEqual(BROWSER_MIC_SIDECAR_FILTERS); expect(getBrowserMicSidecarFilters("no-agc")).toEqual([ "adeclip=threshold=1", diff --git a/electron/ipc/recording/audioFilters.ts b/electron/ipc/recording/audioFilters.ts index b465f5f20..0c9faeb5d 100644 --- a/electron/ipc/recording/audioFilters.ts +++ b/electron/ipc/recording/audioFilters.ts @@ -25,6 +25,10 @@ export const BROWSER_MIC_SIDECAR_FILTERS = [ ]; export function getBrowserMicSidecarFilters(profile?: string | null) { + if (profile === "raw") { + return []; + } + if (profile === "no-agc") { return [...BROWSER_MIC_SIDECAR_BASE_FILTERS, ...BROWSER_MIC_SIDECAR_NO_AGC_GAIN_FILTERS]; } diff --git a/electron/ipc/recording/diagnostics.test.ts b/electron/ipc/recording/diagnostics.test.ts index 9f7ddeb08..a81909c5f 100644 --- a/electron/ipc/recording/diagnostics.test.ts +++ b/electron/ipc/recording/diagnostics.test.ts @@ -134,6 +134,40 @@ describe("getCompanionAudioFallbackPaths", () => { ]); }); + it("recognizes browser microphone WAV sidecars on macOS", async () => { + const videoPath = path.join(tempRoot, "recording.mp4"); + const systemPath = path.join(tempRoot, "recording.system.m4a"); + const micPath = path.join(tempRoot, "recording.mic.wav"); + + await Promise.all([ + fs.writeFile(videoPath, "video"), + fs.writeFile(systemPath, "system"), + fs.writeFile(micPath, "mic"), + ]); + + execFileMock.mockImplementation( + ( + _file: string, + _args: string[], + _options: Record, + callback: ExecFileCallback, + ) => { + const error = new Error("ffmpeg probe found embedded audio") as Error & { + stderr?: string; + }; + error.stderr = "Stream #0:1: Audio: aac"; + callback(error, "", error.stderr); + }, + ); + + const { getCompanionAudioFallbackPaths } = await import("./diagnostics"); + + await expect(getCompanionAudioFallbackPaths(videoPath)).resolves.toEqual([ + videoPath, + micPath, + ]); + }); + it("prefers the mac mic companion alone when embedded audio already exists and no system sidecar is present", async () => { const videoPath = path.join(tempRoot, "recording.mp4"); const micPath = path.join(tempRoot, "recording.mic.m4a"); @@ -187,6 +221,7 @@ describe("getCompanionAudioFallbackPaths", () => { await expect(getCompanionAudioFallbackInfo(videoPath)).resolves.toEqual({ paths: [micPath], + candidatePaths: [micPath], startDelayMsByPath: { [micPath]: 2750, }, diff --git a/electron/ipc/recording/diagnostics.ts b/electron/ipc/recording/diagnostics.ts index edf5f638e..f6b78b67e 100644 --- a/electron/ipc/recording/diagnostics.ts +++ b/electron/ipc/recording/diagnostics.ts @@ -503,32 +503,39 @@ export async function getCompanionAudioFallbackPaths(videoPath: string) { export async function getCompanionAudioFallbackInfo(videoPath: string) { const companionCandidates = await getUsableCompanionAudioCandidates(videoPath); if (companionCandidates.length === 0) { - return { paths: [], startDelayMsByPath: {} }; + return { paths: [], candidatePaths: [], startDelayMsByPath: {} }; } + const candidatePaths = Array.from( + new Set(companionCandidates.flatMap((candidate) => candidate.usablePaths)), + ); let paths: string[]; if (await hasEmbeddedAudioStream(videoPath)) { - const hasUsableMacSystemCompanion = companionCandidates.some( - (candidate) => - candidate.platform === "mac" && - candidate.usablePaths.includes(candidate.systemPath), - ); - const usableMacMicOnlyCompanions = Array.from( + const usableMacMicPaths = Array.from( new Set( companionCandidates.flatMap((candidate) => candidate.platform === "mac" && - !candidate.usablePaths.includes(candidate.systemPath) && candidate.usablePaths.includes(candidate.micPath) ? [candidate.micPath] : [], ), ), ); + const hasUsableMacSystemCompanion = companionCandidates.some( + (candidate) => + candidate.platform === "mac" && + candidate.usablePaths.includes(candidate.systemPath), + ); - if (!hasUsableMacSystemCompanion && usableMacMicOnlyCompanions.length > 0) { - paths = usableMacMicOnlyCompanions; - } else if (hasUsableMacSystemCompanion) { - paths = [videoPath]; + // Always surface the microphone sidecar when it exists: the embedded + // track (inline system audio) can be pure silence while the mic sidecar + // holds the only audible content. + if (hasUsableMacSystemCompanion) { + paths = [videoPath, ...usableMacMicPaths]; + } else if (usableMacMicPaths.length > 0) { + paths = usableMacMicPaths.some((micPath) => micPath.endsWith(".mic.wav")) + ? [videoPath, ...usableMacMicPaths] + : usableMacMicPaths; } else { const companionPaths = Array.from( new Set( @@ -540,7 +547,7 @@ export async function getCompanionAudioFallbackInfo(videoPath: string) { ), ); if (companionPaths.length === 0) { - return { paths: [], startDelayMsByPath: {} }; + return { paths: [], candidatePaths, startDelayMsByPath: {} }; } paths = [videoPath, ...companionPaths]; @@ -552,7 +559,7 @@ export async function getCompanionAudioFallbackInfo(videoPath: string) { } const metadataEntries = await Promise.all( - paths.map(async (audioPath) => { + candidatePaths.map(async (audioPath) => { const startDelayMs = await getCompanionAudioStartDelayMs(audioPath); if (!Number.isFinite(startDelayMs)) { return null; @@ -564,6 +571,7 @@ export async function getCompanionAudioFallbackInfo(videoPath: string) { return { paths, + candidatePaths, startDelayMsByPath: Object.fromEntries( metadataEntries.filter((entry): entry is readonly [string, number] => entry !== null), ), diff --git a/electron/ipc/recording/mac.test.ts b/electron/ipc/recording/mac.test.ts new file mode 100644 index 000000000..9454b4965 --- /dev/null +++ b/electron/ipc/recording/mac.test.ts @@ -0,0 +1,28 @@ +import { EventEmitter } from "node:events"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("electron", () => ({ + app: { getPath: () => "/tmp/recordly-test" }, + BrowserWindow: { getAllWindows: () => [] }, +})); + +import { waitForNativeCaptureStart } from "./mac"; + +describe("waitForNativeCaptureStart", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("returns the time when the helper reports capture readiness", async () => { + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter; + }; + child.stdout = new EventEmitter(); + vi.spyOn(Date, "now").mockReturnValue(123456); + + const ready = waitForNativeCaptureStart(child as never); + child.stdout.emit("data", Buffer.from("Recording started\n")); + + await expect(ready).resolves.toBe(123456); + }); +}); diff --git a/electron/ipc/recording/mac.ts b/electron/ipc/recording/mac.ts index 01951345d..297da9ae6 100644 --- a/electron/ipc/recording/mac.ts +++ b/electron/ipc/recording/mac.ts @@ -34,7 +34,7 @@ import { getFinalMacCompanionAudioPath } from "./macCompanionAudio"; import { pruneAutoRecordings } from "./prune"; export function waitForNativeCaptureStart(process: ChildProcessWithoutNullStreams) { - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { const timer = setTimeout(() => { cleanup(); reject(new Error("Timed out waiting for ScreenCaptureKit recorder to start")); @@ -45,7 +45,7 @@ export function waitForNativeCaptureStart(process: ChildProcessWithoutNullStream stdoutBuffer += chunk.toString(); if (stdoutBuffer.includes("Recording started")) { cleanup(); - resolve(); + resolve(Date.now()); } }; diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index 06a33f84f..ced6711bf 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -677,8 +677,7 @@ export function registerRecordingHandlers( // non-fatal – the helper will report its own TCC status } - // Ensure microphone TCC is granted for this process tree when mic capture - // is requested, so the child helper inherits the grant. + // Warm up microphone permission before renderer-side capture starts. if (options?.capturesMicrophone) { const micStatus = systemPreferences.getMediaAccessStatus("microphone"); if (micStatus !== "granted") { @@ -705,13 +704,13 @@ export function registerRecordingHandlers( const timestamp = Date.now(); const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`); const capturesSystemAudio = Boolean(options?.capturesSystemAudio); - const capturesMicrophone = Boolean(options?.capturesMicrophone); + // Capture the selected mic in the renderer, where getUserMedia provides + // reliable device selection and permission attribution on macOS. + const capturesMicrophone = false; + const browserMicrophoneFallbackRequired = Boolean(options?.capturesMicrophone); const systemAudioOutputPath = capturesSystemAudio ? path.join(recordingsDir, `recording-${timestamp}.system.m4a`) : null; - const microphoneOutputPath = capturesMicrophone - ? path.join(recordingsDir, `recording-${timestamp}.mic.m4a`) - : null; const config: Record = { fps: 60, outputPath, @@ -719,22 +718,10 @@ export function registerRecordingHandlers( capturesMicrophone, }; - if (options?.microphoneDeviceId) { - config.microphoneDeviceId = options.microphoneDeviceId; - } - - if (options?.microphoneLabel) { - config.microphoneLabel = options.microphoneLabel; - } - if (systemAudioOutputPath) { config.systemAudioOutputPath = systemAudioOutputPath; } - if (microphoneOutputPath) { - config.microphoneOutputPath = microphoneOutputPath; - } - const windowId = parseWindowId(source?.id); const screenId = Number(source?.display_id); @@ -749,7 +736,7 @@ export function registerRecordingHandlers( setNativeCaptureOutputBuffer(""); setNativeCaptureTargetPath(outputPath); setNativeCaptureSystemAudioPath(systemAudioOutputPath); - setNativeCaptureMicrophonePath(microphoneOutputPath); + setNativeCaptureMicrophonePath(null); setNativeCaptureStopRequested(false); setNativeCapturePaused(false); captProc = spawn(helperPath, [JSON.stringify(config)], { @@ -766,19 +753,9 @@ export function registerRecordingHandlers( setNativeCaptureOutputBuffer(nativeCaptureOutputBuffer + chunk.toString()); }); - await waitForNativeCaptureStart(captProc); + const captureStartedAtMs = await waitForNativeCaptureStart(captProc); setNativeScreenRecordingActive(true); - // If the native helper reported MICROPHONE_CAPTURE_UNAVAILABLE, it started - // capture without microphone. Clear the mic path so the renderer can fall - // back to a browser-side sidecar recording for the microphone track. - const micUnavailableNatively = nativeCaptureOutputBuffer.includes( - "MICROPHONE_CAPTURE_UNAVAILABLE", - ); - if (micUnavailableNatively) { - setNativeCaptureMicrophonePath(null); - } - recordNativeCaptureDiagnostics({ backend: "mac-screencapturekit", phase: "start", @@ -791,7 +768,11 @@ export function registerRecordingHandlers( microphonePath: nativeCaptureMicrophonePath, processOutput: nativeCaptureOutputBuffer.trim() || undefined, }); - return { success: true, microphoneFallbackRequired: micUnavailableNatively }; + return { + success: true, + captureStartedAtMs, + microphoneFallbackRequired: browserMicrophoneFallbackRequired, + }; } catch (error) { console.error("Failed to start native ScreenCaptureKit recording:", error); const errorStr = String(error); @@ -1622,6 +1603,10 @@ export function registerRecordingHandlers( try { await fs.writeFile(tempWebmPath, Buffer.from(audioData)); + const preRollMs = + typeof options?.startDelayMs === "number" && options.startDelayMs < 0 + ? Math.abs(options.startDelayMs) + : 0; await execFileAsync( getFfmpegBinaryPath(), [ @@ -1638,6 +1623,12 @@ export function registerRecordingHandlers( "48000", "-af", [ + ...(preRollMs > 0 + ? [ + `atrim=start=${(preRollMs / 1000).toFixed(3)}`, + "asetpts=PTS-STARTPTS", + ] + : []), ...getBrowserMicSidecarFilters(options?.browserMicrophoneProfile), "aresample=async=1:first_pts=0", ].join(","), @@ -1655,7 +1646,10 @@ export function registerRecordingHandlers( } else { await fs.rm(tempWebmPath, { force: true }); } - const startDelayMs = options?.startDelayMs; + const startDelayMs = + typeof options?.startDelayMs === "number" && options.startDelayMs < 0 + ? 0 + : options?.startDelayMs; const mediaTrackSettings = pickPrimitiveRecord(options?.mediaTrackSettings); const audioInputDevices = pickAudioInputDevices(options?.audioInputDevices); const mediaRecorder = isRecord(options?.mediaRecorder) diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index c2e16ed60..5525c683b 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -2765,6 +2765,9 @@ export default function VideoEditor() { } setDownloadedWhisperModelPath(null); + setWhisperModelPath((currentPath) => { + return currentPath === result.path ? null : currentPath; + }); setWhisperModelDownloadStatus("idle"); setWhisperModelDownloadProgress(0); })(); diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index c5cd70056..fa3a78ba4 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -30,6 +30,7 @@ const MIN_FRAME_RATE = 30; const CHROME_MEDIA_SOURCE = "desktop"; const RECORDING_FILE_PREFIX = "recording-"; const AUDIO_BITRATE_VOICE = 128_000; +const MIC_FALLBACK_AUDIO_BITRATE = 256_000; const AUDIO_BITRATE_SYSTEM = 192_000; const MIC_GAIN_BOOST = 1.4; const WEBCAM_BITRATE = 8_000_000; @@ -365,6 +366,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const micFallbackRequestedConstraints = useRef(null); const micFallbackAudioInputDevices = useRef(null); const micFallbackRecorderMetadata = useRef(null); + const micFallbackProfile = useRef(null); const micFallbackChunkEvents = useRef([]); const micFallbackRecorderStartedAt = useRef(null); const micFallbackPauseStartedAt = useRef(null); @@ -594,6 +596,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackRequestedConstraints.current = null; micFallbackAudioInputDevices.current = null; micFallbackRecorderMetadata.current = null; + micFallbackProfile.current = null; resetMicFallbackTimingDiagnostics(); } }, [resetMicFallbackTimingDiagnostics]); @@ -629,6 +632,49 @@ export function useScreenRecorder(): UseScreenRecorderReturn { [getMicFallbackRecordedElapsedMs], ); + const prepareMicFallbackRecorder = useCallback( + async ( + profile: BrowserMicrophoneProfile, + audioBitsPerSecond = AUDIO_BITRATE_VOICE, + ) => { + const microphoneConstraints = createProcessedMicrophoneConstraints( + microphoneDeviceId, + profile, + ); + micFallbackRequestedConstraints.current = microphoneConstraints; + const micStream = await navigator.mediaDevices.getUserMedia(microphoneConstraints); + micFallbackTrackSettings.current = createMicrophoneTrackSettingsSnapshot(micStream); + micFallbackAudioInputDevices.current = await createAudioInputDeviceSnapshot().catch( + () => null, + ); + micFallbackChunks.current = []; + const recorder = new MediaRecorder(micStream, { + mimeType: "audio/webm;codecs=opus", + audioBitsPerSecond, + }); + micFallbackRecorderMetadata.current = { + mimeType: recorder.mimeType, + audioBitsPerSecond, + timesliceMs: RECORDER_TIMESLICE_MS, + }; + micFallbackProfile.current = profile; + resetMicFallbackTimingDiagnostics(); + recorder.ondataavailable = appendMicFallbackChunk; + micFallbackRecorder.current = recorder; + return recorder; + }, + [appendMicFallbackChunk, microphoneDeviceId, resetMicFallbackTimingDiagnostics], + ); + + const beginMicFallbackRecorder = useCallback( + (recorder: MediaRecorder, mainStartedAt: number) => { + micFallbackRecorderStartedAt.current = performance.now(); + micFallbackStartDelayMs.current = Math.max(0, Date.now() - mainStartedAt); + recorder.start(RECORDER_TIMESLICE_MS); + }, + [], + ); + const resolveBrowserCaptureSource = useCallback(async (source: ProcessedDesktopSource) => { if (!source?.id?.startsWith("screen:")) { return source; @@ -808,6 +854,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackRequestedConstraints.current = null; micFallbackAudioInputDevices.current = null; micFallbackRecorderMetadata.current = null; + micFallbackProfile.current = null; resetMicFallbackTimingDiagnostics(); return; } @@ -818,10 +865,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const effectiveTrackSettings = mediaTrackSettings ?? micFallbackTrackSettings.current; const sidecarOptions: MicrophoneSidecarOptions = { - ...(Number.isFinite(effectiveStartDelayMs) && (effectiveStartDelayMs ?? 0) >= 0 + ...(Number.isFinite(effectiveStartDelayMs) ? { startDelayMs: effectiveStartDelayMs ?? 0 } : {}), - browserMicrophoneProfile: browserMicrophoneProfile.current, + browserMicrophoneProfile: + micFallbackProfile.current ?? browserMicrophoneProfile.current, ...(requestedBrowserMicrophoneProfile.current ? { requestedBrowserMicrophoneProfile: @@ -877,6 +925,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackRequestedConstraints.current = null; micFallbackAudioInputDevices.current = null; micFallbackRecorderMetadata.current = null; + micFallbackProfile.current = null; resetMicFallbackTimingDiagnostics(); } }, @@ -1129,12 +1178,20 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const finalPath = result.path; - // 1. Finalize the session and switch to editor immediately (Optimistic UI) - // We pass null for webcamPath initially to avoid blocking on webcam disk writes/muxing. + // The editor discovers companion audio only during initialization. Persist + // the mic sidecar before opening it so playback, waveforms, and captions + // all see the same audio that export uses. + if (!isNativeWindows) { + await storeMicrophoneSidecar( + micFallbackBlobPromise, + finalPath, + fallbackStartDelayMs, + fallbackTrackSettings, + ); + } await finalizeRecordingSession(finalPath, null); - // 2. Perform background finalization (webcam, muxing, sidecars) - // We don't await this to keep the UI responsive + // Complete webcam and platform-specific finalization in the background. void (async () => { try { // Await the webcam path in the background @@ -1144,13 +1201,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamPath, ); - // Store sidecars - await storeMicrophoneSidecar( - micFallbackBlobPromise, - finalPath, - fallbackStartDelayMs, - fallbackTrackSettings, - ); + if (isNativeWindows) { + await storeMicrophoneSidecar( + micFallbackBlobPromise, + finalPath, + fallbackStartDelayMs, + fallbackTrackSettings, + ); + } // Perform muxing/renaming if on Windows if (isNativeWindows) { @@ -1162,8 +1220,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { { finalPath, webcamPath }, ); - // Update the session state to notify the editor that all background assets (webcam, mic, etc.) are now ready. - // This broadcasts a 'recording-session-changed' event that the open editor listens to for re-scanning assets. + // Notify the editor when the remaining session assets are ready. await window.electronAPI.setCurrentRecordingSession({ videoPath: finalPath, webcamPath, @@ -1177,8 +1234,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } catch (bgError) { console.error("Error in background finalization:", bgError); } finally { - // After all background tasks are done (webcam, mic sidecars, muxing), - // we can safely close the HUD window to release hardware and resources. + // Release the HUD's camera and overlay resources after finalization. if (typeof window.electronAPI?.hudOverlayClose === "function") { console.log( "[useScreenRecorder] All background tasks finished, closing HUD", @@ -1306,12 +1362,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { void (async () => { setRecording(false); nativeScreenRecording.current = false; - cleanupCapturedMedia(); await window.electronAPI.setRecordingState(false); if (state.reason !== "window-unavailable") { try { - const recoveredPath = await recoverNativeRecordingSession(); + const recoveredPath = await recoverNativeRecordingSession( + stopMicFallbackRecorder(), + micFallbackStartDelayMs.current, + ); if (recoveredPath) { return; } @@ -1323,6 +1381,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } } + cleanupCapturedMedia(); + if (state.reason === "window-unavailable" && !hasPromptedForReselect.current) { hasPromptedForReselect.current = true; alert(state.message); @@ -1353,7 +1413,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { cleanupCapturedMedia(); }; - }, [cleanupCapturedMedia, recoverNativeRecordingSession]); + }, [cleanupCapturedMedia, recoverNativeRecordingSession, stopMicFallbackRecorder]); const startRecording = async () => { if (startInFlight.current) { @@ -1439,6 +1499,24 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } if (useNativeMacScreenCapture || useNativeWindowsCapture) { + let preparedMicRecorder: MediaRecorder | null = null; + let preparedMicStartedAt: number | null = null; + if (useNativeMacScreenCapture && microphoneEnabled) { + try { + // Acquire the selected device before video starts. getUserMedia startup + // is the dominant source of mic/video drift on macOS. + preparedMicRecorder = await prepareMicFallbackRecorder( + "raw", + MIC_FALLBACK_AUDIO_BITRATE, + ); + preparedMicStartedAt = Date.now(); + micFallbackRecorderStartedAt.current = performance.now(); + preparedMicRecorder.start(RECORDER_TIMESLICE_MS); + } catch (micError) { + console.warn("Failed to prepare macOS microphone capture:", micError); + } + } + // Resolve the selected mic label for native capture backends. let micLabel: string | undefined; if (microphoneEnabled) { @@ -1491,8 +1569,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } if (nativeResult.success) { - const mainStartedAt = Date.now(); - micFallbackStartDelayMs.current = null; + const mainStartedAt = nativeResult.captureStartedAtMs ?? Date.now(); + micFallbackStartDelayMs.current = + preparedMicStartedAt === null + ? null + : -Math.max(0, mainStartedAt - preparedMicStartedAt); beginWebcamCapture(); nativeScreenRecording.current = true; nativeWindowsRecording.current = useNativeWindowsCapture; @@ -1506,52 +1587,25 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // record mic via browser getUserMedia as a sidecar file. if (nativeResult.microphoneFallbackRequired && microphoneEnabled) { void logNativeCaptureDiagnostics("start-browser-microphone-fallback"); - console.info("Using browser microphone processing for this recording."); + console.info("Using browser microphone capture for this recording."); try { - const microphoneConstraints = createProcessedMicrophoneConstraints( - microphoneDeviceId, - browserMicrophoneProfile.current, - ); - micFallbackRequestedConstraints.current = microphoneConstraints; - const micStream = - await navigator.mediaDevices.getUserMedia(microphoneConstraints); - micFallbackTrackSettings.current = - createMicrophoneTrackSettingsSnapshot(micStream); - micFallbackAudioInputDevices.current = - await createAudioInputDeviceSnapshot().catch(() => null); - console.info( - "Browser microphone track settings:", - micFallbackTrackSettings.current, - ); - console.info( - "Browser microphone audio input devices:", - micFallbackAudioInputDevices.current, - ); - micFallbackChunks.current = []; - const recorder = new MediaRecorder(micStream, { - mimeType: "audio/webm;codecs=opus", - audioBitsPerSecond: AUDIO_BITRATE_VOICE, - }); - micFallbackRecorderMetadata.current = { - mimeType: recorder.mimeType, - audioBitsPerSecond: AUDIO_BITRATE_VOICE, - timesliceMs: RECORDER_TIMESLICE_MS, - }; - resetMicFallbackTimingDiagnostics(); - micFallbackRecorderStartedAt.current = performance.now(); - recorder.ondataavailable = appendMicFallbackChunk; - micFallbackStartDelayMs.current = Math.max( - 0, - Date.now() - mainStartedAt, - ); - recorder.start(RECORDER_TIMESLICE_MS); - micFallbackRecorder.current = recorder; + const recorder = + preparedMicRecorder ?? + (await prepareMicFallbackRecorder( + useNativeMacScreenCapture + ? "raw" + : browserMicrophoneProfile.current, + )); + if (recorder.state === "inactive") { + beginMicFallbackRecorder(recorder, mainStartedAt); + } } catch (micError) { micFallbackStartDelayMs.current = null; micFallbackTrackSettings.current = null; micFallbackRequestedConstraints.current = null; micFallbackAudioInputDevices.current = null; micFallbackRecorderMetadata.current = null; + micFallbackProfile.current = null; resetMicFallbackTimingDiagnostics(); console.warn("Browser microphone fallback failed:", micError); const permissionDenied = @@ -2050,6 +2104,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { nativeScreenRecording.current = false; nativeWindowsRecording.current = false; setRecording(false); + micFallbackChunks.current = []; + cleanupCapturedMedia(); window.electronAPI?.setRecordingState(false); void (async () => { try {