From a73da08576fcb2360e583deee41396850f0bd157 Mon Sep 17 00:00:00 2001 From: Tadas Petra <60107328+tadaspetra@users.noreply.github.com> Date: Fri, 21 Aug 2026 07:22:46 -0500 Subject: [PATCH] Persist system-audio keep regions on takes System-audio activity ranges captured during recording were only held in a renderer in-memory Map, so an app restart lost them and a later Transcribe & Cut would silence-cut regions with audible screen sound but no mic speech. Add a systemAudioSegments field to Take with defensive normalization (normalizeTakeSpeechSegments), stamp it at recording finalize, and make Transcribe & Cut prefer the persisted ranges with the in-session map as fallback for takes recorded before persistence existed. Co-Authored-By: Claude Fable 5 --- src/renderer/app.ts | 18 ++++++++-- src/shared/domain/project.ts | 33 +++++++++++++++++++ tests/unit/project-domain.test.ts | 55 +++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 3 deletions(-) diff --git a/src/renderer/app.ts b/src/renderer/app.ts index cc656f0..98b3f3a 100644 --- a/src/renderer/app.ts +++ b/src/renderer/app.ts @@ -201,8 +201,9 @@ let systemAudioActivitySegments = []; let systemAudioActivityOpen = null; let systemAudioLastActiveSec = 0; // System-audio "keep" regions captured per take during this app session, -// consumed by the on-demand Transcribe & Cut action. Not persisted: after an -// app restart a cut simply proceeds without them. +// consumed by the on-demand Transcribe & Cut action. The same ranges are +// persisted on the take as `systemAudioSegments`; this map is the fallback +// for takes recorded before persistence existed (or before a project save). const takeSystemAudioActivity = new Map(); const SYSTEM_AUDIO_RMS_ACTIVE_THRESHOLD = 0.01; const SYSTEM_AUDIO_RELEASE_MS = 400; @@ -4152,6 +4153,7 @@ async function stopRecordingImpl() { screenStartOffsetMs: recorderStartOffsetsMs.screen, cameraStartOffsetMs: recorderStartOffsetsMs.camera, audioStartOffsetMs: recorderStartOffsetsMs.audio, + systemAudioSegments: [...systemAudioActivitySegments], sections: sectionsForTimeline }); } @@ -6097,7 +6099,17 @@ async function transcribeAndCutTake() { const { segments: speechSegments, removed: removedTakes } = cutRepeatedTakes(utterances); // Merge system-audio "keep" regions captured while this take recorded so // audible screen sound is never trimmed just because the mic was quiet. - const activeSegments = [...speechSegments, ...(takeSystemAudioActivity.get(takeId) || [])]; + // Prefer the ranges persisted on the take (they survive app restarts); + // fall back to the in-session activity map for takes recorded before + // persistence existed. + const persistedSystemAudio = Array.isArray(take.systemAudioSegments) + ? take.systemAudioSegments + : []; + const systemAudioSegments = + persistedSystemAudio.length > 0 + ? persistedSystemAudio + : takeSystemAudioActivity.get(takeId) || []; + const activeSegments = [...speechSegments, ...systemAudioSegments]; if (activeSegments.length === 0) { setTranscribeCutStatus('No speech detected — nothing to cut', 'warning'); return; diff --git a/src/shared/domain/project.ts b/src/shared/domain/project.ts index 009a02f..76cb2c5 100644 --- a/src/shared/domain/project.ts +++ b/src/shared/domain/project.ts @@ -74,6 +74,15 @@ export interface ProjectSettings { export type AudioSource = 'screen' | 'camera' | 'external'; +// One "keep" region of a take, in take-local seconds. Shares the speech +// segment shape ({ start, end, text }) so system-audio activity ranges can be +// merged directly with transcription segments by the section builder. +export interface TakeSpeechSegment { + start: number; + end: number; + text: string; +} + export interface Take { id: string; createdAt: string; @@ -102,6 +111,10 @@ export interface Take { screenStartOffsetMs: number; cameraStartOffsetMs: number; audioStartOffsetMs: number; + // System-audio "keep" regions detected while this take recorded. Persisted + // so Transcribe & Cut still protects audible screen sound (music, demos) + // after an app restart, when the in-session activity map is gone. + systemAudioSegments: TakeSpeechSegment[]; sections: Section[]; } @@ -305,6 +318,25 @@ export function normalizeExportVideoPreset(value: unknown): ExportVideoPreset { : EXPORT_VIDEO_PRESET_QUALITY; } +export function normalizeTakeSpeechSegments(rawSegments: unknown): TakeSpeechSegment[] { + if (!Array.isArray(rawSegments)) return []; + return rawSegments + .map((rawSegment) => { + if (!isRecord(rawSegment)) return null; + const start = Number(rawSegment.start); + const end = Number(rawSegment.end); + if (!Number.isFinite(start) || !Number.isFinite(end) || end <= start) return null; + + return { + start, + end, + text: typeof rawSegment.text === 'string' ? rawSegment.text.trim() : '' + }; + }) + .filter((segment): segment is TakeSpeechSegment => segment !== null) + .sort((left, right) => left.start - right.start); +} + export function normalizeAudioSource(value: unknown): AudioSource | null { if (value === 'screen' || value === 'camera' || value === 'external') return value; return null; @@ -432,6 +464,7 @@ export function normalizeProjectData(rawProject: unknown, projectFolder?: string screenStartOffsetMs: normalizeRecorderStartOffsetMs(rawTakeRecord.screenStartOffsetMs), cameraStartOffsetMs: normalizeRecorderStartOffsetMs(rawTakeRecord.cameraStartOffsetMs), audioStartOffsetMs: normalizeRecorderStartOffsetMs(rawTakeRecord.audioStartOffsetMs), + systemAudioSegments: normalizeTakeSpeechSegments(rawTakeRecord.systemAudioSegments), sections: normalizeSections(take.sections) }; }), diff --git a/tests/unit/project-domain.test.ts b/tests/unit/project-domain.test.ts index afaab4f..ba80353 100644 --- a/tests/unit/project-domain.test.ts +++ b/tests/unit/project-domain.test.ts @@ -14,6 +14,7 @@ import { normalizeRecorderStartOffsetMs, normalizeExportVideoPreset, normalizeSections, + normalizeTakeSpeechSegments, sanitizeProjectName, toProjectAbsolutePath, toProjectRelativePath @@ -344,6 +345,60 @@ describe('shared/domain/project', () => { expect(sections[2].transcript).toBe('third with dupe id'); }); + test('normalizeTakeSpeechSegments drops invalid entries, coerces text, and sorts by start', () => { + expect(normalizeTakeSpeechSegments(undefined)).toEqual([]); + expect(normalizeTakeSpeechSegments(null)).toEqual([]); + expect(normalizeTakeSpeechSegments('bad')).toEqual([]); + + const segments = normalizeTakeSpeechSegments([ + { start: 4, end: 6, text: ' music ' }, + { start: 0, end: 1.5 }, + { start: 2, end: 2, text: 'zero-length drops' }, + { start: 3, end: 1, text: 'inverted drops' }, + { start: 'bad', end: 9, text: 'non-numeric drops' }, + { start: 7, end: Infinity, text: 'non-finite drops' }, + 'not-a-record', + { start: 8, end: 9, text: 42 } + ]); + + expect(segments).toEqual([ + { start: 0, end: 1.5, text: '' }, + { start: 4, end: 6, text: 'music' }, + { start: 8, end: 9, text: '' } + ]); + }); + + test('normalizeProjectData round-trips take systemAudioSegments and defaults legacy takes to empty', () => { + const project = normalizeProjectData( + { + takes: [ + { + id: 'take-1', + screenPath: 'screen.webm', + cameraPath: null, + duration: 10, + hasSystemAudio: true, + sections: [], + systemAudioSegments: [ + { start: 1.25, end: 3.5, text: '' }, + { start: 5, end: 4, text: 'invalid drops' } + ] + }, + { id: 'take-2', screenPath: 'screen2.webm', cameraPath: null, duration: 5, sections: [] } + ] + }, + '/tmp/my-project' + ); + + expect(project.takes[0].systemAudioSegments).toEqual([{ start: 1.25, end: 3.5, text: '' }]); + // Legacy takes persisted before the field existed hydrate to an empty list. + expect(project.takes[1].systemAudioSegments).toEqual([]); + + // Simulate save → load: re-normalizing the serialized project preserves the ranges. + const reloaded = normalizeProjectData(JSON.parse(JSON.stringify(project)), '/tmp/my-project'); + expect(reloaded.takes[0].systemAudioSegments).toEqual([{ start: 1.25, end: 3.5, text: '' }]); + }); + test('normalizeProjectData converts section imagePath to absolute path', () => { const project = normalizeProjectData( {