Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 26 additions & 9 deletions src/renderer/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -360,8 +360,9 @@ let systemAudioActivitySegments = [];
let systemAudioActivityOpen = null;
let systemAudioLastActiveSec = 0;
// System-audio "keep" regions captured per take during this app session,
// consumed by the on-demand Transcribe & Cut action. Not persisted: after an
// app restart a cut simply proceeds without them.
// consumed by the on-demand Transcribe & Cut action. The same ranges are
// persisted on the take as `systemAudioSegments`; this map is the fallback
// for takes recorded before persistence existed (or before a project save).
const takeSystemAudioActivity = new Map();
const SYSTEM_AUDIO_RMS_ACTIVE_THRESHOLD = 0.01;
const SYSTEM_AUDIO_RELEASE_MS = 400;
Expand Down Expand Up @@ -4539,6 +4540,7 @@ async function stopRecordingImpl() {
screenStartOffsetMs: recorderStartOffsetsMs.screen,
cameraStartOffsetMs: recorderStartOffsetsMs.camera,
audioStartOffsetMs: recorderStartOffsetsMs.audio,
systemAudioSegments: [...systemAudioActivitySegments],
sections: sectionsForTimeline,
transcriptSegments: []
});
Expand Down Expand Up @@ -6625,7 +6627,17 @@ async function transcribeAndCutTake() {
});
// Merge system-audio "keep" regions captured while this take recorded so
// audible screen sound is never trimmed just because the mic was quiet.
const activeSegments = [...speechSegments, ...(takeSystemAudioActivity.get(takeId) || [])];
// Prefer the ranges persisted on the take (they survive app restarts);
// fall back to the in-session activity map for takes recorded before
// persistence existed.
const persistedSystemAudio = Array.isArray(take.systemAudioSegments)
? take.systemAudioSegments
: [];
const systemAudioSegments =
persistedSystemAudio.length > 0
? persistedSystemAudio
: takeSystemAudioActivity.get(takeId) || [];
const activeSegments = [...speechSegments, ...systemAudioSegments];
if (activeSegments.length === 0) {
setTranscribeCutStatus('No speech detected — nothing to cut', 'warning');
return;
Expand Down Expand Up @@ -6792,12 +6804,17 @@ async function removeBadTakesFromTimeline() {
const takeSections = editorState.sections.filter((s) => s.takeId === takeId);
// Words plus system-audio activity: cut edges snap to these so pieces
// don't open with dead air, and pure inter-flub silence is dropped.
// System-audio activity comes from the in-session recording data when
// available, otherwise from the decoded system-audio waveform envelope
// (cached for display). Only when neither exists for a system-audio
// take do we keep plain padded bounds, so screen sound is never trimmed
// on word evidence alone.
let systemAudioKeeps = takeSystemAudioActivity.get(takeId) || [];
// System-audio activity comes from the ranges persisted on the take
// (they survive app restarts), then from the in-session recording data,
// otherwise from the decoded system-audio waveform envelope (cached for
// display). Only when none exist for a system-audio take do we keep
// plain padded bounds, so screen sound is never trimmed on word
// evidence alone.
const persistedKeeps = Array.isArray(take.systemAudioSegments)
? take.systemAudioSegments
: [];
let systemAudioKeeps =
persistedKeeps.length > 0 ? persistedKeeps : takeSystemAudioActivity.get(takeId) || [];
if (take.hasSystemAudio && systemAudioKeeps.length === 0) {
const envelope = takeSystemAudioPeakEnvelopeCache.get(takeId);
if (envelope?.peaks?.length && envelope.duration > 0) {
Expand Down
9 changes: 8 additions & 1 deletion src/shared/domain/project.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,9 @@ export interface ProjectSettings {
export type AudioSource = 'screen' | 'camera' | 'external';

/**
* A stored transcription utterance in take-local recording time.
* A stored speech-shaped segment in take-local recording time: either a
* transcription utterance or a system-audio "keep" region. Sharing the shape
* lets the section builder merge both kinds directly.
*/
export interface TakeSpeechSegment {
start: number;
Expand Down Expand Up @@ -111,6 +113,10 @@ export interface Take {
screenStartOffsetMs: number;
cameraStartOffsetMs: number;
audioStartOffsetMs: number;
// System-audio "keep" regions detected while this take recorded. Persisted
// so Transcribe & Cut still protects audible screen sound (music, demos)
// after an app restart, when the in-session activity map is gone.
systemAudioSegments: TakeSpeechSegment[];
sections: Section[];
// Fine-grained speech utterances stored by Transcribe & Cut so bad-take
// detection and restore can run later without re-transcribing. The removed
Expand Down Expand Up @@ -466,6 +472,7 @@ export function normalizeProjectData(rawProject: unknown, projectFolder?: string
screenStartOffsetMs: normalizeRecorderStartOffsetMs(rawTakeRecord.screenStartOffsetMs),
cameraStartOffsetMs: normalizeRecorderStartOffsetMs(rawTakeRecord.cameraStartOffsetMs),
audioStartOffsetMs: normalizeRecorderStartOffsetMs(rawTakeRecord.audioStartOffsetMs),
systemAudioSegments: normalizeTakeSpeechSegments(rawTakeRecord.systemAudioSegments),
sections: normalizeSections(take.sections),
transcriptSegments: normalizeTakeSpeechSegments(rawTakeRecord.transcriptSegments)
};
Expand Down
54 changes: 54 additions & 0 deletions tests/unit/project-domain.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,60 @@ describe('shared/domain/project', () => {
expect(sections[2].transcript).toBe('third with dupe id');
});

test('normalizeTakeSpeechSegments drops invalid entries, coerces text, and sorts by start', () => {
expect(normalizeTakeSpeechSegments(undefined)).toEqual([]);
expect(normalizeTakeSpeechSegments(null)).toEqual([]);
expect(normalizeTakeSpeechSegments('bad')).toEqual([]);

const segments = normalizeTakeSpeechSegments([
{ start: 4, end: 6, text: ' music ' },
{ start: 0, end: 1.5 },
{ start: 2, end: 2, text: 'zero-length drops' },
{ start: 3, end: 1, text: 'inverted drops' },
{ start: 'bad', end: 9, text: 'non-numeric drops' },
{ start: 7, end: Infinity, text: 'non-finite drops' },
'not-a-record',
{ start: 8, end: 9, text: 42 }
]);

expect(segments).toEqual([
{ start: 0, end: 1.5, text: '' },
{ start: 4, end: 6, text: 'music' },
{ start: 8, end: 9, text: '' }
]);
});

test('normalizeProjectData round-trips take systemAudioSegments and defaults legacy takes to empty', () => {
const project = normalizeProjectData(
{
takes: [
{
id: 'take-1',
screenPath: 'screen.webm',
cameraPath: null,
duration: 10,
hasSystemAudio: true,
sections: [],
systemAudioSegments: [
{ start: 1.25, end: 3.5, text: '' },
{ start: 5, end: 4, text: 'invalid drops' }
]
},
{ id: 'take-2', screenPath: 'screen2.webm', cameraPath: null, duration: 5, sections: [] }
]
},
'/tmp/my-project'
);

expect(project.takes[0].systemAudioSegments).toEqual([{ start: 1.25, end: 3.5, text: '' }]);
// Legacy takes persisted before the field existed hydrate to an empty list.
expect(project.takes[1].systemAudioSegments).toEqual([]);

// Simulate save → load: re-normalizing the serialized project preserves the ranges.
const reloaded = normalizeProjectData(JSON.parse(JSON.stringify(project)), '/tmp/my-project');
expect(reloaded.takes[0].systemAudioSegments).toEqual([{ start: 1.25, end: 3.5, text: '' }]);
});

test('normalizeProjectData converts section imagePath to absolute path', () => {
const project = normalizeProjectData(
{
Expand Down
Loading