diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index db9b46e13..f4d0ea819 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -4,7 +4,7 @@ use crate::ffi::*; use crate::regions::SpeedSegment; -use crate::scene::SceneAudio; +use crate::scene::{SceneAudio, SceneAudioTrack}; use anyhow::{bail, Result}; use std::f32::consts::PI; use std::ffi::CString; @@ -914,6 +914,84 @@ pub fn assemble_concatenated_pcm( output } +/// Mix imported audio tracks (issue #350) over the assembled programme. +/// +/// Each track is decoded across its trim window — already resampled to 48 kHz +/// stereo by `decode_clip_audio`, the same path a clip's own audio takes — scaled +/// by its per-track gain (the same `10^(dB/20)` law as `finish_audio`), and summed +/// into the programme at `start_sec`. The programme length is NOT extended: a +/// track that runs past the video is truncated to it, so the audio and video +/// streams stay the same length for the muxer. +/// +/// The decode window is capped up front at the room left in the programme after +/// `start_sec`, and a track starting at/after the end is skipped without decoding. +/// `decode_clip_audio` preallocates from the window, so this keeps a long track +/// pinned near a short programme's end from buffering (and clamping away) hours of +/// PCM. `trim_end_sec` must therefore be concrete — the renderer sends +/// `trimEnd ?? durationSec`. +/// +/// A track whose file has no decodable audio is skipped — the same degradation a +/// stream-less clip gets. +pub fn mix_external_tracks(mut programme: PlanarPcm, tracks: &[SceneAudioTrack]) -> PlanarPcm { + let programme_len = programme.first().map(Vec::len).unwrap_or(0); + if programme_len == 0 { + return programme; + } + for track in tracks { + let offset = (track.start_sec.max(0.0) * AUDIO_OUTPUT_SAMPLE_RATE as f64).round() as usize; + // A track that starts at or past the programme end contributes nothing — + // skip it before decoding anything. + if offset >= programme_len { + continue; + } + let trim_start = track.trim_start_sec.max(0.0); + let Some(trim_end_full) = track.trim_end_sec else { + // Without a concrete end there is no safe window to decode (see the doc + // comment); the renderer always resolves one, so this only guards a + // hand-written scene. + continue; + }; + // Cap the decode window at the room left in the programme. Everything past + // `offset` that overflows is discarded by `overlay_track_pcm` anyway, so + // decoding it only wastes time and memory — a three-hour track placed at + // second 9 of a ten-second export must not buffer three hours of PCM. + let remaining_sec = (programme_len - offset) as f64 / AUDIO_OUTPUT_SAMPLE_RATE as f64; + let trim_end = trim_end_full.min(trim_start + remaining_sec); + if trim_end <= trim_start { + continue; + } + let decoded = match decode_clip_audio(&track.path, trim_start, trim_end) { + Ok(Some(pcm)) => pcm, + _ => continue, + }; + let gain = 10.0f32.powf(track.gain_db.clamp(-12.0, 12.0) / 20.0); + overlay_track_pcm(&mut programme, &decoded, offset, gain); + } + programme +} + +/// Sum one decoded track into the programme at `offset` samples, scaled by `gain`, +/// truncated at the programme's end. Split out of `mix_external_tracks` so the +/// placement/gain/clamp math is testable without ffmpeg, exactly like +/// `mix_aligned_tracks` is split from the decode above. +fn overlay_track_pcm(programme: &mut PlanarPcm, decoded: &PlanarPcm, offset: usize, gain: f32) { + let programme_len = programme.first().map(Vec::len).unwrap_or(0); + if offset >= programme_len { + return; + } + let room = programme_len - offset; + for channel in 0..AUDIO_OUTPUT_CHANNELS { + let Some(source) = decoded.get(channel) else { + continue; + }; + let count = source.len().min(room); + let dst = &mut programme[channel]; + for k in 0..count { + dst[offset + k] += source[k] * gain; + } + } +} + /// Encodeur AAC attaché au muxer avant son header. Les paquets utilisent le même interleaver /// que la vidéo ; les pts restent en unités échantillon jusqu'au rescale vers l'AVStream. pub(crate) struct AacEncoder { @@ -1044,6 +1122,64 @@ mod tests { assert_eq!(mixed[1], vec![0.25, -0.5, 0.75]); } + // Imported audio track overlay (issue #350). + #[test] + fn overlay_sums_at_offset_with_gain() { + let mut programme = planar(&[0.1, 0.1, 0.1, 0.1]); + // ×2 gain, placed at sample offset 1. + overlay_track_pcm(&mut programme, &planar(&[0.2, 0.2]), 1, 2.0); + assert_eq!(programme[0], vec![0.1, 0.5, 0.5, 0.1]); + assert_eq!(programme[1], vec![0.1, 0.5, 0.5, 0.1]); + } + + #[test] + fn overlay_truncates_a_track_that_runs_past_the_programme() { + let mut programme = planar(&[0.0, 0.0, 0.0]); + // A 4-sample track placed at offset 2 has room for only 1 sample. + overlay_track_pcm(&mut programme, &planar(&[1.0, 1.0, 1.0, 1.0]), 2, 1.0); + assert_eq!(programme[0], vec![0.0, 0.0, 1.0]); + } + + #[test] + fn overlay_past_the_end_is_a_no_op() { + let mut programme = planar(&[0.3, 0.3]); + overlay_track_pcm(&mut programme, &planar(&[1.0]), 5, 1.0); + assert_eq!(programme[0], vec![0.3, 0.3]); + } + + #[test] + fn mix_external_tracks_skips_empty_windows() { + let programme = planar(&[0.4, 0.4]); + let tracks = vec![SceneAudioTrack { + path: "/nope.mp3".into(), + start_sec: 0.0, + gain_db: 0.0, + trim_start_sec: 2.0, + trim_end_sec: Some(1.0), // end <= start: empty window, never decoded + }]; + // The empty window is skipped before any decode, so the programme is + // untouched even though the path does not exist. + let out = mix_external_tracks(programme, &tracks); + assert_eq!(out[0], vec![0.4, 0.4]); + } + + #[test] + fn mix_external_tracks_skips_a_track_that_starts_past_the_programme() { + // 2 samples = ~0.00004 s of programme at 48 kHz; the track starts at 1 s, so + // its offset is past the end. It must be skipped before any decode is + // attempted (the path does not exist), never buffering its window. + let programme = planar(&[0.4, 0.4]); + let tracks = vec![SceneAudioTrack { + path: "/nope.mp3".into(), + start_sec: 1.0, + gain_db: 0.0, + trim_start_sec: 0.0, + trim_end_sec: Some(3600.0), + }]; + let out = mix_external_tracks(programme, &tracks); + assert_eq!(out[0], vec![0.4, 0.4]); + } + #[test] fn single_track_is_not_clamped() { // Promesse de non-régression : une source mono-piste ressort telle quelle, y compris diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs index 910738fc0..8d35c0291 100644 --- a/crates/compositor/src/pipeline_linux.rs +++ b/crates/compositor/src/pipeline_linux.rs @@ -22,7 +22,7 @@ use std::ptr; use crate::audio::{ assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, - stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, + mix_external_tracks, stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, }; use crate::config::Cfg; use crate::d3d::Gpu; @@ -458,6 +458,12 @@ pub fn run_composited_multi( let scene = comp.scene_snapshot(); let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); + // Imported audio tracks (issue #350), cloned out of the borrowed scene so the + // mix step below owns them. Empty for a project with no imported audio. + let audio_tracks = scene + .as_ref() + .map(|scene| scene.audio_tracks.clone()) + .unwrap_or_default(); // Ring de staging a 2 : l'export ne veut que du debit, une frame de latence // ne se voit pas dans un fichier. Voir `Compositor::set_readback_depth` pour // la raison pour laquelle la preview, elle, reste a 1. @@ -535,7 +541,10 @@ pub fn run_composited_multi( let declared_audio: Vec = clips.iter().map(|c| c.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); audio_encoder.encode( - &finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings), + &finish_audio( + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks), + audio_settings, + ), octx, )?; crate::ffi::averr(crate::ffi::av_write_trailer(octx), "write_trailer")?; diff --git a/crates/compositor/src/pipeline_macos.rs b/crates/compositor/src/pipeline_macos.rs index 10de3fac2..1a3ebb9c7 100644 --- a/crates/compositor/src/pipeline_macos.rs +++ b/crates/compositor/src/pipeline_macos.rs @@ -31,7 +31,7 @@ use crate::audio::{ assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, - stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, + mix_external_tracks, stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, }; use crate::compositor::Compositor; use crate::d3d::Gpu; @@ -1077,6 +1077,11 @@ pub fn run_composited_multi( // raconte avoir déjà coûté une fois. let scene = comp.scene_snapshot(); let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); + // Imported audio tracks (issue #350), cloned out of the borrowed scene. + let audio_tracks = scene + .as_ref() + .map(|scene| scene.audio_tracks.clone()) + .unwrap_or_default(); frames = unsafe { crate::timeline_walk::walk_composited_timeline( clips, @@ -1132,7 +1137,10 @@ pub fn run_composited_multi( let declared_audio: Vec = clips.iter().map(|clip| clip.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); audio_encoder.encode( - &finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings), + &finish_audio( + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &plan), &audio_tracks), + audio_settings, + ), octx, )?; diff --git a/crates/compositor/src/pipeline_windows.rs b/crates/compositor/src/pipeline_windows.rs index 11738bcd4..01cbda745 100644 --- a/crates/compositor/src/pipeline_windows.rs +++ b/crates/compositor/src/pipeline_windows.rs @@ -4,7 +4,7 @@ use crate::audio::{ assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, - stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, + mix_external_tracks, stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, }; use crate::compositor::{Compositor, OUT_H, OUT_W}; use crate::config::Cfg; @@ -1343,6 +1343,11 @@ unsafe fn run_multi_inner( // fenêtrage par clip ; `walk_composited_timeline` s'en charge. let scene = comp.scene_snapshot(); let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); + // Imported audio tracks (issue #350), cloned out of the borrowed scene. + let audio_tracks = scene + .as_ref() + .map(|scene| scene.audio_tracks.clone()) + .unwrap_or_default(); // ---- encodeur (choisi à l'exécution, cf. ExportCodec::candidates) + mux ---- // Backend CPU : pas de pool D3D11 du tout. `av_hwdevice_ctx_init(D3D11VA)` échoue sur @@ -1467,7 +1472,7 @@ unsafe fn run_multi_inner( out_fps as f64, ); let assembled_audio = finish_audio( - assemble_concatenated_pcm(&clip_pcm, &audio_plan), + mix_external_tracks(assemble_concatenated_pcm(&clip_pcm, &audio_plan), &audio_tracks), audio_settings, ); audio_encoder.encode(&assembled_audio, octx)?; diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs index 2d20e233b..a9fc994f8 100644 --- a/crates/compositor/src/scene.rs +++ b/crates/compositor/src/scene.rs @@ -420,6 +420,29 @@ pub struct SceneAudio { pub gain_db: f32, } +/// One imported audio track (issue #350) mixed over the assembled programme — +/// voiceover / BGM / SFX. Deliberately a SEPARATE `Scene` field rather than a +/// member of `SceneAudio`, so `SceneAudio` stays `Copy` and the pipelines keep +/// copying it out of a borrow unchanged. +/// +/// `start_sec` is the track's head on the OUTPUT programme; `trim_start_sec` / +/// `trim_end_sec` window the source file (both source seconds). The renderer +/// resolves `start_sec` from the track's raw timeline position — equal to it when +/// the project has no trims/speed, which is the case this first cut mixes exactly. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneAudioTrack { + pub path: String, + #[serde(default)] + pub start_sec: f64, + #[serde(default)] + pub gain_db: f32, + #[serde(default)] + pub trim_start_sec: f64, + #[serde(default)] + pub trim_end_sec: Option, +} + #[derive(Debug, Clone, Copy, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SceneOutput { @@ -451,6 +474,10 @@ pub struct Scene { /// Global audio finishing. Default keeps old scene payloads bit-for-bit compatible. #[serde(default)] pub audio: SceneAudio, + /// Imported audio tracks mixed over the programme (issue #350). `#[serde(default)]`: + /// absent from every scene written before this, and from a project with none. + #[serde(default)] + pub audio_tracks: Vec, /// Crop écran par clip, dans le même ordre que `clips` (`cropByClip` côté TS). #[serde(default)] pub crop_by_clip: Vec>, diff --git a/electron/ai-edition/document-service.test.ts b/electron/ai-edition/document-service.test.ts index 6cbdb97c9..91448d350 100644 --- a/electron/ai-edition/document-service.test.ts +++ b/electron/ai-edition/document-service.test.ts @@ -280,6 +280,48 @@ describe("DocumentService", () => { expect(after.project.primaryAssetId).toBe(first.project.primaryAssetId); expect(after.assets).toHaveLength(2); }); + + // Issue #350 — external audio import (voiceover / BGM / SFX). + it("appends an audio asset without claiming the primary slot", async () => { + const doc = await service.createProject("P"); + const updated = await service.addAsset(doc.project.id, { + path: "/tmp/voiceover.mp3", + kind: "audio", + }); + expect(updated.assets).toHaveLength(1); + expect(updated.assets[0]?.kind).toBe("audio"); + // An audio-only file must never become the project's primary asset, even + // when it is the first file added to an otherwise-empty project. + expect(updated.project.primaryAssetId).toBeUndefined(); + }); + + it("keeps the existing video primary when an audio track is added", async () => { + const doc = await service.createProject("P"); + const withVideo = await service.addAsset(doc.project.id, { path: "/tmp/screen.mp4" }); + const primary = withVideo.project.primaryAssetId; + const withAudio = await service.addAsset(doc.project.id, { + path: "/tmp/bgm.wav", + kind: "audio", + }); + expect(withAudio.project.primaryAssetId).toBe(primary); + expect(withAudio.assets).toHaveLength(2); + }); + + it("rejects unsupported audio extensions", async () => { + const doc = await service.createProject("P"); + await expect( + service.addAsset(doc.project.id, { path: "/tmp/clip.mp4", kind: "audio" }), + ).rejects.toBeInstanceOf(ProjectFileError); + }); + + it("accepts a video extension under the default kind but not as audio", async () => { + const doc = await service.createProject("P"); + // The same extension routing works in reverse: an .mp3 is fine as audio + // but rejected as video (covered above), and an .mp4 is the opposite. + await expect( + service.addAsset(doc.project.id, { path: "/tmp/a.mp3", kind: "audio" }), + ).resolves.toBeDefined(); + }); }); describe("removeAsset", () => { @@ -363,6 +405,50 @@ describe("DocumentService", () => { expect(after.project.primaryAssetId).toBe(b.assets[1]?.id); }); + // Issue #350 — an audio overlay can never be primary. + it("passes primary to the next VIDEO asset, never to an audio asset", async () => { + const doc = await service.createProject("P"); + const video = await service.addAsset(doc.project.id, { path: "/tmp/screen.mp4" }); + await service.addAsset(doc.project.id, { path: "/tmp/music.mp3", kind: "audio" }); + const primaryId = video.project.primaryAssetId; + expect(primaryId).toBeTruthy(); + // Removing the only video leaves just the audio asset; primary must clear, + // not fall to the audio one. + const after = await service.removeAsset(doc.project.id, primaryId ?? ""); + expect(after.project.primaryAssetId).toBeUndefined(); + expect(after.assets).toHaveLength(1); + expect(after.assets[0]?.kind).toBe("audio"); + }); + + it("drops audioTracks that referenced a removed audio asset", async () => { + const doc = await service.createProject("P"); + await service.addAsset(doc.project.id, { path: "/tmp/screen.mp4" }); + const withAudio = await service.addAsset(doc.project.id, { + path: "/tmp/music.mp3", + kind: "audio", + }); + const audioId = withAudio.assets.find((a) => a.kind === "audio")?.id ?? ""; + expect(audioId).toBeTruthy(); + const withTrack = await service.saveProject({ + ...withAudio, + audioTracks: [ + { + id: "trk_1", + assetId: audioId, + timelineStartSec: 0, + durationSec: 10, + trimStartSec: 0, + gainDb: 0, + label: "music", + }, + ], + }); + expect(withTrack.audioTracks).toHaveLength(1); + const after = await service.removeAsset(doc.project.id, audioId); + expect(after.audioTracks).toEqual([]); + expect(after.assets.some((a) => a.id === audioId)).toBe(false); + }); + it("resequences other assets and rederives their anchored regions", async () => { const created = await service.createProject("P"); const withA = await service.addAsset(created.project.id, { path: "/tmp/a.mp4" }); diff --git a/electron/ai-edition/document-service.ts b/electron/ai-edition/document-service.ts index 3c93e3bc0..cf471e4df 100644 --- a/electron/ai-edition/document-service.ts +++ b/electron/ai-edition/document-service.ts @@ -38,6 +38,9 @@ export interface ProjectSummary { export interface AddAssetInput { path: string; label?: string; + // "audio" imports an external voiceover / BGM / SFX file (issue #350). + // Defaults to "video" when omitted, so existing callers are unaffected. + kind?: "video" | "audio"; } export class DocumentNotFoundError extends Error { @@ -72,6 +75,24 @@ function isSupportedVideoPath(filePath: string): boolean { return SUPPORTED_VIDEO_EXTENSIONS.has(ext); } +// Imported audio (issue #350). Decoding is handled downstream by the same +// WebCodecs / ffmpeg paths that read a video's audio track, so this list is the +// container formats decodeAudioData and the compositor can open. +const SUPPORTED_AUDIO_EXTENSIONS = new Set([ + ".mp3", + ".wav", + ".m4a", + ".aac", + ".flac", + ".ogg", + ".opus", +]); + +function isSupportedAudioPath(filePath: string): boolean { + const ext = path.extname(filePath).toLowerCase(); + return SUPPORTED_AUDIO_EXTENSIONS.has(ext); +} + function safeProjectId(raw: string): string { // ponytail: project ids are uuid-prefixed strings (e.g. "proj_"). Reject // anything that smells like path traversal before we ever touch the disk. @@ -283,7 +304,15 @@ export class DocumentService { if (!input.path) { throw new ProjectFileError("Asset path is required.", projectId); } - if (!isSupportedVideoPath(input.path)) { + const kind = input.kind ?? "video"; + if (kind === "audio") { + if (!isSupportedAudioPath(input.path)) { + throw new ProjectFileError( + `Unsupported audio extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_AUDIO_EXTENSIONS].join(", ")})`, + projectId, + ); + } + } else if (!isSupportedVideoPath(input.path)) { throw new ProjectFileError( `Unsupported video extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_VIDEO_EXTENSIONS].join(", ")})`, projectId, @@ -300,18 +329,24 @@ export class DocumentService { } const asset: AxcutAsset = { id: createId("asset"), - kind: "video", + kind, label: input.label?.trim() || path.basename(absolutePath), originalPath: absolutePath, sizeBytes, cameraTrack: null, }; + // An audio import is an overlay, never the thing the timeline is built + // around, so it must not claim the empty primaryAssetId slot — otherwise the + // first file dropped into a fresh project (a BGM track) would become its + // primary asset and the editor would try to lay out clips from a file with + // no video. + const claimsPrimary = kind !== "audio" && !doc.project.primaryAssetId; const next: AxcutDocument = { ...doc, assets: [...doc.assets, asset], project: { ...doc.project, - ...(doc.project.primaryAssetId ? {} : { primaryAssetId: asset.id }), + ...(claimsPrimary ? { primaryAssetId: asset.id } : {}), updatedAt: new Date().toISOString(), }, }; @@ -324,9 +359,13 @@ export class DocumentService { throw new ProjectFileError(`Asset ${assetId} not found in project ${projectId}.`, projectId); } const assets = doc.assets.filter((a) => a.id !== assetId); + // Primary is the thing the timeline is built around, so it must fall to the + // next VIDEO asset — never an audio overlay (issue #350), which can't be + // primary (see addAsset). Falling back to `assets[0]` would hand primary to + // an audio asset when the removed one was the last video. const primaryAssetId = doc.project.primaryAssetId === assetId - ? (assets[0]?.id ?? undefined) + ? (assets.find((a) => a.kind !== "audio")?.id ?? undefined) : doc.project.primaryAssetId; const withoutAssetClips = doc.timeline.clips .filter((clip) => clip.assetId === assetId) @@ -334,6 +373,9 @@ export class DocumentService { const next: AxcutDocument = { ...withoutAssetClips, assets, + // Drop imported audio tracks that referenced the removed asset — they + // would otherwise dangle, pointing at an asset the document no longer has. + audioTracks: withoutAssetClips.audioTracks.filter((t) => t.assetId !== assetId), timeline: { ...withoutAssetClips.timeline, trimRanges: withoutAssetClips.timeline.trimRanges.filter((r) => r.assetId !== assetId), diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 4eb288e14..ea33e78ec 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -286,6 +286,14 @@ interface Window { name?: string; canceled?: boolean; }>; + // Import an external audio file from the timeline toolbar (issue #350). + openAudioFilePicker: () => Promise<{ + success: boolean; + path?: string; + name?: string; + canceled?: boolean; + message?: string; + }>; setCurrentVideoPath: (path: string) => Promise<{ success: boolean }>; setCurrentRecordingSession: ( session: import("../src/lib/recordingSession").RecordingSession | null, diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 958ef6d96..b9b7d23f6 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -183,6 +183,23 @@ function hasAllowedImportVideoExtension(filePath: string): boolean { return ALLOWED_IMPORT_VIDEO_EXTENSIONS.has(path.extname(filePath).toLowerCase()); } +// Imported audio (issue #350). Kept separate from the video set so the two +// pickers stay honest — an audio picker must not approve a video path and vice +// versa. Mirrors SUPPORTED_AUDIO_EXTENSIONS in the document service. +const ALLOWED_IMPORT_AUDIO_EXTENSIONS = new Set([ + ".mp3", + ".wav", + ".m4a", + ".aac", + ".flac", + ".ogg", + ".opus", +]); + +function hasAllowedImportAudioExtension(filePath: string): boolean { + return ALLOWED_IMPORT_AUDIO_EXTENSIONS.has(path.extname(filePath).toLowerCase()); +} + function runProcess( command: string, args: string[], @@ -279,8 +296,13 @@ async function prepareSupplementalPreviewAudioTrack(videoPath: string) { return { success: true, path: pathToFileURL(outputPath).toString() }; } -async function approveReadableVideoPath( - filePath?: string | null, +// Shared core behind the media path approvers. `hasAllowedExtension` is the ONLY +// thing that differs between video and audio imports, so it is the single knob: +// an already-approved path passes regardless, otherwise the extension gate, +// optional trusted-dir confinement, and a stat check decide whether to approve. +async function approveReadableMediaPath( + filePath: string | null | undefined, + hasAllowedExtension: (p: string) => boolean, trustedDirs?: string[], ): Promise { const normalizedPath = normalizeVideoSourcePath(filePath); @@ -292,7 +314,7 @@ async function approveReadableVideoPath( return normalizedPath; } - if (!hasAllowedImportVideoExtension(normalizedPath)) { + if (!hasAllowedExtension(normalizedPath)) { return null; } @@ -319,6 +341,20 @@ async function approveReadableVideoPath( return normalizedPath; } +function approveReadableVideoPath( + filePath?: string | null, + trustedDirs?: string[], +): Promise { + return approveReadableMediaPath(filePath, hasAllowedImportVideoExtension, trustedDirs); +} + +function approveReadableAudioPath( + filePath?: string | null, + trustedDirs?: string[], +): Promise { + return approveReadableMediaPath(filePath, hasAllowedImportAudioExtension, trustedDirs); +} + function resolveRecordingOutputPath(fileName: string): string { const trimmed = fileName.trim(); if (!trimmed) { @@ -3574,6 +3610,8 @@ export function registerIpcHandlers( } }); + // The media tab imports VIDEO (it arranges clips). Audio is imported from the + // timeline toolbar instead (issue #350) — see `open-audio-file-picker` below. ipcMain.handle("open-video-file-picker", async () => { try { const dialogOptions = buildDialogOptions( @@ -3620,6 +3658,55 @@ export function registerIpcHandlers( } }); + // Import an external audio file (voiceover / BGM / SFX) — issue #350. Driven by + // the timeline's "Add audio" tool: audio is a timeline overlay (like an + // annotation), not a media-tab clip, so it has its own audio-only picker and the + // renderer adds it as a kind:"audio" asset + track at the playhead. + ipcMain.handle("open-audio-file-picker", async () => { + try { + const dialogOptions = buildDialogOptions( + { + title: mainT("dialogs", "fileDialogs.selectAudio"), + defaultPath: RECORDINGS_DIR, + filters: [ + { + name: mainT("dialogs", "fileDialogs.audioFiles"), + extensions: ["mp3", "wav", "m4a", "aac", "flac", "ogg", "opus"], + }, + { name: mainT("dialogs", "fileDialogs.allFiles"), extensions: ["*"] }, + ], + properties: ["openFile"], + }, + getMainWindow(), + ); + const result = await dialog.showOpenDialog(dialogOptions); + + if (result.canceled || result.filePaths.length === 0) { + return { success: false, canceled: true }; + } + + const normalizedPath = await approveReadableAudioPath(result.filePaths[0]); + if (!normalizedPath) { + return { + success: false, + message: "Selected file is not a supported readable audio file", + }; + } + + return { + success: true, + path: normalizedPath, + }; + } catch (error) { + console.error("Failed to open audio file picker:", error); + return { + success: false, + message: "Failed to open audio file picker", + error: String(error), + }; + } + }); + ipcMain.handle("reveal-in-folder", async (_, filePath: string) => { try { // showItemInFolder returns nothing, it throws on error diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts index 47d66e270..884b50afd 100644 --- a/electron/ipc/nativeBridge.ts +++ b/electron/ipc/nativeBridge.ts @@ -483,6 +483,7 @@ export function registerNativeBridgeHandlers(context: NativeBridgeContext) { request.payload.projectId, request.payload.path, request.payload.label, + request.payload.kind, ), ); case "document.removeAsset": diff --git a/electron/native-bridge/services/aiEditionService.ts b/electron/native-bridge/services/aiEditionService.ts index 0fbbccc9c..90088781a 100644 --- a/electron/native-bridge/services/aiEditionService.ts +++ b/electron/native-bridge/services/aiEditionService.ts @@ -151,9 +151,17 @@ export class AiEditionService { } } - async addAsset(projectId: string, path: string, label?: string): Promise { - const document = await this.options.documents.addAsset(projectId, { path, label }); - const assetId = document.project.primaryAssetId ?? document.assets.at(-1)?.id ?? ""; + async addAsset( + projectId: string, + path: string, + label?: string, + kind?: "video" | "audio", + ): Promise { + const document = await this.options.documents.addAsset(projectId, { path, label, kind }); + // The just-added asset is always the last one; primaryAssetId is only a + // fallback for the video case and would point at the wrong asset for an + // audio import (which never claims primary), so prefer the tail. + const assetId = document.assets.at(-1)?.id ?? document.project.primaryAssetId ?? ""; return { assetId, document }; } diff --git a/electron/preload.ts b/electron/preload.ts index 6aff16407..16227f110 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -276,6 +276,9 @@ contextBridge.exposeInMainWorld("electronAPI", { openVideoFilePicker: () => { return ipcRenderer.invoke("open-video-file-picker"); }, + openAudioFilePicker: () => { + return ipcRenderer.invoke("open-audio-file-picker"); + }, setCurrentVideoPath: (path: string) => { return ipcRenderer.invoke("set-current-video-path", path); }, diff --git a/src/components/ai-edition/EditorEmptyState.test.tsx b/src/components/ai-edition/EditorEmptyState.test.tsx index acb513e4b..be7f130b7 100644 --- a/src/components/ai-edition/EditorEmptyState.test.tsx +++ b/src/components/ai-edition/EditorEmptyState.test.tsx @@ -53,6 +53,7 @@ const sampleDoc = vi.hoisted( }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }), ); diff --git a/src/components/ai-edition/ExportDialog.showInFolder.test.tsx b/src/components/ai-edition/ExportDialog.showInFolder.test.tsx index 835d9c8fd..db4207c4e 100644 --- a/src/components/ai-edition/ExportDialog.showInFolder.test.tsx +++ b/src/components/ai-edition/ExportDialog.showInFolder.test.tsx @@ -75,6 +75,7 @@ const DOC: AxcutDocument = { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; diff --git a/src/components/ai-edition/ExportDialog.test.ts b/src/components/ai-edition/ExportDialog.test.ts index 3aa8d85b5..ed2f1ab1a 100644 --- a/src/components/ai-edition/ExportDialog.test.ts +++ b/src/components/ai-edition/ExportDialog.test.ts @@ -57,6 +57,7 @@ function doc(assets: AxcutAsset[], clips: AxcutClip[]): AxcutDocument { }, annotations: [], zoomRanges: [], + audioTracks: [], legacyEditor: null, }; } diff --git a/src/components/ai-edition/LeftPanel.tsx b/src/components/ai-edition/LeftPanel.tsx index 2cafaebf2..00ab408ce 100644 --- a/src/components/ai-edition/LeftPanel.tsx +++ b/src/components/ai-edition/LeftPanel.tsx @@ -179,7 +179,11 @@ export function MediaPane() { } }; + // Only video assets are listed here — the media panel arranges clips. Imported + // audio (issue #350) lives on the timeline's audio lane, added from the timeline + // toolbar, and is managed there (select its pill to edit / remove). const filtered = (document?.assets ?? []).filter((a) => { + if (a.kind === "audio") return false; if (!query) return true; const text = `${a.label} ${a.originalPath}`.toLowerCase(); return text.includes(query.toLowerCase()); diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index 64987dea9..a56f3f143 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -427,7 +427,16 @@ export function NewEditorShell() { const handleDropAsset = useCallback( (assetId: string) => enqueueTimelineWrite(() => { - const at = useProjectStore.getState().document?.timeline.clips.length ?? 0; + const doc = useProjectStore.getState().document; + // An audio asset has no video, so it must never become a clip (issue + // #350) — it goes on the audio lane as a track. Adding it "to the + // timeline" reuses its existing track if it already has one (importing + // already placed one) so the same file can't stack up duplicate lanes. + if (doc?.assets.find((a) => a.id === assetId)?.kind === "audio") { + if (doc.audioTracks.some((t) => t.assetId === assetId)) return Promise.resolve(); + return tl.addAudioTrack(assetId).then(() => undefined); + } + const at = doc?.timeline.clips.length ?? 0; return tl.insertClipAt(assetId, at); }).catch((error) => { toast.error(te("mediaStage.couldNotAddAsset"), { @@ -1032,6 +1041,13 @@ export function NewEditorShell() { void tl.addAnnotation(newRegionDurationSec()); return; } + // Unlike its neighbours this opens a file picker rather than dropping a region at + // the playhead — there is nothing to size, so it takes no duration (issue #350). + if (matchesShortcut(e, shortcuts.addAudio, isMac)) { + e.preventDefault(); + void tl.addAudio(); + return; + } if (matchesShortcut(e, shortcuts.addSpeed, isMac)) { e.preventDefault(); void tl.addSpeed(newRegionDurationSec()); @@ -1235,6 +1251,11 @@ export function NewEditorShell() { hasProject={hasProject} hasAsset={hasAsset} videoSources={videoSources} + // Imported audio tracks (issue #350). `videoSources` already + // resolves a URL for every asset (audio included), so it doubles as + // the audio source list; VirtualPreview looks each track up by assetId. + audioTracks={tl.audioTracks} + audioSources={videoSources} clips={clips} zoomRegions={tl.zoomRegions} speedRegions={tl.speedRegions} diff --git a/src/components/ai-edition/Preview.tsx b/src/components/ai-edition/Preview.tsx index aaf04f2db..cd6868e75 100644 --- a/src/components/ai-edition/Preview.tsx +++ b/src/components/ai-edition/Preview.tsx @@ -3,6 +3,7 @@ import type { CameraFullscreenRegion, ZoomFocus } from "@/components/video-edito import { useScopedT } from "@/contexts/I18nContext"; import type { AxcutAnnotationRegion, + AxcutAudioTrack, AxcutClip, AxcutTrimRange, AxcutZoomRegion, @@ -21,6 +22,12 @@ interface PreviewProps { hasProject: boolean; hasAsset: boolean; videoSources: VideoSource[]; + /** Imported audio tracks and the (unfiltered) asset URLs they resolve to + * (issue #350). Passed straight through to VirtualPreview — unlike the video + * `previewSources` below, these are NOT narrowed to clip-referenced assets, + * since an audio track has no clip. */ + audioTracks?: AxcutAudioTrack[]; + audioSources?: VideoSource[]; clips: AxcutClip[]; zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; @@ -52,6 +59,8 @@ export function Preview({ hasProject, hasAsset, videoSources, + audioTracks = [], + audioSources = [], clips, zoomRegions, speedRegions, @@ -178,6 +187,8 @@ export function Preview({ <> ; interface PreviewCanvasProps { videoSources: VideoSource[]; + /** Imported audio tracks + their asset URLs (issue #350), forwarded to + * VirtualPreview. */ + audioTracks?: AxcutAudioTrack[]; + audioSources?: VideoSource[]; clips: AxcutClip[]; zoomRegions?: AxcutZoomRegion[]; speedRegions?: SpeedRegion[]; diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index d12fe248a..cb3ace165 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -13,6 +13,7 @@ import { Layout as LayoutIcon, Loader2, MousePointerClick, + Music, Sliders, Trash2, } from "lucide-react"; @@ -51,6 +52,7 @@ import { } from "@/lib/ai-edition/store/editorSettings"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; +import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; import { buildAggregatedSections, type ClipSection, @@ -2101,6 +2103,98 @@ export function AudioPane() { ); } +type TimelineApi = ReturnType; + +// Per-track controls for the selected imported audio track (issue #350). Shown by +// the inspector in place of the facet when an audio track is selected (see +// FloatingInspector). The header is the generic "Audio track"; the body leads +// with the file name, then the volume (a local live value during the drag, +// committed as one undo step on release), then a delete button styled like the +// region panes' (position and mute are edited on the lane itself). +export function AudioTrackPane({ tl }: { tl: TimelineApi }) { + const ts = useScopedT("settings"); + const trackId = tl.selectedAudioTrackId; + const track = tl.audioTracks.find((t) => t.id === trackId); + const asset = track ? tl.assets.find((a) => a.id === track.assetId) : undefined; + // Live-drag value for the volume slider; null means "show the committed gain". + const [liveGain, setLiveGain] = useState(null); + if (!track) return null; + const fileName = track.label || asset?.label || asset?.originalPath?.split(/[\\/]/).pop() || ""; + + // Match the region panes' danger-outlined delete button (see SelectionPane). + const deleteBtnStyle: CSSProperties = { + display: "flex", + width: "100%", + alignItems: "center", + justifyContent: "center", + gap: 7, + padding: "9px 14px", + borderRadius: 10, + border: "1px solid var(--danger)", + background: "var(--danger-soft)", + color: "var(--danger)", + font: "600 13px var(--font-display)", + cursor: "pointer", + }; + + return ( + } + helpText={ts("audioTrack.help")} + > +
+ {fileName} +
+
+ setLiveGain(value)} + onCommit={() => { + if (liveGain !== null) void tl.setAudioTrackGain(track.id, liveGain); + setLiveGain(null); + }} + /> +
+ + + + ); +} + // ─── Cursor ─────────────────────────────────────────────────────── function safeAssetUrl(relativePath: string): string { diff --git a/src/components/ai-edition/VirtualPreview.audio.test.ts b/src/components/ai-edition/VirtualPreview.audio.test.ts index 208bf05a2..aeb2781e8 100644 --- a/src/components/ai-edition/VirtualPreview.audio.test.ts +++ b/src/components/ai-edition/VirtualPreview.audio.test.ts @@ -1,8 +1,10 @@ import { describe, expect, it } from "vitest"; +import type { AxcutAudioTrack } from "@/lib/ai-edition/schema"; import { applyPreviewAudioSettings, type PreviewAudioGraph, resolveAudioTrackPlayback, + resolveTimelineAudioPlayback, } from "./VirtualPreview"; /** Minimal stand-in: the function only ever touches `gain.gain.value`. */ @@ -80,3 +82,45 @@ describe("applyPreviewAudioSettings", () => { expect(graph.gain.gain.value).toBeCloseTo(0.5, 4); }); }); + +describe("resolveTimelineAudioPlayback", () => { + // A track placed 10s into the RAW timeline, source windowed to 2..8 (6s long). + const track: AxcutAudioTrack = { + id: "t1", + assetId: "a1", + timelineStartSec: 10, + durationSec: 20, + trimStartSec: 2, + trimEndSec: 8, + gainDb: 0, + label: "", + }; + + it("maps the playhead to a source position offset by the trim in-point", () => { + // 3s into the track's span → 2 (trimStart) + 3 = 5s of source. + expect(resolveTimelineAudioPlayback(13, track)).toEqual({ targetTimeSec: 5, shouldPlay: true }); + }); + + it("does not play before the track starts, parked at the in-point", () => { + expect(resolveTimelineAudioPlayback(9, track)).toEqual({ targetTimeSec: 2, shouldPlay: false }); + }); + + it("does not play past the window end, parked at the out-point", () => { + // Window is 6s (2..8), so timeline 10..16. At 16 it has just ended. + expect(resolveTimelineAudioPlayback(16, track)).toEqual({ + targetTimeSec: 8, + shouldPlay: false, + }); + }); + + it("uses the asset duration as the out-point when the tail isn't trimmed", () => { + const untrimmed = { ...track, trimStartSec: 0, trimEndSec: undefined, durationSec: 5 }; + // Span is 10..15. At 14 → 4s of source, still playing. + expect(resolveTimelineAudioPlayback(14, untrimmed)).toEqual({ + targetTimeSec: 4, + shouldPlay: true, + }); + // At 15 the 5s file is over. + expect(resolveTimelineAudioPlayback(15, untrimmed).shouldPlay).toBe(false); + }); +}); diff --git a/src/components/ai-edition/VirtualPreview.playback.test.tsx b/src/components/ai-edition/VirtualPreview.playback.test.tsx index f0389fe2a..13d5121cb 100644 --- a/src/components/ai-edition/VirtualPreview.playback.test.tsx +++ b/src/components/ai-edition/VirtualPreview.playback.test.tsx @@ -207,3 +207,164 @@ describe("VirtualPreview playback across a clip boundary", () => { expect(video.pauseCalls).toHaveLength(0); }); }); + +// Issue #350 — imported audio tracks follow the RAW virtual playhead. The +// decision math is unit-tested in VirtualPreview.audio.test.ts; here we prove the +// rAF loop applies it to the mounted