diff --git a/package.json b/package.json index 1ace2e00a..5a1c577f3 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "build-macapp": "tauri build --target universal-apple-darwin --verbose", "build-winapp": "tauri build --target x86_64-pc-windows-gnu", "copy-pdf-worker": "shx cp node_modules/pdfjs-dist/build/pdf.worker.min.mjs public/pdf.worker.min.mjs", - "copy-ffmpeg-core": "shx mkdir -p public/corelibs/ffmpeg && shx cp node_modules/@ffmpeg/core/dist/umd/ffmpeg-core.js public/corelibs/ffmpeg/ffmpeg-core.js && shx cp node_modules/@ffmpeg/core/dist/umd/ffmpeg-core.wasm public/corelibs/ffmpeg/ffmpeg-core.wasm && shx cp node_modules/@ffmpeg/ffmpeg/dist/umd/ffmpeg.js public/corelibs/ffmpeg/ffmpeg.umd.js && shx cp node_modules/@ffmpeg/ffmpeg/dist/umd/814.ffmpeg.js public/corelibs/ffmpeg/814.ffmpeg.js", + "copy-ffmpeg-core": "shx mkdir -p public/corelibs/ffmpeg && shx cp node_modules/@ffmpeg/core/dist/umd/ffmpeg-core.js public/corelibs/ffmpeg/ffmpeg-core.js && shx cp node_modules/@ffmpeg/core/dist/esm/ffmpeg-core.wasm public/corelibs/ffmpeg/ffmpeg-core.wasm && shx cp node_modules/@ffmpeg/ffmpeg/dist/umd/ffmpeg.js public/corelibs/ffmpeg/ffmpeg.umd.js && shx cp node_modules/@ffmpeg/ffmpeg/dist/umd/814.ffmpeg.js public/corelibs/ffmpeg/814.ffmpeg.js", "eject": "react-scripts eject", "start": "npm run copy-pdf-worker && npm run copy-ffmpeg-core && react-scripts start", "build": "npm run copy-pdf-worker && npm run copy-ffmpeg-core && react-scripts build && echo 'candlestickers.app' > ./build/CNAME", diff --git a/public/corelibs/ffmpeg/ffmpeg-core.wasm b/public/corelibs/ffmpeg/ffmpeg-core.wasm index 246b0fe22..541d6261c 100644 Binary files a/public/corelibs/ffmpeg/ffmpeg-core.wasm and b/public/corelibs/ffmpeg/ffmpeg-core.wasm differ diff --git a/src/Editor/Editor.jsx b/src/Editor/Editor.jsx index f5802f8f1..7d9594408 100644 --- a/src/Editor/Editor.jsx +++ b/src/Editor/Editor.jsx @@ -77,17 +77,28 @@ async function loadPathIntoEditor(editorThis, filePath) { const name = filePath.split('/').pop(); // .WICK/ PROJECT FILE + const VIDEO_MIME = { + '.mp4': 'video/mp4', '.m4v': 'video/x-m4v', + '.mov': 'video/quicktime', + '.webm': 'video/webm', + '.ogv': 'video/ogg', '.ogg': 'video/ogg', + '.avi': 'video/x-msvideo', + '.mkv': 'video/x-matroska', + '.3gp': 'video/3gpp', + '.wmv': 'video/x-ms-wmv', + } + const videoExt = Object.keys(VIDEO_MIME).find(ext => name.endsWith(ext)) + if ( name.endsWith('.wick') || - name.endsWith('.mov') || - name.endsWith('.mp4') + videoExt ) { const bytes = await readFile(filePath, { encoding: null }) const blob = new Blob([bytes]) const file = new File([blob], name, { - type: (name.endsWith('.wick') && 'application/zip') || (name.endsWith('.pdf') && 'application/pdf') || 'video/mp4' + type: (name.endsWith('.wick') && 'application/zip') || (videoExt && VIDEO_MIME[videoExt]) || 'video/mp4' }); @@ -248,13 +259,13 @@ class Editor extends EditorCore { // Wick Project File Input this.openProjectFileFromClient = window.createFileInput({ - accept: '.zip, .wick, .mp4, .pdf', + accept: '.zip, .wick, video/*, .pdf', onChange: this.handleWickFileLoad, }); // Wick file input this.openAssetFileFromClient = window.createFileInput({ - accept: window.Wick.FileAsset.getValidExtensions().join(', '), + accept: window.Wick.FileAsset.getValidExtensions().join(', ') + ', video/*', onChange: this.handleAssetFileImport, multiple: true, }); diff --git a/src/Editor/EditorCore.jsx b/src/Editor/EditorCore.jsx index 0d0dc6f2b..92d003cb7 100644 --- a/src/Editor/EditorCore.jsx +++ b/src/Editor/EditorCore.jsx @@ -1132,6 +1132,24 @@ class EditorCore extends Component { if (options.create) this.createImageFromAsset(gifAsset.uuid, options.location.x || 0, options.location.y || 0); } }); + } else if (acceptedFiles[i].type.startsWith('video/')) { + const mp4File = acceptedFiles[i]; + const fps = Math.max(1, Math.min(60, Number(prompt('Enter FPS for ' + mp4File.name, String(this.project.framerate)) || this.project.framerate))); + const mp4ToastID = this.toast(`Importing ${mp4File.name}…`, 'info', { autoClose: false }); + this.showWaitOverlay(); + this.importMP4AsAsset({ + file: mp4File, + fps, + onProgress: (msg, p) => this.updateToast(mp4ToastID, { text: `${msg} (${Math.round(p || 0)}%)` }), + }).then(({ gifAsset }) => { + this.updateToast(mp4ToastID, { text: `:) Imported ${mp4File.name}`, type: 'success', autoClose: 7000 }); + this.hideWaitOverlay(); + if (options.create) this.createImageFromAsset(gifAsset.uuid, options.location.x || 0, options.location.y || 0); + }).catch(err => { + console.error(err); + this.updateToast(mp4ToastID, { text: `Failed to import ${mp4File.name}`, type: 'error', autoClose: 5000 }); + this.hideWaitOverlay(); + }); } else { var file = acceptedFiles[i]; @@ -1140,6 +1158,87 @@ class EditorCore extends Component { } } + /** + * Extracts frames from an MP4 file, builds a GIFAsset from them, and adds + * it (plus all intermediate image assets) to the current project's asset + * library. -H.A. + * + * @param {File} file - The MP4 file to import. + * @param {number} fps - Frame rate to extract at. + * @param {Function} onProgress - Progress callback (msg, percent). + * @returns {Promise<{ gifAsset, audioBlob, fps, projectName, width, height }>} + */ + importMP4AsAsset = ({ file, fps = 12, onProgress = () => {}, bakeAudio = true }) => { + return new Promise(async (resolve, reject) => { + try { + const projectName = file.name.replace(/\.[^.]+$/, ''); + + const { imageAssets, audioBlob, fps: extractedFps, width, height } = + await MP4ImportPure.importMP4AsSequence({ + mp4File: file, + fps, + projectName, + onProgress, + }); + + // Add individual frame images to the project first + imageAssets.forEach(asset => this.project.addAsset(asset)); + await new Promise(res => this.project.loadAssets(res)); + + // Stitch the frame images into a single animated GIF asset + window.Wick.GIFAsset.fromImages(imageAssets, this.project, (gifAsset) => { + gifAsset.name = projectName; + gifAsset.filename = projectName; + this.project.addAsset(gifAsset); + + if (!audioBlob || !bakeAudio) { + // Project flow: return audioBlob to caller so it can set up root-timeline audio + this.projectDidChange({ actionName: 'Imported MP4 as Asset' }); + resolve({ gifAsset, audioBlob: audioBlob || null, fps: extractedFps, projectName, width, height }); + return; + } + + // Asset flow: import SoundAsset and bake it into the clip's second layer + const audioFile = new File([audioBlob], projectName + '.wav', { type: 'audio/wav' }); + this.importFileAsAsset(audioFile, (soundAsset) => { + if (!soundAsset) { + this.projectDidChange({ actionName: 'Imported MP4 as Asset' }); + resolve({ gifAsset, audioBlob: null, fps: extractedFps, projectName, width, height }); + return; + } + + // Instantiate the clip, add audio layer, re-serialize back into gifAsset.src + gifAsset.createInstance((clip) => { + const audioLayer = new window.Wick.Layer(); + audioLayer.name = 'audio'; + clip.timeline.addLayer(audioLayer); + + const frameEnd = clip.timeline.layers[0].length; + const audioFrame = new window.Wick.Frame({ start: 1, end: frameEnd }); + audioLayer.addFrame(audioFrame); + audioFrame.sound = soundAsset; + + // Temporarily attach clip to project so it can be serialized + this.project.addObject(clip); + window.Wick.WickObjectFile.toWickObjectFile(clip, 'blob', (blobFile) => { + const reader = new FileReader(); + reader.onload = (e) => { + gifAsset.src = e.target.result; + clip.remove(); + this.projectDidChange({ actionName: 'Imported MP4 as Asset' }); + resolve({ gifAsset, audioBlob: null, fps: extractedFps, projectName, width, height }); + }; + reader.readAsDataURL(blobFile); + }); + }, this.project); + }); + }); + } catch (err) { + reject(err); + } + }); + } + /** * Begin interactive object creation process. */ @@ -2399,100 +2498,87 @@ class EditorCore extends Component { } - // if not an mp4 file just load it then - if (file.type !== 'video/mp4') { + // if not a video file just load it then + if (!file.type.startsWith('video/')) { this.importProjectAsWickFile(file); return; } // if an mp4 file then translate it before opening it try { - const toastID = this.toast(`Loading ${file.name}…`, 'info', { autoClose: false }) + const fps = Math.max(1, Math.min(60, Number(prompt("Enter FPS", String(this.project.framerate)) || this.project.framerate))); + const toastID = this.toast(`Loading ${file.name}…`, 'info', { autoClose: false }); - // NOTE need to setup new project BEFORE loading mp4 (otherwise will still work but console will be loaded with error messages) + // NOTE: must set up the new project BEFORE importing so assets have + // somewhere to live (avoids stale-project console errors). this.setupNewProject(); this.projectDidChange({ actionName: 'Reset project' }); + this.showWaitOverlay(); + + // Extract frames + build GIF asset → adds everything to this.project + // bakeAudio: false so we handle audio on the root timeline instead + const { gifAsset, audioBlob, fps: extractedFps, projectName, width, height } = + await this.importMP4AsAsset({ + file, + fps, + bakeAudio: false, + onProgress: (msg, p) => this.updateToast(toastID, { text: `${msg} (${Math.round(p || 0)}%)` }), + }); - this.showWaitOverlay() // disable clicking anywhere - - // GET ALL FRAMES FROM MP4 (run through MP4 import function) - const { imageAssets, audioBlob, fps, projectName, width, height } = - await MP4ImportPure.importMP4AsSequence({ - mp4File: file, - fps: Math.max(1, Math.min(60, Number(prompt("Enter FPS", "12") || 6))), // give user the option of FPS - projectName: file.name.replace(".mp4", ''), - onProgress: (msg, p) => this.updateToast(toastID, { text: `${msg} (${Math.round(p || 0)}%)` }) - }) - - // set up new project + // Fit project to the video this.project.width = width; this.project.height = height; - this.project.framerate = fps; + this.project.framerate = extractedFps; this.project.name = projectName; this.projectDidChange({ actionName: 'Adjusted settings based on MP4' }); - // add image assets to the current project - imageAssets.forEach(asset => this.project.addAsset(asset)); - - // turn the sequence into an asset using the same system used by the GIF import ;-; - await new Promise(res => this.project.loadAssets(res)) - await new Promise((resolve) => { - window.Wick.GIFAsset.fromImages(imageAssets, this.project, (gifAsset) => { - gifAsset.name = projectName; - gifAsset.filename = projectName; - this.project.addAsset(gifAsset); - // extend frame - this.project._children[1].activeFrame.end = this.project._children.length - 3; - - - // add the video asset into the project - this.project.createClipInstanceFromAsset(gifAsset, this.project.width / 2, this.project.height / 2, (clip) => { - this.selectObject(clip) // <-- select the object so we can adjust its settings - - // set to synce and enable play once - this.setSelectionAttribute('animationType', 'playOnce') - this.setSelectionAttribute('isSynced', true) - - // unselect the mp4 clip guy - this.clearSelection(); - - const tl = this.project.activeTimeline; - tl.layers[0].name = "video"; - - // add in the audio file as well (if video has audio) - if (audioBlob) { - tl.addLayer(new window.Wick.Layer()); - tl.layers[1].name = "audio"; - tl.layers[1].addFrame(new window.Wick.Frame()); - tl.layers[1].frames[0].end = tl.layers[0].frames[0].end; - // const soundObj = new window.Wick.Sound({ asset: this.project.assets[1]}); - - const audioFile = new File([audioBlob], projectName + ".wav", { type: 'audio/wav' }) - this.importFileAsAsset(audioFile, () => { - this.project.loadAssets(() => { - // wait for new audio asset to load in - this.setActiveLayerIndex(1); - this.addSoundToActiveFrame(this.project.assets[this.project.assets.length - 1]) - this.setActiveLayerIndex(0); - }) - }) // reuses existing audio import path - } - - this.projectDidChange({ actionName: 'Opened MP4 as GIF sequence' }) - this.updateToast(toastID, { text: `:) Imported ${file.name}`, type: 'success', autoClose: 7000 }) - this.hideWaitOverlay() - resolve() - }) - - // this.project._children[1].activeFrame._children[0]._isSynced = true; + // Extend the root frame and place the clip on the canvas + this.project._children[1].activeFrame.end = this.project._children.length - 3; + + this.project.createClipInstanceFromAsset(gifAsset, this.project.width / 2, this.project.height / 2, (clip) => { + this.selectObject(clip); + this.setSelectionAttribute('animationType', 'playOnce'); + this.setSelectionAttribute('isSynced', true); + this.clearSelection(); + + const tl = this.project.activeTimeline; + tl.layers[0].name = "video"; + + // Add an empty drawing layer and move it to the top + const drawingLayer = new window.Wick.Layer(); + drawingLayer.name = 'Layer'; + tl.addLayer(drawingLayer); + drawingLayer.addFrame(new window.Wick.Frame({ start: 1, end: 1 })); + tl.moveLayer(drawingLayer, 0); + // Layer order: drawing(0), video(1), then audio below + + // Add the audio track if the video had audio + if (audioBlob) { + tl.addLayer(new window.Wick.Layer()); + // Layer order: drawing(0), video(1), audio(2) + tl.layers[2].name = "audio"; + tl.layers[2].addFrame(new window.Wick.Frame()); + tl.layers[2].frames[0].end = tl.layers[1].frames[0].end; + + const audioFile = new File([audioBlob], projectName + ".wav", { type: 'audio/wav' }); + this.importFileAsAsset(audioFile, () => { + this.project.loadAssets(() => { + this.setActiveLayerIndex(2); + this.addSoundToActiveFrame(this.project.assets[this.project.assets.length - 1]); + this.setActiveLayerIndex(0); + }); + }); + } - }) - }) + this.projectDidChange({ actionName: 'Opened MP4 as GIF sequence' }); + this.updateToast(toastID, { text: `:) Imported ${file.name}`, type: 'success', autoClose: 7000 }); + this.hideWaitOverlay(); + }); } catch (err) { - console.error(err) - this.toast('Failed to import MP4.', 'error') - this.hideWaitOverlay() + console.error(err); + this.toast('Failed to import MP4.', 'error'); + this.hideWaitOverlay(); } } diff --git a/src/Editor/Panels/Canvas/Canvas.jsx b/src/Editor/Panels/Canvas/Canvas.jsx index 0047cbe1f..bdede1a3a 100644 --- a/src/Editor/Panels/Canvas/Canvas.jsx +++ b/src/Editor/Panels/Canvas/Canvas.jsx @@ -111,7 +111,7 @@ function CanvasDrop(props) { if (name.endsWith('.wick')) { // Wick Project (.wick file) p.importProjectAsWickFile(file); - } else if (file.type === 'video/mp4' || name.endsWith('.mp4') || + } else if (file.type.startsWith('video/') || /\.(mp4|mov|webm|mkv|avi|3gp|ogv|m4v|wmv)$/i.test(name) || file.type === 'application/pdf' || name.endsWith('.pdf')) { // MP4/PDF → open as new project p.openProjectFile(file); diff --git a/src/Editor/import/MP4Import_Pure.js b/src/Editor/import/MP4Import_Pure.js index 1c2cf82a6..ce043fcfb 100644 --- a/src/Editor/import/MP4Import_Pure.js +++ b/src/Editor/import/MP4Import_Pure.js @@ -11,6 +11,64 @@ function blobToDataURL(blob) { }) } +// WAV encoder worker +// run in a parallel worker thread and trasnfer the results back +const _WAV_WORKER_SRC = ` +self.onmessage = function (e) { + var channels = e.data.channels; + var sampleRate = e.data.sampleRate; + var numSamples = e.data.numSamples; + var numChannels = e.data.numChannels; + + var bytesPerSample = 2; + var blockAlign = numChannels * bytesPerSample; + var byteRate = sampleRate * blockAlign; + var wavDataByteLength = numSamples * blockAlign; + var buffer = new ArrayBuffer(44 + wavDataByteLength); + var view = new DataView(buffer); + + var offset = 0; + function writeStr(s) { for (var i = 0; i < s.length; i++) view.setUint8(offset++, s.charCodeAt(i)); } + function write16(v) { view.setUint16(offset, v, true); offset += 2; } + function write32(v) { view.setUint32(offset, v, true); offset += 4; } + + // RIFF / WAVE header + writeStr('RIFF'); write32(36 + wavDataByteLength); writeStr('WAVE'); + // fmt chunk + writeStr('fmt '); write32(16); write16(1); write16(numChannels); + write32(sampleRate); write32(byteRate); write16(blockAlign); write16(16); + // data chunk + writeStr('data'); write32(wavDataByteLength); + + // Interleave channels — the hot loop + for (var i = 0; i < numSamples; i++) { + for (var ch = 0; ch < numChannels; ch++) { + var s = channels[ch][i]; + if (s > 1) s = 1; + if (s < -1) s = -1; + view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true); + offset += 2; + } + } + + // Transfer the finished ArrayBuffer back (zero-copy) + self.postMessage({ buffer: buffer }, [buffer]); +}; +`; + +function _encodeWavInWorker({ channels, sampleRate, numSamples, numChannels }) { + return new Promise((resolve, reject) => { + const blob = new Blob([_WAV_WORKER_SRC], { type: 'application/javascript' }); + const url = URL.createObjectURL(blob); + const worker = new Worker(url); + URL.revokeObjectURL(url); // worker holds its own internal reference; safe to revoke + worker.onmessage = (e) => { resolve(e.data.buffer); worker.terminate(); }; + worker.onerror = (e) => { reject(new Error('WAV worker: ' + e.message)); worker.terminate(); }; + // Transfer the Float32Arrays into the worker — zero-copy, no serialisation overhead + worker.postMessage({ channels, sampleRate, numSamples, numChannels }, channels.map(ch => ch.buffer)); + }); +} + async function extractAudioWavFromMp4(file) { const arrayBuf = await file.arrayBuffer() const audioCtx = new (window.AudioContext || window.webkitAudioContext)() @@ -18,132 +76,144 @@ async function extractAudioWavFromMp4(file) { try { audioBuf = await audioCtx.decodeAudioData(arrayBuf.slice(0)) // Safari needs a copy } catch (e) { - // no audio, save us all precious time + // no audio track — bail out early audioCtx.close() return null } - const numChannels = audioBuf.numberOfChannels; - const sampleRate = audioBuf.sampleRate; - const numSamples = audioBuf.length; - - // Interleave PCM (16-bit) - const bytesPerSample = 2; - const blockAlign = numChannels * bytesPerSample; - const byteRate = sampleRate * blockAlign; - - const wavDataByteLength = numSamples * blockAlign; - const totalLen = 44 + wavDataByteLength; - const buffer = new ArrayBuffer(totalLen); - const view = new DataView(buffer); - - // RIFF header - let offset = 0; - const writeStr = (s) => { for (let i=0; i { view.setUint16(offset, v, true); offset += 2 } - const write32 = (v) => { view.setUint32(offset, v, true); offset += 4 } - - writeStr('RIFF'); - write32(36 + wavDataByteLength); - writeStr('WAVE'); - - // fmt chunk - writeStr('fmt ') - write32(16) - write16(1) - write16(numChannels) - write32(sampleRate) - write32(byteRate) - write16(blockAlign) - write16(16) - - // data chunk - writeStr('data') - write32(wavDataByteLength) + const numChannels = audioBuf.numberOfChannels + const sampleRate = audioBuf.sampleRate + const numSamples = audioBuf.length - // Interleave channels + // Copy channel data into fresh transferable Float32Arrays before closing the context const channels = [] - for (let ch = 0; ch < numChannels; ch++) channels.push(audioBuf.getChannelData(ch)) - - for (let i = 0; i < numSamples; i++) { - for (let ch = 0; ch < numChannels; ch++) { - let s = Math.max(-1, Math.min(1, channels[ch][i])) - view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true) - offset += 2 - } + for (let ch = 0; ch < numChannels; ch++) { + const src = audioBuf.getChannelData(ch) + const copy = new Float32Array(src.length) + copy.set(src) + channels.push(copy) } - audioCtx.close() - return new Blob([buffer], { type: 'audio/wav' }); + + // Hand off the heavy PCM interleave to the worker thread + const wavBuffer = await _encodeWavInWorker({ channels, sampleRate, numSamples, numChannels }) + return new Blob([wavBuffer], { type: 'audio/wav' }) } async function extractFramesFromMp4(file, { fps = DEFAULT_FPS, maxFrames = Infinity, onProgress = () => {} } = {}) { + const WORKER_COUNT = 4 const url = URL.createObjectURL(file) - const video = document.createElement('video') - video.src = url + + // --- load metadata from a throwaway video element --- // NOTE: do NOT set crossOrigin on blob URLs — blob URLs are same-origin by definition // else WebKitGTK (Linux/Tauri) treats the blob as a CORS fetch, which breaks seeking - video.muted = true - video.playsInline = true - - // Load da metadata stuff + const metaVideo = document.createElement('video') + metaVideo.src = url + metaVideo.muted = true + metaVideo.playsInline = true await new Promise((res, rej) => { const timer = setTimeout(() => rej(new Error('Video metadata load timed out — the file may use an unsupported codec')), 15000) - video.addEventListener('loadedmetadata', () => { clearTimeout(timer); res() }, { once: true }) - video.addEventListener('error', () => { clearTimeout(timer); rej(new Error('Video metadata load failed')) }, { once: true }) + metaVideo.addEventListener('loadedmetadata', () => { clearTimeout(timer); res() }, { once: true }) + metaVideo.addEventListener('error', () => { clearTimeout(timer); rej(new Error('Video metadata load failed')) }, { once: true }) }) + const duration = metaVideo.duration + const w = metaVideo.videoWidth + const h = metaVideo.videoHeight + metaVideo.src = '' // release the metadata decoder slot + + // --- build the full list of frame timestamps upfront --- + const step = 1 / fps + const allTimestamps = [] + for (let t = 0; t <= duration && allTimestamps.length < maxFrames; t += step) { + allTimestamps.push(t) + } + if (!allTimestamps.length) { + URL.revokeObjectURL(url) + throw new Error('No frames could be extracted — the video may use an unsupported codec on this platform') + } + + // --- split timestamps into WORKER_COUNT contiguous chunks --- + // Contiguous chunks = each worker seeks forward through its section, + // which is faster than random-access seeking across the whole timeline. + const numWorkers = Math.min(WORKER_COUNT, allTimestamps.length) + const chunkSize = Math.ceil(allTimestamps.length / numWorkers) + const chunks = [] + for (let i = 0; i < allTimestamps.length; i += chunkSize) { + chunks.push( + allTimestamps.slice(i, i + chunkSize).map((ts, j) => ({ ts, globalIndex: i + j })) + ) + } - const duration = video.duration - const w = video.videoWidth - const h = video.videoHeight - - // create hidden canvas to record frames from it - const canvas = document.createElement('canvas') - canvas.width = w - canvas.height = h - const ctx = canvas.getContext('2d') - - const step = 1 / fps // <-- skip video to recorded frames - const frames = [] - let t = 0 - let frameIndex = 0 - while (t <= duration && frameIndex < maxFrames) { - video.currentTime = t - // Wait for seek — timeout guards against WebKitGTK/GStreamer silently dropping seeked events -H.A. + // results[globalIndex] = blob — keeps frames in the correct order regardless of + // which worker finishes first. + const results = new Array(allTimestamps.length).fill(null) + let framesCompleted = 0 + + // Each chunk runs on its own video + canvas pair, seeking forward through its timestamps. + const extractChunk = async (chunk) => { + const video = document.createElement('video') + video.src = url + video.muted = true + video.playsInline = true + const canvas = document.createElement('canvas') + canvas.width = w + canvas.height = h + const ctx = canvas.getContext('2d') + + // Wait for this worker's video to be ready before seeking await new Promise((res, rej) => { - const timer = setTimeout(() => { cleanup(); rej(new Error('Seek timed out — the video may use an unsupported codec on this platform')) }, 10000) - const onSeeked = () => { clearTimeout(timer); res(); cleanup() } - const onError = () => { clearTimeout(timer); rej(new Error('Seek failed')); cleanup() } - const cleanup = () => { - video.removeEventListener('seeked', onSeeked) - video.removeEventListener('error', onError) - } - video.addEventListener('seeked', onSeeked, { once: true }) - video.addEventListener('error', onError, { once: true }) + const timer = setTimeout(() => rej(new Error('Video metadata load timed out — the file may use an unsupported codec')), 15000) + video.addEventListener('loadedmetadata', () => { clearTimeout(timer); res() }, { once: true }) + video.addEventListener('error', () => { clearTimeout(timer); rej(new Error('Video load failed')) }, { once: true }) }) - // Some browsers need a render tick - await sleep(0) + for (const { ts, globalIndex } of chunk) { + video.currentTime = ts + // Wait for seek — timeout guards against WebKitGTK/GStreamer silently dropping seeked events -H.A. + await new Promise((res, rej) => { + const timer = setTimeout(() => { cleanup(); rej(new Error('Seek timed out — the video may use an unsupported codec on this platform')) }, 10000) + const onSeeked = () => { clearTimeout(timer); res(); cleanup() } + const onError = () => { clearTimeout(timer); rej(new Error('Seek failed')); cleanup() } + const cleanup = () => { + video.removeEventListener('seeked', onSeeked) + video.removeEventListener('error', onError) + } + video.addEventListener('seeked', onSeeked, { once: true }) + video.addEventListener('error', onError, { once: true }) + }) + + // Some browsers need a render tick before drawImage reflects the new frame + await sleep(0) + + ctx.drawImage(video, 0, 0, w, h) + const blob = await new Promise(res => canvas.toBlob(res, 'image/jpeg', 0.9)) + results[globalIndex] = blob + framesCompleted++ + onProgress('Extracting frames…', Math.min(80, Math.round((framesCompleted / allTimestamps.length) * 80))) + } - ctx.drawImage(video, 0, 0, w, h) - const blob = await new Promise(res => canvas.toBlob(res, 'image/jpeg', 0.9)) - frames.push(blob) - frameIndex++ - t += step - onProgress(`Extracting frames…`, Math.min(80, Math.round((t / duration) * 80))) + video.src = '' // release this worker's decoder slot } + // Run all chunks concurrently — 4 seeks in flight at once + await Promise.all(chunks.map(extractChunk)) URL.revokeObjectURL(url) + + const frames = results.filter(Boolean) if (!frames.length) throw new Error('No frames could be extracted — the video may use an unsupported codec on this platform') return { frames, width: w, height: h, duration, fps } } async function mp4ToWickFileBlob({ mp4File, fps = DEFAULT_FPS, projectName = 'Imported Video', onProgress = () => {} }) { - onProgress('Decoding audio…', 5) - const audioBlob = await extractAudioWavFromMp4(mp4File).catch(() => null) + onProgress('Extracting…', 5) + + // Run audio decoding and frame extraction in parallel. + // decodeAudioData is async (browser audio subsystem) and won't block the seek loop. + // The WAV encoding itself runs in a dedicated worker thread (see _encodeWavInWorker). + const audioPromise = extractAudioWavFromMp4(mp4File).catch(() => null) + const framesPromise = extractFramesFromMp4(mp4File, { fps, onProgress }) - onProgress('Extracting frames…', 10) - const { frames, width, height } = await extractFramesFromMp4(mp4File, { fps, onProgress }) + const [audioBlob, { frames, width, height }] = await Promise.all([audioPromise, framesPromise]) if (!frames.length) throw new Error('No frames extracted')