Video importer into asset library + extra video import formats

package.json

@@ -51,7 +51,7 @@
     "build-macapp": "tauri build --target universal-apple-darwin --verbose",
     "build-winapp": "tauri build --target x86_64-pc-windows-gnu",
     "copy-pdf-worker": "shx cp node_modules/pdfjs-dist/build/pdf.worker.min.mjs public/pdf.worker.min.mjs",
-    "copy-ffmpeg-core": "shx mkdir -p public/corelibs/ffmpeg && shx cp node_modules/@ffmpeg/core/dist/umd/ffmpeg-core.js public/corelibs/ffmpeg/ffmpeg-core.js && shx cp node_modules/@ffmpeg/core/dist/umd/ffmpeg-core.wasm public/corelibs/ffmpeg/ffmpeg-core.wasm && shx cp node_modules/@ffmpeg/ffmpeg/dist/umd/ffmpeg.js public/corelibs/ffmpeg/ffmpeg.umd.js && shx cp node_modules/@ffmpeg/ffmpeg/dist/umd/814.ffmpeg.js public/corelibs/ffmpeg/814.ffmpeg.js",
+    "copy-ffmpeg-core": "shx mkdir -p public/corelibs/ffmpeg && shx cp node_modules/@ffmpeg/core/dist/umd/ffmpeg-core.js public/corelibs/ffmpeg/ffmpeg-core.js && shx cp node_modules/@ffmpeg/core/dist/esm/ffmpeg-core.wasm public/corelibs/ffmpeg/ffmpeg-core.wasm && shx cp node_modules/@ffmpeg/ffmpeg/dist/umd/ffmpeg.js public/corelibs/ffmpeg/ffmpeg.umd.js && shx cp node_modules/@ffmpeg/ffmpeg/dist/umd/814.ffmpeg.js public/corelibs/ffmpeg/814.ffmpeg.js",
     "eject": "react-scripts eject",
     "start": "npm run copy-pdf-worker && npm run copy-ffmpeg-core && react-scripts start",
     "build": "npm run copy-pdf-worker && npm run copy-ffmpeg-core && react-scripts build && echo 'candlestickers.app' > ./build/CNAME",

src/Editor/Editor.jsx

@@ -77,17 +77,28 @@ async function loadPathIntoEditor(editorThis, filePath) {
         const name = filePath.split('/').pop();
 
         // .WICK/ PROJECT FILE
+        const VIDEO_MIME = {
+            '.mp4': 'video/mp4', '.m4v': 'video/x-m4v',
+            '.mov': 'video/quicktime',
+            '.webm': 'video/webm',
+            '.ogv': 'video/ogg', '.ogg': 'video/ogg',
+            '.avi': 'video/x-msvideo',
+            '.mkv': 'video/x-matroska',
+            '.3gp': 'video/3gpp',
+            '.wmv': 'video/x-ms-wmv',
+        }
+        const videoExt = Object.keys(VIDEO_MIME).find(ext => name.endsWith(ext))
+
         if (
             name.endsWith('.wick') ||
-            name.endsWith('.mov') ||
-            name.endsWith('.mp4')
+            videoExt
         ) {
 
 
             const bytes = await readFile(filePath, { encoding: null })
             const blob = new Blob([bytes])
             const file = new File([blob], name, {
-                type: (name.endsWith('.wick') && 'application/zip') || (name.endsWith('.pdf') && 'application/pdf') || 'video/mp4'
+                type: (name.endsWith('.wick') && 'application/zip') || (videoExt && VIDEO_MIME[videoExt]) || 'video/mp4'
             });
 
 
@@ -248,13 +259,13 @@ class Editor extends EditorCore {
 
         // Wick Project File Input
         this.openProjectFileFromClient = window.createFileInput({
-            accept: '.zip, .wick, .mp4, .pdf',
+            accept: '.zip, .wick, video/*, .pdf',
             onChange: this.handleWickFileLoad,
         });
 
         // Wick file input
         this.openAssetFileFromClient = window.createFileInput({
-            accept: window.Wick.FileAsset.getValidExtensions().join(', '),
+            accept: window.Wick.FileAsset.getValidExtensions().join(', ') + ', video/*',
             onChange: this.handleAssetFileImport,
             multiple: true,
         });

src/Editor/EditorCore.jsx

@@ -1132,6 +1132,24 @@ class EditorCore extends Component {
                         if (options.create) this.createImageFromAsset(gifAsset.uuid, options.location.x || 0, options.location.y || 0);
                     }
                 });
+            } else if (acceptedFiles[i].type.startsWith('video/')) {
+                const mp4File = acceptedFiles[i];
+                const fps = Math.max(1, Math.min(60, Number(prompt('Enter FPS for ' + mp4File.name, String(this.project.framerate)) || this.project.framerate)));
+                const mp4ToastID = this.toast(`Importing ${mp4File.name}…`, 'info', { autoClose: false });
+                this.showWaitOverlay();
+                this.importMP4AsAsset({
+                    file: mp4File,
+                    fps,
+                    onProgress: (msg, p) => this.updateToast(mp4ToastID, { text: `${msg} (${Math.round(p || 0)}%)` }),
+                }).then(({ gifAsset }) => {
+                    this.updateToast(mp4ToastID, { text: `:) Imported ${mp4File.name}`, type: 'success', autoClose: 7000 });
+                    this.hideWaitOverlay();
+                    if (options.create) this.createImageFromAsset(gifAsset.uuid, options.location.x || 0, options.location.y || 0);
+                }).catch(err => {
+                    console.error(err);
+                    this.updateToast(mp4ToastID, { text: `Failed to import ${mp4File.name}`, type: 'error', autoClose: 5000 });
+                    this.hideWaitOverlay();
+                });
             } else {
                 var file = acceptedFiles[i];
 
@@ -1140,6 +1158,87 @@ class EditorCore extends Component {
         }
     }
 
+    /**
+     * Extracts frames from an MP4 file, builds a GIFAsset from them, and adds
+     * it (plus all intermediate image assets) to the current project's asset
+     * library. -H.A.
+     *
+     * @param {File}     file       - The MP4 file to import.
+     * @param {number}   fps        - Frame rate to extract at.
+     * @param {Function} onProgress - Progress callback (msg, percent).
+     * @returns {Promise<{ gifAsset, audioBlob, fps, projectName, width, height }>}
+     */
+    importMP4AsAsset = ({ file, fps = 12, onProgress = () => {}, bakeAudio = true }) => {
+        return new Promise(async (resolve, reject) => {
+            try {
+                const projectName = file.name.replace(/\.[^.]+$/, '');
+
+                const { imageAssets, audioBlob, fps: extractedFps, width, height } =
+                    await MP4ImportPure.importMP4AsSequence({
+                        mp4File: file,
+                        fps,
+                        projectName,
+                        onProgress,
+                    });
+
+                // Add individual frame images to the project first
+                imageAssets.forEach(asset => this.project.addAsset(asset));
+                await new Promise(res => this.project.loadAssets(res));
+
+                // Stitch the frame images into a single animated GIF asset
+                window.Wick.GIFAsset.fromImages(imageAssets, this.project, (gifAsset) => {
+                    gifAsset.name = projectName;
+                    gifAsset.filename = projectName;
+                    this.project.addAsset(gifAsset);
+
+                    if (!audioBlob || !bakeAudio) {
+                        // Project flow: return audioBlob to caller so it can set up root-timeline audio
+                        this.projectDidChange({ actionName: 'Imported MP4 as Asset' });
+                        resolve({ gifAsset, audioBlob: audioBlob || null, fps: extractedFps, projectName, width, height });
+                        return;
+                    }
+
+                    // Asset flow: import SoundAsset and bake it into the clip's second layer
+                    const audioFile = new File([audioBlob], projectName + '.wav', { type: 'audio/wav' });
+                    this.importFileAsAsset(audioFile, (soundAsset) => {
+                        if (!soundAsset) {
+                            this.projectDidChange({ actionName: 'Imported MP4 as Asset' });
+                            resolve({ gifAsset, audioBlob: null, fps: extractedFps, projectName, width, height });
+                            return;
+                        }
+
+                        // Instantiate the clip, add audio layer, re-serialize back into gifAsset.src
+                        gifAsset.createInstance((clip) => {
+                            const audioLayer = new window.Wick.Layer();
+                            audioLayer.name = 'audio';
+                            clip.timeline.addLayer(audioLayer);
+
+                            const frameEnd = clip.timeline.layers[0].length;
+                            const audioFrame = new window.Wick.Frame({ start: 1, end: frameEnd });
+                            audioLayer.addFrame(audioFrame);
+                            audioFrame.sound = soundAsset;
+
+                            // Temporarily attach clip to project so it can be serialized
+                            this.project.addObject(clip);
+                            window.Wick.WickObjectFile.toWickObjectFile(clip, 'blob', (blobFile) => {
+                                const reader = new FileReader();
+                                reader.onload = (e) => {
+                                    gifAsset.src = e.target.result;
+                                    clip.remove();
+                                    this.projectDidChange({ actionName: 'Imported MP4 as Asset' });
+                                    resolve({ gifAsset, audioBlob: null, fps: extractedFps, projectName, width, height });
+                                };
+                                reader.readAsDataURL(blobFile);
+                            });
+                        }, this.project);
+                    });
+                });
+            } catch (err) {
+                reject(err);
+            }
+        });
+    }
+
     /**
      * Begin interactive object creation process.
      */
@@ -2399,100 +2498,87 @@ class EditorCore extends Component {
         }
 
 
-        // if not an mp4 file just load it then
-        if (file.type !== 'video/mp4') {
+        // if not a video file just load it then
+        if (!file.type.startsWith('video/')) {
             this.importProjectAsWickFile(file);
             return;
         }
 
         // if an mp4 file then translate it before opening it
         try {
-            const toastID = this.toast(`Loading ${file.name}…`, 'info', { autoClose: false })
+            const fps = Math.max(1, Math.min(60, Number(prompt("Enter FPS", String(this.project.framerate)) || this.project.framerate)));
+            const toastID = this.toast(`Loading ${file.name}…`, 'info', { autoClose: false });
 
-            // NOTE need to setup new project BEFORE loading mp4 (otherwise will still work but console will be loaded with error messages)
+            // NOTE: must set up the new project BEFORE importing so assets have
+            // somewhere to live (avoids stale-project console errors).
             this.setupNewProject();
             this.projectDidChange({ actionName: 'Reset project' });
+            this.showWaitOverlay();
+
+            // Extract frames + build GIF asset → adds everything to this.project
+            // bakeAudio: false so we handle audio on the root timeline instead
+            const { gifAsset, audioBlob, fps: extractedFps, projectName, width, height } =
+                await this.importMP4AsAsset({
+                    file,
+                    fps,
+                    bakeAudio: false,
+                    onProgress: (msg, p) => this.updateToast(toastID, { text: `${msg} (${Math.round(p || 0)}%)` }),
+                });
 
-            this.showWaitOverlay() // disable clicking anywhere
-
-            // GET ALL FRAMES FROM MP4 (run through MP4 import function)
-            const { imageAssets, audioBlob, fps, projectName, width, height } =
-                await MP4ImportPure.importMP4AsSequence({
-                    mp4File: file,
-                    fps: Math.max(1, Math.min(60, Number(prompt("Enter FPS", "12") || 6))), // give user the option of FPS
-                    projectName: file.name.replace(".mp4", ''),
-                    onProgress: (msg, p) => this.updateToast(toastID, { text: `${msg} (${Math.round(p || 0)}%)` })
-                })
-
-            // set up new project
+            // Fit project to the video
             this.project.width = width;
             this.project.height = height;
-            this.project.framerate = fps;
+            this.project.framerate = extractedFps;
             this.project.name = projectName;
             this.projectDidChange({ actionName: 'Adjusted settings based on MP4' });
 
-            // add image assets to the current project
-            imageAssets.forEach(asset => this.project.addAsset(asset));
-
-            // turn the sequence into an asset using the same system used by the GIF import ;-;
-            await new Promise(res => this.project.loadAssets(res))
-            await new Promise((resolve) => {
-                window.Wick.GIFAsset.fromImages(imageAssets, this.project, (gifAsset) => {
-                    gifAsset.name = projectName;
-                    gifAsset.filename = projectName;
-                    this.project.addAsset(gifAsset);
-                    // extend frame
-                    this.project._children[1].activeFrame.end = this.project._children.length - 3;
-
-
-                    // add the video asset into the project
-                    this.project.createClipInstanceFromAsset(gifAsset, this.project.width / 2, this.project.height / 2, (clip) => {
-                        this.selectObject(clip) // <-- select the object so we can adjust its settings
-
-                        // set to synce and enable play once
-                        this.setSelectionAttribute('animationType', 'playOnce')
-                        this.setSelectionAttribute('isSynced', true)
-
-                        // unselect the mp4 clip guy
-                        this.clearSelection();
-
-                        const tl = this.project.activeTimeline;
-                        tl.layers[0].name = "video";
-
-                        // add in the audio file as well (if video has audio)
-                        if (audioBlob) {
-                            tl.addLayer(new window.Wick.Layer());
-                            tl.layers[1].name = "audio";
-                            tl.layers[1].addFrame(new window.Wick.Frame());
-                            tl.layers[1].frames[0].end = tl.layers[0].frames[0].end;
-                            // const soundObj = new window.Wick.Sound({ asset: this.project.assets[1]});
-
-                            const audioFile = new File([audioBlob], projectName + ".wav", { type: 'audio/wav' })
-                            this.importFileAsAsset(audioFile, () => {
-                                this.project.loadAssets(() => {
-                                    // wait for new audio asset to load in
-                                    this.setActiveLayerIndex(1);
-                                    this.addSoundToActiveFrame(this.project.assets[this.project.assets.length - 1])
-                                    this.setActiveLayerIndex(0);
-                                })
-                            })  // reuses existing audio import path
-                        }
-
-                        this.projectDidChange({ actionName: 'Opened MP4 as GIF sequence' })
-                        this.updateToast(toastID, { text: `:) Imported ${file.name}`, type: 'success', autoClose: 7000 })
-                        this.hideWaitOverlay()
-                        resolve()
-                    })
-
-                    // this.project._children[1].activeFrame._children[0]._isSynced = true;
+            // Extend the root frame and place the clip on the canvas
+            this.project._children[1].activeFrame.end = this.project._children.length - 3;
+
+            this.project.createClipInstanceFromAsset(gifAsset, this.project.width / 2, this.project.height / 2, (clip) => {
+                this.selectObject(clip);
+                this.setSelectionAttribute('animationType', 'playOnce');
+                this.setSelectionAttribute('isSynced', true);
+                this.clearSelection();
+
+                const tl = this.project.activeTimeline;
+                tl.layers[0].name = "video";
+
+                // Add an empty drawing layer and move it to the top
+                const drawingLayer = new window.Wick.Layer();
+                drawingLayer.name = 'Layer';
+                tl.addLayer(drawingLayer);
+                drawingLayer.addFrame(new window.Wick.Frame({ start: 1, end: 1 }));
+                tl.moveLayer(drawingLayer, 0);
+                // Layer order: drawing(0), video(1), then audio below
+
+                // Add the audio track if the video had audio
+                if (audioBlob) {
+                    tl.addLayer(new window.Wick.Layer());
+                    // Layer order: drawing(0), video(1), audio(2)
+                    tl.layers[2].name = "audio";
+                    tl.layers[2].addFrame(new window.Wick.Frame());
+                    tl.layers[2].frames[0].end = tl.layers[1].frames[0].end;
+
+                    const audioFile = new File([audioBlob], projectName + ".wav", { type: 'audio/wav' });
+                    this.importFileAsAsset(audioFile, () => {
+                        this.project.loadAssets(() => {
+                            this.setActiveLayerIndex(2);
+                            this.addSoundToActiveFrame(this.project.assets[this.project.assets.length - 1]);
+                            this.setActiveLayerIndex(0);
+                        });
+                    });
+                }
 
-                })
-            })
+                this.projectDidChange({ actionName: 'Opened MP4 as GIF sequence' });
+                this.updateToast(toastID, { text: `:) Imported ${file.name}`, type: 'success', autoClose: 7000 });
+                this.hideWaitOverlay();
+            });
 
         } catch (err) {
-            console.error(err)
-            this.toast('Failed to import MP4.', 'error')
-            this.hideWaitOverlay()
+            console.error(err);
+            this.toast('Failed to import MP4.', 'error');
+            this.hideWaitOverlay();
         }
 
     }

src/Editor/Panels/Canvas/Canvas.jsx

@@ -111,7 +111,7 @@ function CanvasDrop(props) {
         if (name.endsWith('.wick')) {
           // Wick Project (.wick file)
           p.importProjectAsWickFile(file);
-        } else if (file.type === 'video/mp4' || name.endsWith('.mp4') ||
+        } else if (file.type.startsWith('video/') || /\.(mp4|mov|webm|mkv|avi|3gp|ogv|m4v|wmv)$/i.test(name) ||
                    file.type === 'application/pdf' || name.endsWith('.pdf')) {
           // MP4/PDF → open as new project
           p.openProjectFile(file);