From 2cdabc663d53e4990c8c85ea4eb7e2587db9fdcb Mon Sep 17 00:00:00 2001 From: VINAYAK KHEMKA <216440511+Radioactive012@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:29:17 +0530 Subject: [PATCH 01/12] feat(recorder): add floating macOS self-view --- electron/electron-env.d.ts | 7 + electron/floatingSelfView.ts | 36 ++++ electron/ipc/handlers.ts | 8 + electron/preload.ts | 5 + src/components/launch/HudControls.tsx | 30 ++- src/components/launch/HudDeviceSettings.tsx | 24 +++ src/components/launch/LaunchWindow.module.css | 42 ++++ src/components/launch/LaunchWindow.tsx | 56 ++++++ src/hooks/useFloatingSelfView.ts | 180 ++++++++++++++++++ src/hooks/useScreenRecorder.ts | 20 +- src/i18n/locales/ar/launch.json | 7 + src/i18n/locales/en/launch.json | 7 + src/i18n/locales/es/launch.json | 7 + src/i18n/locales/fr/launch.json | 7 + src/i18n/locales/it/launch.json | 7 + src/i18n/locales/ja-JP/launch.json | 7 + src/i18n/locales/ko-KR/launch.json | 7 + src/i18n/locales/pt-BR/launch.json | 7 + src/i18n/locales/ru/launch.json | 7 + src/i18n/locales/tr/launch.json | 7 + src/i18n/locales/vi/launch.json | 7 + src/i18n/locales/zh-CN/launch.json | 7 + src/i18n/locales/zh-TW/launch.json | 7 + src/lib/userPreferences.ts | 7 + 24 files changed, 502 insertions(+), 4 deletions(-) create mode 100644 electron/floatingSelfView.ts create mode 100644 src/hooks/useFloatingSelfView.ts diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 4eb288e14..c5c4f7b3f 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -23,6 +23,9 @@ declare namespace NodeJS { // Used in Renderer process, expose in `preload.ts` interface Window { + __openscreenRequestFloatingSelfView?: () => Promise< + import("../src/hooks/useFloatingSelfView").FloatingSelfViewRequestResult + >; electronAPI: { invokeNativeBridge: ( request: import("../src/native/contracts").NativeBridgeRequest, @@ -84,6 +87,10 @@ interface Window { status: string; error?: string; }>; + requestFloatingSelfViewAutoOpen: () => Promise<{ + success: boolean; + error?: string; + }>; assetBaseUrl: string; storeRecordedVideo: ( videoData: ArrayBuffer, diff --git a/electron/floatingSelfView.ts b/electron/floatingSelfView.ts new file mode 100644 index 000000000..036a48f10 --- /dev/null +++ b/electron/floatingSelfView.ts @@ -0,0 +1,36 @@ +import type { BrowserWindow, WebContents } from "electron"; + +export interface AutoFloatingSelfViewResult { + success: boolean; + error?: "unauthorized-sender" | "hud-unavailable" | "request-failed"; +} + +const AUTO_PIP_RENDERER_CALL = "window.__openscreenRequestFloatingSelfView?.()"; + +/** Execute only a fixed renderer entry point, and only in the current HUD. */ +export async function requestAutoFloatingSelfView( + sender: WebContents, + currentHud: BrowserWindow | null, +): Promise { + if (!currentHud || currentHud.isDestroyed() || currentHud.webContents.isDestroyed()) { + return { success: false, error: "hud-unavailable" }; + } + if (sender !== currentHud.webContents) { + return { success: false, error: "unauthorized-sender" }; + } + + try { + const result = (await sender.executeJavaScript(AUTO_PIP_RENDERER_CALL, true)) as unknown; + if ( + typeof result === "object" && + result !== null && + "success" in result && + typeof result.success === "boolean" + ) { + return result.success ? { success: true } : { success: false, error: "request-failed" }; + } + return { success: false, error: "request-failed" }; + } catch { + return { success: false, error: "request-failed" }; + } +} diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 958ef6d96..bd41f2725 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -53,6 +53,7 @@ import type { CursorTelemetryReader } from "../ai-edition/deep-agent/service"; import { DocumentService } from "../ai-edition/document-service"; import { LlmConfigStore } from "../ai-edition/llm-config-store"; import { isDiagnosticModeEnabled, mainLogBuffer } from "../diagnostics/main-log-buffer"; +import { requestAutoFloatingSelfView } from "../floatingSelfView"; import { mainT } from "../i18n"; import { getInstallChannel } from "../install-channel"; import { RECORDINGS_DIR } from "../main"; @@ -1736,6 +1737,13 @@ export function registerIpcHandlers( onRecordingStateChange?: (recording: boolean, sourceName: string) => void, _switchToHud?: () => void, ) { + ipcMain.handle("request-floating-self-view-auto-open", async (event) => { + if (process.platform !== "darwin") { + return { success: false, error: "unsupported-platform" }; + } + return requestAutoFloatingSelfView(event.sender, getMainWindow()); + }); + async function requestScreenAccess() { if (process.platform !== "darwin") { return { success: true, granted: true, status: "granted" }; diff --git a/electron/preload.ts b/electron/preload.ts index 6aff16407..e19cf33e7 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -156,6 +156,11 @@ contextBridge.exposeInMainWorld("electronAPI", { requestNativeMacCursorAccess: () => { return ipcRenderer.invoke("request-native-mac-cursor-access"); }, + requestFloatingSelfViewAutoOpen: () => + ipcRenderer.invoke("request-floating-self-view-auto-open") as Promise<{ + success: boolean; + error?: string; + }>, storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => { return ipcRenderer.invoke("store-recorded-video", videoData, fileName); }, diff --git a/src/components/launch/HudControls.tsx b/src/components/launch/HudControls.tsx index 360499c79..5044c693e 100644 --- a/src/components/launch/HudControls.tsx +++ b/src/components/launch/HudControls.tsx @@ -1,4 +1,4 @@ -import { Check, Languages, NotepadText, Settings } from "lucide-react"; +import { Check, Languages, NotepadText, PictureInPicture2, Settings } from "lucide-react"; import { memo } from "react"; import { formatTimePadded } from "../../utils/timeUtils"; import { Button } from "../ui/button"; @@ -435,6 +435,34 @@ export const HudRecordingControls = memo(function HudRecordingControls({ ); }); +export const HudSelfViewButton = memo(function HudSelfViewButton({ + open, + disabled, + label, + onClick, +}: { + open: boolean; + disabled: boolean; + label: string; + onClick: () => void; +}) { + return ( + + + + ); +}); + export const HudLanguageButton = memo(function HudLanguageButton({ vertical, code, diff --git a/src/components/launch/HudDeviceSettings.tsx b/src/components/launch/HudDeviceSettings.tsx index 4b40206d7..126252975 100644 --- a/src/components/launch/HudDeviceSettings.tsx +++ b/src/components/launch/HudDeviceSettings.tsx @@ -22,6 +22,8 @@ export interface HudDeviceSettingsLabels { cameraUnavailable: string; preview: string; previewUnavailable: string; + floatingSelfView: string; + floatingSelfViewHint: string; about: string; checkForUpdates: string; checkingForUpdates: string; @@ -95,9 +97,12 @@ export const HudDeviceSettings = memo(function HudDeviceSettings({ versionLabel, canCheckForUpdates, checkingForUpdates, + showFloatingSelfViewSetting, + floatingSelfViewEnabled, onSelectMic, onSelectCamera, onCheckForUpdates, + onFloatingSelfViewEnabledChange, onClose, panelRef, }: { @@ -113,9 +118,12 @@ export const HudDeviceSettings = memo(function HudDeviceSettings({ versionLabel: string | null; canCheckForUpdates: boolean; checkingForUpdates: boolean; + showFloatingSelfViewSetting: boolean; + floatingSelfViewEnabled: boolean; onSelectMic: (device: MicrophoneDevice) => void; onSelectCamera: (device: CameraDevice) => void; onCheckForUpdates: () => void; + onFloatingSelfViewEnabledChange: (enabled: boolean) => void; onClose: () => void; panelRef: (el: HTMLDivElement | null) => void; }) { @@ -218,6 +226,22 @@ export const HudDeviceSettings = memo(function HudDeviceSettings({ ) : null} + {showFloatingSelfViewSetting ? ( + + ) : null} + {/* The HUD has no other settings surface, and an app the user cannot ask "which version am I running?" is an app whose bug reports arrive without one. The update button is absent — not disabled — where a package manager owns the diff --git a/src/components/launch/LaunchWindow.module.css b/src/components/launch/LaunchWindow.module.css index 8821194b1..17760b739 100644 --- a/src/components/launch/LaunchWindow.module.css +++ b/src/components/launch/LaunchWindow.module.css @@ -133,6 +133,48 @@ margin: 1px 0; } +.hudPreferenceRow { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + margin: 8px 6px 4px; + padding: 10px; + border-radius: 10px; + background: rgba(255, 255, 255, 0.035); + cursor: pointer; +} + +.hudPreferenceLabel, +.hudPreferenceHint { + display: block; +} + +.hudPreferenceLabel { + font-size: 12px; + font-weight: 600; + color: #e9edf3; +} + +.hudPreferenceHint { + margin-top: 2px; + font-size: 10px; + line-height: 1.35; + color: #828c99; +} + +.hudPreferenceCheckbox { + width: 16px; + height: 16px; + flex: 0 0 auto; + accent-color: #10b981; +} + +.hudPreferenceCheckbox:focus-visible { + outline: 2px solid #10b981; + outline-offset: 3px; +} + /* Device settings. Same shell as the popovers, and deliberately the same width as the notice column (HUD_STACK_WIDTH) so opening it only ever changes the overlay window's height, never its width. */ diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index 8a2427fc1..eeec79b60 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -1,9 +1,11 @@ import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; +import { toast } from "sonner"; import { useI18n, useScopedT } from "@/contexts/I18nContext"; import { getAvailableLocales, getLocaleName } from "@/i18n/loader"; import { loadUserPreferences, saveUserPreferences } from "@/lib/userPreferences"; import { nativeBridgeClient } from "@/native"; import { type CameraDevice, useCameraDevices } from "../../hooks/useCameraDevices"; +import { useFloatingSelfView } from "../../hooks/useFloatingSelfView"; import { type MicrophoneDevice, useMicrophoneDevices } from "../../hooks/useMicrophoneDevices"; import { usePortalOwnsSource } from "../../hooks/usePortalOwnsSource"; import { useScreenRecorder } from "../../hooks/useScreenRecorder"; @@ -20,6 +22,7 @@ import { HudNotice, HudRecordButton, HudRecordingControls, + HudSelfViewButton, HudSettingsButton, HudSourceButton, HudStudioButton, @@ -100,6 +103,7 @@ export function LaunchWindow() { setSystemAudioEnabled, webcamEnabled, setWebcamEnabled, + webcamPreviewStream, webcamDeviceId, setWebcamDeviceId, setWebcamDeviceName, @@ -122,6 +126,21 @@ export function LaunchWindow() { ); const [supportsCursorModeToggle, setSupportsCursorModeToggle] = useState(false); const [isLinuxHud, setIsLinuxHud] = useState(false); + const isMacHud = window.electronAPI?.getPlatform?.() === "darwin"; + const [floatingSelfViewEnabled, setFloatingSelfViewEnabled] = useState( + () => loadUserPreferences().floatingSelfViewEnabled, + ); + const handleSelfViewUnavailable = useCallback(() => { + toast.error(t("selfView.unavailable")); + }, [t]); + const floatingSelfView = useFloatingSelfView({ + recording, + webcamEnabled, + stream: webcamPreviewStream, + autoShowEnabled: floatingSelfViewEnabled, + isMac: isMacHud, + onUnavailable: handleSelfViewUnavailable, + }); // The running version, and whether this copy may offer an update check at all — a // Store/Flathub/Snap/Nix install is kept current by its package manager and is offered // nothing (electron/install-channel.ts). Asked once: neither answer changes while the app @@ -690,6 +709,11 @@ export function LaunchWindow() { }); }, [closePopovers]); + const handleFloatingSelfViewPreference = useCallback((enabled: boolean) => { + setFloatingSelfViewEnabled(enabled); + saveUserPreferences({ floatingSelfViewEnabled: enabled }); + }, []); + const toggleSystemAudio = useCallback(() => { if (controlsLocked) return; setSystemAudioEnabled(!systemAudioEnabled); @@ -897,6 +921,8 @@ export function LaunchWindow() { cameraUnavailable: t("webcam.unavailable"), preview: t("deviceSettings.preview"), previewUnavailable: t("deviceSettings.previewUnavailable"), + floatingSelfView: t("selfView.autoShow"), + floatingSelfViewHint: t("selfView.autoShowHint"), about: t("deviceSettings.about"), checkForUpdates: tCommon("actions.checkForUpdates"), checkingForUpdates: t("deviceSettings.checkingForUpdates"), @@ -923,6 +949,15 @@ export function LaunchWindow() { onPointerMove={handleRootPointerMove} onPointerLeave={handlePointerLeave} > + {/* The recorder owns this stream. PiP borrows it without ever stopping tracks. */} +