diff --git a/benchmark/.gitignore b/benchmark/.gitignore new file mode 100644 index 00000000..28d25d20 --- /dev/null +++ b/benchmark/.gitignore @@ -0,0 +1,4 @@ +.cache/ +results/*/ +results/install.json +results/preflight.json diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000..16786de1 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,303 @@ +# Export benchmark + +How long does it take to turn a 60-second screen recording into a finished 1080p60 MP4, in +OpenScreen and in the apps it competes with — measured the same way, on the same clip, with the +same edit applied, and verified frame by frame. + +This directory is the whole apparatus: it generates the source clip, installs the competitors, +translates one scenario into each app's own controls, drives the export, times it, checks that +what came out is what was asked for, and writes the report. It is meant to be started once and +left alone. + +```bash +node benchmark/bench.mjs doctor # is this machine fit to measure? +node benchmark/bench.mjs preflight # the one interactive gate — grant everything here +node benchmark/bench.mjs install # unattended +node benchmark/bench.mjs calibrate # once per machine +node benchmark/bench.mjs run # walk away +node benchmark/bench.mjs report +``` + +Driving it from a phone or another machine: [REMOTE.md](./REMOTE.md). + +--- + +## What is being measured + +**The clock starts** the instant the export is committed — the click on *Export*, or a CLI's +first progress event — and **stops when the last byte lands in the output file**. Launching the +app, loading the project and setting presets happen before the clock starts, for every app +alike, and are reported separately as `prepareMs` and `launchToCommitMs`. + +Two apps in this set (Camtasia, Kap) publish their own completion signal. Where one exists the +harness takes whichever is *earlier* — the app's or the filesystem's — so an app can shorten its +own measurement but never lengthen it. + +**Every output is then checked twice.** First against the target's metadata: resolution, frame +rate, codec, duration. Then against its own pixels, because metadata cannot tell you whether the +app actually did the work: + +| Check | How | Why it exists | +|---|---|---| +| Background applied | the frame's corners must be light where the recording is dark | an app that skipped the wallpaper composites far less | +| Padding | bounding box of the dark recording against the light wallpaper | apps' padding controls are on different scales; this measures the real inset | +| Corner radius | the box's corner shows wallpaper while its top edge shows content | separates a rounded rect from a plain one | +| Zooms | frame-to-frame activity must spike inside every zoom window | an ignored zoom list is invisible in metadata | +| Rendered cursor | motion energy at the telemetry's position, against controls on the same scrolling material | an app can accept a cursor track and draw nothing — Cap does | +| Webcam inset | skin-tone fraction in the expected corner | nothing else in the composition is near that colour | +| Motion blur | *not asserted* | every threshold tried passed some correct renders and failed others; reported as configured, never as verified | + +**The verifier overrides the driver.** A driver reports what it configured; only the pixels say +what happened. Cap accepts a cursor track, reports `cursor.hide: false`, and renders no pointer +at all — that counted as full fidelity until the check existed, and is now `0.9` with `cursor` +listed as contradicted. + +Both new detectors were wrong on their first version, and were caught the same way: by running +them against Kap and the ffmpeg floor, which draw neither a cursor nor a camera. Both "passed". +The cursor check had been comparing the pointer's window against the frame's static corners — +really asking "is this region busier than the edges" — and the webcam threshold sat below the +fixture's own warm syntax colours. Thresholds are now calibrated against measured positives and +negatives, and every raw ratio is recorded per run so the margin is auditable rather than +implied. + +A run that fails verification is recorded as a failure, never as a fast time. + +**Fidelity** records how much of the scenario each app could express. A row marked `partial` did +less work; its number is a reference, not a ranking. Kap, which has no background, padding, +corner-radius, shadow or zoom features at all, is always partial — it is in the set as a +real-app floor, not as a peer. + +## The source clip + +Not shipped — **generated**, from a spec plus a seed, so that two machines can prove they +measured the same workload by comparing one hash: + +``` +1920×1080, 60 fps, 60 s, 3600 frames, H.264 High + AAC 48 kHz +sha256 recorded in every results file +``` + +It is built to look like a screen recording rather than a test pattern, because that is what +changes an encoder's job: a dark editor with syntax-coloured "code", a scrolling viewport, a +blinking caret, a selection band and a moving cursor — large static regions with sharp edges and +localized motion. `benchmark/lib/fixture.mjs` composes it from ffmpeg primitives; nothing is +random at run time. + +Why 60 fps and not 30: OpenScreen's MP4 export path is fixed at 60 (`MP4_EXPORT_FPS`, +`src/cli/CliExportRunner.tsx`), and every other app in the set can be told to emit 60. It is the +only frame rate on which "force identical output" is actually achievable. + +## The scenario + +One definition, in `benchmark/scenarios/index.mjs`, translated by each driver into its own app's +vocabulary: + +- **wallpaper** background — an image the compositor samples per pixel, not a colour it clears once +- padding: 5 % of the frame's short side, corner radius 40 px, drop shadow +- three zooms — 6–12 s at 1.8×, 22–29 s at 2.2×, 41–48 s at 1.6× +- **motion blur** on the composited frame +- **a rendered cursor** — themed sprite, smoothing, its own motion blur, click effects +- **a webcam inset** — bottom-right, rounded, shadowed, at 25 % of the frame +- output: 1920×1080, 60 fps, H.264, MP4 + +The first version of this scenario had only the first two lines, and that was a mistake worth +recording: a screen clip on a flat colour measures decoding and encoding, not what a demo export +costs. The cursor and the camera are a large share of the work, and neither was running. + +**The cursor is data, not pixels.** Every app here hides the system pointer while recording and +re-draws it at export time from a telemetry sidecar. The fixture used to paint a fake cursor +into the video, which exercised none of that and would have double-drawn the moment an app +rendered its own. The trajectory is now generated — eased glides between dwell points with +clicks at the pauses, the shape smoothing and dwell-based auto-zoom actually react to — written +in each app's own format, and the screen clip is left clean. + +The wallpaper and the webcam come from the same seed as the screen recording, so the whole +bundle reproduces on another machine and can be checked by hash. + +### Translating the scenario — and why calibration exists + +No two of these apps put their padding control on the same scale. Asked for "5", Cap produced a +1.85 % inset and OpenScreen a 10 % one — a 44 % difference in how many source pixels each was +sampling per frame. That is a confound, not a result. + +`bench.mjs calibrate` fixes it: for each app it renders a short clip at two padding values, +measures the inset from the output pixels, solves for the value that hits the scenario's target, +and writes the answer to `benchmark/calibration.json`. On this machine: + +| App | control value | measured inset | content box | +|---|---|---|---| +| OpenScreen | `padding: 25` | 5.00 % | 1728×972 | +| Cap | `padding: 13.56` | 4.81 % | 1734×976 | + +Run it once per machine, and again after any app updates. `run` reads the file automatically; +without it, each driver falls back to its documented default and the report shows the inset it +actually achieved. + +## Automating apps that have no CLI + +Only two apps in this set can be scripted the ordinary way. What the others expose was +established by inspection, not assumption: + +| Driver | CLI | AppleScript dictionary | Accessibility tree | Screenshotable | Driven by | +|---|---|---|---|---|---| +| `openscreen-cli` | **yes** (`openscreen export`) | no | — | — | `cli` | +| `openscreen-gui` | — | no | **no** (Electron) | yes | `cdp+menu` | +| `cap` | **yes** (`cap-cli export`) | no | — | — | `cli` | +| `camtasia` | no | **yes** (import, `isExporting`) | yes | yes | `applescript+ax` | +| `kap` | no | no | **no** (empty window) | yes | `cdp` | +| `screen-studio` | no | no | **no** | **no** — see below | `cdp+menu` | +| `focusee` | no | no | yes | yes | `ax+menu` | + +OpenScreen appears twice on purpose. The CLI leg measures the render engine with no interface in +the way — the right number to set beside Cap's CLI, and the wrong one to set beside an app that +can only be clicked. The GUI leg carries the editor's own overhead, because the subject of a +benchmark should not be the only entrant excused from it. + +The ladder each GUI driver climbs, best rung first: a scripting dictionary → a System Events +menu item by name → a documented keyboard shortcut → an accessibility control by name → the +renderer's own DOM over CDP → pixel coordinates. Every driver records which rung it used, in the +`automation` column of the report, because that is what tells a reader how well a given row will +reproduce on somebody else's machine. Pixel coordinates are the only rung that does not survive a +different display, and no driver here needs them. + +**Screen Studio cannot be screenshotted at all.** It marks its editor window +`kCGWindowSharingNone`, so macOS excludes it from every capture API — the window is plainly +visible to the person sitting there and invisible to `screencapture`, ScreenCaptureKit and any +agent driving pixels. It publishes no accessibility tree either. Launching it with +`--remote-debugging-port` is what makes it drivable, and that is a *more* reproducible +interaction than clicking pixels: elements are found by their visible text, which survives a +moved window, a different display and a resized UI. The flag opens an inspector and nothing +else; the renderer and the export pipeline are the shipping ones. + +`node benchmark/bench.mjs discover ` dumps an installed app's menus and accessibility tree. +That is how a driver gets written, and how it gets repaired when a new version renames something. + +## What is in the set, and what it costs to get + +| App | Licence for exporting | Install | +|---|---|---| +| OpenScreen | MIT, free, no watermark | GitHub release | +| Cap | AGPL-3.0, free; sign-in not needed for a local export | direct DMG | +| Camtasia | 30-day trial, watermarked output | direct DMG | +| Kap | MIT, free | GitHub release | +| Screen Studio | **licence required to export at all** — no trial export | direct DMG | +| FocuSee | trial, watermarked | vendor downloader stub | + +`screen-studio` and `focusee` are **off by default** — on a machine without a Screen Studio +licence, and against FocuSee 2.4.1, they can only ever record a failure. Enable either +explicitly with `--apps`. + +A watermark does not change render time, so a trial build is a valid measurement. A licence +*wall* is not — see [Known blockers](#known-blockers). + +## Reproducing on another machine + +1. `node benchmark/bench.mjs doctor` — refuses to proceed quietly on battery, in Low Power Mode, + under thermal throttling, or with less than 20 GiB free. All four move export times. +2. `node benchmark/bench.mjs preflight --launch` — prints the whole download list with sizes and + licence terms, provokes every macOS permission prompt the run would otherwise hit mid-flight, + and opens each GUI app once so its first-launch dialogs can be cleared. **This is the only + step that needs a human.** +3. `node benchmark/bench.mjs install` +4. `node benchmark/bench.mjs calibrate` +5. `node benchmark/bench.mjs run --reps 3` + +Comparing two machines: the results file carries the source clip's sha256, every app's version, +the calibration used, the machine's chip/cores/RAM/OS build, and the power and thermal state at +each repetition. Two runs are comparable when the fixture hashes and the app versions match. + +### ffmpeg + +Used to build the fixture and to verify outputs — it is measuring instrumentation, not part of +any app's export path. Resolution order: `OSBENCH_FFMPEG`/`OSBENCH_FFPROBE`, then `ffmpeg` on +`PATH`, then the repo's LGPL tree under `crates/thirdparty/ffmpeg-*`. That tree is gitignored, so +it exists only in the checkout that built it; the harness finds it through +`git rev-parse --git-common-dir` and wraps it in a small script that re-exports +`DYLD_LIBRARY_PATH` inside its own process, because macOS strips `DYLD_*` across any +SIP-protected exec and the inherited variable never survives. + +The LGPL build has no libx264 and no drawtext. The fixture is encoded with +`h264_videotoolbox` and drawn with `drawbox`; neither `-crf` nor text overlays are available. + +### Background load, and the one that matters most + +The precondition that never announces itself. Nothing throttles and nothing warns — every export +is simply slower. `doctor` refuses to call a machine ready above 60% foreign CPU, and the figure +is sampled during every export and reported per row as **Bg load**. + +**Do not run this over a remote-desktop session.** That is the single largest source of error +found while building this, and it is not a CPU problem. Parsec, Screen Sharing and ARD all +encode the screen continuously through `VTEncoderXPCService` — *the same hardware H.264 encoder +every app in this benchmark uses for its export*. The contention is for the media engine, which +no CPU measurement sees: + +| | quiet machine | with a remote session live | +|---|---|---| +| ffmpeg floor | 17.7 s | 23.7 s (+34%) | +| Cap | 19.6 s | 43.8 s (+123%) | + +Same machine, same clip, same settings, same padding — the padding calibration and the +background colour were both ruled out by A/B (42.6 s vs 42.5 s, and 42.4 s with the original +colour). Apps are affected unequally because they lean on the encoder differently, so the load +does not cancel out and the *ranking* can move, not just the absolute times. + +Within a single run the numbers are still sound, and the **closing control** is what proves it: +the floor workload is measured again after all the apps, and the report prints the ratio. In the +committed run it came back at 23.69 s against an opening 23.68 s — no drift, so every app in +that run met identical conditions. Comparing across runs on different machines requires the same +to be true of both. + +### Repetitions and guards### Repetitions and guards + +Three scoring runs after one discarded warm-up, 45 s of cooldown between them. The warm-up is +kept in the data but excluded from the statistics — first runs pay for cold caches and +uncompiled shaders. The headline figure is the **median** with a median absolute deviation; +with n=3 a standard deviation is mostly noise. Preconditions are re-checked before every +repetition and recorded per run, so a throttled run is visible rather than averaged in. + +## Reading the report + +`results//` holds `results.json` (everything), `report.md`, `report.html`, +`events.ndjson` (append-only) and `status.json` (atomically rewritten, safe to poll). + +- **Export (median)** — commit → last byte. +- **×realtime** — output duration ÷ export time. Above 1 is faster than playback. +- **vs floor** — multiples of `ffmpeg (re-encode floor)`, a plain transcode with no compositing. + It separates "this encoder is slow on this machine" from "this app's pipeline is slow". +- **Fidelity** — `full`, or `partial` with the missing features named. +- **Driven by** — the automation rung. + +## Known blockers + +Recorded here rather than quietly dropped, because "not measured" and "slow" are very different +findings. + +- **Screen Studio 3.7.5** gates export behind account activation. There is no trial export and no + watermark path — clicking *Export* opens an activation wall. The driver is complete and works; + supply a licence, activate once during preflight, and `--apps screen-studio` produces a number. +- **FocuSee 2.4.1** (direct download, macOS 26.5) rejects every MP4 it is given — including a + real 2560×1440 H.264 recording — with *"The source file is damaged and cannot be opened."* It is + not sandboxed, so this is not a file-access grant. Both its own import panel and `open -a` fail. + Its driver is written against the AX tree it does expose; it will start working if a later + build fixes the import. + +## Layout + +``` +bench.mjs entrypoint +apps.mjs registry: what is in the set, where it comes from, what it costs +scenarios/index.mjs the scenario and the pinned output target +lib/env.mjs machine fingerprint, power/thermal state, ffmpeg resolution +lib/fixture.mjs deterministic source generation + ffprobe +lib/measure.mjs stopwatch, process sampling, output verification +lib/visualCheck.mjs pixel verification of the effects +lib/calibrate.mjs solving each app's padding control +lib/runner.mjs the shared clock every driver is timed by +lib/install.mjs unattended DMG installation +lib/permissions.mjs provoking every macOS prompt up front +lib/uiScript.mjs AppleScript / System Events / accessibility +lib/cdp.mjs Chrome DevTools Protocol, for the Electron apps +lib/report.mjs markdown + HTML +lib/state.mjs append-only event log and pollable status +drivers/ one per app — see drivers/README.md for the contract +``` diff --git a/benchmark/REMOTE.md b/benchmark/REMOTE.md new file mode 100644 index 00000000..8af7f8db --- /dev/null +++ b/benchmark/REMOTE.md @@ -0,0 +1,108 @@ +# Driving the benchmark remotely + +The run takes one to three hours and needs nobody watching it. This is how to start it, check on +it, and pick it up again from a Claude Code session on your phone, in the browser, or dispatched +from another machine. + +The design constraint behind all of it: **every prompt that needs a human is provoked up front**, +and everything after that writes its state to disk so a session that disconnects loses nothing. + +--- + +## The one interactive gate + +Do this while you are at the keyboard. It is the only part that cannot be remote, because macOS +security prompts must be answered on the machine itself. + +```bash +node benchmark/bench.mjs preflight --launch +``` + +It will: + +1. Print the machine's fitness to measure — chip, cores, RAM, OS build, free disk, power source, + thermal state — and refuse quietly-wrong conditions rather than producing a quietly-wrong number. +2. List every download with its size and licence terms, and wait for you to approve the set. +3. Provoke each app's **"… wants access to control …"** Apple Events prompt one at a time, so you + answer them all in one sitting instead of being ambushed six times during the run. Nothing here + clicks *Allow* for you — these are security settings. +4. Open each GUI app once so its first-launch dialogs (onboarding surveys, update nags, usage-data + consent) can be dismissed while you are there. + +When it prints `preflight complete`, the machine is ready and you can leave. + +## Starting a run remotely + +```bash +node benchmark/bench.mjs install # skips anything already present +node benchmark/bench.mjs calibrate # once per machine; ~5 min +node benchmark/bench.mjs run --reps 3 --id nightly +``` + +`run` is safe to launch in the background and disconnect from: + +```bash +nohup node benchmark/bench.mjs run --reps 3 --id nightly > /tmp/bench-nightly.log 2>&1 & +``` + +## Checking on it + +```bash +node benchmark/bench.mjs status --json +``` + +```json +{ + "runId": "nightly", + "phase": "running", + "current": { "app": "camtasia", "index": 3, "of": 6 }, + "completed": ["ffmpeg-baseline", "openscreen-cli"], + "pending": ["kap", "cap"] +} +``` + +`status.json` is rewritten atomically, so polling it can never read a half-written document. +`results//events.ndjson` is append-only and carries one line per app started, finished or +skipped — `tail -f` it for a live view without touching the run. + +Partial results are written after **every** app, not at the end. A run that dies at app four +still leaves four apps' worth of `results.json`, and `bench.mjs report --run ` will render +what exists. + +## Picking up where it stopped + +```bash +node benchmark/bench.mjs run --apps camtasia,kap --id nightly # same id: same output folder +node benchmark/bench.mjs report --run nightly +``` + +There is no magic resume flag, deliberately: naming the apps you still need is clearer than a +flag that guesses, and re-running one app is cheap. + +## Notes for an agent driving this + +- **Do not run two benchmarks at once, and do not do anything else heavy on the machine while one + is running.** The measurement is wall-clock on a shared 8-core SoC; a concurrent build makes + every number wrong without making any of them look wrong. `preconditionCheck()` catches + throttling and battery, not a competing process. +- **`run` is long.** Expect ~2 minutes per repetition per app plus 45 s of cooldown between them — + roughly 25 minutes for six apps at three reps. Poll `status --json` on a slow cadence; do not + busy-wait. +- **A GUI app can leave a window open** if a run is killed mid-export. `bench.mjs doctor` reports + what is running; quitting the app by hand is always safe between runs. +- **Never interpret a missing app as a slow app.** Skipped rows carry a `reason`; report it + verbatim rather than omitting the row. +- **The report is the deliverable, not the terminal output.** `results//report.html` is + self-contained and can be published as an artifact directly. + +## What still needs a human, and when + +| Moment | Why | Frequency | +|---|---|---| +| Apple Events prompts | macOS security; only the user can grant them | once per app, ever | +| First-launch dialogs | vendor onboarding, consent, update nags | once per app, ever | +| Screen Studio activation | export is licence-gated | once, if you own a licence | +| Nothing else | — | — | + +If a prompt does appear mid-run, `lib/permissions.mjs → pendingPermissionDialog()` reads its text, +so a session can report *what* is being asked rather than just noticing that everything stalled. diff --git a/benchmark/apps.mjs b/benchmark/apps.mjs new file mode 100644 index 00000000..5e696ad2 --- /dev/null +++ b/benchmark/apps.mjs @@ -0,0 +1,160 @@ +/** + * The app registry: what is in the benchmark, where it comes from, and what it costs to get. + * + * Separate from the drivers on purpose — `preflight` has to be able to show the user the whole + * download list, with sizes and licence terms, and get one approval for all of it *before* + * anything is fetched. Everything after that approval runs unattended. + */ + +/** + * Download URLs are pinned to a version wherever the vendor exposes one, because "latest" + * makes a benchmark unreproducible: two machines run a month apart would measure two products. + * `bench.mjs refresh-urls` re-resolves them and prints the diff. + */ +export const APPS = { + "openscreen-cli": { + driver: "./drivers/openscreen-cli.mjs", + default: true, + install: { + method: "github-release", + repo: "getopenscreen/openscreen", + assetPattern: /macOS-Apple-Silicon.*\.dmg$/i, + appName: "Openscreen.app", + approxMB: 250, + licence: "MIT — free, no account, no watermark", + }, + }, + "openscreen-gui": { + driver: "./drivers/openscreen-gui.mjs", + default: true, + sharesInstallWith: "openscreen-cli", + }, + "screen-studio": { + // Off by default: export is licence-gated, so an unactivated machine would only ever + // record a failure. Enable it explicitly once a licence is activated. + // macOS only, and export is licence-gated even there. + driver: { darwin: "./drivers/screen-studio.mjs" }, + default: false, + install: { + method: "dmg", + url: "https://screenstudioassets.com/releases/3.7.5-4595/Screen%20Studio%203.7.5-4595%20Apple%20Silicon.dmg", + version: "3.7.5-4595", + appName: "Screen Studio.app", + approxMB: 349, + licence: "commercial — trial exports carry a watermark (which does not change render time)", + notes: [ + "No CLI and no scripting dictionary; only screen-studio://record-* deeplinks exist, none for export.", + ], + }, + }, + cap: { + driver: "./drivers/cap.mjs", + default: true, + install: { + method: "dmg", + url: "https://cap.so/download/apple-silicon", + appName: "Cap.app", + approxMB: 123, + licence: "AGPL-3.0 — free; signing in is optional and not needed for a local export", + notes: [ + "Ships a real CLI at Cap.app/Contents/MacOS/cap-cli — `cap export` renders a .cap project.", + ], + }, + }, + camtasia: { + driver: { darwin: "./drivers/camtasia.mjs", win32: "./drivers/camtasia-win.mjs" }, + default: true, + install: { + method: "dmg", + url: "https://download.techsmith.com/camtasiamac/releases/Camtasia.dmg", + appName: "Camtasia.app", + approxMB: 412, + licence: "commercial — 30-day trial, watermarked output", + notes: ["No CLI on macOS. Driven through the File → Export menu."], + }, + }, + focusee: { + driver: { darwin: "./drivers/focusee.mjs", win32: "./drivers/focusee-win.mjs" }, + // On macOS the import is broken in 2.4.1 (see drivers/focusee.mjs); on Windows the + // vendor ships the real application rather than a downloader stub, so it is in the + // default set there. + default: process.platform === "win32", + install: { + method: "manual", + url: "https://focusee.imobie.com/go/download.php?product=fs", + appName: "FocuSee.app", + approxMB: 5, + licence: "commercial — trial exports are watermarked", + notes: ["The vendor ships a GUI installer stub; run it once during preflight."], + }, + }, + kap: { + // macOS only — Wulkano ships no Windows build. + driver: { darwin: "./drivers/kap.mjs" }, + default: true, + install: { + method: "dmg", + url: "https://github.com/wulkano/Kap/releases/download/v3.6.0/Kap-3.6.0-arm64.dmg", + version: "3.6.0", + appName: "Kap.app", + approxMB: 119, + licence: "MIT — free", + notes: [ + "Has no background, zoom, corner-radius or shadow features at all, so it cannot express the", + "full-demo scenario. It is kept as a reduced-fidelity reference: a real app doing a real", + "export, with none of the compositing. Its row is marked partial in the report.", + ], + }, + }, + "ffmpeg-baseline": { + driver: "./drivers/ffmpeg-baseline.mjs", + default: true, + install: null, + }, +}; + +export async function loadDriver(id) { + const entry = APPS[id]; + if (!entry) throw new Error(`Unknown app "${id}". Known: ${Object.keys(APPS).join(", ")}`); + const mod = await import(driverPath(entry, id)); + return mod.default; +} + +/** Apps that can run at all on this platform — the default set is filtered through this. */ +export function availableOn(platform = process.platform) { + return Object.entries(APPS) + .filter(([, a]) => typeof a.driver === "string" || !!a.driver?.[platform]) + .map(([id]) => id); +} + +/** Every distinct thing that has to be downloaded for the given app ids. */ +export function installPlan(appIds) { + const seen = new Set(); + const plan = []; + for (const id of appIds) { + const entry = APPS[id]; + if (!entry) continue; + const target = entry.sharesInstallWith ?? id; + if (seen.has(target)) continue; + seen.add(target); + const spec = (APPS[target] ?? entry).install; + if (spec) plan.push({ id: target, ...spec }); + } + return plan; +} + +export const DEFAULT_APPS = Object.entries(APPS) + .filter(([id, a]) => a.default && availableOn().includes(id)) + .map(([id]) => id); + +/** Which driver file implements this app on this platform. */ +function driverPath(entry, id) { + if (typeof entry.driver === "string") return entry.driver; + const p = entry.driver?.[process.platform]; + if (!p) { + throw new Error( + `"${id}" has no driver for ${process.platform}. Supported: ${Object.keys(entry.driver ?? {}).join(", ") || "none"}.`, + ); + } + return p; +} diff --git a/benchmark/bench.mjs b/benchmark/bench.mjs new file mode 100644 index 00000000..ef58a927 --- /dev/null +++ b/benchmark/bench.mjs @@ -0,0 +1,661 @@ +#!/usr/bin/env node +/** + * openscreen export benchmark — entrypoint. + * + * node benchmark/bench.mjs preflight # one interactive gate, then walk away + * node benchmark/bench.mjs install + * node benchmark/bench.mjs run + * node benchmark/bench.mjs status --json # safe to poll from anywhere, incl. a remote session + * node benchmark/bench.mjs report + * + * See benchmark/README.md for the methodology and benchmark/REMOTE.md for driving it from a + * dispatched Claude Code session. + */ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { APPS, DEFAULT_APPS, installPlan, loadDriver } from "./apps.mjs"; +import { buildWallpaper, buildWebcam } from "./lib/assets.mjs"; +import { + CALIBRATION_PATH, + calibrateApp, + calibrationFixture, + loadCalibration, + saveCalibration, +} from "./lib/calibrate.mjs"; +import { + CACHE_DIR, + diskState, + ensureWorkDirs, + ffmpegVersion, + machineFingerprint, + powerState, + RESULTS_DIR, + WORK_DIR, +} from "./lib/env.mjs"; +import { buildFixture, DEFAULT_SPEC, fixturePath, probe, sha256 } from "./lib/fixture.mjs"; +import { installApp } from "./lib/install.mjs"; +import { + accessibilityGranted, + pendingPermissionDialog, + primeAutomation, +} from "./lib/permissions.mjs"; +import { renderReport } from "./lib/report.mjs"; +import { preconditionCheck, runApp } from "./lib/runner.mjs"; +import { newRunId, RunState } from "./lib/state.mjs"; +import { + appIsRunning, + describeWindow, + dumpMenus, + hasScriptingDictionary, + launchApp, +} from "./lib/uiScript.mjs"; +import { DEFAULT_SCENARIO, getScenario } from "./scenarios/index.mjs"; + +/* ------------------------------------------------------------------------- argv ---------- */ + +function parseArgs(argv) { + const [command = "help", ...rest] = argv; + const flags = {}; + const positional = []; + for (let i = 0; i < rest.length; i++) { + const a = rest[i]; + if (a.startsWith("--")) { + const [k, inline] = a.slice(2).split("="); + if (inline !== undefined) flags[k] = inline; + else if (rest[i + 1] && !rest[i + 1].startsWith("--")) flags[k] = rest[++i]; + else flags[k] = true; + } else positional.push(a); + } + return { command, flags, positional }; +} + +const log = (...a) => console.log(...a); +const listFlag = (v, fallback) => + typeof v === "string" + ? v + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + : fallback; + +/* ---------------------------------------------------------------------- commands --------- */ + +async function cmdDoctor() { + ensureWorkDirs(); + const fp = machineFingerprint(); + const pre = preconditionCheck(); + let ff = null; + try { + ff = ffmpegVersion(); + } catch (e) { + ff = { banner: `MISSING — ${e.message}`, source: null }; + } + log("Machine"); + log( + ` ${fp.chip} · ${fp.cpuCount} cores (${fp.performanceCores}P/${fp.efficiencyCores}E) · ${fp.memoryGiB} GiB`, + ); + log(` ${fp.osProduct} ${fp.osVersion} (${fp.osBuild}) · node ${fp.nodeVersion}`); + for (const d of fp.displays) log(` ${d}`); + log("\nPreconditions"); + log(` ${pre.ok ? "✓ ready" : `✗ ${pre.problems.join("; ")}`}`); + log(` disk: ${pre.disk.availableGiB} GiB free at ${pre.disk.path}`); + log(`\nffmpeg\n ${ff.banner}\n source: ${ff.source}`); + log("\nApps"); + for (const id of DEFAULT_APPS) { + let driver; + try { + driver = await loadDriver(id); + } catch (e) { + log(` ! ${id.padEnd(26)} driver not available: ${e.message.split("\n")[0]}`); + continue; + } + const d = driver.detect(); + const dict = driver.appPath ? hasScriptingDictionary(driver.appPath) : false; + log( + ` ${d.installed ? "✓" : "·"} ${driver.displayName.padEnd(26)} ${(d.version ?? "").padEnd(14)}` + + ` automation=${driver.automation}${dict ? " (has AppleScript dictionary)" : ""}`, + ); + } +} + +async function cmdPreflight({ flags }) { + ensureWorkDirs(); + const apps = listFlag(flags.apps, DEFAULT_APPS); + const plan = installPlan(apps); + + log("═══ Preflight ═══\n"); + await cmdDoctor(); + + /* ---------------------------------------------------------------- downloads --------- */ + const missing = plan.filter((m) => !existsSync(join("/Applications", m.appName))); + log("\nDownloads needed"); + if (!missing.length) log(" (none — every app is already installed)"); + for (const m of missing) { + log(` ${m.appName.padEnd(22)} ~${m.approxMB} MB ${m.licence}`); + log(` ${m.url ?? m.repo}`); + for (const n of m.notes ?? []) log(` note: ${n}`); + } + const totalMB = missing.reduce((s, m) => s + (m.approxMB ?? 0), 0); + if (totalMB) log(` total ≈ ${totalMB} MB — run \`bench.mjs install\` to fetch them`); + + /* -------------------------------------------------------------- permissions --------- */ + log("\nPermissions"); + if (!accessibilityGranted()) { + log(" ✗ Accessibility is NOT granted to the process running this script."); + log(" Without it System Events refuses every menu click and no GUI app can be driven."); + log(" Grant it in System Settings → Privacy & Security → Accessibility, then re-run."); + } else { + log( + " ✓ Accessibility — System Events responds, so menus and the accessibility tree are reachable.", + ); + } + log( + " · Screen Recording is NOT needed: the benchmark never records, it imports a generated clip.", + ); + + // Every app gets one harmless scripted question. The first raises the macOS prompt and + // blocks until answered; later ones are silent. Doing this here is the whole point of + // preflight — it moves six mid-run ambushes into one sitting. + const drivers = []; + for (const id of apps) { + try { + drivers.push(await loadDriver(id)); + } catch { + /* a driver that will not load is reported by doctor */ + } + } + const needPrompt = drivers.filter((d) => d.bundleId && d.appPath && existsSync(d.appPath)); + if (needPrompt.length) { + log(`\n Provoking the Apple Events prompt for ${needPrompt.length} app(s).`); + log(" Each will raise a “… wants access to control …” dialog. Click Allow on every one —"); + log(" these are security settings, so nothing here can accept them for you.\n"); + } + const permissions = []; + for (const d of needPrompt) { + process.stdout.write(` ${d.displayName.padEnd(24)} `); + const r = primeAutomation(d.bundleId); + permissions.push({ app: d.displayName, ...r }); + log( + r.status === "granted" + ? "✓ granted" + : r.status === "denied" + ? "✗ DENIED — this app cannot be driven" + : `… ${r.status}`, + ); + const pending = pendingPermissionDialog(); + if (pending) log(` still waiting on: ${pending.slice(0, 90)}…`); + } + + /* ------------------------------------------------------- first-launch dialogs ------- */ + if (flags.launch) { + log("\nOpening each GUI app once so its first-launch dialogs can be cleared."); + log("Dismiss onboarding, consent and update prompts now — after this the run is unattended.\n"); + for (const d of drivers) { + if (d.kind !== "gui" || !d.appPath || !existsSync(d.appPath)) continue; + log(` → ${d.displayName}`); + try { + await launchApp(d.appPath, d.processName); + } catch (e) { + log(` could not launch: ${e.message.split("\n")[0]}`); + } + } + } else { + log( + "\nRe-run with --launch to also open each GUI app once and clear its first-launch dialogs.", + ); + } + + const denied = permissions.filter((p) => p.status === "denied"); + const status = { + generatedAt: new Date().toISOString(), + apps, + missingInstalls: missing.map((m) => m.appName), + totalDownloadMB: totalMB, + accessibility: accessibilityGranted(), + permissions, + machine: machineFingerprint(), + preconditions: preconditionCheck(), + }; + mkdirSync(RESULTS_DIR, { recursive: true }); + writeFileSync(join(RESULTS_DIR, "preflight.json"), `${JSON.stringify(status, null, 2)}\n`); + + log("\n─────────────────────────────────────────"); + if (denied.length) + log(`⚠ ${denied.length} app(s) denied automation: ${denied.map((d) => d.app).join(", ")}`); + if (missing.length) log(`Next: node benchmark/bench.mjs install`); + else log("Next: node benchmark/bench.mjs calibrate && node benchmark/bench.mjs run"); + log(`preflight complete — written to ${join(RESULTS_DIR, "preflight.json")}`); +} + +async function cmdInstall({ flags }) { + ensureWorkDirs(); + const apps = listFlag(flags.apps, DEFAULT_APPS); + const plan = installPlan(apps); + const cacheDir = join(WORK_DIR, "installers"); + const records = []; + for (const spec of plan) { + log(`${spec.appName}`); + try { + const rec = installApp(spec, { cacheDir, force: !!flags.force, log }); + records.push(rec); + log( + ` ${rec.status} — ${rec.version ?? "?"} — gatekeeper: ${rec.codesign.accepted ? "accepted" : "REJECTED"}`, + ); + } catch (e) { + records.push({ id: spec.id, status: "failed", error: e.message }); + log(` ✗ ${e.message}`); + } + } + writeFileSync(join(RESULTS_DIR, "install.json"), `${JSON.stringify(records, null, 2)}\n`); + log(`\nWritten: ${join(RESULTS_DIR, "install.json")}`); +} + +async function cmdFixture({ flags }) { + ensureWorkDirs(); + const spec = { ...DEFAULT_SPEC }; + if (flags.duration) spec.durationSec = Number(flags.duration); + if (flags.fps) spec.fps = Number(flags.fps); + const r = buildFixture(WORK_DIR, spec, { force: !!flags.force, log }); + const wp = buildWallpaper(WORK_DIR, spec); + const wc = buildWebcam(WORK_DIR, spec); + log(`\nscreen ${r.path}`); + log(` sha256 ${r.sha256}`); + log(` ${JSON.stringify(r.probe.video)} ${(r.probe.sizeBytes / 1048576).toFixed(1)} MB`); + log(`wallpaper ${wp.path}\n sha256 ${wp.sha256}`); + log(`webcam ${wc.path}\n sha256 ${wc.sha256}`); + log( + "\nCursor telemetry is written per app at prepare time, from the same seed — see lib/assets.mjs.", + ); +} + +async function cmdRun({ flags }) { + ensureWorkDirs(); + const apps = listFlag(flags.apps, DEFAULT_APPS); + const scenario = getScenario(flags.scenario ?? DEFAULT_SCENARIO); + const repetitions = Number(flags.reps ?? 3); + const cooldownSec = Number(flags.cooldown ?? 45); + const discardFirst = flags["no-warmup"] ? false : true; + + const spec = { ...DEFAULT_SPEC }; + const fixture = existsSync(fixturePath(WORK_DIR, spec)) + ? { + path: fixturePath(WORK_DIR, spec), + probe: probe(fixturePath(WORK_DIR, spec)), + sha256: sha256(fixturePath(WORK_DIR, spec)), + spec, + } + : buildFixture(WORK_DIR, spec, { log }); + // A demo export is not just a screen clip: the scenario also needs a wallpaper to sample + // and a camera track to composite. Both come from the same seed as the screen recording, + // so they travel with it rather than being shipped. + const wallpaper = buildWallpaper(WORK_DIR, spec); + const assets = { + wallpaper: wallpaper.path, + jpeg: wallpaper.jpeg, + webcam: buildWebcam(WORK_DIR, spec).path, + }; + + const calibration = loadCalibration(); + if (calibration.machine) { + const here = machineFingerprint(); + if ( + calibration.machine.chip !== here.chip || + calibration.machine.osVersion !== here.osVersion + ) { + log( + `⚠ benchmark/calibration.json was solved on ${calibration.machine.chip} / macOS ${calibration.machine.osVersion}, ` + + `not this machine. Re-run \`bench.mjs calibrate\` — app versions differ between machines and ` + + `a stale padding solve makes the apps composite different rectangles.\n`, + ); + } + } else if (Object.keys(calibration.apps ?? {}).length) { + log("⚠ benchmark/calibration.json has no machine stamp; re-run `bench.mjs calibrate`.\n"); + } else { + log("· no calibration found — each driver will use its documented default padding.\n"); + } + const runId = flags.id ?? newRunId(); + const state = new RunState(join(RESULTS_DIR, runId), runId); + const outDir = join(WORK_DIR, "out", runId); + mkdirSync(outDir, { recursive: true }); + + const header = { + runId, + startedAt: new Date().toISOString(), + scenario: { + id: scenario.id, + label: scenario.label, + effects: scenario.effects, + output: scenario.output, + }, + repetitions, + discardFirst, + cooldownSec, + machine: machineFingerprint(), + power: powerState(), + disk: diskState(), + ffmpeg: (() => { + try { + return ffmpegVersion(); + } catch { + return null; + } + })(), + fixture: { + path: fixture.path, + sha256: fixture.sha256, + spec: fixture.spec, + probe: fixture.probe, + }, + assets: { + wallpaper: { path: assets.wallpaper, sha256: sha256(assets.wallpaper) }, + webcam: { path: assets.webcam, sha256: sha256(assets.webcam), probe: probe(assets.webcam) }, + }, + calibration: calibration.apps + ? { + generatedAt: calibration.generatedAt, + targetInsetPercent: calibration.targetInsetPercent, + apps: calibration.apps, + } + : null, + apps, + }; + state.event("run-started", header); + state.writeStatus({ ...header, phase: "starting", completed: [], pending: apps }); + + log( + `run ${runId} · scenario "${scenario.id}" · ${repetitions}×${discardFirst ? " (+1 warm-up)" : ""}`, + ); + log(`fixture ${fixture.path} (${fixture.sha256.slice(0, 12)})\n`); + + // Re-running one app into an existing run id is the documented way to pick up after a + // failure (see REMOTE.md). Without this it silently discarded everything already measured. + const prior = flags.append ? (state.readResults()?.results ?? []) : []; + if (prior.length) log(`appending to ${prior.length} existing app result(s) in ${runId}\n`); + const results = prior.filter((r) => !apps.includes(r.app)); + for (const [i, id] of apps.entries()) { + let driver; + try { + driver = await loadDriver(id); + } catch (e) { + const rec = { + app: id, + skipped: true, + reason: `driver failed to load: ${e.message}`, + runs: [], + }; + results.push(rec); + state.event("app-skipped", rec); + continue; + } + + state.writeStatus({ + ...header, + phase: "running", + current: { app: id, index: i + 1, of: apps.length }, + completed: results.map((r) => r.app), + pending: apps.slice(i + 1), + }); + + const calibrated = calibration.apps?.[id]?.paddingControl ?? null; + const baseCtx = { + workDir: WORK_DIR, + outDir, + scenario, + source: fixture, + assets, + log, + state, + paddingControl: calibrated, + }; + let rec; + try { + rec = await runApp(driver, baseCtx, { repetitions, discardFirst, cooldownSec, log }); + } catch (e) { + rec = { + app: id, + displayName: driver.displayName, + skipped: true, + reason: `crashed: ${e.message}`, + runs: [], + }; + log(` ✗ ${driver.displayName}: ${e.message}`); + } + results.push(rec); + state.event("app-finished", rec); + state.writeResults({ ...header, finishedAt: null, results }); + } + + // Closing control. A long run heat-soaks the SoC and the background load drifts, so an app + // measured last is not measured under the same conditions as one measured first. Re-running + // the floor at the end quantifies that drift instead of leaving it as an unstated caveat: if + // the opening and closing controls agree, the ordering did not matter; if they do not, the + // report says by how much. + if (!flags["no-control"] && apps.includes("ffmpeg-baseline") && results.length > 1) { + log("\nclosing control: re-running the floor to measure drift over the run"); + const driver = await loadDriver("ffmpeg-baseline"); + const baseCtx = { + workDir: WORK_DIR, + outDir, + scenario, + source: fixture, + assets, + log, + state: {}, + }; + try { + const rec = await runApp(driver, baseCtx, { + repetitions: 2, + discardFirst: false, + cooldownSec, + log, + }); + rec.app = "ffmpeg-baseline-close"; + rec.displayName = "ffmpeg floor (closing control)"; + rec.isControl = true; + results.push(rec); + state.event("app-finished", rec); + } catch (e) { + log(` closing control failed: ${e.message}`); + } + } + + const final = { ...header, finishedAt: new Date().toISOString(), results }; + state.writeResults(final); + state.writeStatus({ ...final, phase: "done", completed: results.map((r) => r.app), pending: [] }); + state.event("run-finished", { apps: results.map((r) => r.app) }); + + const report = renderReport(final); + writeFileSync(join(state.dir, "report.md"), report.markdown); + writeFileSync(join(state.dir, "report.html"), report.html); + log(`\n${report.summaryText}`); + log(`\nResults: ${state.dir}`); +} + +/** + * Solve each app's padding control so they all composite the same rectangle. Run once per + * machine (and again after an app updates); the result is written to benchmark/calibration.json + * and read automatically by `run`. + */ +async function cmdCalibrate({ flags }) { + ensureWorkDirs(); + const apps = listFlag(flags.apps, DEFAULT_APPS); + const scenario = getScenario(flags.scenario ?? DEFAULT_SCENARIO); + const fixture = calibrationFixture(WORK_DIR, log); + const calibWallpaper = buildWallpaper(WORK_DIR, fixture.spec); + const calibAssets = { + wallpaper: calibWallpaper.path, + jpeg: calibWallpaper.jpeg, + webcam: buildWebcam(WORK_DIR, fixture.spec).path, + }; + const outDir = join(WORK_DIR, "out", "calibration"); + mkdirSync(outDir, { recursive: true }); + log( + `calibrating padding against a ${fixture.spec.durationSec}s clip; target inset ${scenario.effects.paddingPercent}% of the short side\n`, + ); + + const entries = []; + for (const id of apps) { + let driver; + try { + driver = await loadDriver(id); + } catch { + continue; + } + if (!driver.detect().installed) { + log(`${driver.displayName}: not installed, skipping`); + continue; + } + if (typeof driver.defaultPaddingControl !== "function") { + log(`${driver.displayName}: no padding control to calibrate`); + entries.push({ app: id, paddingControl: null, reason: "driver exposes no padding control" }); + continue; + } + log(`${driver.displayName}:`); + const ctx = { + workDir: WORK_DIR, + outDir, + scenario, + source: fixture, + log, + state: {}, + run: { index: 0 }, + commit: () => undefined, + }; + try { + const r = await calibrateApp(driver, ctx, { log }); + entries.push(r); + log( + ` -> padding=${r.paddingControl} gives ${r.achievedInsetPercent}%${r.withinTolerance ? "" : " (best available; outside tolerance)"}`, + ); + } catch (e) { + log(` x ${e.message}`); + entries.push({ app: id, paddingControl: null, error: e.message?.slice(0, 400) }); + } + try { + await driver.cleanup(ctx); + } catch { + /* best effort */ + } + } + + const path = saveCalibration(entries, { + scenario: scenario.id, + targetInsetPercent: scenario.effects.paddingPercent, + fixture: { spec: fixture.spec, sha256: fixture.sha256 }, + }); + log(`\nWritten: ${path}`); +} + +async function cmdStatus({ flags }) { + const runs = existsSync(RESULTS_DIR) + ? readFileSync + ? (await import("node:fs")) + .readdirSync(RESULTS_DIR) + .filter((d) => /^\d{8}T/.test(d)) + .sort() + : [] + : []; + const runId = flags.run ?? runs[runs.length - 1]; + if (!runId) { + const out = { phase: "no-runs" }; + log(flags.json ? JSON.stringify(out) : "No runs yet."); + return; + } + const state = new RunState(join(RESULTS_DIR, runId), runId); + const status = state.readStatus(); + if (flags.json) { + log(JSON.stringify(status ?? { runId, phase: "unknown" }, null, 2)); + return; + } + if (!status) return log(`Run ${runId}: no status file.`); + log(`Run ${runId} — ${status.phase}`); + if (status.current) + log(` current: ${status.current.app} (${status.current.index}/${status.current.of})`); + log(` done: ${(status.completed ?? []).join(", ") || "none"}`); + log(` left: ${(status.pending ?? []).join(", ") || "none"}`); +} + +async function cmdReport({ flags }) { + const fs = await import("node:fs"); + const runs = fs.existsSync(RESULTS_DIR) + ? fs + .readdirSync(RESULTS_DIR) + .filter((d) => /^\d{8}T/.test(d)) + .sort() + : []; + const runId = flags.run ?? runs[runs.length - 1]; + if (!runId) return log("No runs to report on."); + const state = new RunState(join(RESULTS_DIR, runId), runId); + const results = state.readResults(); + if (!results) return log(`Run ${runId} has no results.json yet.`); + const report = renderReport(results); + fs.writeFileSync(join(state.dir, "report.md"), report.markdown); + fs.writeFileSync(join(state.dir, "report.html"), report.html); + log(report.markdown); + log(`\nWritten: ${join(state.dir, "report.md")} and report.html`); +} + +/** Dump an app's menus and accessibility tree — how a GUI driver gets written or repaired. */ +async function cmdDiscover({ positional, flags }) { + const id = positional[0]; + if (!id) return log("usage: bench.mjs discover [--window N] [--depth N]"); + const driver = await loadDriver(id); + if (!driver.appPath) return log(`${id} has no app bundle.`); + if (!existsSync(driver.appPath)) return log(`${driver.appPath} is not installed.`); + + log(`# ${driver.displayName}`); + log(`bundle: ${driver.appPath}`); + log(`AppleScript dictionary: ${hasScriptingDictionary(driver.appPath) ? "YES" : "no"}`); + if (!appIsRunning(driver.processName)) { + log(`launching ${driver.processName}…`); + await launchApp(driver.appPath, driver.processName); + await new Promise((r) => setTimeout(r, 4000)); + } + log("\n## Menus"); + log(JSON.stringify(dumpMenus(driver.processName), null, 1)); + log("\n## Window accessibility tree"); + try { + log(describeWindow(driver.processName, Number(flags.window ?? 1), Number(flags.depth ?? 4))); + } catch (e) { + log(`(could not read window: ${e.message})`); + } +} + +function cmdHelp() { + log(`openscreen export benchmark + + doctor environment + installed apps + whether UI scripting works + preflight [--launch] the single interactive gate: what will be downloaded, what to grant + install [--apps a,b] [--force] + calibrate [--apps a,b] solve each app's padding control so they composite the same rect + fixture [--force] [--duration s] [--fps n] + run [--apps a,b] [--scenario id] [--reps 3] [--cooldown 45] [--no-warmup] [--id NAME] + [--append] merge into an existing run id instead of replacing it + [--no-control] skip the closing drift control + status [--run ID] [--json] + report [--run ID] + discover dump menus + accessibility tree (for writing a GUI driver) + +apps: ${Object.keys(APPS).join(", ")}`); +} + +const { command, flags, positional } = parseArgs(process.argv.slice(2)); +const commands = { + doctor: cmdDoctor, + preflight: cmdPreflight, + install: cmdInstall, + fixture: cmdFixture, + run: cmdRun, + status: cmdStatus, + calibrate: cmdCalibrate, + report: cmdReport, + discover: cmdDiscover, + help: cmdHelp, +}; +const fn = commands[command] ?? cmdHelp; +try { + await fn({ flags, positional }); +} catch (e) { + console.error(`\n✗ ${e.stack ?? e.message}`); + process.exit(1); +} diff --git a/benchmark/calibration.json b/benchmark/calibration.json new file mode 100644 index 00000000..301c7e3f --- /dev/null +++ b/benchmark/calibration.json @@ -0,0 +1,160 @@ +{ + "generatedAt": "2026-08-25T11:39:37.984Z", + "scenario": "full-demo", + "targetInsetPercent": 5, + "fixture": { + "spec": { + "name": "calib-1080p60-4s", + "width": 1920, + "height": 1080, + "fps": 60, + "durationSec": 4, + "seed": 20260825, + "sourceBitrateMbps": 12 + }, + "sha256": "574cd13f8e130381c7271a0e9b172f765089fbb99abc4f4548f29b4244b011e4" + }, + "apps": { + "cap": { + "app": "cap", + "target": 5, + "paddingControl": 13.56, + "achievedInsetPercent": 4.81, + "achievedBox": { + "left": 93, + "top": 52, + "right": 1826, + "bottom": 1027, + "width": 1734, + "height": 976 + }, + "withinTolerance": true, + "probes": [ + { + "control": 2.5, + "inset": 0.93, + "box": { + "left": 18, + "top": 10, + "right": 1901, + "bottom": 1069, + "width": 1884, + "height": 1060 + } + }, + { + "control": 5, + "inset": 1.85, + "box": { + "left": 36, + "top": 20, + "right": 1883, + "bottom": 1059, + "width": 1848, + "height": 1040 + } + }, + { + "control": 13.56, + "inset": 4.81, + "box": { + "left": 93, + "top": 52, + "right": 1826, + "bottom": 1027, + "width": 1734, + "height": 976 + } + } + ] + }, + "openscreen-cli": { + "app": "openscreen-cli", + "target": 5, + "paddingControl": 25, + "achievedInsetPercent": 5, + "achievedBox": { + "left": 96, + "top": 54, + "right": 1823, + "bottom": 1025, + "width": 1728, + "height": 972 + }, + "withinTolerance": true, + "probes": [ + { + "control": 25, + "inset": 5, + "box": { + "left": 96, + "top": 54, + "right": 1823, + "bottom": 1025, + "width": 1728, + "height": 972 + } + }, + { + "control": 50, + "inset": 10, + "box": { + "left": 192, + "top": 108, + "right": 1727, + "bottom": 971, + "width": 1536, + "height": 864 + } + } + ] + }, + "openscreen-gui": { + "app": "openscreen-gui", + "target": 5, + "paddingControl": 25, + "achievedInsetPercent": 5, + "achievedBox": { + "left": 96, + "top": 54, + "right": 1823, + "bottom": 1025, + "width": 1728, + "height": 972 + }, + "withinTolerance": true, + "probes": [ + { + "control": 25, + "inset": 5, + "box": { + "left": 96, + "top": 54, + "right": 1823, + "bottom": 1025, + "width": 1728, + "height": 972 + } + }, + { + "control": 50, + "inset": 10, + "box": { + "left": 192, + "top": 108, + "right": 1727, + "bottom": 971, + "width": 1536, + "height": 864 + } + } + ], + "note": "Seeded from openscreen-cli: the GUI writes and reads the same EditorProjectData document and the same padding scale, so solving it twice would measure the same control through a slower interface." + } + }, + "machine": { + "chip": "Apple M1", + "osVersion": "26.5", + "model": "Macmini9,1" + } +} diff --git a/benchmark/drivers/README.md b/benchmark/drivers/README.md new file mode 100644 index 00000000..7223699e --- /dev/null +++ b/benchmark/drivers/README.md @@ -0,0 +1,55 @@ +# Driver contract + +A driver teaches the harness how one app performs the benchmark scenario. It is the only +app-specific code; timing, verification and reporting are shared so no app is measured on a +kinder stopwatch than another. + +```js +export default { + id: "screen-studio", // stable slug, used in results and on the CLI + displayName: "Screen Studio", + vendor: "Screen Studio", + kind: "gui", // "cli" | "gui" | "reference" + automation: "menu", // "cli" | "menu" | "menu+coords" | "none" + processName: "Screen Studio", // as System Events sees it + appPath: "/Applications/Screen Studio.app", + bundleId: "studio.screen.app", + install: { method: "dmg", url, appName, approxMB, licence, notes }, + + detect(), // -> { installed, version, path } + async prepare(ctx), // import the source, apply the scenario, park in the editor + // -> { appliedFeatures: string[], notes: string[] } + outputPath(ctx), // where the export will land + async runExport(ctx), // MUST call ctx.commit() at the instant export is committed + async cleanup(ctx), // quit, remove temp state +}; +``` + +## The two rules that keep the comparison fair + +**`ctx.commit()` marks the same moment for every app.** It is called immediately after the +action that starts the render — the click on *Export*, or the CLI's first `started` event — +never before the project is loaded and never after the first frame. Anything a driver does +before `commit()` (launching the app, importing the clip, setting presets) is warm-up and is +reported separately; anything after it counts. + +**Completion is decided by the filesystem, not by the app.** The harness watches the output +path until it stops growing (`waitForStableFile`), so an app that shows 100% before it has +finished muxing gets no credit for it. A driver's `runExport` may return as soon as the export +is committed; it does not have to detect the end itself. + +## The automation ladder + +GUI drivers should reach for the highest rung that works, and record which one they used — +`automation` in the results is what tells a reader how reproducible a given row is. + +| Rung | Mechanism | Reproducible across machines? | +|---|---|---| +| 1 | AppleScript dictionary (`sdef`) | yes — none of these apps has one | +| 2 | System Events menu item by name | yes, until the app renames the item | +| 3 | Documented keyboard shortcut | yes | +| 4 | Accessibility control by name/description | mostly — names drift between versions | +| 5 | Pixel coordinates | no — flagged as reduced reproducibility | + +`node bench.mjs discover ` dumps the menus and the accessibility tree of an installed app, +which is how a driver gets written or repaired when a new version moves something. diff --git a/benchmark/drivers/camtasia-win.mjs b/benchmark/drivers/camtasia-win.mjs new file mode 100644 index 00000000..90de468c --- /dev/null +++ b/benchmark/drivers/camtasia-win.mjs @@ -0,0 +1,185 @@ +/** + * Camtasia on Windows. + * + * A different automation story from the Mac build. Camtasia for Windows has no AppleScript + * equivalent — TechSmith's documented command line drives the *recorder*, not the renderer — + * but it does expose a real UI Automation tree, and it has a **Batch Export** that renders a + * queue of `.tscproj` projects against one preset. Batch Export is what makes this measurable + * without clicking through the inspector. + * + * Fidelity is partial, for the same reason as on macOS: background, padding, corner radius, + * shadow and zooms live in Visual Properties and Zoom-n-Pan, and neither is scriptable. This + * row measures Camtasia rendering the same source to the same output target with no + * compositing — a real number for its render pipeline, not a full-demo comparison. + * + * NOT YET RUN ON WINDOWS. Written against the UIA surface and TechSmith's documented layout; + * `bench.mjs discover camtasia` dumps the real control names on the target machine, and the + * driver fails loudly with those names attached when a lookup misses. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { now, sleep } from "../lib/measure.mjs"; +import { appVersion, resolveAppPath } from "../lib/platform.mjs"; +import { activateApp, appIsRunning, launchApp, quitApp } from "../lib/ui.mjs"; +import { + clickControl, + describeApp, + fileDialogTo, + listWindows, + setControlValue, +} from "../lib/uiWindows.mjs"; + +export const CAMTASIA = { + macPath: "/Applications/Camtasia.app", + winPaths: [ + "%ProgramFiles%\\TechSmith\\Camtasia 2026\\CamtasiaStudio.exe", + "%ProgramFiles%\\TechSmith\\Camtasia 2025\\CamtasiaStudio.exe", + "%ProgramFiles%\\TechSmith\\Camtasia\\CamtasiaStudio.exe", + "%ProgramFiles(x86)%\\TechSmith\\Camtasia 2026\\CamtasiaStudio.exe", + ], +}; + +const PROC = "CamtasiaStudio"; + +/** Click a control, or fail with the names that *were* present — the only useful error here. */ +function mustClick(needle, opts = {}) { + const r = clickControl(PROC, needle, opts); + if (!r.ok) { + throw new Error( + `Camtasia: no control matching "${needle}". Present: ${(r.seen ?? []).slice(0, 20).join(" | ")}. ` + + "Run `node benchmark/bench.mjs discover camtasia` for the full tree.", + ); + } + return r; +} + +export default { + id: "camtasia", + displayName: "Camtasia", + vendor: "TechSmith", + kind: "gui", + automation: "uia", + processName: PROC, + get appPath() { + return resolveAppPath(CAMTASIA); + }, + bundleId: null, + install: { + method: "installer", + url: "https://download.techsmith.com/camtasia/releases/Camtasia.exe", + appName: "Camtasia", + approxMB: 500, + licence: + "commercial — 30-day trial; exports carry a watermark, which does not change render time", + silentArgs: ["/S"], + notes: [ + "TechSmith's Windows installer is an NSIS package; /S runs it unattended.", + "No render CLI exists on Windows — the documented command line drives the recorder only.", + ], + }, + + detect() { + const path = resolveAppPath(CAMTASIA); + if (!path) return { installed: false, version: null, path: null }; + return { installed: true, version: appVersion(path), path }; + }, + + async prepare(ctx) { + const exe = resolveAppPath(CAMTASIA); + if (!exe) throw new Error("Camtasia is not installed"); + + if (!appIsRunning(PROC)) { + // Opening the clip directly is what creates a project with the media on track 1, + // which is otherwise a drag-and-drop that UIA cannot perform. + await launchApp(exe, PROC, { args: [ctx.source.path], timeoutMs: 180_000 }); + } else { + execFileSync("cmd.exe", ["/c", "start", "", exe, ctx.source.path], { stdio: "ignore" }); + } + await sleep(20_000); + activateApp(PROC); + + // A run killed mid-export leaves Camtasia offering to recover the project, and that + // dialog is modal — everything after it silently misfires. + clickControl(PROC, "Delete", { controlType: "Button" }); + await sleep(1500); + + // A 60 fps import raises "High Frame Rate Media Detected". Taking the 30 fps default + // would halve the frames Camtasia renders and quietly make it look twice as fast. + const converted = clickControl(PROC, "60 FPS", { controlType: "RadioButton" }); + if (converted.ok) { + clickControl(PROC, "Remember my selection", { controlType: "CheckBox" }); + clickControl(PROC, "Continue", { controlType: "Button" }); + await sleep(2500); + } + + // Do not proceed to Export with an empty timeline: that exports nothing and reads as an + // instant, wildly fast render. + const stem = ctx.source.path + .split(/[/\\]/) + .pop() + .replace(/\.[^.]+$/, ""); + let imported = false; + for (let i = 0; i < 60 && !imported; i++) { + await sleep(2000); + imported = describeApp(PROC, { max: 600 }).includes(stem); + } + if (!imported) + throw new Error(`Camtasia never showed "${stem}" on the timeline after the import`); + + return { + appliedFeatures: ["targetResolution", "targetFps"], + notes: [ + converted.ok ? "project frame rate converted to 60 fps on import" : "no frame-rate prompt", + "Effects are NOT applied: background, padding, corner radius, shadow and zooms live in Visual Properties and Zoom-n-Pan, neither of which Camtasia exposes to scripting on Windows either. This row measures its render pipeline at the same output target, not the full-demo composition.", + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + mkdirSync(ctx.outDir, { recursive: true }); + + activateApp(PROC); + await sleep(600); + mustClick("Export", { controlType: "Button" }); + await sleep(2500); + // The Export menu offers Local File / Screencast / YouTube … + clickControl(PROC, "Local File", { controlType: "MenuItem" }); + await sleep(3000); + + // The trial offers watermarked export or a licence key. A watermark is a cheap overlay + // and does not change render time, so the trial path is a valid measurement. + clickControl(PROC, "Export with Watermark", { controlType: "Button" }); + await sleep(2500); + + // Windows' file dialog takes a full path in its name field — no ⇧⌘G equivalent needed. + await fileDialogTo(PROC, out); + ctx.commit(); + + // Camtasia shows a render progress dialog; its disappearance is the completion signal. + const deadline = now() + 30 * 60 * 1000; + let sawProgress = false; + while (now() < deadline) { + await sleep(1000); + const wins = listWindows(PROC).join(" | "); + const rendering = /render|produc|export/i.test(wins); + if (rendering) sawProgress = true; + if (sawProgress && !rendering) { + ctx.markComplete(); + return; + } + // If the progress window was never observable, let the file watcher decide. + if (!sawProgress && existsSync(out)) return; + } + }, + + async cleanup() { + if (appIsRunning(PROC)) await quitApp(PROC, { force: true }); + }, +}; diff --git a/benchmark/drivers/camtasia.mjs b/benchmark/drivers/camtasia.mjs new file mode 100644 index 00000000..3e6f0679 --- /dev/null +++ b/benchmark/drivers/camtasia.mjs @@ -0,0 +1,291 @@ +/** + * Camtasia — the traditional screencast editor. + * + * A different generation of tool from the rest of the set: Camtasia is a general video editor + * that happens to record screens, rather than a demo-maker. That shows up in what can be + * automated and in what the row means. + * + * **Fidelity is partial, and deliberately so.** Camtasia can express every effect in the + * scenario, but not through any scripted interface: + * + * · background — a colour clip on a lower track, or the canvas colour in Properties + * · padding — Visual Properties → Scale on the selected media + * · corner radius— Visual Effects → Border + * · shadow — Visual Effects → Drop Shadow + * · zooms — Edit → Zoom-n-Pan, a panel with no scripting surface + * + * Its AppleScript dictionary exposes `add file`, `addAction` (transitions only) and a readable + * `isExporting`, and nothing that reaches Visual Properties or Zoom-n-Pan. Driving those means + * clicking through the inspector, which is the least reproducible rung of the ladder and would + * make this row depend on Camtasia's panel layout not moving between releases. + * + * So this driver measures Camtasia rendering the *same source to the same output target*, with + * no compositing, and the report marks it partial. That is a real and useful number — it is the + * cost of Camtasia's render pipeline on this machine — but it is not comparable to a full-demo + * row, and the report does not rank it against one. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { now, sleep } from "../lib/measure.mjs"; +import { + activateApp, + appIsRunning, + clickMenuItem, + jxa, + launchApp, + osa, + quitApp, +} from "../lib/uiScript.mjs"; + +const APP = "/Applications/Camtasia.app"; +const PROC = "Camtasia"; + +/** Click the first AXButton/AXRadioButton/AXCheckBox anywhere in the app whose name matches. */ +function clickByName(pattern, { roles = ["AXButton"], required = true } = {}) { + const res = jxa(` + const se = Application("System Events"); + const p = se.processes["${PROC}"]; + p.frontmost = true; + const roles = ${JSON.stringify(roles)}; + function findAll(el, d, out) { + if (d > 9) return out; + try { + if (roles.includes(el.role())) { + let n = ""; try { n = el.name() || ""; } catch (e) {} + if (n) out.push([String(n), el]); + } + } catch (e) {} + try { for (const k of el.uiElements()) findAll(k, d + 1, out); } catch (e) {} + return out; + } + let all = []; + for (const w of p.windows()) { + findAll(w, 0, all); + try { for (const s of w.sheets()) findAll(s, 0, all); } catch (e) {} + } + const re = new RegExp(${JSON.stringify(pattern)}, "i"); + const hit = all.find(([n]) => re.test(n)); + if (!hit) JSON.stringify({ ok: false, seen: all.map(a => a[0]).slice(0, 20) }); + else { hit[1].click(); JSON.stringify({ ok: true, matched: hit[0] }); } + `); + const parsed = JSON.parse(res); + if (!parsed.ok && required) { + throw new Error( + `Camtasia: no control matching /${pattern}/. Present: ${(parsed.seen ?? []).join(", ")}`, + ); + } + return parsed; +} + +/** Is a modal sheet — the save panel — actually up? */ +async function sheetPresent() { + try { + return ( + jxa(` + const se = Application("System Events"); + const p = se.processes["${PROC}"]; + let n = 0; + for (const w of p.windows()) { try { n += w.sheets().length; } catch (e) {} } + String(n > 0); + `) === "true" + ); + } catch { + return false; + } +} + +export default { + id: "camtasia", + displayName: "Camtasia", + vendor: "TechSmith", + kind: "gui", + automation: "applescript+ax", + processName: PROC, + appPath: APP, + bundleId: "com.techsmith.camtasia", + install: { + method: "dmg", + url: "https://download.techsmith.com/camtasiamac/releases/Camtasia.dmg", + appName: "Camtasia.app", + approxMB: 412, + licence: + "commercial — 30-day trial; exports carry a watermark, which does not change render time", + }, + + detect() { + if (!existsSync(APP)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* keep null */ + } + return { installed: true, version, path: APP }; + }, + + async prepare(ctx) { + if (!appIsRunning(PROC)) await launchApp(APP, PROC); + await sleep(6000); + activateApp(PROC); + await sleep(800); + + // A run that was interrupted mid-export leaves Camtasia offering to recover the project + // on next launch. That dialog is modal, so everything after it silently misfires — + // keystrokes meant for a save panel end up naming markers in the timeline instead. + clickByName("^Delete$", { required: false }); + await sleep(1500); + + clickMenuItem(PROC, "File", ["New Project"]); + await sleep(5000); + + // `add file` needs a real file reference, not a POSIX path string. + osa(`tell application "Camtasia" to add file (POSIX file "${ctx.source.path}") at time 0`, { + timeoutMs: 180_000, + }); + + // Importing a minute of 1080p60 takes Camtasia a while, and `add file` returns before it + // has finished. Wait for the media bin to actually show the clip rather than guessing at + // a delay — a driver that proceeds to Export with an empty timeline exports nothing and + // looks like a timeout. + const stem = ctx.source.path + .split("/") + .pop() + .replace(/\.[^.]+$/, ""); + let imported = false; + for (let i = 0; i < 60 && !imported; i++) { + await sleep(2000); + try { + imported = + jxa(` + const se = Application("System Events"); + const p = se.processes["${PROC}"]; + function has(el, d) { + if (d > 9) return false; + try { + const n = (el.name() || "") + " " + (el.value() || ""); + if (n.includes(${JSON.stringify(stem)})) return true; + } catch (e) {} + try { for (const k of el.uiElements()) if (has(k, d + 1)) return true; } catch (e) {} + return false; + } + String(p.windows().some(w => has(w, 0))); + `) === "true"; + } catch { + /* the window may be mid-layout */ + } + } + if (!imported) { + throw new Error(`Camtasia never showed "${stem}" on the timeline after the import`); + } + + // A 60 fps import raises "High Frame Rate Media Detected". Taking the 30 fps default + // would halve the frames Camtasia renders and quietly make it look twice as fast, so the + // answer is forced here and the choice is remembered for later runs. + const converted = clickByName("Convert the entire project to 60", { + roles: ["AXRadioButton"], + required: false, + }); + if (converted.ok) { + clickByName("Remember my selection", { roles: ["AXCheckBox"], required: false }); + clickByName("^Continue$", { required: false }); + await sleep(2500); + } + + return { + appliedFeatures: ["targetResolution", "targetFps"], + notes: [ + converted.ok + ? "project frame rate converted to 60 fps on import" + : "no frame-rate prompt (already remembered from an earlier run)", + "Effects are NOT applied: background, padding, corner radius, shadow and zooms live in Visual Properties and Zoom-n-Pan, neither of which Camtasia exposes to scripting. This row measures its render pipeline at the same output target, not the full-demo composition.", + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + mkdirSync(ctx.outDir, { recursive: true }); + + activateApp(PROC); + await sleep(600); + clickMenuItem(PROC, "Export", ["Local File"]); + await sleep(4000); + + // The trial offers watermarked export or a licence key. A watermark is a cheap overlay + // and does not change render time, so the trial path is a valid measurement. + clickByName("Export with Watermark", { required: false }); + await sleep(3000); + + // Never type blind. If the export sheet did not open, ⇧⌘G and the filename would land in + // the editor — which is exactly how an interrupted run once created timeline markers + // named after the output file. + if (!(await sheetPresent())) { + throw new Error( + "Camtasia: Export → Local File did not raise a save sheet. Something modal is in the " + + "way (a recovery prompt, an upsell, or an unfinished export).", + ); + } + + // Camtasia's export sheet is a standard save panel: ⇧⌘G reaches it. + const dir = out.replace(/\/[^/]+$/, ""); + const stem = out + .split("/") + .pop() + .replace(/\.mp4$/i, ""); + osa(`tell application "System Events" to tell process "${PROC}" + set frontmost to true + keystroke "g" using {command down, shift down} + delay 0.9 + keystroke "${dir}" + delay 0.6 + key code 36 + delay 1.2 + keystroke "a" using {command down} + keystroke "${stem}" + delay 0.5 + end tell`); + await sleep(1200); + + clickByName("^Export$"); + ctx.commit(); + + // Camtasia publishes its own progress; `isExporting` going false is a cleaner stop than + // the filesystem, which sees the file appear before the muxer is finished with it. + const deadline = now() + 30 * 60 * 1000; + let sawExporting = false; + while (now() < deadline) { + await sleep(1000); + let exporting = null; + try { + exporting = osa(`tell application "Camtasia" to return isExporting of front project`, { + timeoutMs: 8000, + }); + } catch { + exporting = null; // the property is not always readable; the file watcher covers it + } + if (exporting === "true") sawExporting = true; + if (sawExporting && exporting === "false") { + ctx.markComplete(); + return; + } + if (existsSync(out) && !sawExporting) { + // isExporting was never readable on this build — let the runner's watcher decide. + return; + } + } + }, + + async cleanup() { + if (appIsRunning(PROC)) await quitApp(PROC, { force: true }); + }, +}; diff --git a/benchmark/drivers/cap.mjs b/benchmark/drivers/cap.mjs new file mode 100644 index 00000000..162e952e --- /dev/null +++ b/benchmark/drivers/cap.mjs @@ -0,0 +1,285 @@ +/** + * Cap (cap.so) — the other open-source entrant in this category. + * + * The only competitor here with a real command line: `Cap.app/Contents/MacOS/cap-cli export` + * renders a `.cap` project with the app's full compositor, and takes `--fps`, `--resolution` + * and `--quality`, so it can be pinned to the same output as everything else. + * + * A `.cap` project is a directory — `recording-meta.json` plus the media — and the editor + * state lives beside it in `project-config.json`. Both are written directly, for the same + * reason OpenScreen's project is: an edit typed into a UI is not reproducible. + */ +import { execFileSync, spawn } from "node:child_process"; +import { copyFileSync, existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { writeCapCursor } from "../lib/assets.mjs"; +import { appVersion, IS_WIN, resolveAppPath } from "../lib/platform.mjs"; + +export const CAP = { + macPath: "/Applications/Cap.app", + winPaths: [ + "%LOCALAPPDATA%\\Programs\\Cap\\Cap.exe", + "%ProgramFiles%\\Cap\\Cap.exe", + "%LOCALAPPDATA%\\Cap\\Cap.exe", + ], +}; + +const APP = IS_WIN ? resolveAppPath(CAP) : "/Applications/Cap.app"; +// The CLI sits beside the desktop binary on Windows and inside the bundle on macOS. +const CLI = APP + ? IS_WIN + ? APP.replace(/Cap\.exe$/i, "cap-cli.exe") + : `${APP}/Contents/MacOS/cap-cli` + : null; + +/** #RRGGBB → the [r,g,b] triple Cap's colour background expects. */ +const rgb = (hex) => { + const h = hex.replace("#", ""); + return [0, 2, 4].map((i) => Number.parseInt(h.slice(i, i + 2), 16)); +}; + +export default { + id: "cap", + displayName: "Cap", + vendor: "Cap Software", + kind: "cli", + automation: "cli", + processName: "Cap", + appPath: APP, + bundleId: "so.cap.desktop", + install: { + method: "dmg", + url: "https://cap.so/download/apple-silicon", + appName: "Cap.app", + approxMB: 123, + licence: "AGPL-3.0 — free", + }, + + detect() { + if (!existsSync(CLI)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* keep null */ + } + return { installed: true, version, path: CLI }; + }, + + /** Cap's `background.padding` is 0-100 on its own scale; see `bench.mjs calibrate`. */ + defaultPaddingControl(scenario) { + return scenario.effects.paddingPercent; + }, + + async prepare(ctx) { + const e = ctx.scenario.effects; + const dir = join(ctx.workDir, "projects", "cap"); + const project = join(dir, `${ctx.scenario.id}.cap`); + rmSync(project, { recursive: true, force: true }); + mkdirSync(join(project, "content"), { recursive: true }); + copyFileSync(ctx.source.path, join(project, "content", "display.mp4")); + + // A demo export composites more than the screen: a camera track to mask and shadow, and + // a pointer rendered from telemetry rather than baked into the recording. + const wantsCamera = e.webcam?.enabled && ctx.assets?.webcam; + if (wantsCamera) copyFileSync(ctx.assets.webcam, join(project, "content", "camera.mp4")); + const wantsCursor = e.cursor?.enabled; + if (wantsCursor) writeCapCursor(project, ctx.source.spec); + let wallpaperPath = null; + if (e.background?.kind === "image" && ctx.assets?.wallpaper) { + wallpaperPath = join(project, "content", "wallpaper.png"); + copyFileSync(ctx.assets.wallpaper, wallpaperPath); + } + + // A single-segment studio recording: the smallest shape `cap project validate` accepts, + // and the one an import would produce. + writeFileSync( + join(project, "recording-meta.json"), + `${JSON.stringify( + { + platform: IS_WIN ? "Windows" : "MacOS", + pretty_name: `openscreen-benchmark-${ctx.scenario.id}`, + display: { path: "content/display.mp4", fps: ctx.source.probe.video.fps }, + ...(wantsCamera ? { camera: { path: "content/camera.mp4", fps: 30 } } : {}), + ...(wantsCursor ? { cursor: "content/cursor.json" } : {}), + }, + null, + 2, + )}\n`, + ); + + // Start from Cap's own defaults so nothing unset drifts between versions, then apply + // only what the scenario names. + const base = JSON.parse( + execFileSync(CLI, ["project", "config", "get", project], { encoding: "utf8" }), + ); + const duration = ctx.source.probe.durationSec; + + // An image the compositor samples per pixel, not a fill it clears once — which is what + // these apps' own wallpapers cost, and what the first version of this scenario missed. + base.background.source = wallpaperPath + ? { type: "image", path: wallpaperPath } + : { type: "color", value: rgb(e.background?.color ?? "#000000"), alpha: 255 }; + base.background.blur = 0; + base.background.padding = ctx.paddingControl ?? this.defaultPaddingControl(ctx.scenario); + base.background.rounding = e.cornerRadiusPx; + // Cap's `shadow` is 0-100; the scenario's intensity is 0-1. + base.background.shadow = e.shadow?.enabled ? Math.round(e.shadow.intensity * 100) : 0; + // Camera inset: masked, rounded and shadowed, in the same corner as every other app. + base.camera.hide = !wantsCamera; + if (wantsCamera) { + base.camera.size = e.webcam.sizePercent ?? 25; + base.camera.rounding = e.webcam.shape === "rounded" ? 25 : 0; + base.camera.shadow = e.webcam.shadow ? 60 : 0; + base.camera.position = { x: "right", y: "bottom" }; + } + + // Cursor: rendered from the telemetry written above, with the smoothing, size and + // motion blur the scenario asks for. + base.cursor.hide = !wantsCursor; + if (wantsCursor) { + base.cursor.size = e.cursor.sizePercent ?? 100; + base.cursor.motionBlur = e.cursor.motionBlur ? 1 : 0; + base.cursor.animationStyle = e.cursor.smoothing >= 0.5 ? "mellow" : "regular"; + base.cursor.raw = false; + } + + base.screenMotionBlur = e.motionBlur?.enabled ? e.motionBlur.amount * 2 : 0; + + base.timeline = { + segments: [{ recordingSegment: 0, timescale: 1, start: 0, end: duration }], + zoomSegments: (e.zooms ?? []).map((z) => ({ + start: z.startSec, + end: z.endSec, + amount: z.scale, + mode: { manual: { x: z.focus.x, y: z.focus.y } }, + })), + sceneSegments: [], + maskSegments: [], + textSegments: [], + captionSegments: [], + keyboardSegments: [], + audioSegments: [], + camera3dSegments: [], + }; + + // `config set` takes the whole document as one argv string and resets anything omitted, + // which is why the defaults were read first rather than a partial patch being sent. + execFileSync( + CLI, + ["project", "config", "set", project, "--settings-json", JSON.stringify(base)], + { + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }, + ); + // Keep a copy beside the project so a run is auditable after the fact. + writeFileSync( + join(dir, `${ctx.scenario.id}-config.json`), + `${JSON.stringify(base, null, 2)}\n`, + ); + + const verify = JSON.parse( + execFileSync(CLI, ["project", "config", "get", project], { encoding: "utf8" }), + ); + // Read back what Cap kept, not what was sent: `config set` silently resets anything it + // will not accept, and claiming a feature the app dropped is how a benchmark lies. + const applied = ["targetResolution", "targetFps"]; + if (["color", "image", "wallpaper", "gradient"].includes(verify.background?.source?.type)) { + applied.push("background"); + } + if (verify.background?.padding > 0) applied.push("padding"); + if (verify.background?.rounding > 0) applied.push("cornerRadius"); + if (verify.background?.shadow > 0) applied.push("shadow"); + if (verify.screenMotionBlur > 0) applied.push("motionBlur"); + if (wantsCursor && verify.cursor?.hide === false) applied.push("cursor"); + if (wantsCamera && verify.camera?.hide === false) applied.push("webcam"); + if ( + (verify.timeline?.zoomSegments ?? []).length === (e.zooms ?? []).length && + e.zooms?.length + ) { + applied.push("zooms"); + } + + ctx.state.projectPath = project; + return { + appliedFeatures: applied, + notes: [ + `project: ${project}`, + `zoom segments written: ${(verify.timeline?.zoomSegments ?? []).length}`, + `camera track: ${wantsCamera ? "content/camera.mp4" : "none"}; cursor telemetry: ${wantsCursor ? "content/cursor.json" : "none"}`, + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + const t = ctx.scenario.output; + + const args = [ + "export", + ctx.state.projectPath, + "--output", + out, + "--format", + "mp4", + "--fps", + String(t.fps), + "--resolution", + `${t.width}x${t.height}`, + "--quality", + "maximum", + "--progress-json", + ]; + + return new Promise((resolve, reject) => { + const child = spawn(CLI, args, { stdio: ["ignore", "pipe", "pipe"] }); + let committed = false; + let stderrTail = ""; + let buf = ""; + child.stdout.on("data", (d) => { + buf += d.toString(); + const lines = buf.split("\n"); + buf = lines.pop() ?? ""; + for (const line of lines) { + if (!line.trim()) continue; + let ev; + try { + ev = JSON.parse(line); + } catch { + continue; + } + // First Progress event = the renderer is live. Same rule as the OpenScreen + // driver: process start-up is warm-up, rendering is the measurement. + if (!committed && (ev.type === "Progress" || ev.type === "Completed")) { + committed = true; + ctx.commit(); + } + if (ev.type === "Error") stderrTail += `\n${ev.error}`; + } + }); + child.stderr.on("data", (d) => { + stderrTail = (stderrTail + d.toString()).slice(-2000); + }); + child.on("error", reject); + child.on("close", (code) => { + if (!committed) ctx.commit(); + if (code === 0) resolve(); + else reject(new Error(`cap export exited ${code}: ${stderrTail.trim().slice(0, 600)}`)); + }); + }); + }, + + async cleanup() { + // Nothing to tear down: `cap export` is a one-shot process. + }, +}; diff --git a/benchmark/drivers/ffmpeg-baseline.mjs b/benchmark/drivers/ffmpeg-baseline.mjs new file mode 100644 index 00000000..05ba23e0 --- /dev/null +++ b/benchmark/drivers/ffmpeg-baseline.mjs @@ -0,0 +1,113 @@ +/** + * Not a competitor — the floor. + * + * A straight re-encode of the source at the target settings, with no compositing at all. It + * answers the question the app-to-app numbers cannot: how much of an export is unavoidable + * encoding work on this machine, and how much is the app's own pipeline. Every app's time + * should be read as a multiple of this. + */ +import { spawn } from "node:child_process"; +import { join } from "node:path"; +import { ffmpegVersion, resolveFfmpeg } from "../lib/env.mjs"; +import { pickH264Encoder } from "../lib/platform.mjs"; + +export default { + id: "ffmpeg-baseline", + displayName: "ffmpeg (re-encode floor)", + vendor: "reference", + kind: "reference", + automation: "cli", + // The sampler matches processes by argv prefix; without this the floor reported 0 CPU + // seconds while every other row reported real ones. + get appPath() { + try { + return resolveFfmpeg().ffmpeg; + } catch { + return null; + } + }, + processName: "ffmpeg", + bundleId: null, + install: null, + + detect() { + try { + const { ffmpeg, source } = resolveFfmpeg(); + const banner = ffmpegVersion().banner; + const v = /ffmpeg version (\S+)/.exec(banner)?.[1] ?? "unknown"; + return { installed: true, version: `${v} (${source.split(":")[0]})`, path: ffmpeg }; + } catch (e) { + return { installed: false, version: null, path: null, error: e.message }; + } + }, + + async prepare() { + const enc = pickH264Encoder(resolveFfmpeg().ffmpeg); + return { + // The floor deliberately applies nothing. Listing the two output features it *does* + // honour keeps the fidelity score honest rather than showing a bare zero. + appliedFeatures: ["targetResolution", "targetFps"], + notes: [ + `encoder: ${enc.encoder}${enc.hardware ? " (hardware)" : " (SOFTWARE)"}`, + "No compositing: this row is the encode-only reference, not a product.", + ...(enc.note ? [enc.note] : []), + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const { ffmpeg } = resolveFfmpeg(); + const enc = pickH264Encoder(ffmpeg); + const out = this.outputPath(ctx); + const t = ctx.scenario.output; + + const args = [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + ctx.source.path, + "-vf", + `scale=${t.width}:${t.height}:flags=bicubic,format=yuv420p`, + "-r", + String(t.fps), + "-c:v", + "h264_videotoolbox", + "-b:v", + "20M", + "-profile:v", + "high", + "-c:a", + "aac", + "-b:a", + "128k", + "-movflags", + "+faststart", + out, + ]; + + return new Promise((resolve, reject) => { + const child = spawn(ffmpeg, args, { stdio: ["ignore", "ignore", "pipe"] }); + let stderr = ""; + child.stderr.on("data", (d) => { + stderr += d.toString(); + }); + // The process is the export: commit the instant it is live. + ctx.commit(); + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) resolve(); + else reject(new Error(`ffmpeg exited ${code}: ${stderr.trim().slice(0, 500)}`)); + }); + }); + }, + + async cleanup() { + // Nothing to tear down: the floor spawns one ffmpeg and it exits. + }, +}; diff --git a/benchmark/drivers/focusee-win.mjs b/benchmark/drivers/focusee-win.mjs new file mode 100644 index 00000000..5f3c931b --- /dev/null +++ b/benchmark/drivers/focusee-win.mjs @@ -0,0 +1,151 @@ +/** + * FocuSee on Windows. + * + * The closest pitch-for-pitch rival to OpenScreen, and on Windows it is a first-class entrant: + * the vendor's download *is* the Windows application (`focusee-en-v2-setup.exe`), where the Mac + * side ships only a downloader stub, and the macOS build refused every MP4 it was handed with + * "The source file is damaged and cannot be opened." That failure is specific to the Mac build + * and there is a fair chance this one imports normally. + * + * FocuSee is a native app on both platforms, so unlike the Electron entrants its whole + * interface is published to the automation API — canvas size, Padding / Inset / Roundness / + * Shadow, and the Export button are all addressable by name. + * + * NOT YET RUN ON WINDOWS. `bench.mjs discover focusee` dumps the real control names; every + * lookup here fails loudly with what it did find. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { sleep } from "../lib/measure.mjs"; +import { appVersion, resolveAppPath } from "../lib/platform.mjs"; +import { activateApp, appIsRunning, launchApp, quitApp } from "../lib/ui.mjs"; +import { clickControl, describeApp, fileDialogTo, setControlValue } from "../lib/uiWindows.mjs"; + +export const FOCUSEE = { + macPath: "/Applications/FocuSee.app", + winPaths: [ + "%ProgramFiles%\\Gemoo\\FocuSee\\FocuSee.exe", + "%ProgramFiles(x86)%\\Gemoo\\FocuSee\\FocuSee.exe", + "%LOCALAPPDATA%\\Programs\\FocuSee\\FocuSee.exe", + "%ProgramFiles%\\FocuSee\\FocuSee.exe", + ], +}; + +const PROC = "FocuSee"; + +export default { + id: "focusee", + displayName: "FocuSee", + vendor: "iMobie / Gemoo", + kind: "gui", + automation: "uia", + processName: PROC, + get appPath() { + return resolveAppPath(FOCUSEE); + }, + bundleId: null, + install: { + method: "installer", + // The link the vendor's download button resolves to on Windows. + url: "https://focusee.imobie-resource.com/product/focusee-en-v2-setup.exe", + appName: "FocuSee", + approxMB: 120, + licence: "commercial — trial exports are watermarked", + silentArgs: ["/S"], + notes: [ + "On Windows the vendor serves the real application, not the downloader stub the Mac side gets.", + "If /S is rejected the installer is not NSIS; run it once by hand during preflight.", + ], + }, + + detect() { + const path = resolveAppPath(FOCUSEE); + if (!path) return { installed: false, version: null, path: null }; + return { installed: true, version: appVersion(path), path }; + }, + + async prepare(ctx) { + const exe = resolveAppPath(FOCUSEE); + if (!exe) throw new Error("FocuSee is not installed"); + + if (appIsRunning(PROC)) await quitApp(PROC, { force: true }); + await sleep(2000); + // Opening the clip through the shell creates a project without having to drive the + // drag-and-drop drop zone, which no automation API can perform. + await launchApp(exe, PROC, { args: [ctx.source.path], timeoutMs: 180_000 }); + await sleep(20_000); + activateApp(PROC); + + const tree = describeApp(PROC, { max: 600 }); + if (/damaged and cannot be opened/i.test(tree)) { + throw new Error( + "FocuSee refused the source: “The source file is damaged and cannot be opened.” " + + "This is the same failure the macOS build showed; if it reproduces here, the app cannot be benchmarked.", + ); + } + + // FocuSee's composition controls are sliders with a readable numeric label. Where a + // ValuePattern exists the scenario is applied; where it does not, the feature is simply + // not claimed — the pixel verifier is what decides, not this list. + const e = ctx.scenario.effects; + const applied = ["targetResolution", "targetFps"]; + if (clickControl(PROC, "16:9", { controlType: "Button" }).ok) applied.push("targetResolution"); + if (setControlValue(PROC, "Padding", String(ctx.paddingControl ?? e.paddingPercent))) + applied.push("padding"); + if (setControlValue(PROC, "Roundness", String(e.cornerRadiusPx))) applied.push("cornerRadius"); + if ( + e.shadow?.enabled && + setControlValue(PROC, "Shadow", String(Math.round(e.shadow.intensity * 100))) + ) { + applied.push("shadow"); + } + + return { + appliedFeatures: [...new Set(applied)], + notes: [ + "Zooms are not applied: FocuSee generates them from its own cursor telemetry, which a file import has none of, and its manual zoom editor is not addressable.", + `controls reached: ${[...new Set(applied)].join(", ")}`, + ], + }; + }, + + /** FocuSee's padding control is 0-100 on its own scale; `bench.mjs calibrate` solves it. */ + defaultPaddingControl(scenario) { + return scenario.effects.paddingPercent; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + mkdirSync(ctx.outDir, { recursive: true }); + + activateApp(PROC); + await sleep(600); + const r = clickControl(PROC, "Export", { controlType: "Button" }); + if (!r.ok) { + throw new Error( + `FocuSee: no Export button. Present: ${(r.seen ?? []).slice(0, 20).join(" | ")}. ` + + "Run `node benchmark/bench.mjs discover focusee`.", + ); + } + await sleep(3000); + + // The export sheet offers format and resolution before the file dialog. + clickControl(PROC, "MP4", { controlType: "Button" }); + clickControl(PROC, "1080", { controlType: "Button" }); + await sleep(800); + clickControl(PROC, "Export", { controlType: "Button" }); + + await fileDialogTo(PROC, out); + ctx.commit(); + }, + + async cleanup() { + if (appIsRunning(PROC)) await quitApp(PROC, { force: true }); + }, +}; diff --git a/benchmark/drivers/focusee.mjs b/benchmark/drivers/focusee.mjs new file mode 100644 index 00000000..ce97d01b --- /dev/null +++ b/benchmark/drivers/focusee.mjs @@ -0,0 +1,176 @@ +/** + * FocuSee — the closest pitch-for-pitch rival, currently unmeasurable. + * + * **FocuSee 2.4.1 rejects every MP4 it is given.** Its own import panel and `open -a` both end + * at *"The source file is damaged and cannot be opened."* — for the benchmark fixture and for a + * real 2560×1440 H.264 screen recording alike. The app is not sandboxed (no + * `com.apple.security.app-sandbox` entitlement), so this is not a file-access grant that could + * be fixed by choosing the file through a picker. Verified on macOS 26.5 with the direct + * download from imobie; the Mac App Store build may differ. + * + * The rest of the driver is written and works: FocuSee is a native Cocoa app, so unlike the + * Electron entrants its whole interface is published to the accessibility API — the canvas-size + * buttons, the Padding / Inset / Roundness / Shadow values and the Export button are all + * addressable by name. If a later build fixes the import, this driver should measure it as-is. + * + * Install note: the vendor ships a ~5 MB downloader stub rather than the app. It is notarised + * (iMobie Inc., team 2QJGLWL8Y6) and installs FocuSee.app into /Applications on launch, but it + * is a GUI installer, so `bench.mjs install` cannot fetch this one unattended. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { sleep } from "../lib/measure.mjs"; +import { activateApp, appIsRunning, jxa, launchApp, osa, quitApp } from "../lib/uiScript.mjs"; + +const APP = "/Applications/FocuSee.app"; +const PROC = "FocuSee"; + +/** Read every static-text value in FocuSee's edit window — how its state is inspected. */ +function editorText() { + return JSON.parse( + jxa(` + const se = Application("System Events"); + const p = se.processes["${PROC}"]; + const win = p.windows().find(w => { try { return w.name() === "edit"; } catch (e) { return false; } }); + function txt(el, d) { + if (d > 10) return []; + let out = []; + try { + const r = el.role(); + if (r === "AXStaticText" || r === "AXButton") { + const v = el.value() || el.name(); + if (v && String(v) !== "button") out.push(String(v).slice(0, 60)); + } + } catch (e) {} + try { for (const k of el.uiElements()) out = out.concat(txt(k, d + 1)); } catch (e) {} + return out; + } + JSON.stringify(win ? txt(win, 0) : []); + `), + ); +} + +export default { + id: "focusee", + displayName: "FocuSee", + vendor: "iMobie", + kind: "gui", + automation: "ax+menu", + processName: PROC, + appPath: APP, + bundleId: "com.imobie.FocuSee", + install: { + method: "manual", + url: "https://focusee.imobie.com/go/download.php?product=fs", + appName: "FocuSee.app", + approxMB: 5, + licence: "commercial — trial exports are watermarked", + notes: [ + "The download is a GUI installer stub, not the app, so this one cannot be installed unattended.", + "Run the stub once during preflight; it places FocuSee.app in /Applications itself.", + ], + }, + + detect() { + if (!existsSync(APP)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* keep null */ + } + return { installed: true, version, path: APP }; + }, + + async prepare(ctx) { + if (appIsRunning(PROC)) await quitApp(PROC, { force: true }); + await sleep(2000); + // FocuSee registers as an MP4 handler; opening the file this way is what creates a + // project without having to drive its drag-and-drop drop zone. + execFileSync("/usr/bin/open", ["-a", APP, ctx.source.path]); + await sleep(18000); + activateApp(PROC); + + const text = editorText(); + const damaged = text.some((t) => /damaged and cannot be opened/i.test(t)); + if (damaged) { + throw new Error( + "FocuSee refused the source: “The source file is damaged and cannot be opened.” " + + "Reproduced with a real 1440p H.264 recording too, so it is not specific to the benchmark " + + "fixture. The app is not sandboxed, so this is not a file-access grant.", + ); + } + if (!text.length) throw new Error("FocuSee did not open an edit window for the source clip"); + + // The composition controls are AX static texts paired with sliders; FocuSee exposes their + // values but not setters, so what the scenario can reach here is limited to the canvas + // aspect ratio. Whatever is applied is reported, never assumed. + const applied = ["targetResolution", "targetFps"]; + return { + appliedFeatures: applied, + notes: [`editor state: ${text.slice(0, 20).join(" · ")}`], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + activateApp(PROC); + await sleep(600); + + const clicked = JSON.parse( + jxa(` + const se = Application("System Events"); + const p = se.processes["${PROC}"]; + p.frontmost = true; + function findAll(el, d, out) { + if (d > 10) return out; + try { if (el.role() === "AXButton") { const n = el.name() || ""; if (n) out.push([String(n), el]); } } catch (e) {} + try { for (const k of el.uiElements()) findAll(k, d + 1, out); } catch (e) {} + return out; + } + let all = []; + for (const w of p.windows()) findAll(w, 0, all); + const hit = all.find(([n]) => /^export$/i.test(n)); + if (!hit) JSON.stringify({ ok: false, seen: all.map(a => a[0]).slice(0, 20) }); + else { hit[1].click(); JSON.stringify({ ok: true }); } + `), + ); + if (!clicked.ok) + throw new Error(`FocuSee: no Export button. Present: ${(clicked.seen ?? []).join(", ")}`); + await sleep(3000); + + const dir = out.replace(/\/[^/]+$/, ""); + const stem = out + .split("/") + .pop() + .replace(/\.mp4$/i, ""); + osa(`tell application "System Events" to tell process "${PROC}" + set frontmost to true + keystroke "g" using {command down, shift down} + delay 0.8 + keystroke "${dir}" + delay 0.5 + key code 36 + delay 1.0 + keystroke "a" using {command down} + keystroke "${stem}" + delay 0.4 + key code 36 + end tell`); + ctx.commit(); + }, + + async cleanup() { + if (appIsRunning(PROC)) await quitApp(PROC, { force: true }); + }, +}; diff --git a/benchmark/drivers/kap.mjs b/benchmark/drivers/kap.mjs new file mode 100644 index 00000000..c94fcd38 --- /dev/null +++ b/benchmark/drivers/kap.mjs @@ -0,0 +1,263 @@ +/** + * Kap — the open-source minimum. + * + * Kap has no background, no padding, no corner radius, no shadow and no zooms: it trims and + * re-encodes, and that is all it claims to do. It cannot express the full-demo scenario and is + * not a peer of the other apps here. It is kept because it answers a question the synthetic + * ffmpeg floor cannot — what a *real, shipping app* costs to get a frame from disk to disk on + * this machine, Electron shell and all. Its row is always marked partial. + * + * Automation: Kap is Electron with no accessibility tree, so `System Events` sees an empty + * window. Launched with `--remote-debugging-port` its renderer is reachable, and the export + * button, the settings fields and the progress text are all plain DOM. + * + * Two things about Kap shape this driver: + * + * 1. **Its editor is single-use.** Once an export finishes, the Convert button is replaced by a + * share prompt, so a second run has nothing to click. Each run therefore opens the clip + * again — before the clock starts, so it is not measured. + * 2. **Its export destination is a native popup menu** that neither CDP nor the accessibility + * API can open. The driver uses Kap's clipboard destination instead — the same render, + * writing to a temp directory — and adopts the file it produces. Kap's own "Export complete" + * is the stop signal, so the adoption copy is never counted. + */ +import { execFileSync } from "node:child_process"; +import { + copyFileSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { join } from "node:path"; +import { CdpSession, DOM_HELPERS, listTargets } from "../lib/cdp.mjs"; +import { now, sleep } from "../lib/measure.mjs"; +import { appIsRunning, quitApp } from "../lib/uiScript.mjs"; + +const APP = "/Applications/Kap.app"; +const BIN = `${APP}/Contents/MacOS/Kap`; +const PORT = 9334; +const HISTORY = join( + homedir(), + "Library", + "Application Support", + "Kap", + "export-usage-history.json", +); + +/** Every `//*.mp4` Kap could have written. */ +function tempExports() { + const base = process.env.TMPDIR || tmpdir(); + const out = new Map(); + let dirs = []; + try { + dirs = readdirSync(base); + } catch { + return out; + } + for (const d of dirs) { + const dir = join(base, d); + let entries = []; + try { + if (!statSync(dir).isDirectory()) continue; + entries = readdirSync(dir); + } catch { + continue; + } + for (const f of entries) { + if (!f.endsWith(".mp4")) continue; + const p = join(dir, f); + try { + out.set(p, statSync(p).mtimeMs); + } catch { + /* vanished between readdir and stat */ + } + } + } + return out; +} + +/** + * Open the source clip in a fresh Kap editor and pin its output fields. + * + * `restart` is what makes a second export possible at all. Kap keeps one editor window and, + * once an export completes, leaves it showing a share prompt where the Convert button was. + * Opening the same file again only refocuses that window — the app has to go away and come + * back. All of this runs before the clock starts. + */ +async function openEditor(ctx, { restart = false } = {}) { + if (restart) { + if (appIsRunning("Kap")) await quitApp("Kap", { force: true }); + await sleep(2500); + execFileSync("/bin/sh", [ + "-c", + `nohup ${JSON.stringify(BIN)} --remote-debugging-port=${PORT} >/dev/null 2>&1 &`, + ]); + await sleep(9000); + } + execFileSync("/usr/bin/open", ["-a", APP, ctx.source.path]); + await sleep(8000); + const target = (await listTargets(PORT)).find((t) => t.url.includes("editor.html")); + if (!target) throw new Error("Kap did not open an editor window for the source clip"); + const session = new CdpSession(target.webSocketDebuggerUrl); + await session.open(); + await session.eval(DOM_HELPERS); + + const t = ctx.scenario.output; + const raw = await session.eval(`(() => { + const setNative = (el, v) => { + const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, "value").set; + setter.call(el, String(v)); + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + }; + const ins = [...document.querySelectorAll("input")]; + const w = ins.find(i => i.value === "1920") || ins[ins.length - 3]; + const h = ins.find(i => i.value === "1080") || ins[ins.length - 2]; + const f = ins[ins.length - 1]; + if (w) setNative(w, ${t.width}); + if (h) setNative(h, ${t.height}); + if (f) setNative(f, ${t.fps}); + return JSON.stringify({ + format: (document.querySelector(".format") || {}).innerText, + plugin: (document.querySelector(".plugin") || {}).innerText, + values: [...document.querySelectorAll("input")].map(i => i.value), + }); + })()`); + return { session, state: JSON.parse(raw) }; +} + +export default { + id: "kap", + displayName: "Kap", + vendor: "Wulkano", + kind: "gui", + automation: "cdp", + processName: "Kap", + appPath: APP, + bundleId: "com.wulkano.kap", + install: { + method: "dmg", + url: "https://github.com/wulkano/Kap/releases/download/v3.6.0/Kap-3.6.0-arm64.dmg", + version: "3.6.0", + appName: "Kap.app", + approxMB: 119, + licence: "MIT — free", + }, + + detect() { + if (!existsSync(BIN)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* keep null */ + } + return { installed: true, version, path: BIN }; + }, + + async prepare(ctx) { + // Kap picks its default format from a usage ledger rather than a setting. Promoting mp4 + // there is how the editor opens on MP4 instead of GIF without touching the UI. + if (existsSync(HISTORY)) { + try { + const h = JSON.parse(readFileSync(HISTORY, "utf8")); + for (const k of Object.keys(h)) h[k].lastUsed = 1; + h.mp4 = { lastUsed: 99, plugins: { default: 99 } }; + writeFileSync(HISTORY, JSON.stringify(h, null, 1)); + } catch { + /* a fresh install has no ledger; the default is fine */ + } + } + + if (appIsRunning(this.processName)) await quitApp(this.processName, { force: true }); + await sleep(1500); + execFileSync("/bin/sh", [ + "-c", + `nohup ${JSON.stringify(BIN)} --remote-debugging-port=${PORT} >/dev/null 2>&1 &`, + ]); + await sleep(9000); + + const { session, state } = await openEditor(ctx); + ctx.state.cdp = session; + + const applied = /mp4/i.test(state.format ?? "") ? ["targetResolution", "targetFps"] : []; + return { + appliedFeatures: applied, + notes: [ + `format selector reads "${state.format}", destination "${state.plugin}"`, + "Kap has no background, padding, corner-radius, shadow or zoom features — the full-demo scenario cannot be expressed, so this row is a re-encode reference rather than a competitor.", + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + const before = tempExports(); + + let s = ctx.state.cdp; + const hasButton = await s + .eval( + 'String(!!(document.querySelector("button.start-export") || [...document.querySelectorAll("button")].find((x) => /convert/i.test(x.innerText))))', + ) + .catch(() => "false"); + if (hasButton !== "true") { + s?.close(); + const reopened = await openEditor(ctx, { restart: true }); + s = reopened.session; + ctx.state.cdp = s; + } + + const clicked = await s.eval(`(() => { + const b = document.querySelector("button.start-export") + || [...document.querySelectorAll("button")].find(x => /convert/i.test(x.innerText)); + if (!b) return "no-button"; + b.click(); + return "clicked"; + })()`); + if (clicked !== "clicked") + throw new Error(`Kap: could not find the Convert button (${clicked})`); + ctx.commit(); + + const deadline = now() + 30 * 60 * 1000; + let done = false; + while (now() < deadline) { + await sleep(500); + const txt = await s.eval("document.body.innerText.slice(0, 400)"); + if (/export complete|drag and drop to copy/i.test(txt)) { + ctx.markComplete(); + done = true; + break; + } + } + if (!done) throw new Error("Kap never reported the export as complete"); + + const after = tempExports(); + const fresh = [...after.entries()] + .filter(([p, m]) => !before.has(p) || before.get(p) !== m) + .sort((a, b) => b[1] - a[1]); + if (!fresh.length) + throw new Error("Kap reported completion but wrote no file into the temp tree"); + mkdirSync(ctx.outDir, { recursive: true }); + copyFileSync(fresh[0][0], out); + ctx.state.kapTempPath = fresh[0][0]; + }, + + async cleanup(ctx) { + ctx.state?.cdp?.close(); + if (appIsRunning(this.processName)) await quitApp(this.processName, { force: true }); + }, +}; diff --git a/benchmark/drivers/openscreen-cli.mjs b/benchmark/drivers/openscreen-cli.mjs new file mode 100644 index 00000000..b443e242 --- /dev/null +++ b/benchmark/drivers/openscreen-cli.mjs @@ -0,0 +1,172 @@ +/** + * OpenScreen, headless. + * + * The subject of the benchmark, driven through its own `openscreen export` command. The + * project is written as JSON rather than built in the editor, so the scenario is exact and + * byte-reproducible — see lib/openscreenProject.mjs. + * + * Because this path skips the UI entirely it is *not* directly comparable to Screen Studio's + * or Camtasia's numbers; `openscreen-gui` exists for that comparison, and the report keeps the + * two rows apart. + */ +import { execFileSync, spawn } from "node:child_process"; +import { existsSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { buildProject } from "../lib/openscreenProject.mjs"; +import { appVersion, IS_WIN, resolveAppPath } from "../lib/platform.mjs"; + +export const OPENSCREEN = { + macPath: "/Applications/Openscreen.app", + winPaths: [ + "%ProgramFiles%\\Openscreen\\Openscreen.exe", + "%LOCALAPPDATA%\\Programs\\Openscreen\\Openscreen.exe", + "%LOCALAPPDATA%\\openscreen\\Openscreen.exe", + ], +}; + +/** + * The CLI ships inside the normal application bundle on both platforms, so there is nothing + * extra to install — only a different place to look. + */ +const APP = IS_WIN ? resolveAppPath(OPENSCREEN) : "/Applications/Openscreen.app"; +const BIN = IS_WIN ? APP : `${APP}/Contents/MacOS/Openscreen`; + +export default { + id: "openscreen-cli", + displayName: "OpenScreen (CLI)", + vendor: "OpenScreen", + kind: "cli", + automation: "cli", + processName: "Openscreen", + appPath: APP, + bundleId: "com.etiennelescot.openscreen", + install: { + method: "dmg", + // Resolved at install time from the GitHub release feed; see lib/install.mjs. + url: "github:getopenscreen/openscreen", + appName: "Openscreen.app", + approxMB: 250, + licence: "MIT — free, no account, no watermark", + notes: ["The CLI ships inside the normal app bundle; there is nothing extra to install."], + }, + + detect() { + if (!existsSync(BIN)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* unreadable plist — report installed without a version */ + } + return { installed: true, version, path: BIN }; + }, + + /** + * OpenScreen's `padding` is 0-100 on its own scale, not a percentage of the frame. The + * default below is a starting point; `bench.mjs calibrate` measures the inset it actually + * produces and solves for the value that matches every other app. + */ + defaultPaddingControl(scenario) { + return Math.round(Math.min(100, Math.max(0, scenario.effects.paddingPercent * 10))); + }, + + async prepare(ctx) { + const outDir = join(ctx.workDir, "projects", "openscreen-cli"); + const { projectPath } = buildProject({ + sourcePath: ctx.source.path, + scenario: ctx.scenario, + outDir, + title: ctx.scenario.id, + paddingControl: ctx.paddingControl ?? this.defaultPaddingControl(ctx.scenario), + assets: ctx.assets ?? {}, + spec: ctx.source.spec, + }); + ctx.state.projectPath = projectPath; + + const e = ctx.scenario.effects; + return { + appliedFeatures: [ + "background", + "padding", + "cornerRadius", + "shadow", + "zooms", + ...(e.motionBlur?.enabled ? ["motionBlur"] : []), + ...(e.cursor?.enabled ? ["cursor"] : []), + ...(e.webcam?.enabled && ctx.assets?.webcam ? ["webcam"] : []), + "targetResolution", + "targetFps", + ], + notes: [ + `project: ${projectPath}`, + "MP4 export is fixed at 60 fps (MP4_EXPORT_FPS, src/cli/CliExportRunner.tsx) — which is why the pinned target is 60.", + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + + const args = ["export", ctx.state.projectPath, "-o", out, "--quality", "good", "--json"]; + return new Promise((resolve, reject) => { + const child = spawn(BIN, args, { stdio: ["ignore", "pipe", "pipe"] }); + let committed = false; + let stderrTail = ""; + let buf = ""; + + child.stdout.on("data", (d) => { + buf += d.toString(); + const lines = buf.split("\n"); + buf = lines.pop() ?? ""; + for (const line of lines) { + if (!line.trim()) continue; + let ev; + try { + ev = JSON.parse(line); + } catch { + continue; + } + // `started` fires once the hidden renderer is up and the render begins. Taking + // t0 here rather than at spawn keeps Electron's cold boot out of the export + // number — the GUI apps are warm when their clock starts, so this one is too. + // The boot cost is still recorded, as launchToCommitMs. + if (!committed && (ev.event === "started" || ev.event === "progress")) { + committed = true; + ctx.commit(); + } + if (ev.event === "progress") ctx.progress?.(ev.percentage); + if (ev.event === "done") ctx.state.reportedOutput = ev.outputPath; + } + }); + child.stderr.on("data", (d) => { + stderrTail = (stderrTail + d.toString()).slice(-2000); + }); + child.on("error", reject); + child.on("close", (code) => { + if (!committed) ctx.commit(); // never leave the run without a t0 + if (code === 0) resolve(); + else + reject(new Error(`openscreen export exited ${code}: ${stderrTail.trim().slice(0, 600)}`)); + }); + }); + }, + + async cleanup() { + // CLI exports leave the on-device STT server running; on an 8 GB machine those orphans + // distort the next run's memory figures and the process sampler's totals. + try { + execFileSync("/usr/bin/pkill", ["-f", "whisper-stt-server"], { stdio: "ignore" }); + } catch { + /* none running */ + } + }, +}; diff --git a/benchmark/drivers/openscreen-gui.mjs b/benchmark/drivers/openscreen-gui.mjs new file mode 100644 index 00000000..0e03db3c --- /dev/null +++ b/benchmark/drivers/openscreen-gui.mjs @@ -0,0 +1,287 @@ +/** + * OpenScreen, through its own editor. + * + * The CLI leg (`openscreen-cli`) measures the render engine with no interface in the way. That + * is the right number to compare against Cap's CLI, and the wrong one to compare against an app + * that can only be driven by clicking — a UI leg carries the editor's own overhead, and the + * subject of a benchmark should not be the only entrant excused from it. + * + * So this driver does what a person does: opens the project in the editor, opens the export + * dialog, picks MP4 / 1080p / 60 / H.264, presses Export and answers the save panel. + * + * OpenScreen is Electron, so the editor is reached over CDP and every control is found by its + * visible text. The two native surfaces on the path — the File menu and the save panel — are + * driven through System Events. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { CdpSession, DOM_HELPERS, listTargets } from "../lib/cdp.mjs"; +import { sleep } from "../lib/measure.mjs"; +import { buildProject } from "../lib/openscreenProject.mjs"; +import { + activateApp, + appIsRunning, + clickMenuItem, + listWindows, + osa, + quitApp, +} from "../lib/uiScript.mjs"; + +const APP = "/Applications/Openscreen.app"; +const BIN = `${APP}/Contents/MacOS/Openscreen`; +const PORT = 9335; + +const editorTarget = async () => + (await listTargets(PORT)).find((t) => t.url.includes("windowType=editor")); + +export default { + id: "openscreen-gui", + displayName: "OpenScreen (GUI)", + vendor: "OpenScreen", + kind: "gui", + automation: "cdp+menu", + processName: "Openscreen", + appPath: APP, + bundleId: "com.etiennelescot.openscreen", + install: null, // shares the install with openscreen-cli + + detect() { + if (!existsSync(BIN)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* keep null */ + } + return { installed: true, version, path: BIN }; + }, + + defaultPaddingControl(scenario) { + return Math.round(Math.min(100, Math.max(0, scenario.effects.paddingPercent * 10))); + }, + + async prepare(ctx) { + const outDir = join(ctx.workDir, "projects", "openscreen-gui"); + const { projectPath } = buildProject({ + sourcePath: ctx.source.path, + scenario: ctx.scenario, + outDir, + title: ctx.scenario.id, + paddingControl: ctx.paddingControl ?? this.defaultPaddingControl(ctx.scenario), + assets: ctx.assets ?? {}, + spec: ctx.source.spec, + }); + ctx.state.projectPath = projectPath; + + // A single-instance lock keys on the userData path, so a stale instance must go before + // the debugging port can be opened on a fresh one. + if (appIsRunning(this.processName)) await quitApp(this.processName, { force: true }); + try { + execFileSync("/usr/bin/pkill", ["-f", "whisper-stt-server"], { stdio: "ignore" }); + } catch { + /* none running */ + } + await sleep(2000); + execFileSync("/bin/sh", [ + "-c", + `nohup ${JSON.stringify(BIN)} --remote-debugging-port=${PORT} >/dev/null 2>&1 &`, + ]); + await sleep(12000); + + // The launcher opens on the HUD; the editor is a separate window. switchToEditor never + // resolves its promise, so it is fired and then waited for by polling the target list. + const hud = (await listTargets(PORT)).find((t) => t.url.includes("hud-overlay")); + if (hud) { + const h = new CdpSession(hud.webSocketDebuggerUrl); + await h.open(); + // switchToEditor tears the HUD renderer down, so the CDP reply for this evaluate may + // never arrive. Fire it, give it a moment, move on. + try { + await h.send( + "Runtime.evaluate", + { expression: "window.electronAPI.switchToEditor()", awaitPromise: false }, + { timeoutMs: 5000 }, + ); + } catch { + /* expected when the page goes away mid-call */ + } + h.close(); + } + let ed = null; + for (let i = 0; i < 30 && !ed; i++) { + await sleep(1000); + ed = await editorTarget(); + } + if (!ed) throw new Error("the OpenScreen editor window never appeared"); + + const s = new CdpSession(ed.webSocketDebuggerUrl); + await s.open(); + await s.eval(DOM_HELPERS); + ctx.state.cdp = s; + + // File → Load Project… raises an in-app picker whose "Browse files…" button is what + // opens the real panel. Both steps are needed; the in-app list ignores ⇧⌘G. + activateApp(this.processName); + await sleep(700); + clickMenuItem(this.processName, "File", ["Load Project"]); + await sleep(2000); + await s.eval(`JSON.stringify(window.__osbench.click("Browse files"))`); + await sleep(2000); + osa(`tell application "System Events" to tell process "${this.processName}" + set frontmost to true + keystroke "g" using {command down, shift down} + delay 0.8 + keystroke "${projectPath}" + delay 0.6 + key code 36 + delay 1.4 + key code 36 + end tell`); + await sleep(10000); + + // Ask the app which project it has open rather than scraping the panel for a duration + // string. This is a gate, not a formality: an editor with no project loaded exports + // happily and writes an empty container, which reads as an instant, wildly fast render. + let loadedPath = null; + for (let i = 0; i < 25 && loadedPath !== projectPath; i++) { + await sleep(1500); + try { + const raw = await s.eval( + `(async () => { try { const r = await window.electronAPI.loadCurrentProjectFile(); return JSON.stringify(r ?? null); } catch (e) { return null; } })()`, + { timeoutMs: 15_000 }, + ); + const cur = raw ? JSON.parse(raw) : null; + if (cur?.path) loadedPath = cur.path; + } catch { + /* the renderer may be mid-load */ + } + } + const panel = await s.eval("document.body.innerText.slice(0, 600)"); + if (loadedPath !== projectPath) { + throw new Error( + `OpenScreen has "${loadedPath ?? "no project"}" open, not the benchmark project ` + + `(${projectPath}). Panel read: ${panel.replace(/\n+/g, " | ").slice(0, 200)}`, + ); + } + + const applied = ["targetResolution", "targetFps"]; + const e = ctx.scenario.effects; + if (panel.includes(e.background.color)) applied.push("background"); + if (/Padding/.test(panel)) applied.push("padding"); + if (new RegExp(`Roundness\\s*\\|?\\s*${e.cornerRadiusPx}`).test(panel.replace(/\n/g, " "))) { + applied.push("cornerRadius"); + } + if (/Shadow\s*\|?\s*(?!0%)\d+%/.test(panel.replace(/\n/g, " "))) applied.push("shadow"); + // Zooms live on the timeline rather than the composition panel; the project was written + // with them and the pixel verifier is what confirms they rendered. + if (e.zooms?.length) applied.push("zooms"); + + return { + appliedFeatures: applied, + notes: [ + `project: ${projectPath}`, + `composition panel after load: ${panel.replace(/\n+/g, " | ").slice(0, 220)}`, + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const s = ctx.state.cdp; + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + const t = ctx.scenario.output; + + await s.eval(`JSON.stringify(window.__osbench.click("Export", { exact: true }))`); + await sleep(2000); + + // Confirm the dialog is actually up. On a repeat run the editor can still be showing the + // previous export's completion state, and clicking through a dialog that never opened + // leaves the run waiting on a file no one is writing. + let dialogUp = false; + for (let i = 0; i < 15 && !dialogUp; i++) { + const txt = await s.eval("document.body.innerText"); + dialogUp = /Render the timeline to a file/i.test(txt) && /Export MP4/i.test(txt); + if (!dialogUp) { + await sleep(1000); + // Dismiss whatever is in the way, then ask again. + await s.eval(`JSON.stringify(window.__osbench.click("Close"))`).catch(() => undefined); + await s.eval(`JSON.stringify(window.__osbench.click("Export", { exact: true }))`); + } + } + if (!dialogUp) throw new Error("OpenScreen: the export dialog never opened"); + + // The dialog's controls are plain buttons labelled with their value. + for (const label of ["MP4", `${t.height}p`, String(t.fps), "H.264"]) { + const r = JSON.parse( + await s.eval( + `JSON.stringify(window.__osbench.click(${JSON.stringify(label)}, { exact: true }))`, + ), + ); + if (!r.ok) throw new Error(`OpenScreen export dialog: no control labelled "${label}"`); + await sleep(350); + } + + const go = JSON.parse(await s.eval(`JSON.stringify(window.__osbench.click("Export MP4"))`)); + if (!go.ok) throw new Error("OpenScreen export dialog: no “Export MP4” button"); + + // Pressing Export raises the system save panel; the render starts when it is answered. + await sleep(2500); + const dir = out.replace(/\/[^/]+$/, ""); + // The save panel appends the format's extension itself, so a name that already carries + // one comes back as "…run0.mp4.mp4" and the watcher waits forever on a path that will + // never exist. Type the stem only. + const file = out + .split("/") + .pop() + .replace(/\.mp4$/i, ""); + osa(`tell application "System Events" to tell process "${this.processName}" + set frontmost to true + keystroke "g" using {command down, shift down} + delay 0.7 + keystroke "${dir}" + delay 0.5 + key code 36 + delay 1.0 + keystroke "a" using {command down} + keystroke "${file}" + delay 0.4 + key code 36 + delay 0.8 + end tell`); + ctx.commit(); + + // Answer a replace-confirmation if one appears, then let the runner's file watcher decide + // when the render is done. + await sleep(1200); + try { + osa(`tell application "System Events" to tell process "${this.processName}" + repeat with w in windows + try + if exists (button "Replace" of sheet 1 of w) then click button "Replace" of sheet 1 of w + end try + end repeat + end tell`); + } catch { + /* the common case: no alert */ + } + }, + + async cleanup(ctx) { + ctx.state?.cdp?.close(); + if (appIsRunning(this.processName)) await quitApp(this.processName, { force: true }); + try { + execFileSync("/usr/bin/pkill", ["-f", "whisper-stt-server"], { stdio: "ignore" }); + } catch { + /* none running */ + } + }, +}; diff --git a/benchmark/drivers/screen-studio.mjs b/benchmark/drivers/screen-studio.mjs new file mode 100644 index 00000000..563c7c51 --- /dev/null +++ b/benchmark/drivers/screen-studio.mjs @@ -0,0 +1,237 @@ +/** + * Screen Studio — the app that defined this category. + * + * Fully automated up to the point where it stops being possible: **export is gated behind + * account activation.** Pressing Export on an unactivated install opens an activation wall + * asking for an email or licence key. There is no trial export and no watermark path — the + * bundle contains no "free trial" strings at all. With a licence activated once (during + * preflight), every step below runs unattended and the app becomes a full peer in the table. + * + * Getting to that point took the most work of any app here, and the findings are worth stating + * because they shape the driver: + * + * 1. **It cannot be screenshotted.** The editor window is marked `kCGWindowSharingNone`, so + * macOS excludes it from every capture API. It is plainly visible to the person at the + * machine and invisible to `screencapture`, ScreenCaptureKit, and any agent driving pixels. + * 2. **It publishes no accessibility tree.** `System Events` sees a window containing three + * traffic-light buttons and nothing else. + * 3. Its only documented automation is three `screen-studio://record-*` deeplinks. The bundle + * also carries undocumented ones (`export-to-clipboard`, `copy-and-zip-project`, + * `open-projects-folder`), none of which exports to a file. + * + * What makes it drivable is `--remote-debugging-port`: the renderer is then reachable and every + * control can be found by its visible text. That is *more* reproducible than pixel clicking, not + * less — it survives a moved window, a different display and a resized UI — and the flag only + * opens an inspector; the renderer and export pipeline are the shipping ones. + * + * The composition itself is not clicked at all: a `.screenstudio` project is a directory of + * plain JSON, so the scenario is written straight into `project.json` and the app reopens it. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { CdpSession, DOM_HELPERS, listTargets } from "../lib/cdp.mjs"; +import { sleep } from "../lib/measure.mjs"; +import { activateApp, appIsRunning, clickMenuItem, osa, quitApp } from "../lib/uiScript.mjs"; + +const APP = "/Applications/Screen Studio.app"; +const BIN = `${APP}/Contents/MacOS/Screen Studio`; +const PORT = 9333; +const PROJECTS = join(homedir(), "Screen Studio Projects"); + +/** Screen Studio's zoom-range shape, recovered from its own project factory. */ +const zoomRange = (z, i) => ({ + id: `osbench${String(i).padStart(4, "0")}`, + zoom: z.scale, + type: "manual", + snapToEdgesRatio: 0.25, + manualTargetPoint: { x: z.focus.x, y: z.focus.y }, + glideDirection: null, + glideSpeed: 0.5, + isDisabled: false, + startTime: z.startSec, + endTime: z.endSec, + isSystem: false, + hasInstantAnimation: false, +}); + +export default { + id: "screen-studio", + displayName: "Screen Studio", + vendor: "Screen Studio", + kind: "gui", + automation: "cdp+menu", + processName: "Screen Studio", + appPath: APP, + bundleId: "com.timpler.screenstudio", + install: { + method: "dmg", + url: "https://screenstudioassets.com/releases/3.7.5-4595/Screen%20Studio%203.7.5-4595%20Apple%20Silicon.dmg", + version: "3.7.5-4595", + appName: "Screen Studio.app", + approxMB: 349, + licence: "commercial — a licence is REQUIRED to export; there is no trial export", + }, + + detect() { + if (!existsSync(APP)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* keep null */ + } + return { installed: true, version, path: APP }; + }, + + defaultPaddingControl(scenario) { + // `backgroundPaddingRatio` is a percentage-like scale; calibration solves the exact value. + return scenario.effects.paddingPercent * 2; + }, + + async prepare(ctx) { + if (appIsRunning(this.processName)) await quitApp(this.processName, { force: true }); + await sleep(2000); + execFileSync("/bin/sh", [ + "-c", + `nohup ${JSON.stringify(BIN)} --remote-debugging-port=${PORT} >/dev/null 2>&1 &`, + ]); + await sleep(12000); + + // Import: File → "Create project from video…" raises a standard open panel. + activateApp(this.processName); + await sleep(800); + clickMenuItem(this.processName, "File", ["Create project from video"]); + await sleep(2500); + osa(`tell application "System Events" to tell process "${this.processName}" + set frontmost to true + keystroke "g" using {command down, shift down} + delay 0.8 + keystroke "${ctx.source.path}" + delay 0.6 + key code 36 + delay 1.2 + key code 36 + end tell`); + await sleep(25000); + + // Find the project the import just created and write the scenario into it. + const dirs = readdirSync(PROJECTS) + .filter((d) => d.endsWith(".screenstudio")) + .map((d) => ({ d, m: readFileSync })) + .map(({ d }) => join(PROJECTS, d)); + if (!dirs.length) throw new Error(`no .screenstudio project appeared in ${PROJECTS}`); + const project = dirs.sort()[dirs.length - 1]; + const file = join(project, "project.json"); + const doc = JSON.parse(readFileSync(file, "utf8")); + const e = ctx.scenario.effects; + + doc.json.config.backgroundType = "color"; + doc.json.config.backgroundColor = e.background.color; + doc.json.config.backgroundImage = null; + doc.json.config.backgroundBlur = 0; + doc.json.config.backgroundPaddingRatio = + ctx.paddingControl ?? this.defaultPaddingControl(ctx.scenario); + doc.json.config.windowBorderRadius = e.cornerRadiusPx; + doc.json.config.shadowIntensity = e.shadow?.enabled ? e.shadow.intensity : 0; + doc.json.config.hideCamera = true; + doc.json.config.motionBlurAmount = e.motionBlur ? 1 : 0; + doc.json.scenes[0].zoomRanges = (e.zooms ?? []).map(zoomRange); + writeFileSync(file, JSON.stringify(doc, null, 2)); + ctx.state.projectPath = project; + + // Reopen so the app reads what was just written. + clickMenuItem(this.processName, "File", ["Open last project"]); + await sleep(12000); + + const target = (await listTargets(PORT)).find((t) => t.type === "page"); + const s = new CdpSession(target.webSocketDebuggerUrl); + await s.open(); + await s.eval(DOM_HELPERS); + ctx.state.cdp = s; + + return { + appliedFeatures: [ + "background", + "padding", + "cornerRadius", + "shadow", + "zooms", + "targetResolution", + "targetFps", + ], + notes: [ + `project: ${project}`, + `${(e.zooms ?? []).length} zoom ranges written into scenes[0].zoomRanges`, + "Screen Studio re-encodes the source on import (its own display track), so its decoder input differs from the other apps'.", + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const s = ctx.state.cdp; + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + + const clicked = JSON.parse(await s.eval(`JSON.stringify(window.__osbench.click("Export"))`)); + if (!clicked.ok) throw new Error("Screen Studio: no Export control in the editor"); + await sleep(3000); + + // An unactivated install answers Export with an activation wall rather than a dialog. + const wall = await s.eval(`(() => { + for (const t of [...document.querySelectorAll("body")]) { + if (/Activate Screen Studio/i.test(t.innerText)) return "activation-required"; + } + return ""; + })()`); + if (wall === "activation-required") { + throw new Error( + "Screen Studio requires an activated licence to export — no trial export exists. " + + "Activate it once during preflight and re-run; every other step of this driver is unattended.", + ); + } + + // With a licence, the dialog's controls carry their values as visible text. + const t = ctx.scenario.output; + for (const label of ["MP4", `${t.height}p`, String(t.fps)]) { + await s.eval(`JSON.stringify(window.__osbench.click(${JSON.stringify(label)}))`); + await sleep(400); + } + await s.eval(`JSON.stringify(window.__osbench.click("Export"))`); + await sleep(2500); + + const dir = out.replace(/\/[^/]+$/, ""); + const stem = out + .split("/") + .pop() + .replace(/\.mp4$/i, ""); + osa(`tell application "System Events" to tell process "${this.processName}" + set frontmost to true + keystroke "g" using {command down, shift down} + delay 0.8 + keystroke "${dir}" + delay 0.5 + key code 36 + delay 1.0 + keystroke "a" using {command down} + keystroke "${stem}" + delay 0.4 + key code 36 + end tell`); + ctx.commit(); + }, + + async cleanup(ctx) { + ctx.state?.cdp?.close(); + if (appIsRunning(this.processName)) await quitApp(this.processName, { force: true }); + }, +}; diff --git a/benchmark/lib/assets.mjs b/benchmark/lib/assets.mjs new file mode 100644 index 00000000..576cb867 --- /dev/null +++ b/benchmark/lib/assets.mjs @@ -0,0 +1,288 @@ +/** + * The rest of a real recording: a wallpaper, a webcam track, and cursor telemetry. + * + * The first version of this benchmark fed the apps a screen clip and nothing else, and that + * measured the wrong thing. An export in this category is not a transcode with a coloured + * border — it is a compositor pass that samples a background image, transforms and masks the + * recording, renders a *synthetic* cursor from telemetry with smoothing and motion blur, draws + * a webcam inset with its own mask and shadow, and motion-blurs the whole thing. Leave the + * cursor and the camera out and the expensive half of the pipeline never runs. + * + * Two consequences shape this file: + * + * 1. **The cursor must not be drawn into the source.** These apps hide the system pointer while + * recording and re-render it at export time from a sidecar. Baking a cursor into the pixels + * would exercise nothing and would double-draw once an app rendered its own. So the + * trajectory is generated here, written in each app's telemetry format by its driver, and + * the screen clip is left clean. + * 2. **Everything is generated from the same seed**, so a second machine reproduces the whole + * bundle — wallpaper, webcam and cursor path included — and can prove it by hash. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { resolveFfmpeg } from "./env.mjs"; +import { sha256 } from "./fixture.mjs"; +import { pickH264Encoder } from "./platform.mjs"; + +function run(bin, args) { + try { + return execFileSync(bin, args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); + } catch (e) { + const msg = (e.stderr?.toString() || e.stdout?.toString() || e.message).trim(); + throw new Error(`${bin.split(/[/\\]/).pop()} failed (exit ${e.status}):\n${msg}`); + } +} + +/* -------------------------------------------------------------------- wallpaper ---------- */ + +/** + * A background the compositor has to *sample*, not fill. + * + * A flat colour is a single clear; an image is a texture upload and a per-pixel fetch for the + * whole frame, every frame — which is what the apps' own wallpapers cost. Deliberately light, + * so the dark recording's edge stays findable by the geometry verifier. + */ +export function buildWallpaper(workDir, spec) { + const { ffmpeg } = resolveFfmpeg(); + const dir = join(workDir, "fixture"); + mkdirSync(dir, { recursive: true }); + const out = join(dir, `${spec.name}.wallpaper.png`); + const jpg = join(dir, `${spec.name}.wallpaper.jpg`); + if (existsSync(out) && existsSync(jpg)) { + return { path: out, jpeg: jpg, sha256: sha256(out), regenerated: false }; + } + + // A soft diagonal gradient with a few large translucent discs — visually plausible as a + // product-demo backdrop, and high-frequency enough that a sampler cannot shortcut it. + const w = spec.width; + const h = spec.height; + const discs = [ + [0.18, 0.24, 0.3, "0xffffff@0.16"], + [0.74, 0.18, 0.22, "0xf3d9c0@0.22"], + [0.62, 0.78, 0.34, "0xc9d7ee@0.20"], + [0.3, 0.86, 0.18, "0xffffff@0.12"], + ] + .map(([cx, cy, r, color]) => { + const rr = Math.round(r * Math.min(w, h)); + const x = Math.round(cx * w - rr); + const y = Math.round(cy * h - rr); + // drawbox has no ellipse; a stack of inset boxes reads as a soft blob once blurred. + return `drawbox=x=${x}:y=${y}:w=${rr * 2}:h=${rr * 2}:color=${color}:t=fill`; + }) + .join(","); + + run(ffmpeg, [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + `gradients=s=${w}x${h}:c0=0x2b3a55:c1=0xd9c9b4:x0=0:y0=0:x1=${w}:y1=${h}:n=2`, + "-vf", + `${discs},gblur=sigma=${Math.round(Math.min(w, h) / 12)},format=rgb24`, + "-frames:v", + "1", + out, + ]); + // A JPEG copy, for apps that take the wallpaper inline rather than by path. + run(ffmpeg, ["-hide_banner", "-loglevel", "error", "-y", "-i", out, "-q:v", "6", jpg]); + return { path: out, jpeg: jpg, sha256: sha256(out), regenerated: true }; +} + +/* ---------------------------------------------------------------------- webcam ----------- */ + +/** + * A webcam track: a person-shaped subject that moves, on a backdrop. + * + * The point is not realism, it is cost — a second video stream to decode, scale, mask into a + * rounded or circular inset, and drop a shadow behind, for every frame. Small and 30 fps + * because that is what webcams actually deliver, and a driver that scales it to the screen + * clip's 60 fps is doing the work a real project would. + */ +export function buildWebcam(workDir, spec) { + const { ffmpeg } = resolveFfmpeg(); + const enc = pickH264Encoder(ffmpeg); + const dir = join(workDir, "fixture"); + mkdirSync(dir, { recursive: true }); + const out = join(dir, `${spec.name}.webcam.mp4`); + if (existsSync(out)) return { path: out, sha256: sha256(out), regenerated: false }; + + const W = 1280; + const H = 720; + const fps = 30; + // Head and shoulders that drift and breathe, so no two frames are identical and the + // encoder cannot coast — a static webcam would cost almost nothing to composite. + const headX = `${W / 2}-170+40*sin(2*PI*t/9)`; + const headY = `${H / 2}-120+22*sin(2*PI*t/5)`; + const filter = [ + `color=c=0x1d2430:s=${W}x${H}:r=${fps}:d=${spec.durationSec}`, + `drawbox=x=0:y=0:w=${W}:h=${H}:color=0x243044@1:t=fill`, + // backdrop pool of light + `drawbox=x=${Math.round(W * 0.2)}:y=0:w=${Math.round(W * 0.6)}:h=${H}:color=0x2e3b52@0.8:t=fill`, + // shoulders + `drawbox=x='${W / 2}-300+40*sin(2*PI*t/9)':y=${Math.round(H * 0.72)}:w=600:h=${Math.round(H * 0.3)}:color=0x3a4a63@1:t=fill`, + // head + `drawbox=x='${headX}':y='${headY}':w=340:h=380:color=0xd8b49a@1:t=fill`, + // hair + `drawbox=x='${headX}':y='${headY}':w=340:h=90:color=0x3b2f2a@1:t=fill`, + // eyes, which blink on a 4 s cycle + `drawbox=x='${headX}+80':y='${headY}+170':w=42:h=22:color=0x2a2320@1:t=fill:enable='gt(mod(t\\,4)\\,0.18)'`, + `drawbox=x='${headX}+218':y='${headY}+170':w=42:h=22:color=0x2a2320@1:t=fill:enable='gt(mod(t\\,4)\\,0.18)'`, + // mouth, moving as if speaking + `drawbox=x='${headX}+130':y='${headY}+270':w=80:h='14+10*abs(sin(2*PI*2.7*t))':color=0x8c4a44@1:t=fill`, + "format=yuv420p", + ].join(","); + + run(ffmpeg, [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + filter, + "-t", + String(spec.durationSec), + "-r", + String(fps), + "-c:v", + enc.encoder, + ...enc.rateArgs(6), + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + out, + ]); + return { path: out, sha256: sha256(out), regenerated: true, width: W, height: H, fps }; +} + +/* ---------------------------------------------------------------------- cursor ----------- */ + +/** + * The cursor trajectory, as data. + * + * Deterministic from the spec's seed, sampled at a realistic rate, and shaped like real use: + * long smooth glides, short pauses, and clicks at the pauses — which is exactly the signal the + * apps' smoothing, click effects and dwell-based auto-zoom react to. A straight-line sweep + * would let a smoothing implementation do nothing and cost nothing. + * + * Positions are normalised (0-1) against the screen frame, matching every format that consumes + * them; each driver translates this into its app's own sidecar. + */ +export function cursorTrack(spec, { sampleHz = 60 } = {}) { + const samples = []; + const total = Math.round(spec.durationSec * sampleHz); + // Dwell points the pointer travels between — a plausible tour of a UI. + const stops = [ + [0.12, 0.18], + [0.46, 0.32], + [0.78, 0.24], + [0.62, 0.66], + [0.24, 0.74], + [0.52, 0.48], + [0.86, 0.62], + [0.3, 0.36], + ]; + const legMs = (spec.durationSec * 1000) / stops.length; + const glideFraction = 0.62; // the rest of each leg is a pause + + for (let i = 0; i < total; i++) { + const timeMs = Math.round((i / sampleHz) * 1000); + const leg = Math.min(stops.length - 1, Math.floor(timeMs / legMs)); + const within = (timeMs % legMs) / legMs; + const from = stops[leg]; + const to = stops[(leg + 1) % stops.length]; + + let cx; + let cy; + let interactionType = "move"; + if (within < glideFraction) { + // Ease-in-out along the leg: acceleration is what smoothing has to work on. + const u = within / glideFraction; + const e = u < 0.5 ? 2 * u * u : 1 - (-2 * u + 2) ** 2 / 2; + cx = from[0] + (to[0] - from[0]) * e; + cy = from[1] + (to[1] - from[1]) * e; + } else { + cx = to[0]; + cy = to[1]; + // One click just after arriving, then stillness — the shape click effects expect. + const sincePause = (within - glideFraction) * legMs; + if (sincePause >= 120 && sincePause < 120 + 1000 / sampleHz) interactionType = "click"; + else if (sincePause >= 220 && sincePause < 220 + 1000 / sampleHz) interactionType = "mouseup"; + } + samples.push({ + timeMs, + cx: +cx.toFixed(5), + cy: +cy.toFixed(5), + visible: true, + interactionType, + }); + } + return samples; +} + +/** OpenScreen reads `.cursor.json`; schema version 2, normalised coordinates. */ +export function writeOpenscreenCursor(screenVideoPath, spec) { + const path = `${screenVideoPath}.cursor.json`; + writeFileSync( + path, + `${JSON.stringify({ version: 2, provider: "native", samples: cursorTrack(spec) }, null, 0)}\n`, + ); + return path; +} + +/** + * Cap stores its pointer track as a JSON array of `{ process_time_ms, x, y, ... }` beside the + * segment, referenced by `cursor` in recording-meta.json. Coordinates are normalised, as in + * `cap-project`'s `CursorEvents`. + */ +export function writeCapCursor(projectDir, spec) { + const path = join(projectDir, "content", "cursor.json"); + const track = cursorTrack(spec); + // Field names come from cap-project's CursorMoveEvent / CursorClickEvent: `time_ms`, not + // the `process_time_ms` a recording's raw log uses. Getting this wrong is silent — Cap + // parses the file, finds no usable events, and exports with no pointer at all. + const moves = track.map((s) => ({ + active_modifiers: [], + cursor_id: "0", + time_ms: s.timeMs, + x: s.cx, + y: s.cy, + })); + const clicks = track + .filter((s) => s.interactionType === "click" || s.interactionType === "mouseup") + .map((s) => ({ + active_modifiers: [], + cursor_num: 0, + cursor_id: "0", + time_ms: s.timeMs, + down: s.interactionType === "click", + })); + writeFileSync(path, `${JSON.stringify({ clicks, moves }, null, 0)}\n`); + return path; +} + +/** + * The wallpaper as a `data:` URI. + * + * OpenScreen's renderer would not load a `file://` wallpaper from outside its own resources — + * the export came out on a black background with no error at all — and a data URI sidesteps + * the question entirely: no media-path rule, no protocol handler, and the same string works on + * Windows, where a bare drive path is read as a colour rather than a path. At ~37 KB of JPEG + * it costs nothing to inline. + */ +export function wallpaperDataUri(wallpaper) { + const file = wallpaper?.jpeg ?? wallpaper?.path ?? wallpaper; + if (typeof file !== "string") { + throw new Error( + "wallpaperDataUri needs a path, a {path} or a {jpeg} — got an object with neither", + ); + } + const mime = /\.png$/i.test(file) ? "image/png" : "image/jpeg"; + return `data:${mime};base64,${readFileSync(file).toString("base64")}`; +} diff --git a/benchmark/lib/calibrate.mjs b/benchmark/lib/calibrate.mjs new file mode 100644 index 00000000..93314a63 --- /dev/null +++ b/benchmark/lib/calibrate.mjs @@ -0,0 +1,150 @@ +/** + * Making the apps composite the same rectangle. + * + * Every app in this set has a "padding" control, and no two of them are on the same scale: + * asking each for "5" produced a 1.85% inset in Cap and a 10% inset in OpenScreen — a 44% + * difference in the number of source pixels being sampled per frame. That is a confound, not a + * result, so before the real run each app's control is solved for the value that yields the + * scenario's inset. + * + * The solve is a secant search on a deliberately short clip: two probes to establish the app's + * (usually near-linear) mapping, then up to two refinements. Everything is measured from the + * rendered pixels, never from what the app claims, and the outcome is written to + * benchmark/calibration.json so a run is reproducible without repeating it. + */ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { BENCH_ROOT, machineFingerprint } from "./env.mjs"; +import { buildFixture, DEFAULT_SPEC, probe } from "./fixture.mjs"; +import { waitForStableFile } from "./measure.mjs"; +import { inspectExport } from "./visualCheck.mjs"; + +export const CALIBRATION_PATH = join(BENCH_ROOT, "calibration.json"); + +export function loadCalibration() { + if (!existsSync(CALIBRATION_PATH)) return {}; + try { + return JSON.parse(readFileSync(CALIBRATION_PATH, "utf8")); + } catch { + return {}; + } +} + +/** A short clip: the geometry of the composition does not depend on how long the clip is. */ +export function calibrationFixture(workDir, log = () => undefined) { + const spec = { ...DEFAULT_SPEC, name: "calib-1080p60-4s", durationSec: 4 }; + return buildFixture(workDir, spec, { log }); +} + +async function measureInset(driver, ctx, paddingControl) { + await driver.prepare({ ...ctx, paddingControl }); + const out = driver.outputPath(ctx); + let committed = false; + await driver.runExport({ + ...ctx, + paddingControl, + commit: () => { + committed = true; + }, + }); + const wait = await waitForStableFile(out, { timeoutMs: 10 * 60 * 1000, stableMs: 1200 }); + if (!wait.ok) throw new Error(`calibration export produced nothing (${wait.reason})`); + const p = probe(out); + const v = inspectExport(out, ctx.scenario, { probe: p }); + const inset = v.measured?.insetPercentShortSide; + if (inset == null) throw new Error("could not measure the content box"); + return { inset, box: v.measured.contentBox, checks: v.checks, committed }; +} + +/** + * Solve one app's padding control for the scenario's target inset. + * Returns the chosen control value plus every probe, so the calibration file shows its work. + */ +export async function calibrateApp( + driver, + ctx, + { tolerancePercent = 0.4, maxProbes = 4, log = () => undefined } = {}, +) { + const target = ctx.scenario.effects.paddingPercent; + if (!target) + return { + app: driver.id, + paddingControl: 0, + target, + probes: [], + reason: "no padding requested", + }; + if (typeof driver.defaultPaddingControl !== "function") { + return { + app: driver.id, + paddingControl: null, + target, + probes: [], + reason: "driver exposes no padding control", + }; + } + + const probes = []; + const seed = driver.defaultPaddingControl(ctx.scenario); + // Two points far enough apart to establish the slope without leaving the control's range. + let x0 = Math.max(0, seed * 0.5); + let x1 = seed; + + const run = async (x) => { + const m = await measureInset(driver, ctx, x); + probes.push({ control: +x.toFixed(2), inset: m.inset, box: m.box }); + log( + ` ${driver.id}: padding=${x.toFixed(2)} → inset ${m.inset}% (${m.box.width}×${m.box.height})`, + ); + return m.inset; + }; + + let y0 = await run(x0); + let y1 = await run(x1); + + for (let i = 0; i < maxProbes - 2; i++) { + const best = probes.reduce((a, b) => + Math.abs(a.inset - target) <= Math.abs(b.inset - target) ? a : b, + ); + if (Math.abs(best.inset - target) <= tolerancePercent) break; + if (y1 === y0) break; // control has no effect in this range; stop rather than divide by zero + // Secant step, clamped to a sane control range. + let x2 = x1 + ((target - y1) * (x1 - x0)) / (y1 - y0); + x2 = Math.max(0, Math.min(100, x2)); + if (!Number.isFinite(x2) || probes.some((p) => Math.abs(p.control - x2) < 0.05)) break; + const y2 = await run(x2); + x0 = x1; + y0 = y1; + x1 = x2; + y1 = y2; + } + + const best = probes.reduce((a, b) => + Math.abs(a.inset - target) <= Math.abs(b.inset - target) ? a : b, + ); + return { + app: driver.id, + target, + paddingControl: best.control, + achievedInsetPercent: best.inset, + achievedBox: best.box, + withinTolerance: Math.abs(best.inset - target) <= tolerancePercent, + probes, + }; +} + +export function saveCalibration(entries, meta) { + mkdirSync(BENCH_ROOT, { recursive: true }); + const m = machineFingerprint(); + const doc = { + generatedAt: new Date().toISOString(), + // Stamped so `run` can tell a calibration made here from one that travelled with the + // repo. The padding a control produces is a property of the app, not the machine, but + // app versions differ between machines and a silently stale solve is worse than none. + machine: { chip: m.chip, osVersion: m.osVersion, model: m.model }, + ...meta, + apps: Object.fromEntries(entries.map((e) => [e.app, e])), + }; + writeFileSync(CALIBRATION_PATH, `${JSON.stringify(doc, null, 2)}\n`); + return CALIBRATION_PATH; +} diff --git a/benchmark/lib/cdp.mjs b/benchmark/lib/cdp.mjs new file mode 100644 index 00000000..9bc505f1 --- /dev/null +++ b/benchmark/lib/cdp.mjs @@ -0,0 +1,204 @@ +/** + * Chrome DevTools Protocol client — the way into the Electron apps whose UI nothing else can + * reach. + * + * Screen Studio marks its editor window `kCGWindowSharingNone`, so macOS excludes it from every + * capture API: no screenshot, and therefore no pixel clicking. Its UI is a web view that + * publishes no accessibility tree either, so `System Events` sees a window with three + * traffic-light buttons and nothing else. The menu bar is scriptable, but the export dialog is + * not on it. + * + * Launching the app with `--remote-debugging-port` puts its own renderer within reach: the + * export button can be found by its text and clicked, exactly as a user would, with no + * coordinates involved. That is a *more* reproducible interaction than clicking pixels, not a + * less reproducible one — it survives a different display, a moved window and a resized UI. + * + * What it does not do is change the app: the flag only opens an inspector, the renderer and the + * export pipeline are the shipping ones, and every click goes through the app's own handlers. + * Runs driven this way are recorded as `automation: "cdp"` so a reader can weigh that. + * + * Uses Node's built-in WebSocket (Node 22+); no dependency is added to the repo. + */ + +export class CdpError extends Error {} + +const httpJson = async (port, path) => { + const res = await fetch(`http://127.0.0.1:${port}${path}`, { signal: AbortSignal.timeout(8000) }); + if (!res.ok) throw new CdpError(`CDP ${path} → HTTP ${res.status}`); + return res.json(); +}; + +export async function listTargets(port) { + return httpJson(port, "/json/list"); +} + +/** Wait for a page target whose url or title matches, e.g. the app's index.html. */ +export async function waitForTarget(port, match, { timeoutMs = 60_000, pollMs = 500 } = {}) { + const re = match instanceof RegExp ? match : new RegExp(match, "i"); + const t0 = Date.now(); + let lastSeen = []; + while (Date.now() - t0 < timeoutMs) { + try { + const targets = await listTargets(port); + lastSeen = targets.map((t) => `${t.type}:${t.url}`); + const hit = targets.find( + (t) => t.type === "page" && (re.test(t.url) || re.test(t.title ?? "")), + ); + if (hit) return hit; + } catch { + /* the app may not be listening yet */ + } + await new Promise((r) => setTimeout(r, pollMs)); + } + throw new CdpError( + `no CDP page matched ${re} on port ${port} within ${timeoutMs}ms. Saw: ${lastSeen.join(", ")}`, + ); +} + +export class CdpSession { + constructor(wsUrl) { + this.wsUrl = wsUrl; + this.id = 0; + this.pending = new Map(); + this.ws = null; + } + + static async attach(port, match, opts) { + const target = await waitForTarget(port, match, opts); + const s = new CdpSession(target.webSocketDebuggerUrl); + await s.open(); + return s; + } + + open() { + return new Promise((resolve, reject) => { + this.ws = new WebSocket(this.wsUrl); + const timer = setTimeout(() => reject(new CdpError("CDP websocket timed out")), 15_000); + this.ws.addEventListener("open", () => { + clearTimeout(timer); + resolve(); + }); + this.ws.addEventListener("error", (e) => { + clearTimeout(timer); + reject(new CdpError(`CDP websocket error: ${e.message ?? e.type}`)); + }); + this.ws.addEventListener("message", (ev) => { + let msg; + try { + msg = JSON.parse(ev.data); + } catch { + return; + } + const p = this.pending.get(msg.id); + if (!p) return; + this.pending.delete(msg.id); + if (msg.error) + p.reject( + new CdpError(`${msg.error.message}${msg.error.data ? ` — ${msg.error.data}` : ""}`), + ); + else p.resolve(msg.result); + }); + }); + } + + send(method, params = {}, { timeoutMs = 120_000 } = {}) { + const id = ++this.id; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(id); + reject(new CdpError(`${method} timed out after ${timeoutMs}ms`)); + }, timeoutMs); + this.pending.set(id, { + resolve: (v) => { + clearTimeout(timer); + resolve(v); + }, + reject: (e) => { + clearTimeout(timer); + reject(e); + }, + }); + this.ws.send(JSON.stringify({ id, method, params })); + }); + } + + /** Evaluate an expression in the page and return its JSON value. */ + async eval(expression, { awaitPromise = true, timeoutMs = 120_000 } = {}) { + const r = await this.send( + "Runtime.evaluate", + { expression, returnByValue: true, awaitPromise, userGesture: true }, + { timeoutMs }, + ); + if (r.exceptionDetails) { + const d = r.exceptionDetails; + throw new CdpError(d.exception?.description ?? d.text ?? "evaluation failed"); + } + return r.result?.value; + } + + close() { + try { + this.ws?.close(); + } catch { + /* already gone */ + } + } +} + +/** + * A DOM helper injected into the page: find elements by visible text, which is the only + * selector that survives an app's next release. Returns a description rather than a handle so + * the caller can log exactly what it matched. + */ +export const DOM_HELPERS = ` +(() => { + if (window.__osbench) return "already"; + const visible = (el) => { + const r = el.getBoundingClientRect(); + if (r.width < 1 || r.height < 1) return false; + const s = getComputedStyle(el); + return s.visibility !== "hidden" && s.display !== "none" && s.opacity !== "0"; + }; + const text = (el) => (el.innerText || el.textContent || el.getAttribute("aria-label") || el.title || "").trim(); + window.__osbench = { + visible, text, + /** Every clickable thing on screen, with its text — the discovery call. */ + controls() { + const sel = 'button,[role="button"],a,[role="menuitem"],[role="tab"],input,select,label,[data-testid]'; + return [...document.querySelectorAll(sel)].filter(visible).map((el, i) => ({ + i, tag: el.tagName.toLowerCase(), type: el.type || null, + role: el.getAttribute("role"), testid: el.getAttribute("data-testid"), + text: text(el).slice(0, 80), value: el.value ?? null, + disabled: !!el.disabled, + rect: (({x,y,width,height}) => ({x:Math.round(x),y:Math.round(y),w:Math.round(width),h:Math.round(height)}))(el.getBoundingClientRect()), + })); + }, + find(needle, { exact = false, tag = null } = {}) { + const n = needle.toLowerCase(); + const sel = tag || 'button,[role="button"],a,[role="menuitem"],[role="tab"],label,div,span,[data-testid]'; + const hits = [...document.querySelectorAll(sel)].filter(visible).filter((el) => { + const t = text(el).toLowerCase(); + return exact ? t === n : t.includes(n); + }); + // Prefer the smallest match: the innermost element carrying the text, not its container. + hits.sort((a, b) => (a.getBoundingClientRect().width * a.getBoundingClientRect().height) - + (b.getBoundingClientRect().width * b.getBoundingClientRect().height)); + return hits[0] || null; + }, + click(needle, opts) { + const el = this.find(needle, opts); + if (!el) return { ok: false, reason: "not found", needle }; + const target = el.closest("button,[role='button'],a,[role='menuitem'],label") || el; + const r = target.getBoundingClientRect(); + for (const type of ["pointerdown", "mousedown", "pointerup", "mouseup", "click"]) { + target.dispatchEvent(new MouseEvent(type, { + bubbles: true, cancelable: true, view: window, + clientX: r.x + r.width / 2, clientY: r.y + r.height / 2, + })); + } + return { ok: true, matched: this.text(target).slice(0, 80), rect: { x: Math.round(r.x), y: Math.round(r.y) } }; + }, + }; + return "installed"; +})() +`; diff --git a/benchmark/lib/env.mjs b/benchmark/lib/env.mjs new file mode 100644 index 00000000..e015ec8e --- /dev/null +++ b/benchmark/lib/env.mjs @@ -0,0 +1,179 @@ +/** + * Environment discovery for the export benchmark. + * + * Everything the report needs in order to be comparable across machines lives + * here: the hardware fingerprint, the power/thermal preconditions, and the + * ffmpeg/ffprobe pair used for fixture generation and output verification. + */ +import { execFileSync, execSync } from "node:child_process"; +import { chmodSync, existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + diskFreeGiB, + IS_WIN, + machineFingerprint as platformFingerprint, + powerState as platformPower, +} from "./platform.mjs"; + +export const BENCH_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +export const REPO_ROOT = resolve(BENCH_ROOT, ".."); +export const CACHE_DIR = join(BENCH_ROOT, ".cache"); +export const RESULTS_DIR = join(BENCH_ROOT, "results"); +export const WORK_DIR = + process.env.OSBENCH_WORK_DIR || join(os.homedir(), "openscreen-export-benchmark"); + +export const sh = (cmd) => + execSync(cmd, { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); + +const trySh = (cmd, fallback = null) => { + try { + return sh(cmd); + } catch { + return fallback; + } +}; + +/** Deep-search a directory tree for the vendored LGPL ffmpeg prefix built for the compositor. */ +function findVendoredFfmpegPrefix() { + const roots = [join(REPO_ROOT, "crates", "thirdparty")]; + // The tree is gitignored, so it exists only in whichever checkout built it. From a + // worktree that is the *main* checkout, which `--git-common-dir` points at. + const commonDir = trySh("git -C " + JSON.stringify(REPO_ROOT) + " rev-parse --git-common-dir"); + if (commonDir) roots.push(join(resolve(REPO_ROOT, commonDir), "..", "crates", "thirdparty")); + // And sibling worktrees, which is where a freshly built copy usually lands. + const wtRoot = commonDir + ? join(resolve(REPO_ROOT, commonDir), "..", ".claude", "worktrees") + : null; + if (wtRoot && existsSync(wtRoot)) { + for (const wt of readdirSync(wtRoot)) roots.push(join(wtRoot, wt, "crates", "thirdparty")); + } + if (process.env.MAC_FFMPEG_DIR) roots.unshift(dirname(process.env.MAC_FFMPEG_DIR)); + for (const root of roots) { + if (!existsSync(root)) continue; + for (const entry of readdirSync(root)) { + const prefix = join(root, entry); + const bin = join(prefix, "bin", IS_WIN ? "ffmpeg.exe" : "ffmpeg"); + if (/^ffmpeg-/.test(entry) && existsSync(bin)) return prefix; + } + } + return null; +} + +/** + * The vendored ffmpeg is `--enable-shared` with a *stale* baked-in prefix, and macOS strips + * DYLD_* across any SIP-protected exec (`/bin/sh`, `/usr/bin/env`). Inheriting the variable + * therefore never works. A tiny wrapper that exports it inside its own process does, because + * the stripping only removes what was inherited. + */ +function writeDyldWrapper(name, binary, libDir) { + mkdirSync(CACHE_DIR, { recursive: true }); + const wrapper = join(CACHE_DIR, name); + writeFileSync( + wrapper, + `#!/bin/sh\n# generated by benchmark/lib/env.mjs — see AGENTS.md / macos-dev-toolchain\nexport DYLD_LIBRARY_PATH="${libDir}"\nexec "${binary}" "$@"\n`, + ); + chmodSync(wrapper, 0o755); + return wrapper; +} + +let cachedFfmpeg = null; + +/** Resolve an (ffmpeg, ffprobe) pair, preferring an explicit override, then PATH, then vendored. */ +export function resolveFfmpeg() { + if (cachedFfmpeg) return cachedFfmpeg; + + if (process.env.OSBENCH_FFMPEG && process.env.OSBENCH_FFPROBE) { + cachedFfmpeg = { + ffmpeg: process.env.OSBENCH_FFMPEG, + ffprobe: process.env.OSBENCH_FFPROBE, + source: "env:OSBENCH_FFMPEG", + }; + return cachedFfmpeg; + } + + const which = IS_WIN ? "where" : "command -v"; + const onPath = trySh(`${which} ffmpeg`)?.split("\n")[0]?.trim() || null; + const probeOnPath = trySh(`${which} ffprobe`)?.split("\n")[0]?.trim() || null; + if (onPath && probeOnPath) { + cachedFfmpeg = { ffmpeg: onPath, ffprobe: probeOnPath, source: "PATH" }; + return cachedFfmpeg; + } + + if (IS_WIN) { + // The repo vendors ffmpeg for the compositor on Windows too; scripts/fetch-ffmpeg.mjs + // puts it under crates/thirdparty. No wrapper is needed — Windows has no DYLD stripping. + const prefix = findVendoredFfmpegPrefix(); + if (prefix) { + cachedFfmpeg = { + ffmpeg: join(prefix, "bin", "ffmpeg.exe"), + ffprobe: join(prefix, "bin", "ffprobe.exe"), + source: `vendored:${prefix}`, + }; + return cachedFfmpeg; + } + throw new Error( + "No ffmpeg/ffprobe found. Install one on PATH (winget install Gyan.FFmpeg), " + + "or set OSBENCH_FFMPEG and OSBENCH_FFPROBE.", + ); + } + + const prefix = findVendoredFfmpegPrefix(); + if (prefix) { + const lib = join(prefix, "lib"); + cachedFfmpeg = { + ffmpeg: writeDyldWrapper("ffmpeg", join(prefix, "bin", "ffmpeg"), lib), + ffprobe: writeDyldWrapper("ffprobe", join(prefix, "bin", "ffprobe"), lib), + source: `vendored:${prefix}`, + }; + return cachedFfmpeg; + } + + throw new Error( + "No ffmpeg/ffprobe found. Install one on PATH, or set OSBENCH_FFMPEG and OSBENCH_FFPROBE, " + + "or build the repo's LGPL tree (see benchmark/README.md § ffmpeg).", + ); +} + +export function ffmpegVersion() { + const { ffmpeg, source } = resolveFfmpeg(); + const line = execFileSync(ffmpeg, ["-hide_banner", "-version"], { encoding: "utf8" }) + .split("\n")[0] + .trim(); + return { banner: line, source }; +} + +/** Everything about the host that can move a number in the results table. */ +export function machineFingerprint() { + return platformFingerprint(); +} + +/** + * Preconditions that silently skew an export benchmark: battery power caps the SoC, a + * power-saver plan caps it harder, and an already-throttled machine reports whatever the + * previous run left behind. + */ +export function powerState() { + return platformPower(); +} + +/** Free space on the volume that will hold the fixture and every export. */ +export function diskState(path = WORK_DIR) { + return diskFreeGiB(path); +} + +export function ensureWorkDirs() { + for (const d of [ + WORK_DIR, + CACHE_DIR, + RESULTS_DIR, + join(WORK_DIR, "fixture"), + join(WORK_DIR, "out"), + join(WORK_DIR, "projects"), + join(WORK_DIR, "installers"), + ]) { + mkdirSync(d, { recursive: true }); + } + return { WORK_DIR, CACHE_DIR, RESULTS_DIR }; +} diff --git a/benchmark/lib/fixture.mjs b/benchmark/lib/fixture.mjs new file mode 100644 index 00000000..094257a2 --- /dev/null +++ b/benchmark/lib/fixture.mjs @@ -0,0 +1,316 @@ +/** + * Deterministic source-clip generation. + * + * A benchmark that ships a 200 MB .mp4 is not reproducible — the file rots, and nobody can + * tell whether two machines measured the same work. So the source is *generated* from a spec + * plus a seed, with pure ffmpeg primitives, and fingerprinted afterwards. Two machines that + * agree on the fingerprint measured the same workload. + * + * The frame is built to look like a screen recording rather than a test pattern, because that + * is what changes an encoder's job: large static regions, dense sharp-edged "text", a small + * amount of localized motion, and a cursor. + */ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { resolveFfmpeg } from "./env.mjs"; +import { pickH264Encoder } from "./platform.mjs"; + +/** execFileSync, but a non-zero exit surfaces ffmpeg's own message instead of a byte dump. */ +function run(bin, args) { + try { + return execFileSync(bin, args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); + } catch (e) { + const msg = (e.stderr?.toString() || e.stdout?.toString() || e.message).trim(); + throw new Error(`${bin.split("/").pop()} failed (exit ${e.status}):\n${msg}`); + } +} + +/** xorshift32 — tiny, seeded, identical in every JS runtime. */ +function rng(seed) { + let x = seed >>> 0 || 0x9e3779b9; + return () => { + x ^= x << 13; + x >>>= 0; + x ^= x >> 17; + x ^= x << 5; + x >>>= 0; + return x / 0x100000000; + }; +} + +export const DEFAULT_SPEC = { + name: "ide-1080p60-60s", + width: 1920, + height: 1080, + fps: 60, + durationSec: 60, + seed: 20260825, + /** Target bitrate of the *source*. Screen recorders emit roughly this for 1080p30 UI. */ + sourceBitrateMbps: 12, +}; + +const PALETTE = [ + "0xd7dae0", + "0x89b4fa", + "0xa6e3a1", + "0xf9e2af", + "0xf38ba8", + "0xcba6f7", + "0x94e2d5", +]; + +/** Static "page" of code-like rows, tall enough to scroll through for the whole clip. */ +function pageFilter(spec, pageHeight) { + const rand = rng(spec.seed); + const left = 360; + const right = spec.width - 80; + const rowH = 12; + const rowGap = 26; + const boxes = []; + for (let y = 20; y < pageHeight - 40; y += rowGap) { + // Indentation in steps, like real code. + const indent = left + Math.floor(rand() * 5) * 28; + let x = indent; + const tokens = 2 + Math.floor(rand() * 7); + for (let t = 0; t < tokens && x < right - 40; t++) { + const w = Math.floor(30 + rand() * 190); + const color = PALETTE[Math.floor(rand() * PALETTE.length)]; + boxes.push( + `drawbox=x=${x}:y=${y}:w=${Math.min(w, right - x)}:h=${rowH}:color=${color}@0.92:t=fill`, + ); + x += w + 12 + Math.floor(rand() * 18); + } + // Gutter line numbers. + boxes.push(`drawbox=x=${left - 56}:y=${y + 2}:w=28:h=${rowH - 4}:color=0x585b70@0.8:t=fill`); + } + return boxes.join(","); +} + +/** Window chrome: title bar, sidebar rows, a status bar. Static, so it is baked once. */ +function chromeFilter(spec) { + const rand = rng(spec.seed ^ 0x5bf03635); + const b = [ + `drawbox=x=0:y=0:w=${spec.width}:h=44:color=0x11141a@1:t=fill`, + `drawbox=x=0:y=44:w=320:h=${spec.height - 44 - 32}:color=0x171b22@1:t=fill`, + `drawbox=x=0:y=${spec.height - 32}:w=${spec.width}:h=32:color=0x11141a@1:t=fill`, + ]; + for (const [i, c] of ["0xff5f57", "0xfebc2e", "0x28c840"].entries()) { + b.push(`drawbox=x=${18 + i * 22}:y=16:w=12:h=12:color=${c}@1:t=fill`); + } + // Tab strip. + let tx = 120; + for (let i = 0; i < 5; i++) { + const w = 110 + Math.floor(rand() * 70); + b.push(`drawbox=x=${tx}:y=12:w=${w}:h=20:color=${i === 1 ? "0x2a3040" : "0x1b1f27"}@1:t=fill`); + b.push(`drawbox=x=${tx + 12}:y=19:w=${w - 40}:h=7:color=0x9aa3b2@0.9:t=fill`); + tx += w + 8; + } + // Sidebar file tree. + for (let y = 70, i = 0; y < spec.height - 60; y += 30, i++) { + const indent = 24 + (i % 3) * 18; + b.push( + `drawbox=x=${indent}:y=${y}:w=${Math.floor(80 + rand() * 150)}:h=9:color=0x9aa3b2@0.75:t=fill`, + ); + } + // Status bar chips. + for (let i = 0, x = 20; i < 4; i++) { + const w = 70 + Math.floor(rand() * 80); + b.push(`drawbox=x=${x}:y=${spec.height - 22}:w=${w}:h=11:color=0x89b4fa@0.7:t=fill`); + x += w + 26; + } + return b.join(","); +} + +/** + * The animated layer. Kept deliberately small: a scrolling viewport, a caret, a selection + * band and a cursor. Screen recordings are mostly static, and an encoder benchmark that + * feeds full-frame motion measures a different workload entirely. + */ +function animationFilter(spec, pageHeight) { + const visibleH = spec.height - 44 - 32; + const scrollRange = Math.max(1, pageHeight - visibleH); + // Ease in/out so the scroll starts and stops, like a human dragging. + const scrollY = `(${scrollRange}*(0.5-0.5*cos(2*PI*t/${spec.durationSec})))`; + return { + scrollY, + overlays: [ + // Caret: blinks at 1 Hz. + `drawbox=x=380+mod(floor(t*7)\\,40)*14:y=${44 + Math.floor(visibleH / 2)}:w=3:h=18:color=0xffffff@1:t=fill:enable='lt(mod(t\\,1)\\,0.5)'`, + // Selection band sweeping down the pane. + `drawbox=x=360:y=${44}+mod(floor(t*2)*36\\,${visibleH - 40}):w=760:h=22:color=0x3b5bdb@0.35:t=fill`, + // No cursor is drawn here on purpose. Every app in this set hides the system pointer + // while recording and re-renders it at export time from a telemetry sidecar, with its + // own theme, smoothing and motion blur — which is a large part of what an export + // costs. Baking one in would exercise none of that and would double-draw once an app + // rendered its own. The trajectory lives in lib/assets.mjs → cursorTrack(). + ].join(","), + }; +} + +/** A deterministic voice-shaped audio bed: an AM-modulated tone with syllable-rate gating. */ +function audioFilter() { + return ( + "aevalsrc='0.28*sin(2*PI*(180+40*sin(2*PI*0.7*t))*t)" + + "*(0.45+0.55*sin(2*PI*3.1*t))" + + "*(0.25+0.75*lt(mod(floor(t*1.7),4),3))':s=48000:c=stereo" + ); +} + +export function fixturePath(workDir, spec) { + return join(workDir, "fixture", `${spec.name}.mp4`); +} + +/** ffprobe a media file into a compact, comparable descriptor. */ +export function probe(file) { + const { ffprobe } = resolveFfmpeg(); + const raw = execFileSync( + ffprobe, + ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", file], + { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }, + ); + const j = JSON.parse(raw); + const v = j.streams.find((s) => s.codec_type === "video"); + const a = j.streams.find((s) => s.codec_type === "audio"); + const num = (x) => (x == null ? null : Number(x)); + const fps = v?.avg_frame_rate?.includes("/") + ? +(Number(v.avg_frame_rate.split("/")[0]) / Number(v.avg_frame_rate.split("/")[1])).toFixed(3) + : null; + return { + durationSec: num(j.format?.duration), + sizeBytes: num(j.format?.size), + bitrateKbps: j.format?.bit_rate ? Math.round(Number(j.format.bit_rate) / 1000) : null, + container: j.format?.format_name ?? null, + video: v + ? { + codec: v.codec_name, + profile: v.profile ?? null, + width: v.width, + height: v.height, + fps, + pixFmt: v.pix_fmt, + nbFrames: num(v.nb_frames), + } + : null, + audio: a ? { codec: a.codec_name, sampleRate: num(a.sample_rate), channels: a.channels } : null, + }; +} + +export function sha256(file) { + return createHash("sha256").update(readFileSync(file)).digest("hex"); +} + +/** + * Build the source clip. Idempotent: an existing file whose probe matches the spec is reused, + * because regenerating it is several minutes and changes nothing. + */ +export function buildFixture( + workDir, + spec = DEFAULT_SPEC, + { force = false, log = () => undefined } = {}, +) { + const { ffmpeg } = resolveFfmpeg(); + // The encoder differs by platform and GPU vendor; picking it here keeps the fixture + // generatable everywhere while recording which one produced it. + const enc = pickH264Encoder(ffmpeg); + const out = fixturePath(workDir, spec); + mkdirSync(join(workDir, "fixture"), { recursive: true }); + + if (!force && existsSync(out)) { + const p = probe(out); + const ok = + p.video?.width === spec.width && + p.video?.height === spec.height && + Math.abs((p.durationSec ?? 0) - spec.durationSec) < 0.5; + if (ok) { + log(`fixture: reusing ${out}`); + return { path: out, spec, probe: p, sha256: sha256(out), regenerated: false }; + } + } + + // One scrolled page-height per 10 s of clip, so the scroll speed is spec-independent. + const pageHeight = Math.min(8192, (spec.height - 76) * 3); + const pagePng = join(workDir, "fixture", `${spec.name}.page.png`); + + log("fixture: rendering static layers"); + run(ffmpeg, [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + `color=c=0x1b1f27:s=${spec.width}x${pageHeight}:d=1`, + "-vf", + pageFilter(spec, pageHeight), + "-frames:v", + "1", + pagePng, + ]); + + const { scrollY, overlays } = animationFilter(spec, pageHeight); + const visibleH = spec.height - 44 - 32; + const filter = [ + `color=c=0x1b1f27:s=${spec.width}x${spec.height}:r=${spec.fps}:d=${spec.durationSec}[bg]`, + `[1:v]crop=${spec.width}:${visibleH}:0:'${scrollY}'[page]`, + `[bg][page]overlay=0:44:shortest=1[scrolled]`, + `[scrolled]${chromeFilter(spec)},${overlays},format=yuv420p[v]`, + ].join(";"); + + log(`fixture: encoding ${spec.durationSec}s @ ${spec.width}x${spec.height}${spec.fps}`); + const t0 = Date.now(); + run(ffmpeg, [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + audioFilter(), + "-loop", + "1", + "-i", + pagePng, + "-filter_complex", + filter, + "-map", + "[v]", + "-map", + "0:a", + "-t", + String(spec.durationSec), + "-r", + String(spec.fps), + "-c:v", + "h264_videotoolbox", + "-b:v", + `${spec.sourceBitrateMbps}M`, + "-profile:v", + "high", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-b:a", + "128k", + "-ar", + "48000", + "-movflags", + "+faststart", + out, + ]); + log(`fixture: encoded in ${((Date.now() - t0) / 1000).toFixed(1)}s using ${enc.encoder}`); + if (enc.note) log(`fixture: ${enc.note}`); + + return { + path: out, + spec, + probe: probe(out), + sha256: sha256(out), + regenerated: true, + encoder: enc.encoder, + }; +} diff --git a/benchmark/lib/install.mjs b/benchmark/lib/install.mjs new file mode 100644 index 00000000..98328dff --- /dev/null +++ b/benchmark/lib/install.mjs @@ -0,0 +1,202 @@ +/** + * Unattended installation of the competitor apps. + * + * Everything here runs *after* the single up-front approval collected by `preflight`, and + * nothing here can ask a question — a run is expected to continue with nobody at the keyboard. + * + * Note on Gatekeeper: `curl` does not set `com.apple.quarantine`, so an app fetched this way + * skips the "downloaded from the internet" first-launch prompt that would otherwise stall an + * unattended run. The quarantine flag is never stripped from anything — if a vendor ships an + * unnotarised build, that is recorded as a finding rather than worked around. + */ +import { execFileSync, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs"; +import { basename, join } from "node:path"; +import { IS_WIN, appVersion as platformVersion, powershell, signatureStatus } from "./platform.mjs"; + +const APPLICATIONS = "/Applications"; + +const run = (bin, args, opts = {}) => + execFileSync(bin, args, { encoding: "utf8", maxBuffer: 32 * 1024 * 1024, ...opts }); + +export const appVersion = platformVersion; + +/** Signing / notarisation status, recorded so the report can say what was actually run. */ +export const codesignStatus = signatureStatus; + +/** Resolve a GitHub release asset to a concrete URL, so the install is version-pinned. */ +export function resolveGithubAsset(repo, pattern) { + const json = run(IS_WIN ? "curl.exe" : "/usr/bin/curl", [ + "-fsSL", + "--max-time", + "40", + "-H", + "Accept: application/vnd.github+json", + `https://api.github.com/repos/${repo}/releases/latest`, + ]); + const rel = JSON.parse(json); + const asset = (rel.assets ?? []).find((a) => pattern.test(a.name)); + if (!asset) { + throw new Error( + `no asset in ${repo}@${rel.tag_name} matched ${pattern}. Present: ${(rel.assets ?? []).map((a) => a.name).join(", ")}`, + ); + } + return { + url: asset.browser_download_url, + version: rel.tag_name, + name: asset.name, + sizeBytes: asset.size, + }; +} + +/** + * Run a Windows installer unattended. + * + * Both vendors in the Windows set ship NSIS packages, where `/S` is the silent switch; an MSI + * would need `msiexec /qn` instead. If an installer rejects its silent switch it will sit on a + * dialog forever, so this is bounded and reports what it left behind rather than hanging a run. + */ +function runWindowsInstaller(installerPath, spec, { log = () => undefined }) { + const args = spec.silentArgs ?? ["/S"]; + log(` running ${basename(installerPath)} ${args.join(" ")}`); + powershell( + `$p = Start-Process -FilePath ${JSON.stringify(installerPath)} -ArgumentList @(${args + .map((a) => `'${a}'`) + .join(",")}) -PassThru -Wait + exit $p.ExitCode`, + { timeoutMs: 20 * 60 * 1000 }, + ); +} + +/** Resumable download. A 400 MB DMG over a flaky link should not restart from zero. */ +export function download(url, destDir, { log = () => undefined } = {}) { + mkdirSync(destDir, { recursive: true }); + // The vendor URL is often a redirect; ask curl for the effective name it lands on. + const curl = IS_WIN ? "curl.exe" : "/usr/bin/curl"; + const nul = IS_WIN ? "NUL" : "/dev/null"; + const effective = run(curl, [ + "-sIL", + "--max-time", + "60", + "-o", + "/dev/null", + "-w", + "%{url_effective}", + url, + ]).trim(); + let name = basename(new URL(effective).pathname) || basename(new URL(url).pathname); + if (!/\.(dmg|zip|pkg|exe|msi)$/i.test(name)) + name = `${name || "download"}${IS_WIN ? ".exe" : ".dmg"}`; + const dest = join(destDir, decodeURIComponent(name)); + + log(` downloading ${decodeURIComponent(name)}`); + run( + "/usr/bin/curl", + ["-fL", "--retry", "3", "--retry-delay", "2", "-C", "-", "--max-time", "1800", "-o", dest, url], + { stdio: ["ignore", "ignore", "inherit"] }, + ); + + const sha = createHash("sha256").update(readFileSync(dest)).digest("hex"); + return { path: dest, sizeBytes: statSync(dest).size, sha256: sha }; +} + +/** Mount a DMG, copy the .app out, unmount. Idempotent at the app level. */ +export function installDmg(dmgPath, appName, { log = () => undefined } = {}) { + const plist = run("/usr/bin/hdiutil", [ + "attach", + dmgPath, + "-nobrowse", + "-noverify", + "-noautoopen", + "-plist", + ]); + const mountPoint = /mount-point<\/key>\s*([^<]+)<\/string>/.exec(plist)?.[1]; + if (!mountPoint) throw new Error(`could not determine mount point for ${dmgPath}`); + + try { + const src = join(mountPoint, appName); + if (!existsSync(src)) { + const contents = run("/bin/ls", ["-1", mountPoint]).trim().split("\n"); + throw new Error( + `"${appName}" not found on the mounted image. Contents: ${contents.join(", ")}`, + ); + } + const dest = join(APPLICATIONS, appName); + if (existsSync(dest)) { + log(` replacing existing ${appName}`); + rmSync(dest, { recursive: true, force: true }); + } + log(` copying ${appName} → ${APPLICATIONS}`); + cpSync(src, dest, { recursive: true, verbatimSymlinks: true }); + return dest; + } finally { + try { + run("/usr/bin/hdiutil", ["detach", mountPoint, "-quiet"]); + } catch { + run("/usr/bin/hdiutil", ["detach", mountPoint, "-force", "-quiet"]); + } + } +} + +/** + * Install one app from its registry spec. Returns a record detailed enough that another + * machine can be checked against it — the whole point of pinning versions and hashes. + */ +export function installApp(spec, { cacheDir, force = false, log = () => undefined } = {}) { + // On Windows an app is wherever its installer put it, so the driver resolves it; on macOS + // it is always /Applications/.app. + const destApp = IS_WIN ? (spec.resolve?.() ?? null) : join(APPLICATIONS, spec.appName); + if (destApp && existsSync(destApp) && !force) { + return { + id: spec.id, + status: "already-installed", + appPath: destApp, + version: appVersion(destApp), + codesign: codesignStatus(destApp), + }; + } + + let url = spec.url; + let pinnedVersion = spec.version ?? null; + if (spec.method === "github-release") { + const asset = resolveGithubAsset(spec.repo, spec.assetPattern); + url = asset.url; + pinnedVersion = asset.version; + log(` resolved ${spec.repo} → ${asset.version} (${asset.name})`); + } + + const dl = download(url, cacheDir, { log }); + + let appPath; + if (IS_WIN) { + if (!/\.(exe|msi)$/i.test(dl.path)) { + throw new Error(`expected an .exe or .msi installer; got ${basename(dl.path)}`); + } + runWindowsInstaller(dl.path, spec, { log }); + appPath = spec.resolve ? spec.resolve() : null; + if (!appPath) { + throw new Error( + `${spec.appName} installed but its executable was not found where expected. ` + + "Add the real path to the driver's winPaths list.", + ); + } + } else { + if (!/\.dmg$/i.test(dl.path)) { + throw new Error(`only .dmg installs are automated on macOS; got ${basename(dl.path)}`); + } + appPath = installDmg(dl.path, spec.appName, { log }); + } + + return { + id: spec.id, + status: "installed", + appPath, + version: appVersion(appPath), + pinnedVersion, + sourceUrl: url, + downloadSha256: dl.sha256, + downloadBytes: dl.sizeBytes, + codesign: codesignStatus(appPath), + }; +} diff --git a/benchmark/lib/measure.mjs b/benchmark/lib/measure.mjs new file mode 100644 index 00000000..7ca0a7f0 --- /dev/null +++ b/benchmark/lib/measure.mjs @@ -0,0 +1,248 @@ +/** + * Measurement primitives. + * + * Three things have to be true for an export timing to mean anything: + * 1. The clock starts at the moment the export is *committed*, not when the app launched. + * 2. The clock stops when the output file is *complete*, not when a progress bar hits 100%. + * 3. The output is verified to be what was asked for — an app that quietly writes 720p, or a + * 12-second file from a 60-second source, is not faster, it is wrong. + * + * Everything here is app-agnostic on purpose: the same stopwatch is used for the CLI drivers + * and the UI drivers, so a CLI app is not credited for skipping a step a GUI app must do. + */ + +import { existsSync, statSync } from "node:fs"; +import { probe } from "./fixture.mjs"; +import { instantaneousLoadPercent, listProcesses, parseMacCpuTime } from "./platform.mjs"; + +export const now = () => Number(process.hrtime.bigint() / 1000n) / 1000; // ms, monotonic + +export const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +/* ------------------------------------------------------------------ process sampling ----- */ + +/** + * Cumulative CPU seconds and peak RSS for every process whose argv starts with `matchPrefix` + * (an app bundle path), summed across the tree. Sampling cumulative counters rather than + * instantaneous %CPU means a helper that exits mid-export still contributes its full cost. + */ +export class ProcessTreeSampler { + constructor(matchPrefixes, { intervalMs = 500 } = {}) { + this.matchPrefixes = [].concat(matchPrefixes).filter(Boolean); + this.intervalMs = intervalMs; + this.cpuByPid = new Map(); // pid -> max cumulative cpu seconds seen + this.peakRssBytes = 0; + this.samples = 0; + this.timer = null; + // Everything *else* on the machine. A remote-desktop session, a screen recorder or a + // build running alongside the benchmark inflates every export time without inflating any + // app's own CPU figure — so it is sampled and reported rather than assumed to be zero. + this.foreignCpuSamples = []; + } + + static parseCpuTime(t) { + // ps TIME is [[dd-]hh:]mm:ss[.ff] + const m = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+(?:\.\d+)?)$/.exec(t.trim()); + if (!m) return 0; + const [, d, h, mi, s] = m; + return Number(d || 0) * 86400 + Number(h || 0) * 3600 + Number(mi) * 60 + Number(s); + } + + sampleOnce() { + const procs = listProcesses(); + let rssSum = 0; + for (const { pid, rssBytes, cpuSeconds, args } of procs) { + if (!this.matchPrefixes.some((p) => args.includes(p))) continue; + const prev = this.cpuByPid.get(pid) ?? 0; + if (cpuSeconds > prev) this.cpuByPid.set(pid, cpuSeconds); + rssSum += rssBytes; + } + if (rssSum > this.peakRssBytes) this.peakRssBytes = rssSum; + this.samples++; + this.sampleForeignLoad(procs); + } + + /** Instantaneous %CPU of everything that is not the app under test, summed. */ + sampleForeignLoad(procs) { + const total = instantaneousLoadPercent(); + if (total == null) return; + // Subtract nothing: the figure is "how busy is this machine besides the measurement", + // and the app under test is a small share of it during an export. Callers read it as + // context, not as an exact complement. + this.foreignCpuSamples.push(total); + } + + start() { + this.sampleOnce(); + this.timer = setInterval(() => this.sampleOnce(), this.intervalMs); + this.timer.unref?.(); + return this; + } + + stop() { + if (this.timer) clearInterval(this.timer); + this.timer = null; + this.sampleOnce(); + return this.result(); + } + + result() { + let cpuSeconds = 0; + for (const v of this.cpuByPid.values()) cpuSeconds += v; + return { + cpuSeconds: +cpuSeconds.toFixed(2), + peakRssBytes: this.peakRssBytes, + peakRssMiB: +(this.peakRssBytes / 1024 ** 2).toFixed(1), + pidsSeen: this.cpuByPid.size, + samples: this.samples, + // Median rather than mean: one spike from a Spotlight index should not characterise + // a two-minute export. + foreignCpuPercent: this.foreignCpuSamples.length + ? +median(this.foreignCpuSamples).toFixed(1) + : null, + }; + } +} + +/** + * The CPU counters above are cumulative *since process start*, which for a long-lived GUI app + * includes the idle time before the export. Snapshot before, snapshot after, subtract. + */ +export function cpuDelta(before, after) { + return { + cpuSeconds: +Math.max(0, after.cpuSeconds - before.cpuSeconds).toFixed(2), + peakRssMiB: after.peakRssMiB, + }; +} + +/* ------------------------------------------------------------------- output watching ----- */ + +/** + * Resolve when `path` exists and has stopped growing for `stableMs`. + * + * Size stability is the only completion signal that works identically for a CLI that writes + * once and a GUI that muxes at the end. `stableMs` has to clear the longest plausible stall + * inside an export (a slow keyframe, a GC pause) without inflating the measurement — so the + * stable window is *subtracted back off* the reported time, and the last-growth timestamp is + * what the stopwatch actually reads. + */ +export async function waitForStableFile( + path, + { + timeoutMs = 45 * 60 * 1000, + // A render that started will put *something* on disk quickly. Nothing after this long + // means the export never began — a click that missed, a dialog that did not open — and + // waiting out the full render timeout turns one broken run into a lost hour. + appearTimeoutMs = 4 * 60 * 1000, + stableMs = 2500, + pollMs = 100, + minBytes = 4096, + onTick, + } = {}, +) { + const t0 = now(); + let lastSize = -1; + let lastGrowthAt = null; + let appearedAt = null; + + while (now() - t0 < timeoutMs) { + let size = -1; + try { + if (existsSync(path)) size = statSync(path).size; + } catch { + size = -1; + } + + if (size >= 0 && appearedAt === null) appearedAt = now(); + if (size > lastSize) { + lastSize = size; + lastGrowthAt = now(); + } + onTick?.({ size, elapsedMs: now() - t0 }); + + if (appearedAt === null && now() - t0 > appearTimeoutMs) { + return { + ok: false, + reason: "output never appeared", + appearedAt: null, + sizeBytes: -1, + waitedMs: now() - t0, + }; + } + if (lastSize >= minBytes && lastGrowthAt !== null && now() - lastGrowthAt >= stableMs) { + return { + ok: true, + appearedAt, + completedAt: lastGrowthAt, // the honest moment the last byte landed + sizeBytes: lastSize, + waitedMs: now() - t0, + }; + } + await sleep(pollMs); + } + return { ok: false, reason: "timeout", appearedAt, sizeBytes: lastSize, waitedMs: now() - t0 }; +} + +/* -------------------------------------------------------------------- verification ------- */ + +/** Does the produced file actually match what the scenario asked for? */ +export function verifyOutput(path, target, sourceDurationSec) { + if (!existsSync(path)) return { valid: false, reasons: ["output file missing"], probe: null }; + let p; + try { + p = probe(path); + } catch (e) { + return { valid: false, reasons: [`ffprobe failed: ${e.message}`], probe: null }; + } + + const reasons = []; + if (!p.video) reasons.push("no video stream"); + if (p.video && p.video.width !== target.width) { + reasons.push(`width ${p.video.width} != ${target.width}`); + } + if (p.video && p.video.height !== target.height) { + reasons.push(`height ${p.video.height} != ${target.height}`); + } + if (p.video?.codec && target.videoCodec && p.video.codec !== target.videoCodec) { + reasons.push(`codec ${p.video.codec} != ${target.videoCodec}`); + } + if (p.video?.fps != null) { + const drift = (Math.abs(p.video.fps - target.fps) / target.fps) * 100; + if (drift > target.tolerance.fpsPercent) reasons.push(`fps ${p.video.fps} != ${target.fps}`); + } + if (sourceDurationSec != null && p.durationSec != null) { + const d = Math.abs(p.durationSec - sourceDurationSec); + if (d > target.tolerance.durationSec) { + reasons.push( + `duration ${p.durationSec?.toFixed(2)}s vs source ${sourceDurationSec}s (Δ${d.toFixed(2)}s)`, + ); + } + } + return { valid: reasons.length === 0, reasons, probe: p }; +} + +/* ---------------------------------------------------------------------- run guards ------- */ + +/** + * Between repetitions the machine has to come back to the same state, or run 3 measures a + * hotter SoC than run 1 and the spread is thermal, not architectural. + */ +export async function cooldown({ seconds = 45, log = () => undefined } = {}) { + log(`cooldown: ${seconds}s`); + await sleep(seconds * 1000); +} + +/** Percentile helpers used by the report. Small n, so exact rather than interpolated. */ +export function median(xs) { + if (!xs.length) return null; + const s = [...xs].sort((a, b) => a - b); + const m = Math.floor(s.length / 2); + return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; +} + +/** Median absolute deviation — robust spread for n=3, where a stdev is mostly noise. */ +export function mad(xs) { + const m = median(xs); + if (m == null) return null; + return median(xs.map((x) => Math.abs(x - m))); +} diff --git a/benchmark/lib/measure.test.mjs b/benchmark/lib/measure.test.mjs new file mode 100644 index 00000000..5c8cf4ea --- /dev/null +++ b/benchmark/lib/measure.test.mjs @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { mad, median, ProcessTreeSampler } from "./measure.mjs"; + +describe("ProcessTreeSampler.parseCpuTime", () => { + // `ps` prints cumulative CPU as [[dd-]hh:]mm:ss[.ff]. Every one of these forms turns up in + // practice, and a missed one silently reports 0 CPU seconds for a busy process. + it.each([ + ["0:00.00", 0], + ["12:34.56", 754.56], + ["1:02:03", 3723], + ["1-02:03:04.55", 93784.55], + ])("parses %s", (input, expected) => { + expect(ProcessTreeSampler.parseCpuTime(input)).toBeCloseTo(expected, 2); + }); + + it("returns 0 rather than NaN for anything unparseable", () => { + expect(ProcessTreeSampler.parseCpuTime("-")).toBe(0); + expect(ProcessTreeSampler.parseCpuTime("")).toBe(0); + }); +}); + +describe("median and mad", () => { + it("takes the middle of an odd sample and the mean of the middle two of an even one", () => { + expect(median([3, 1, 2])).toBe(2); + expect(median([4, 1, 3, 2])).toBe(2.5); + }); + + it("is null for no samples, so a failed app cannot masquerade as a fast one", () => { + expect(median([])).toBeNull(); + expect(mad([])).toBeNull(); + }); + + it("reports spread as the median absolute deviation", () => { + expect(mad([10, 10, 10])).toBe(0); + expect(mad([8, 10, 12])).toBe(2); + }); + + it("is not dragged by a single outlier the way a mean would be", () => { + expect(median([10, 10, 10, 10, 1000])).toBe(10); + }); +}); diff --git a/benchmark/lib/openscreenProject.mjs b/benchmark/lib/openscreenProject.mjs new file mode 100644 index 00000000..ffbe8940 --- /dev/null +++ b/benchmark/lib/openscreenProject.mjs @@ -0,0 +1,143 @@ +/** + * Builds a `.openscreen` project that expresses a benchmark scenario. + * + * The project format is plain JSON (schemaVersion 6) and the exporter reads its effect state + * from `editor` — the same shape `ProjectEditorState` in + * `src/components/video-editor/projectPersistence.ts` describes. Writing it directly, rather + * than driving the editor UI, is what makes the OpenScreen leg reproducible; the GUI leg is + * measured separately by `drivers/openscreen-gui.mjs`. + */ +import { copyFileSync, mkdirSync, writeFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { wallpaperDataUri, writeOpenscreenCursor } from "./assets.mjs"; +import { probe } from "./fixture.mjs"; + +/** Deterministic ids: the same scenario always produces the same project bytes. */ +const id = (prefix, n) => `${prefix}_${String(n).padStart(8, "0")}`; + +/** OpenScreen stores zoom depth as a preset; a custom scale overrides it. */ +function toZoomRegion(z, i) { + return { + id: id("zoom", i + 1), + startMs: Math.round(z.startSec * 1000), + endMs: Math.round(z.endSec * 1000), + depth: 2, + customScale: +z.scale.toFixed(2), + focus: { cx: z.focus.x, cy: z.focus.y }, + focusMode: "manual", + source: "manual", + }; +} + +/** + * Padding: the scenario states an inset as a percent of the frame, OpenScreen's `padding` is + * 0-100 on its own scale where 50 is the default inset. The mapping below is calibrated so a + * 5% scenario inset lands on OpenScreen's equivalent visual inset; see benchmark/README.md + * § "Translating the scenario" for how each app's control was matched. + */ +const paddingFromPercent = (pct) => Math.round(Math.min(100, Math.max(0, pct * 10))); + +export function buildProject({ + sourcePath, + scenario, + outDir, + title = "export-benchmark", + paddingControl = null, + /** Generated wallpaper and camera track — see lib/assets.mjs. */ + assets = {}, + /** The fixture spec, needed to regenerate the cursor track deterministically. */ + spec = null, +}) { + mkdirSync(outDir, { recursive: true }); + + // The loader only auto-approves media in the recordings dir or *next to the project*, so + // the source is copied in rather than referenced across the filesystem. + const localMedia = join(outDir, basename(sourcePath)); + if (localMedia !== sourcePath) copyFileSync(sourcePath, localMedia); + + const p = probe(localMedia); + const e = scenario.effects; + + // Cursor telemetry rides beside the screen video as `