From eba3e51521e0fca8c76f0dc351a4b10921308df0 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Tue, 25 Aug 2026 16:14:51 +0200 Subject: [PATCH 01/12] bench: add a reproducible export-time benchmark against the competition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measures how long OpenScreen takes to turn a 60 s screen recording into a finished 1080p60 MP4, next to the apps it competes with, on one clip with one edit applied and one stopwatch. What makes the numbers defensible: - The source is generated from a seed rather than shipped, so two machines can prove they measured the same workload by comparing one sha256. It is built to look like a screen recording — static regions, sharp edges, localized motion — because that is what changes an encoder's job. - One clock for every app. It starts when the export is committed and stops when the last byte lands; app launch, project loading and presets are warm-up and are reported separately. Apps that publish their own completion signal can shorten their measurement, never lengthen it. - Outputs are verified as pixels, not just metadata: corner colour proves the background, a content bounding box measures the real inset, an inset corner against its own edge proves the radius, and a frame-to-frame activity trace proves each zoom actually rendered. A run that fails verification is a failure, not a fast time. - Padding controls are on a different scale in every app — asked for "5", Cap gave a 1.85 % inset and OpenScreen 10 %, a 44 % difference in pixels sampled per frame. `bench.mjs calibrate` solves each app's control against measured output so they composite the same rectangle. - Background CPU is sampled per run and printed in the table. It is the one precondition that never announces itself. Fidelity is tracked per app and partial rows are ranked separately: an app that skipped the compositing did less work and is a reference, not a competitor. Only two of the apps can be scripted the ordinary way. Screen Studio marks its editor kCGWindowSharingNone — invisible to every capture API — and publishes no accessibility tree, so it and the other Electron apps are driven over CDP by element text, which reproduces better than coordinates. Camtasia has an AppleScript dictionary; FocuSee is native and exposes its AX tree. Two competitors are gated by their vendors rather than by the automation, and both are documented in place: Screen Studio requires an activated licence to export at all, and FocuSee 2.4.1 rejects every MP4 as "damaged", including real recordings. See benchmark/README.md for the method and benchmark/REMOTE.md for driving a run from a dispatched session. --- benchmark/.gitignore | 4 + benchmark/README.md | 250 +++++++++++ benchmark/REMOTE.md | 108 +++++ benchmark/apps.mjs | 136 ++++++ benchmark/bench.mjs | 593 ++++++++++++++++++++++++++ benchmark/calibration.json | 160 +++++++ benchmark/drivers/README.md | 55 +++ benchmark/drivers/camtasia.mjs | 291 +++++++++++++ benchmark/drivers/cap.mjs | 230 ++++++++++ benchmark/drivers/ffmpeg-baseline.mjs | 96 +++++ benchmark/drivers/focusee.mjs | 176 ++++++++ benchmark/drivers/kap.mjs | 234 ++++++++++ benchmark/drivers/openscreen-cli.mjs | 152 +++++++ benchmark/drivers/openscreen-gui.mjs | 246 +++++++++++ benchmark/drivers/screen-studio.mjs | 237 ++++++++++ benchmark/lib/calibrate.mjs | 150 +++++++ benchmark/lib/cdp.mjs | 204 +++++++++ benchmark/lib/env.mjs | 205 +++++++++ benchmark/lib/fixture.mjs | 302 +++++++++++++ benchmark/lib/install.mjs | 178 ++++++++ benchmark/lib/measure.mjs | 253 +++++++++++ benchmark/lib/openscreenProject.mjs | 101 +++++ benchmark/lib/permissions.mjs | 107 +++++ benchmark/lib/report.mjs | 333 +++++++++++++++ benchmark/lib/runner.mjs | 277 ++++++++++++ benchmark/lib/state.mjs | 78 ++++ benchmark/lib/uiScript.mjs | 341 +++++++++++++++ benchmark/lib/visualCheck.mjs | 214 ++++++++++ benchmark/scenarios/index.mjs | 136 ++++++ 29 files changed, 5847 insertions(+) create mode 100644 benchmark/.gitignore create mode 100644 benchmark/README.md create mode 100644 benchmark/REMOTE.md create mode 100644 benchmark/apps.mjs create mode 100644 benchmark/bench.mjs create mode 100644 benchmark/calibration.json create mode 100644 benchmark/drivers/README.md create mode 100644 benchmark/drivers/camtasia.mjs create mode 100644 benchmark/drivers/cap.mjs create mode 100644 benchmark/drivers/ffmpeg-baseline.mjs create mode 100644 benchmark/drivers/focusee.mjs create mode 100644 benchmark/drivers/kap.mjs create mode 100644 benchmark/drivers/openscreen-cli.mjs create mode 100644 benchmark/drivers/openscreen-gui.mjs create mode 100644 benchmark/drivers/screen-studio.mjs create mode 100644 benchmark/lib/calibrate.mjs create mode 100644 benchmark/lib/cdp.mjs create mode 100644 benchmark/lib/env.mjs create mode 100644 benchmark/lib/fixture.mjs create mode 100644 benchmark/lib/install.mjs create mode 100644 benchmark/lib/measure.mjs create mode 100644 benchmark/lib/openscreenProject.mjs create mode 100644 benchmark/lib/permissions.mjs create mode 100644 benchmark/lib/report.mjs create mode 100644 benchmark/lib/runner.mjs create mode 100644 benchmark/lib/state.mjs create mode 100644 benchmark/lib/uiScript.mjs create mode 100644 benchmark/lib/visualCheck.mjs create mode 100644 benchmark/scenarios/index.mjs diff --git a/benchmark/.gitignore b/benchmark/.gitignore new file mode 100644 index 00000000..28d25d20 --- /dev/null +++ b/benchmark/.gitignore @@ -0,0 +1,4 @@ +.cache/ +results/*/ +results/install.json +results/preflight.json diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000..7ef30623 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,250 @@ +# Export benchmark + +How long does it take to turn a 60-second screen recording into a finished 1080p60 MP4, in +OpenScreen and in the apps it competes with — measured the same way, on the same clip, with the +same edit applied, and verified frame by frame. + +This directory is the whole apparatus: it generates the source clip, installs the competitors, +translates one scenario into each app's own controls, drives the export, times it, checks that +what came out is what was asked for, and writes the report. It is meant to be started once and +left alone. + +```bash +node benchmark/bench.mjs doctor # is this machine fit to measure? +node benchmark/bench.mjs preflight # the one interactive gate — grant everything here +node benchmark/bench.mjs install # unattended +node benchmark/bench.mjs calibrate # once per machine +node benchmark/bench.mjs run # walk away +node benchmark/bench.mjs report +``` + +Driving it from a phone or another machine: [REMOTE.md](./REMOTE.md). + +--- + +## What is being measured + +**The clock starts** the instant the export is committed — the click on *Export*, or a CLI's +first progress event — and **stops when the last byte lands in the output file**. Launching the +app, loading the project and setting presets happen before the clock starts, for every app +alike, and are reported separately as `prepareMs` and `launchToCommitMs`. + +Two apps in this set (Camtasia, Kap) publish their own completion signal. Where one exists the +harness takes whichever is *earlier* — the app's or the filesystem's — so an app can shorten its +own measurement but never lengthen it. + +**Every output is then checked twice.** First against the target's metadata: resolution, frame +rate, codec, duration. Then against its own pixels, because metadata cannot tell you whether the +app actually did the work: + +| Check | How | Why it exists | +|---|---|---| +| Background applied | the frame's four corners must be the scenario's colour | an app that skipped the background composites far less | +| Padding | bounding box of everything that is not background | apps' padding controls are on different scales; this measures the real inset | +| Corner radius | the box's corner is background while its top edge is not | separates a rounded rect from a plain one | +| Zooms | frame-to-frame activity must spike inside every zoom window | an ignored zoom list is invisible in metadata | + +A run that fails verification is recorded as a failure, never as a fast time. + +**Fidelity** records how much of the scenario each app could express. A row marked `partial` did +less work; its number is a reference, not a ranking. Kap, which has no background, padding, +corner-radius, shadow or zoom features at all, is always partial — it is in the set as a +real-app floor, not as a peer. + +## The source clip + +Not shipped — **generated**, from a spec plus a seed, so that two machines can prove they +measured the same workload by comparing one hash: + +``` +1920×1080, 60 fps, 60 s, 3600 frames, H.264 High + AAC 48 kHz +sha256 recorded in every results file +``` + +It is built to look like a screen recording rather than a test pattern, because that is what +changes an encoder's job: a dark editor with syntax-coloured "code", a scrolling viewport, a +blinking caret, a selection band and a moving cursor — large static regions with sharp edges and +localized motion. `benchmark/lib/fixture.mjs` composes it from ffmpeg primitives; nothing is +random at run time. + +Why 60 fps and not 30: OpenScreen's MP4 export path is fixed at 60 (`MP4_EXPORT_FPS`, +`src/cli/CliExportRunner.tsx`), and every other app in the set can be told to emit 60. It is the +only frame rate on which "force identical output" is actually achievable. + +## The scenario + +One definition, in `benchmark/scenarios/index.mjs`, translated by each driver into its own app's +vocabulary: + +- background: solid `#C9CDD6` +- padding: 5 % of the frame's short side +- corner radius: 40 px +- drop shadow +- three zooms — 6–12 s at 1.8×, 22–29 s at 2.2×, 41–48 s at 1.6× +- output: 1920×1080, 60 fps, H.264, MP4 + +The colour is deliberately far from anything in the generated source: the verifier finds the +composited video's edge by colour distance, and a background close to the recording's own dark +chrome makes that boundary unfindable. (It did, the first time.) + +### Translating the scenario — and why calibration exists + +No two of these apps put their padding control on the same scale. Asked for "5", Cap produced a +1.85 % inset and OpenScreen a 10 % one — a 44 % difference in how many source pixels each was +sampling per frame. That is a confound, not a result. + +`bench.mjs calibrate` fixes it: for each app it renders a short clip at two padding values, +measures the inset from the output pixels, solves for the value that hits the scenario's target, +and writes the answer to `benchmark/calibration.json`. On this machine: + +| App | control value | measured inset | content box | +|---|---|---|---| +| OpenScreen | `padding: 25` | 5.00 % | 1728×972 | +| Cap | `padding: 13.56` | 4.81 % | 1734×976 | + +Run it once per machine, and again after any app updates. `run` reads the file automatically; +without it, each driver falls back to its documented default and the report shows the inset it +actually achieved. + +## Automating apps that have no CLI + +Only two apps in this set can be scripted the ordinary way. What the others expose was +established by inspection, not assumption: + +| App | CLI | AppleScript dictionary | Accessibility tree | Screenshotable | Driven by | +|---|---|---|---|---|---| +| OpenScreen | **yes** (`openscreen export`) | no | — | — | `cli` | +| Cap | **yes** (`cap-cli export`) | no | — | — | `cli` | +| Camtasia | no | **yes** (import, `isExporting`) | yes | yes | `applescript+menu` | +| Kap | no | no | **no** (empty window) | yes | `cdp` | +| Screen Studio | no | no | **no** | **no** — see below | `cdp` | +| FocuSee | no | no | yes | yes | `ax+menu` | + +The ladder each GUI driver climbs, best rung first: a scripting dictionary → a System Events +menu item by name → a documented keyboard shortcut → an accessibility control by name → the +renderer's own DOM over CDP → pixel coordinates. Every driver records which rung it used, in the +`automation` column of the report, because that is what tells a reader how well a given row will +reproduce on somebody else's machine. Pixel coordinates are the only rung that does not survive a +different display, and no driver here needs them. + +**Screen Studio cannot be screenshotted at all.** It marks its editor window +`kCGWindowSharingNone`, so macOS excludes it from every capture API — the window is plainly +visible to the person sitting there and invisible to `screencapture`, ScreenCaptureKit and any +agent driving pixels. It publishes no accessibility tree either. Launching it with +`--remote-debugging-port` is what makes it drivable, and that is a *more* reproducible +interaction than clicking pixels: elements are found by their visible text, which survives a +moved window, a different display and a resized UI. The flag opens an inspector and nothing +else; the renderer and the export pipeline are the shipping ones. + +`node benchmark/bench.mjs discover ` dumps an installed app's menus and accessibility tree. +That is how a driver gets written, and how it gets repaired when a new version renames something. + +## What is in the set, and what it costs to get + +| App | Licence for exporting | Install | +|---|---|---| +| OpenScreen | MIT, free, no watermark | GitHub release | +| Cap | AGPL-3.0, free; sign-in not needed for a local export | direct DMG | +| Camtasia | 30-day trial, watermarked output | direct DMG | +| Kap | MIT, free | GitHub release | +| Screen Studio | **licence required to export at all** — no trial export | direct DMG | +| FocuSee | trial, watermarked | vendor downloader stub | + +A watermark does not change render time, so a trial build is a valid measurement. A licence +*wall* is not — see [Known blockers](#known-blockers). + +## Reproducing on another machine + +1. `node benchmark/bench.mjs doctor` — refuses to proceed quietly on battery, in Low Power Mode, + under thermal throttling, or with less than 20 GiB free. All four move export times. +2. `node benchmark/bench.mjs preflight --launch` — prints the whole download list with sizes and + licence terms, provokes every macOS permission prompt the run would otherwise hit mid-flight, + and opens each GUI app once so its first-launch dialogs can be cleared. **This is the only + step that needs a human.** +3. `node benchmark/bench.mjs install` +4. `node benchmark/bench.mjs calibrate` +5. `node benchmark/bench.mjs run --reps 3` + +Comparing two machines: the results file carries the source clip's sha256, every app's version, +the calibration used, the machine's chip/cores/RAM/OS build, and the power and thermal state at +each repetition. Two runs are comparable when the fixture hashes and the app versions match. + +### ffmpeg + +Used to build the fixture and to verify outputs — it is measuring instrumentation, not part of +any app's export path. Resolution order: `OSBENCH_FFMPEG`/`OSBENCH_FFPROBE`, then `ffmpeg` on +`PATH`, then the repo's LGPL tree under `crates/thirdparty/ffmpeg-*`. That tree is gitignored, so +it exists only in the checkout that built it; the harness finds it through +`git rev-parse --git-common-dir` and wraps it in a small script that re-exports +`DYLD_LIBRARY_PATH` inside its own process, because macOS strips `DYLD_*` across any +SIP-protected exec and the inherited variable never survives. + +The LGPL build has no libx264 and no drawtext. The fixture is encoded with +`h264_videotoolbox` and drawn with `drawbox`; neither `-crf` nor text overlays are available. + +### Background load + +The one precondition that never announces itself. Nothing throttles and nothing warns — every +export is simply slower. On this machine, reached over a remote-desktop session, the screen +encoder alone holds 100–200 % of a core permanently, and `doctor` refuses to call the machine +ready above 60 %. + +It is sampled during every export and reported per row as **Bg load**. Because it is the same +for every app in one run, the *comparison* survives it; the absolute times do not. Two runs are +only comparable at similar background load, which is why the figure is in the table rather than +in a footnote. + +### Repetitions and guards + +Three scoring runs after one discarded warm-up, 45 s of cooldown between them. The warm-up is +kept in the data but excluded from the statistics — first runs pay for cold caches and +uncompiled shaders. The headline figure is the **median** with a median absolute deviation; +with n=3 a standard deviation is mostly noise. Preconditions are re-checked before every +repetition and recorded per run, so a throttled run is visible rather than averaged in. + +## Reading the report + +`results//` holds `results.json` (everything), `report.md`, `report.html`, +`events.ndjson` (append-only) and `status.json` (atomically rewritten, safe to poll). + +- **Export (median)** — commit → last byte. +- **×realtime** — output duration ÷ export time. Above 1 is faster than playback. +- **vs floor** — multiples of `ffmpeg (re-encode floor)`, a plain transcode with no compositing. + It separates "this encoder is slow on this machine" from "this app's pipeline is slow". +- **Fidelity** — `full`, or `partial` with the missing features named. +- **Driven by** — the automation rung. + +## Known blockers + +Recorded here rather than quietly dropped, because "not measured" and "slow" are very different +findings. + +- **Screen Studio 3.7.5** gates export behind account activation. There is no trial export and no + watermark path — clicking *Export* opens an activation wall. The driver is complete and works; + supply a licence, activate once during preflight, and `--apps screen-studio` produces a number. +- **FocuSee 2.4.1** (direct download, macOS 26.5) rejects every MP4 it is given — including a + real 2560×1440 H.264 recording — with *"The source file is damaged and cannot be opened."* It is + not sandboxed, so this is not a file-access grant. Both its own import panel and `open -a` fail. + Its driver is written against the AX tree it does expose; it will start working if a later + build fixes the import. + +## Layout + +``` +bench.mjs entrypoint +apps.mjs registry: what is in the set, where it comes from, what it costs +scenarios/index.mjs the scenario and the pinned output target +lib/env.mjs machine fingerprint, power/thermal state, ffmpeg resolution +lib/fixture.mjs deterministic source generation + ffprobe +lib/measure.mjs stopwatch, process sampling, output verification +lib/visualCheck.mjs pixel verification of the effects +lib/calibrate.mjs solving each app's padding control +lib/runner.mjs the shared clock every driver is timed by +lib/install.mjs unattended DMG installation +lib/permissions.mjs provoking every macOS prompt up front +lib/uiScript.mjs AppleScript / System Events / accessibility +lib/cdp.mjs Chrome DevTools Protocol, for the Electron apps +lib/report.mjs markdown + HTML +lib/state.mjs append-only event log and pollable status +drivers/ one per app — see drivers/README.md for the contract +``` diff --git a/benchmark/REMOTE.md b/benchmark/REMOTE.md new file mode 100644 index 00000000..8af7f8db --- /dev/null +++ b/benchmark/REMOTE.md @@ -0,0 +1,108 @@ +# Driving the benchmark remotely + +The run takes one to three hours and needs nobody watching it. This is how to start it, check on +it, and pick it up again from a Claude Code session on your phone, in the browser, or dispatched +from another machine. + +The design constraint behind all of it: **every prompt that needs a human is provoked up front**, +and everything after that writes its state to disk so a session that disconnects loses nothing. + +--- + +## The one interactive gate + +Do this while you are at the keyboard. It is the only part that cannot be remote, because macOS +security prompts must be answered on the machine itself. + +```bash +node benchmark/bench.mjs preflight --launch +``` + +It will: + +1. Print the machine's fitness to measure — chip, cores, RAM, OS build, free disk, power source, + thermal state — and refuse quietly-wrong conditions rather than producing a quietly-wrong number. +2. List every download with its size and licence terms, and wait for you to approve the set. +3. Provoke each app's **"… wants access to control …"** Apple Events prompt one at a time, so you + answer them all in one sitting instead of being ambushed six times during the run. Nothing here + clicks *Allow* for you — these are security settings. +4. Open each GUI app once so its first-launch dialogs (onboarding surveys, update nags, usage-data + consent) can be dismissed while you are there. + +When it prints `preflight complete`, the machine is ready and you can leave. + +## Starting a run remotely + +```bash +node benchmark/bench.mjs install # skips anything already present +node benchmark/bench.mjs calibrate # once per machine; ~5 min +node benchmark/bench.mjs run --reps 3 --id nightly +``` + +`run` is safe to launch in the background and disconnect from: + +```bash +nohup node benchmark/bench.mjs run --reps 3 --id nightly > /tmp/bench-nightly.log 2>&1 & +``` + +## Checking on it + +```bash +node benchmark/bench.mjs status --json +``` + +```json +{ + "runId": "nightly", + "phase": "running", + "current": { "app": "camtasia", "index": 3, "of": 6 }, + "completed": ["ffmpeg-baseline", "openscreen-cli"], + "pending": ["kap", "cap"] +} +``` + +`status.json` is rewritten atomically, so polling it can never read a half-written document. +`results//events.ndjson` is append-only and carries one line per app started, finished or +skipped — `tail -f` it for a live view without touching the run. + +Partial results are written after **every** app, not at the end. A run that dies at app four +still leaves four apps' worth of `results.json`, and `bench.mjs report --run ` will render +what exists. + +## Picking up where it stopped + +```bash +node benchmark/bench.mjs run --apps camtasia,kap --id nightly # same id: same output folder +node benchmark/bench.mjs report --run nightly +``` + +There is no magic resume flag, deliberately: naming the apps you still need is clearer than a +flag that guesses, and re-running one app is cheap. + +## Notes for an agent driving this + +- **Do not run two benchmarks at once, and do not do anything else heavy on the machine while one + is running.** The measurement is wall-clock on a shared 8-core SoC; a concurrent build makes + every number wrong without making any of them look wrong. `preconditionCheck()` catches + throttling and battery, not a competing process. +- **`run` is long.** Expect ~2 minutes per repetition per app plus 45 s of cooldown between them — + roughly 25 minutes for six apps at three reps. Poll `status --json` on a slow cadence; do not + busy-wait. +- **A GUI app can leave a window open** if a run is killed mid-export. `bench.mjs doctor` reports + what is running; quitting the app by hand is always safe between runs. +- **Never interpret a missing app as a slow app.** Skipped rows carry a `reason`; report it + verbatim rather than omitting the row. +- **The report is the deliverable, not the terminal output.** `results//report.html` is + self-contained and can be published as an artifact directly. + +## What still needs a human, and when + +| Moment | Why | Frequency | +|---|---|---| +| Apple Events prompts | macOS security; only the user can grant them | once per app, ever | +| First-launch dialogs | vendor onboarding, consent, update nags | once per app, ever | +| Screen Studio activation | export is licence-gated | once, if you own a licence | +| Nothing else | — | — | + +If a prompt does appear mid-run, `lib/permissions.mjs → pendingPermissionDialog()` reads its text, +so a session can report *what* is being asked rather than just noticing that everything stalled. diff --git a/benchmark/apps.mjs b/benchmark/apps.mjs new file mode 100644 index 00000000..2e328ad3 --- /dev/null +++ b/benchmark/apps.mjs @@ -0,0 +1,136 @@ +/** + * The app registry: what is in the benchmark, where it comes from, and what it costs to get. + * + * Separate from the drivers on purpose — `preflight` has to be able to show the user the whole + * download list, with sizes and licence terms, and get one approval for all of it *before* + * anything is fetched. Everything after that approval runs unattended. + */ + +/** + * Download URLs are pinned to a version wherever the vendor exposes one, because "latest" + * makes a benchmark unreproducible: two machines run a month apart would measure two products. + * `bench.mjs refresh-urls` re-resolves them and prints the diff. + */ +export const APPS = { + "openscreen-cli": { + driver: "./drivers/openscreen-cli.mjs", + default: true, + install: { + method: "github-release", + repo: "getopenscreen/openscreen", + assetPattern: /macOS-Apple-Silicon.*\.dmg$/i, + appName: "Openscreen.app", + approxMB: 250, + licence: "MIT — free, no account, no watermark", + }, + }, + "openscreen-gui": { + driver: "./drivers/openscreen-gui.mjs", + default: true, + sharesInstallWith: "openscreen-cli", + }, + "screen-studio": { + // Off by default: export is licence-gated, so an unactivated machine would only ever + // record a failure. Enable it explicitly once a licence is activated. + driver: "./drivers/screen-studio.mjs", + default: false, + install: { + method: "dmg", + url: "https://screenstudioassets.com/releases/3.7.5-4595/Screen%20Studio%203.7.5-4595%20Apple%20Silicon.dmg", + version: "3.7.5-4595", + appName: "Screen Studio.app", + approxMB: 349, + licence: "commercial — trial exports carry a watermark (which does not change render time)", + notes: [ + "No CLI and no scripting dictionary; only screen-studio://record-* deeplinks exist, none for export.", + ], + }, + }, + cap: { + driver: "./drivers/cap.mjs", + default: true, + install: { + method: "dmg", + url: "https://cap.so/download/apple-silicon", + appName: "Cap.app", + approxMB: 123, + licence: "AGPL-3.0 — free; signing in is optional and not needed for a local export", + notes: [ + "Ships a real CLI at Cap.app/Contents/MacOS/cap-cli — `cap export` renders a .cap project.", + ], + }, + }, + camtasia: { + driver: "./drivers/camtasia.mjs", + default: true, + install: { + method: "dmg", + url: "https://download.techsmith.com/camtasiamac/releases/Camtasia.dmg", + appName: "Camtasia.app", + approxMB: 412, + licence: "commercial — 30-day trial, watermarked output", + notes: ["No CLI on macOS. Driven through the File → Export menu."], + }, + }, + focusee: { + driver: "./drivers/focusee.mjs", + default: false, // see drivers/focusee.mjs — import is broken in 2.4.1 + install: { + method: "manual", + url: "https://focusee.imobie.com/go/download.php?product=fs", + appName: "FocuSee.app", + approxMB: 5, + licence: "commercial — trial exports are watermarked", + notes: ["The vendor ships a GUI installer stub; run it once during preflight."], + }, + }, + kap: { + driver: "./drivers/kap.mjs", + default: true, + install: { + method: "dmg", + url: "https://github.com/wulkano/Kap/releases/download/v3.6.0/Kap-3.6.0-arm64.dmg", + version: "3.6.0", + appName: "Kap.app", + approxMB: 119, + licence: "MIT — free", + notes: [ + "Has no background, zoom, corner-radius or shadow features at all, so it cannot express the", + "full-demo scenario. It is kept as a reduced-fidelity reference: a real app doing a real", + "export, with none of the compositing. Its row is marked partial in the report.", + ], + }, + }, + "ffmpeg-baseline": { + driver: "./drivers/ffmpeg-baseline.mjs", + default: true, + install: null, + }, +}; + +export const DEFAULT_APPS = Object.entries(APPS) + .filter(([, a]) => a.default) + .map(([id]) => id); + +export async function loadDriver(id) { + const entry = APPS[id]; + if (!entry) throw new Error(`Unknown app "${id}". Known: ${Object.keys(APPS).join(", ")}`); + const mod = await import(entry.driver); + return mod.default; +} + +/** Every distinct thing that has to be downloaded for the given app ids. */ +export function installPlan(appIds) { + const seen = new Set(); + const plan = []; + for (const id of appIds) { + const entry = APPS[id]; + if (!entry) continue; + const target = entry.sharesInstallWith ?? id; + if (seen.has(target)) continue; + seen.add(target); + const spec = (APPS[target] ?? entry).install; + if (spec) plan.push({ id: target, ...spec }); + } + return plan; +} diff --git a/benchmark/bench.mjs b/benchmark/bench.mjs new file mode 100644 index 00000000..cef7a028 --- /dev/null +++ b/benchmark/bench.mjs @@ -0,0 +1,593 @@ +#!/usr/bin/env node +/** + * openscreen export benchmark — entrypoint. + * + * node benchmark/bench.mjs preflight # one interactive gate, then walk away + * node benchmark/bench.mjs install + * node benchmark/bench.mjs run + * node benchmark/bench.mjs status --json # safe to poll from anywhere, incl. a remote session + * node benchmark/bench.mjs report + * + * See benchmark/README.md for the methodology and benchmark/REMOTE.md for driving it from a + * dispatched Claude Code session. + */ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { APPS, DEFAULT_APPS, installPlan, loadDriver } from "./apps.mjs"; +import { + CALIBRATION_PATH, + calibrateApp, + calibrationFixture, + loadCalibration, + saveCalibration, +} from "./lib/calibrate.mjs"; +import { + CACHE_DIR, + diskState, + ensureWorkDirs, + ffmpegVersion, + machineFingerprint, + powerState, + RESULTS_DIR, + WORK_DIR, +} from "./lib/env.mjs"; +import { buildFixture, DEFAULT_SPEC, fixturePath, probe, sha256 } from "./lib/fixture.mjs"; +import { installApp } from "./lib/install.mjs"; +import { + accessibilityGranted, + pendingPermissionDialog, + primeAutomation, +} from "./lib/permissions.mjs"; +import { renderReport } from "./lib/report.mjs"; +import { preconditionCheck, runApp } from "./lib/runner.mjs"; +import { newRunId, RunState } from "./lib/state.mjs"; +import { + appIsRunning, + describeWindow, + dumpMenus, + hasScriptingDictionary, + launchApp, +} from "./lib/uiScript.mjs"; +import { DEFAULT_SCENARIO, getScenario } from "./scenarios/index.mjs"; + +/* ------------------------------------------------------------------------- argv ---------- */ + +function parseArgs(argv) { + const [command = "help", ...rest] = argv; + const flags = {}; + const positional = []; + for (let i = 0; i < rest.length; i++) { + const a = rest[i]; + if (a.startsWith("--")) { + const [k, inline] = a.slice(2).split("="); + if (inline !== undefined) flags[k] = inline; + else if (rest[i + 1] && !rest[i + 1].startsWith("--")) flags[k] = rest[++i]; + else flags[k] = true; + } else positional.push(a); + } + return { command, flags, positional }; +} + +const log = (...a) => console.log(...a); +const listFlag = (v, fallback) => + typeof v === "string" + ? v + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + : fallback; + +/* ---------------------------------------------------------------------- commands --------- */ + +async function cmdDoctor() { + ensureWorkDirs(); + const fp = machineFingerprint(); + const pre = preconditionCheck(); + let ff = null; + try { + ff = ffmpegVersion(); + } catch (e) { + ff = { banner: `MISSING — ${e.message}`, source: null }; + } + log("Machine"); + log( + ` ${fp.chip} · ${fp.cpuCount} cores (${fp.performanceCores}P/${fp.efficiencyCores}E) · ${fp.memoryGiB} GiB`, + ); + log(` ${fp.osProduct} ${fp.osVersion} (${fp.osBuild}) · node ${fp.nodeVersion}`); + for (const d of fp.displays) log(` ${d}`); + log("\nPreconditions"); + log(` ${pre.ok ? "✓ ready" : `✗ ${pre.problems.join("; ")}`}`); + log(` disk: ${pre.disk.availableGiB} GiB free at ${pre.disk.path}`); + log(`\nffmpeg\n ${ff.banner}\n source: ${ff.source}`); + log("\nApps"); + for (const id of DEFAULT_APPS) { + let driver; + try { + driver = await loadDriver(id); + } catch (e) { + log(` ! ${id.padEnd(26)} driver not available: ${e.message.split("\n")[0]}`); + continue; + } + const d = driver.detect(); + const dict = driver.appPath ? hasScriptingDictionary(driver.appPath) : false; + log( + ` ${d.installed ? "✓" : "·"} ${driver.displayName.padEnd(26)} ${(d.version ?? "").padEnd(14)}` + + ` automation=${driver.automation}${dict ? " (has AppleScript dictionary)" : ""}`, + ); + } +} + +async function cmdPreflight({ flags }) { + ensureWorkDirs(); + const apps = listFlag(flags.apps, DEFAULT_APPS); + const plan = installPlan(apps); + + log("═══ Preflight ═══\n"); + await cmdDoctor(); + + /* ---------------------------------------------------------------- downloads --------- */ + const missing = plan.filter((m) => !existsSync(join("/Applications", m.appName))); + log("\nDownloads needed"); + if (!missing.length) log(" (none — every app is already installed)"); + for (const m of missing) { + log(` ${m.appName.padEnd(22)} ~${m.approxMB} MB ${m.licence}`); + log(` ${m.url ?? m.repo}`); + for (const n of m.notes ?? []) log(` note: ${n}`); + } + const totalMB = missing.reduce((s, m) => s + (m.approxMB ?? 0), 0); + if (totalMB) log(` total ≈ ${totalMB} MB — run \`bench.mjs install\` to fetch them`); + + /* -------------------------------------------------------------- permissions --------- */ + log("\nPermissions"); + if (!accessibilityGranted()) { + log(" ✗ Accessibility is NOT granted to the process running this script."); + log(" Without it System Events refuses every menu click and no GUI app can be driven."); + log(" Grant it in System Settings → Privacy & Security → Accessibility, then re-run."); + } else { + log( + " ✓ Accessibility — System Events responds, so menus and the accessibility tree are reachable.", + ); + } + log( + " · Screen Recording is NOT needed: the benchmark never records, it imports a generated clip.", + ); + + // Every app gets one harmless scripted question. The first raises the macOS prompt and + // blocks until answered; later ones are silent. Doing this here is the whole point of + // preflight — it moves six mid-run ambushes into one sitting. + const drivers = []; + for (const id of apps) { + try { + drivers.push(await loadDriver(id)); + } catch { + /* a driver that will not load is reported by doctor */ + } + } + const needPrompt = drivers.filter((d) => d.bundleId && d.appPath && existsSync(d.appPath)); + if (needPrompt.length) { + log(`\n Provoking the Apple Events prompt for ${needPrompt.length} app(s).`); + log(" Each will raise a “… wants access to control …” dialog. Click Allow on every one —"); + log(" these are security settings, so nothing here can accept them for you.\n"); + } + const permissions = []; + for (const d of needPrompt) { + process.stdout.write(` ${d.displayName.padEnd(24)} `); + const r = primeAutomation(d.bundleId); + permissions.push({ app: d.displayName, ...r }); + log( + r.status === "granted" + ? "✓ granted" + : r.status === "denied" + ? "✗ DENIED — this app cannot be driven" + : `… ${r.status}`, + ); + const pending = pendingPermissionDialog(); + if (pending) log(` still waiting on: ${pending.slice(0, 90)}…`); + } + + /* ------------------------------------------------------- first-launch dialogs ------- */ + if (flags.launch) { + log("\nOpening each GUI app once so its first-launch dialogs can be cleared."); + log("Dismiss onboarding, consent and update prompts now — after this the run is unattended.\n"); + for (const d of drivers) { + if (d.kind !== "gui" || !d.appPath || !existsSync(d.appPath)) continue; + log(` → ${d.displayName}`); + try { + await launchApp(d.appPath, d.processName); + } catch (e) { + log(` could not launch: ${e.message.split("\n")[0]}`); + } + } + } else { + log( + "\nRe-run with --launch to also open each GUI app once and clear its first-launch dialogs.", + ); + } + + const denied = permissions.filter((p) => p.status === "denied"); + const status = { + generatedAt: new Date().toISOString(), + apps, + missingInstalls: missing.map((m) => m.appName), + totalDownloadMB: totalMB, + accessibility: accessibilityGranted(), + permissions, + machine: machineFingerprint(), + preconditions: preconditionCheck(), + }; + mkdirSync(RESULTS_DIR, { recursive: true }); + writeFileSync(join(RESULTS_DIR, "preflight.json"), `${JSON.stringify(status, null, 2)}\n`); + + log("\n─────────────────────────────────────────"); + if (denied.length) + log(`⚠ ${denied.length} app(s) denied automation: ${denied.map((d) => d.app).join(", ")}`); + if (missing.length) log(`Next: node benchmark/bench.mjs install`); + else log("Next: node benchmark/bench.mjs calibrate && node benchmark/bench.mjs run"); + log(`preflight complete — written to ${join(RESULTS_DIR, "preflight.json")}`); +} + +async function cmdInstall({ flags }) { + ensureWorkDirs(); + const apps = listFlag(flags.apps, DEFAULT_APPS); + const plan = installPlan(apps); + const cacheDir = join(WORK_DIR, "installers"); + const records = []; + for (const spec of plan) { + log(`${spec.appName}`); + try { + const rec = installApp(spec, { cacheDir, force: !!flags.force, log }); + records.push(rec); + log( + ` ${rec.status} — ${rec.version ?? "?"} — gatekeeper: ${rec.codesign.accepted ? "accepted" : "REJECTED"}`, + ); + } catch (e) { + records.push({ id: spec.id, status: "failed", error: e.message }); + log(` ✗ ${e.message}`); + } + } + writeFileSync(join(RESULTS_DIR, "install.json"), `${JSON.stringify(records, null, 2)}\n`); + log(`\nWritten: ${join(RESULTS_DIR, "install.json")}`); +} + +async function cmdFixture({ flags }) { + ensureWorkDirs(); + const spec = { ...DEFAULT_SPEC }; + if (flags.duration) spec.durationSec = Number(flags.duration); + if (flags.fps) spec.fps = Number(flags.fps); + const r = buildFixture(WORK_DIR, spec, { force: !!flags.force, log }); + log(`\n${r.path}`); + log(` sha256 ${r.sha256}`); + log(` ${JSON.stringify(r.probe.video)} ${(r.probe.sizeBytes / 1048576).toFixed(1)} MB`); +} + +async function cmdRun({ flags }) { + ensureWorkDirs(); + const apps = listFlag(flags.apps, DEFAULT_APPS); + const scenario = getScenario(flags.scenario ?? DEFAULT_SCENARIO); + const repetitions = Number(flags.reps ?? 3); + const cooldownSec = Number(flags.cooldown ?? 45); + const discardFirst = flags["no-warmup"] ? false : true; + + const spec = { ...DEFAULT_SPEC }; + const fixture = existsSync(fixturePath(WORK_DIR, spec)) + ? { + path: fixturePath(WORK_DIR, spec), + probe: probe(fixturePath(WORK_DIR, spec)), + sha256: sha256(fixturePath(WORK_DIR, spec)), + spec, + } + : buildFixture(WORK_DIR, spec, { log }); + + const calibration = loadCalibration(); + if (calibration.machine) { + const here = machineFingerprint(); + if ( + calibration.machine.chip !== here.chip || + calibration.machine.osVersion !== here.osVersion + ) { + log( + `⚠ benchmark/calibration.json was solved on ${calibration.machine.chip} / macOS ${calibration.machine.osVersion}, ` + + `not this machine. Re-run \`bench.mjs calibrate\` — app versions differ between machines and ` + + `a stale padding solve makes the apps composite different rectangles.\n`, + ); + } + } else if (Object.keys(calibration.apps ?? {}).length) { + log("⚠ benchmark/calibration.json has no machine stamp; re-run `bench.mjs calibrate`.\n"); + } else { + log("· no calibration found — each driver will use its documented default padding.\n"); + } + const runId = flags.id ?? newRunId(); + const state = new RunState(join(RESULTS_DIR, runId), runId); + const outDir = join(WORK_DIR, "out", runId); + mkdirSync(outDir, { recursive: true }); + + const header = { + runId, + startedAt: new Date().toISOString(), + scenario: { + id: scenario.id, + label: scenario.label, + effects: scenario.effects, + output: scenario.output, + }, + repetitions, + discardFirst, + cooldownSec, + machine: machineFingerprint(), + power: powerState(), + disk: diskState(), + ffmpeg: (() => { + try { + return ffmpegVersion(); + } catch { + return null; + } + })(), + fixture: { + path: fixture.path, + sha256: fixture.sha256, + spec: fixture.spec, + probe: fixture.probe, + }, + calibration: calibration.apps + ? { + generatedAt: calibration.generatedAt, + targetInsetPercent: calibration.targetInsetPercent, + apps: calibration.apps, + } + : null, + apps, + }; + state.event("run-started", header); + state.writeStatus({ ...header, phase: "starting", completed: [], pending: apps }); + + log( + `run ${runId} · scenario "${scenario.id}" · ${repetitions}×${discardFirst ? " (+1 warm-up)" : ""}`, + ); + log(`fixture ${fixture.path} (${fixture.sha256.slice(0, 12)})\n`); + + const results = []; + for (const [i, id] of apps.entries()) { + let driver; + try { + driver = await loadDriver(id); + } catch (e) { + const rec = { + app: id, + skipped: true, + reason: `driver failed to load: ${e.message}`, + runs: [], + }; + results.push(rec); + state.event("app-skipped", rec); + continue; + } + + state.writeStatus({ + ...header, + phase: "running", + current: { app: id, index: i + 1, of: apps.length }, + completed: results.map((r) => r.app), + pending: apps.slice(i + 1), + }); + + const calibrated = calibration.apps?.[id]?.paddingControl ?? null; + const baseCtx = { + workDir: WORK_DIR, + outDir, + scenario, + source: fixture, + log, + state, + paddingControl: calibrated, + }; + let rec; + try { + rec = await runApp(driver, baseCtx, { repetitions, discardFirst, cooldownSec, log }); + } catch (e) { + rec = { + app: id, + displayName: driver.displayName, + skipped: true, + reason: `crashed: ${e.message}`, + runs: [], + }; + log(` ✗ ${driver.displayName}: ${e.message}`); + } + results.push(rec); + state.event("app-finished", rec); + state.writeResults({ ...header, finishedAt: null, results }); + } + + const final = { ...header, finishedAt: new Date().toISOString(), results }; + state.writeResults(final); + state.writeStatus({ ...final, phase: "done", completed: results.map((r) => r.app), pending: [] }); + state.event("run-finished", { apps: results.map((r) => r.app) }); + + const report = renderReport(final); + writeFileSync(join(state.dir, "report.md"), report.markdown); + writeFileSync(join(state.dir, "report.html"), report.html); + log(`\n${report.summaryText}`); + log(`\nResults: ${state.dir}`); +} + +/** + * Solve each app's padding control so they all composite the same rectangle. Run once per + * machine (and again after an app updates); the result is written to benchmark/calibration.json + * and read automatically by `run`. + */ +async function cmdCalibrate({ flags }) { + ensureWorkDirs(); + const apps = listFlag(flags.apps, DEFAULT_APPS); + const scenario = getScenario(flags.scenario ?? DEFAULT_SCENARIO); + const fixture = calibrationFixture(WORK_DIR, log); + const outDir = join(WORK_DIR, "out", "calibration"); + mkdirSync(outDir, { recursive: true }); + log( + `calibrating padding against a ${fixture.spec.durationSec}s clip; target inset ${scenario.effects.paddingPercent}% of the short side\n`, + ); + + const entries = []; + for (const id of apps) { + let driver; + try { + driver = await loadDriver(id); + } catch { + continue; + } + if (!driver.detect().installed) { + log(`${driver.displayName}: not installed, skipping`); + continue; + } + if (typeof driver.defaultPaddingControl !== "function") { + log(`${driver.displayName}: no padding control to calibrate`); + entries.push({ app: id, paddingControl: null, reason: "driver exposes no padding control" }); + continue; + } + log(`${driver.displayName}:`); + const ctx = { + workDir: WORK_DIR, + outDir, + scenario, + source: fixture, + log, + state: {}, + run: { index: 0 }, + commit: () => undefined, + }; + try { + const r = await calibrateApp(driver, ctx, { log }); + entries.push(r); + log( + ` -> padding=${r.paddingControl} gives ${r.achievedInsetPercent}%${r.withinTolerance ? "" : " (best available; outside tolerance)"}`, + ); + } catch (e) { + log(` x ${e.message}`); + entries.push({ app: id, paddingControl: null, error: e.message?.slice(0, 400) }); + } + try { + await driver.cleanup(ctx); + } catch { + /* best effort */ + } + } + + const path = saveCalibration(entries, { + scenario: scenario.id, + targetInsetPercent: scenario.effects.paddingPercent, + fixture: { spec: fixture.spec, sha256: fixture.sha256 }, + }); + log(`\nWritten: ${path}`); +} + +async function cmdStatus({ flags }) { + const runs = existsSync(RESULTS_DIR) + ? readFileSync + ? (await import("node:fs")) + .readdirSync(RESULTS_DIR) + .filter((d) => /^\d{8}T/.test(d)) + .sort() + : [] + : []; + const runId = flags.run ?? runs[runs.length - 1]; + if (!runId) { + const out = { phase: "no-runs" }; + log(flags.json ? JSON.stringify(out) : "No runs yet."); + return; + } + const state = new RunState(join(RESULTS_DIR, runId), runId); + const status = state.readStatus(); + if (flags.json) { + log(JSON.stringify(status ?? { runId, phase: "unknown" }, null, 2)); + return; + } + if (!status) return log(`Run ${runId}: no status file.`); + log(`Run ${runId} — ${status.phase}`); + if (status.current) + log(` current: ${status.current.app} (${status.current.index}/${status.current.of})`); + log(` done: ${(status.completed ?? []).join(", ") || "none"}`); + log(` left: ${(status.pending ?? []).join(", ") || "none"}`); +} + +async function cmdReport({ flags }) { + const fs = await import("node:fs"); + const runs = fs.existsSync(RESULTS_DIR) + ? fs + .readdirSync(RESULTS_DIR) + .filter((d) => /^\d{8}T/.test(d)) + .sort() + : []; + const runId = flags.run ?? runs[runs.length - 1]; + if (!runId) return log("No runs to report on."); + const state = new RunState(join(RESULTS_DIR, runId), runId); + const results = state.readResults(); + if (!results) return log(`Run ${runId} has no results.json yet.`); + const report = renderReport(results); + fs.writeFileSync(join(state.dir, "report.md"), report.markdown); + fs.writeFileSync(join(state.dir, "report.html"), report.html); + log(report.markdown); + log(`\nWritten: ${join(state.dir, "report.md")} and report.html`); +} + +/** Dump an app's menus and accessibility tree — how a GUI driver gets written or repaired. */ +async function cmdDiscover({ positional, flags }) { + const id = positional[0]; + if (!id) return log("usage: bench.mjs discover [--window N] [--depth N]"); + const driver = await loadDriver(id); + if (!driver.appPath) return log(`${id} has no app bundle.`); + if (!existsSync(driver.appPath)) return log(`${driver.appPath} is not installed.`); + + log(`# ${driver.displayName}`); + log(`bundle: ${driver.appPath}`); + log(`AppleScript dictionary: ${hasScriptingDictionary(driver.appPath) ? "YES" : "no"}`); + if (!appIsRunning(driver.processName)) { + log(`launching ${driver.processName}…`); + await launchApp(driver.appPath, driver.processName); + await new Promise((r) => setTimeout(r, 4000)); + } + log("\n## Menus"); + log(JSON.stringify(dumpMenus(driver.processName), null, 1)); + log("\n## Window accessibility tree"); + try { + log(describeWindow(driver.processName, Number(flags.window ?? 1), Number(flags.depth ?? 4))); + } catch (e) { + log(`(could not read window: ${e.message})`); + } +} + +function cmdHelp() { + log(`openscreen export benchmark + + doctor environment + installed apps + whether UI scripting works + preflight [--launch] the single interactive gate: what will be downloaded, what to grant + install [--apps a,b] [--force] + calibrate [--apps a,b] solve each app's padding control so they composite the same rect + fixture [--force] [--duration s] [--fps n] + run [--apps a,b] [--scenario id] [--reps 3] [--cooldown 45] [--no-warmup] [--id NAME] + status [--run ID] [--json] + report [--run ID] + discover dump menus + accessibility tree (for writing a GUI driver) + +apps: ${Object.keys(APPS).join(", ")}`); +} + +const { command, flags, positional } = parseArgs(process.argv.slice(2)); +const commands = { + doctor: cmdDoctor, + preflight: cmdPreflight, + install: cmdInstall, + fixture: cmdFixture, + run: cmdRun, + status: cmdStatus, + calibrate: cmdCalibrate, + report: cmdReport, + discover: cmdDiscover, + help: cmdHelp, +}; +const fn = commands[command] ?? cmdHelp; +try { + await fn({ flags, positional }); +} catch (e) { + console.error(`\n✗ ${e.stack ?? e.message}`); + process.exit(1); +} diff --git a/benchmark/calibration.json b/benchmark/calibration.json new file mode 100644 index 00000000..301c7e3f --- /dev/null +++ b/benchmark/calibration.json @@ -0,0 +1,160 @@ +{ + "generatedAt": "2026-08-25T11:39:37.984Z", + "scenario": "full-demo", + "targetInsetPercent": 5, + "fixture": { + "spec": { + "name": "calib-1080p60-4s", + "width": 1920, + "height": 1080, + "fps": 60, + "durationSec": 4, + "seed": 20260825, + "sourceBitrateMbps": 12 + }, + "sha256": "574cd13f8e130381c7271a0e9b172f765089fbb99abc4f4548f29b4244b011e4" + }, + "apps": { + "cap": { + "app": "cap", + "target": 5, + "paddingControl": 13.56, + "achievedInsetPercent": 4.81, + "achievedBox": { + "left": 93, + "top": 52, + "right": 1826, + "bottom": 1027, + "width": 1734, + "height": 976 + }, + "withinTolerance": true, + "probes": [ + { + "control": 2.5, + "inset": 0.93, + "box": { + "left": 18, + "top": 10, + "right": 1901, + "bottom": 1069, + "width": 1884, + "height": 1060 + } + }, + { + "control": 5, + "inset": 1.85, + "box": { + "left": 36, + "top": 20, + "right": 1883, + "bottom": 1059, + "width": 1848, + "height": 1040 + } + }, + { + "control": 13.56, + "inset": 4.81, + "box": { + "left": 93, + "top": 52, + "right": 1826, + "bottom": 1027, + "width": 1734, + "height": 976 + } + } + ] + }, + "openscreen-cli": { + "app": "openscreen-cli", + "target": 5, + "paddingControl": 25, + "achievedInsetPercent": 5, + "achievedBox": { + "left": 96, + "top": 54, + "right": 1823, + "bottom": 1025, + "width": 1728, + "height": 972 + }, + "withinTolerance": true, + "probes": [ + { + "control": 25, + "inset": 5, + "box": { + "left": 96, + "top": 54, + "right": 1823, + "bottom": 1025, + "width": 1728, + "height": 972 + } + }, + { + "control": 50, + "inset": 10, + "box": { + "left": 192, + "top": 108, + "right": 1727, + "bottom": 971, + "width": 1536, + "height": 864 + } + } + ] + }, + "openscreen-gui": { + "app": "openscreen-gui", + "target": 5, + "paddingControl": 25, + "achievedInsetPercent": 5, + "achievedBox": { + "left": 96, + "top": 54, + "right": 1823, + "bottom": 1025, + "width": 1728, + "height": 972 + }, + "withinTolerance": true, + "probes": [ + { + "control": 25, + "inset": 5, + "box": { + "left": 96, + "top": 54, + "right": 1823, + "bottom": 1025, + "width": 1728, + "height": 972 + } + }, + { + "control": 50, + "inset": 10, + "box": { + "left": 192, + "top": 108, + "right": 1727, + "bottom": 971, + "width": 1536, + "height": 864 + } + } + ], + "note": "Seeded from openscreen-cli: the GUI writes and reads the same EditorProjectData document and the same padding scale, so solving it twice would measure the same control through a slower interface." + } + }, + "machine": { + "chip": "Apple M1", + "osVersion": "26.5", + "model": "Macmini9,1" + } +} diff --git a/benchmark/drivers/README.md b/benchmark/drivers/README.md new file mode 100644 index 00000000..7223699e --- /dev/null +++ b/benchmark/drivers/README.md @@ -0,0 +1,55 @@ +# Driver contract + +A driver teaches the harness how one app performs the benchmark scenario. It is the only +app-specific code; timing, verification and reporting are shared so no app is measured on a +kinder stopwatch than another. + +```js +export default { + id: "screen-studio", // stable slug, used in results and on the CLI + displayName: "Screen Studio", + vendor: "Screen Studio", + kind: "gui", // "cli" | "gui" | "reference" + automation: "menu", // "cli" | "menu" | "menu+coords" | "none" + processName: "Screen Studio", // as System Events sees it + appPath: "/Applications/Screen Studio.app", + bundleId: "studio.screen.app", + install: { method: "dmg", url, appName, approxMB, licence, notes }, + + detect(), // -> { installed, version, path } + async prepare(ctx), // import the source, apply the scenario, park in the editor + // -> { appliedFeatures: string[], notes: string[] } + outputPath(ctx), // where the export will land + async runExport(ctx), // MUST call ctx.commit() at the instant export is committed + async cleanup(ctx), // quit, remove temp state +}; +``` + +## The two rules that keep the comparison fair + +**`ctx.commit()` marks the same moment for every app.** It is called immediately after the +action that starts the render — the click on *Export*, or the CLI's first `started` event — +never before the project is loaded and never after the first frame. Anything a driver does +before `commit()` (launching the app, importing the clip, setting presets) is warm-up and is +reported separately; anything after it counts. + +**Completion is decided by the filesystem, not by the app.** The harness watches the output +path until it stops growing (`waitForStableFile`), so an app that shows 100% before it has +finished muxing gets no credit for it. A driver's `runExport` may return as soon as the export +is committed; it does not have to detect the end itself. + +## The automation ladder + +GUI drivers should reach for the highest rung that works, and record which one they used — +`automation` in the results is what tells a reader how reproducible a given row is. + +| Rung | Mechanism | Reproducible across machines? | +|---|---|---| +| 1 | AppleScript dictionary (`sdef`) | yes — none of these apps has one | +| 2 | System Events menu item by name | yes, until the app renames the item | +| 3 | Documented keyboard shortcut | yes | +| 4 | Accessibility control by name/description | mostly — names drift between versions | +| 5 | Pixel coordinates | no — flagged as reduced reproducibility | + +`node bench.mjs discover ` dumps the menus and the accessibility tree of an installed app, +which is how a driver gets written or repaired when a new version moves something. diff --git a/benchmark/drivers/camtasia.mjs b/benchmark/drivers/camtasia.mjs new file mode 100644 index 00000000..3e6f0679 --- /dev/null +++ b/benchmark/drivers/camtasia.mjs @@ -0,0 +1,291 @@ +/** + * Camtasia — the traditional screencast editor. + * + * A different generation of tool from the rest of the set: Camtasia is a general video editor + * that happens to record screens, rather than a demo-maker. That shows up in what can be + * automated and in what the row means. + * + * **Fidelity is partial, and deliberately so.** Camtasia can express every effect in the + * scenario, but not through any scripted interface: + * + * · background — a colour clip on a lower track, or the canvas colour in Properties + * · padding — Visual Properties → Scale on the selected media + * · corner radius— Visual Effects → Border + * · shadow — Visual Effects → Drop Shadow + * · zooms — Edit → Zoom-n-Pan, a panel with no scripting surface + * + * Its AppleScript dictionary exposes `add file`, `addAction` (transitions only) and a readable + * `isExporting`, and nothing that reaches Visual Properties or Zoom-n-Pan. Driving those means + * clicking through the inspector, which is the least reproducible rung of the ladder and would + * make this row depend on Camtasia's panel layout not moving between releases. + * + * So this driver measures Camtasia rendering the *same source to the same output target*, with + * no compositing, and the report marks it partial. That is a real and useful number — it is the + * cost of Camtasia's render pipeline on this machine — but it is not comparable to a full-demo + * row, and the report does not rank it against one. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { now, sleep } from "../lib/measure.mjs"; +import { + activateApp, + appIsRunning, + clickMenuItem, + jxa, + launchApp, + osa, + quitApp, +} from "../lib/uiScript.mjs"; + +const APP = "/Applications/Camtasia.app"; +const PROC = "Camtasia"; + +/** Click the first AXButton/AXRadioButton/AXCheckBox anywhere in the app whose name matches. */ +function clickByName(pattern, { roles = ["AXButton"], required = true } = {}) { + const res = jxa(` + const se = Application("System Events"); + const p = se.processes["${PROC}"]; + p.frontmost = true; + const roles = ${JSON.stringify(roles)}; + function findAll(el, d, out) { + if (d > 9) return out; + try { + if (roles.includes(el.role())) { + let n = ""; try { n = el.name() || ""; } catch (e) {} + if (n) out.push([String(n), el]); + } + } catch (e) {} + try { for (const k of el.uiElements()) findAll(k, d + 1, out); } catch (e) {} + return out; + } + let all = []; + for (const w of p.windows()) { + findAll(w, 0, all); + try { for (const s of w.sheets()) findAll(s, 0, all); } catch (e) {} + } + const re = new RegExp(${JSON.stringify(pattern)}, "i"); + const hit = all.find(([n]) => re.test(n)); + if (!hit) JSON.stringify({ ok: false, seen: all.map(a => a[0]).slice(0, 20) }); + else { hit[1].click(); JSON.stringify({ ok: true, matched: hit[0] }); } + `); + const parsed = JSON.parse(res); + if (!parsed.ok && required) { + throw new Error( + `Camtasia: no control matching /${pattern}/. Present: ${(parsed.seen ?? []).join(", ")}`, + ); + } + return parsed; +} + +/** Is a modal sheet — the save panel — actually up? */ +async function sheetPresent() { + try { + return ( + jxa(` + const se = Application("System Events"); + const p = se.processes["${PROC}"]; + let n = 0; + for (const w of p.windows()) { try { n += w.sheets().length; } catch (e) {} } + String(n > 0); + `) === "true" + ); + } catch { + return false; + } +} + +export default { + id: "camtasia", + displayName: "Camtasia", + vendor: "TechSmith", + kind: "gui", + automation: "applescript+ax", + processName: PROC, + appPath: APP, + bundleId: "com.techsmith.camtasia", + install: { + method: "dmg", + url: "https://download.techsmith.com/camtasiamac/releases/Camtasia.dmg", + appName: "Camtasia.app", + approxMB: 412, + licence: + "commercial — 30-day trial; exports carry a watermark, which does not change render time", + }, + + detect() { + if (!existsSync(APP)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* keep null */ + } + return { installed: true, version, path: APP }; + }, + + async prepare(ctx) { + if (!appIsRunning(PROC)) await launchApp(APP, PROC); + await sleep(6000); + activateApp(PROC); + await sleep(800); + + // A run that was interrupted mid-export leaves Camtasia offering to recover the project + // on next launch. That dialog is modal, so everything after it silently misfires — + // keystrokes meant for a save panel end up naming markers in the timeline instead. + clickByName("^Delete$", { required: false }); + await sleep(1500); + + clickMenuItem(PROC, "File", ["New Project"]); + await sleep(5000); + + // `add file` needs a real file reference, not a POSIX path string. + osa(`tell application "Camtasia" to add file (POSIX file "${ctx.source.path}") at time 0`, { + timeoutMs: 180_000, + }); + + // Importing a minute of 1080p60 takes Camtasia a while, and `add file` returns before it + // has finished. Wait for the media bin to actually show the clip rather than guessing at + // a delay — a driver that proceeds to Export with an empty timeline exports nothing and + // looks like a timeout. + const stem = ctx.source.path + .split("/") + .pop() + .replace(/\.[^.]+$/, ""); + let imported = false; + for (let i = 0; i < 60 && !imported; i++) { + await sleep(2000); + try { + imported = + jxa(` + const se = Application("System Events"); + const p = se.processes["${PROC}"]; + function has(el, d) { + if (d > 9) return false; + try { + const n = (el.name() || "") + " " + (el.value() || ""); + if (n.includes(${JSON.stringify(stem)})) return true; + } catch (e) {} + try { for (const k of el.uiElements()) if (has(k, d + 1)) return true; } catch (e) {} + return false; + } + String(p.windows().some(w => has(w, 0))); + `) === "true"; + } catch { + /* the window may be mid-layout */ + } + } + if (!imported) { + throw new Error(`Camtasia never showed "${stem}" on the timeline after the import`); + } + + // A 60 fps import raises "High Frame Rate Media Detected". Taking the 30 fps default + // would halve the frames Camtasia renders and quietly make it look twice as fast, so the + // answer is forced here and the choice is remembered for later runs. + const converted = clickByName("Convert the entire project to 60", { + roles: ["AXRadioButton"], + required: false, + }); + if (converted.ok) { + clickByName("Remember my selection", { roles: ["AXCheckBox"], required: false }); + clickByName("^Continue$", { required: false }); + await sleep(2500); + } + + return { + appliedFeatures: ["targetResolution", "targetFps"], + notes: [ + converted.ok + ? "project frame rate converted to 60 fps on import" + : "no frame-rate prompt (already remembered from an earlier run)", + "Effects are NOT applied: background, padding, corner radius, shadow and zooms live in Visual Properties and Zoom-n-Pan, neither of which Camtasia exposes to scripting. This row measures its render pipeline at the same output target, not the full-demo composition.", + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + mkdirSync(ctx.outDir, { recursive: true }); + + activateApp(PROC); + await sleep(600); + clickMenuItem(PROC, "Export", ["Local File"]); + await sleep(4000); + + // The trial offers watermarked export or a licence key. A watermark is a cheap overlay + // and does not change render time, so the trial path is a valid measurement. + clickByName("Export with Watermark", { required: false }); + await sleep(3000); + + // Never type blind. If the export sheet did not open, ⇧⌘G and the filename would land in + // the editor — which is exactly how an interrupted run once created timeline markers + // named after the output file. + if (!(await sheetPresent())) { + throw new Error( + "Camtasia: Export → Local File did not raise a save sheet. Something modal is in the " + + "way (a recovery prompt, an upsell, or an unfinished export).", + ); + } + + // Camtasia's export sheet is a standard save panel: ⇧⌘G reaches it. + const dir = out.replace(/\/[^/]+$/, ""); + const stem = out + .split("/") + .pop() + .replace(/\.mp4$/i, ""); + osa(`tell application "System Events" to tell process "${PROC}" + set frontmost to true + keystroke "g" using {command down, shift down} + delay 0.9 + keystroke "${dir}" + delay 0.6 + key code 36 + delay 1.2 + keystroke "a" using {command down} + keystroke "${stem}" + delay 0.5 + end tell`); + await sleep(1200); + + clickByName("^Export$"); + ctx.commit(); + + // Camtasia publishes its own progress; `isExporting` going false is a cleaner stop than + // the filesystem, which sees the file appear before the muxer is finished with it. + const deadline = now() + 30 * 60 * 1000; + let sawExporting = false; + while (now() < deadline) { + await sleep(1000); + let exporting = null; + try { + exporting = osa(`tell application "Camtasia" to return isExporting of front project`, { + timeoutMs: 8000, + }); + } catch { + exporting = null; // the property is not always readable; the file watcher covers it + } + if (exporting === "true") sawExporting = true; + if (sawExporting && exporting === "false") { + ctx.markComplete(); + return; + } + if (existsSync(out) && !sawExporting) { + // isExporting was never readable on this build — let the runner's watcher decide. + return; + } + } + }, + + async cleanup() { + if (appIsRunning(PROC)) await quitApp(PROC, { force: true }); + }, +}; diff --git a/benchmark/drivers/cap.mjs b/benchmark/drivers/cap.mjs new file mode 100644 index 00000000..7c355c07 --- /dev/null +++ b/benchmark/drivers/cap.mjs @@ -0,0 +1,230 @@ +/** + * Cap (cap.so) — the other open-source entrant in this category. + * + * The only competitor here with a real command line: `Cap.app/Contents/MacOS/cap-cli export` + * renders a `.cap` project with the app's full compositor, and takes `--fps`, `--resolution` + * and `--quality`, so it can be pinned to the same output as everything else. + * + * A `.cap` project is a directory — `recording-meta.json` plus the media — and the editor + * state lives beside it in `project-config.json`. Both are written directly, for the same + * reason OpenScreen's project is: an edit typed into a UI is not reproducible. + */ +import { execFileSync, spawn } from "node:child_process"; +import { copyFileSync, existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const APP = "/Applications/Cap.app"; +const CLI = `${APP}/Contents/MacOS/cap-cli`; + +/** #RRGGBB → the [r,g,b] triple Cap's colour background expects. */ +const rgb = (hex) => { + const h = hex.replace("#", ""); + return [0, 2, 4].map((i) => Number.parseInt(h.slice(i, i + 2), 16)); +}; + +export default { + id: "cap", + displayName: "Cap", + vendor: "Cap Software", + kind: "cli", + automation: "cli", + processName: "Cap", + appPath: APP, + bundleId: "so.cap.desktop", + install: { + method: "dmg", + url: "https://cap.so/download/apple-silicon", + appName: "Cap.app", + approxMB: 123, + licence: "AGPL-3.0 — free", + }, + + detect() { + if (!existsSync(CLI)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* keep null */ + } + return { installed: true, version, path: CLI }; + }, + + /** Cap's `background.padding` is 0-100 on its own scale; see `bench.mjs calibrate`. */ + defaultPaddingControl(scenario) { + return scenario.effects.paddingPercent; + }, + + async prepare(ctx) { + const e = ctx.scenario.effects; + const dir = join(ctx.workDir, "projects", "cap"); + const project = join(dir, `${ctx.scenario.id}.cap`); + rmSync(project, { recursive: true, force: true }); + mkdirSync(join(project, "content"), { recursive: true }); + copyFileSync(ctx.source.path, join(project, "content", "display.mp4")); + + // A single-segment studio recording: the smallest shape `cap project validate` accepts, + // and the one an import would produce. + writeFileSync( + join(project, "recording-meta.json"), + `${JSON.stringify( + { + platform: "MacOS", + pretty_name: `openscreen-benchmark-${ctx.scenario.id}`, + display: { path: "content/display.mp4", fps: ctx.source.probe.video.fps }, + }, + null, + 2, + )}\n`, + ); + + // Start from Cap's own defaults so nothing unset drifts between versions, then apply + // only what the scenario names. + const base = JSON.parse( + execFileSync(CLI, ["project", "config", "get", project], { encoding: "utf8" }), + ); + const duration = ctx.source.probe.durationSec; + + base.background.source = { + type: "color", + value: rgb(e.background?.color ?? "#000000"), + alpha: 255, + }; + base.background.blur = 0; + base.background.padding = ctx.paddingControl ?? this.defaultPaddingControl(ctx.scenario); + base.background.rounding = e.cornerRadiusPx; + // Cap's `shadow` is 0-100; the scenario's intensity is 0-1. + base.background.shadow = e.shadow?.enabled ? Math.round(e.shadow.intensity * 100) : 0; + base.camera.hide = true; + base.cursor.hide = !e.cursorEffects; + base.screenMotionBlur = e.motionBlur ? 1 : 0; + + base.timeline = { + segments: [{ recordingSegment: 0, timescale: 1, start: 0, end: duration }], + zoomSegments: (e.zooms ?? []).map((z) => ({ + start: z.startSec, + end: z.endSec, + amount: z.scale, + mode: { manual: { x: z.focus.x, y: z.focus.y } }, + })), + sceneSegments: [], + maskSegments: [], + textSegments: [], + captionSegments: [], + keyboardSegments: [], + audioSegments: [], + camera3dSegments: [], + }; + + // `config set` takes the whole document as one argv string and resets anything omitted, + // which is why the defaults were read first rather than a partial patch being sent. + execFileSync( + CLI, + ["project", "config", "set", project, "--settings-json", JSON.stringify(base)], + { + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }, + ); + // Keep a copy beside the project so a run is auditable after the fact. + writeFileSync( + join(dir, `${ctx.scenario.id}-config.json`), + `${JSON.stringify(base, null, 2)}\n`, + ); + + const verify = JSON.parse( + execFileSync(CLI, ["project", "config", "get", project], { encoding: "utf8" }), + ); + const applied = ["targetResolution", "targetFps"]; + if (verify.background?.source?.type === "color") applied.push("background"); + if (verify.background?.padding > 0) applied.push("padding"); + if (verify.background?.rounding > 0) applied.push("cornerRadius"); + if (verify.background?.shadow > 0) applied.push("shadow"); + if ( + (verify.timeline?.zoomSegments ?? []).length === (e.zooms ?? []).length && + e.zooms?.length + ) { + applied.push("zooms"); + } + + ctx.state.projectPath = project; + return { + appliedFeatures: applied, + notes: [ + `project: ${project}`, + `zoom segments written: ${(verify.timeline?.zoomSegments ?? []).length}`, + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + const t = ctx.scenario.output; + + const args = [ + "export", + ctx.state.projectPath, + "--output", + out, + "--format", + "mp4", + "--fps", + String(t.fps), + "--resolution", + `${t.width}x${t.height}`, + "--quality", + "maximum", + "--progress-json", + ]; + + return new Promise((resolve, reject) => { + const child = spawn(CLI, args, { stdio: ["ignore", "pipe", "pipe"] }); + let committed = false; + let stderrTail = ""; + let buf = ""; + child.stdout.on("data", (d) => { + buf += d.toString(); + const lines = buf.split("\n"); + buf = lines.pop() ?? ""; + for (const line of lines) { + if (!line.trim()) continue; + let ev; + try { + ev = JSON.parse(line); + } catch { + continue; + } + // First Progress event = the renderer is live. Same rule as the OpenScreen + // driver: process start-up is warm-up, rendering is the measurement. + if (!committed && (ev.type === "Progress" || ev.type === "Completed")) { + committed = true; + ctx.commit(); + } + if (ev.type === "Error") stderrTail += `\n${ev.error}`; + } + }); + child.stderr.on("data", (d) => { + stderrTail = (stderrTail + d.toString()).slice(-2000); + }); + child.on("error", reject); + child.on("close", (code) => { + if (!committed) ctx.commit(); + if (code === 0) resolve(); + else reject(new Error(`cap export exited ${code}: ${stderrTail.trim().slice(0, 600)}`)); + }); + }); + }, + + async cleanup() { + // Nothing to tear down: `cap export` is a one-shot process. + }, +}; diff --git a/benchmark/drivers/ffmpeg-baseline.mjs b/benchmark/drivers/ffmpeg-baseline.mjs new file mode 100644 index 00000000..38c973b0 --- /dev/null +++ b/benchmark/drivers/ffmpeg-baseline.mjs @@ -0,0 +1,96 @@ +/** + * Not a competitor — the floor. + * + * A straight re-encode of the source at the target settings, with no compositing at all. It + * answers the question the app-to-app numbers cannot: how much of an export is unavoidable + * encoding work on this machine, and how much is the app's own pipeline. Every app's time + * should be read as a multiple of this. + */ +import { spawn } from "node:child_process"; +import { join } from "node:path"; +import { resolveFfmpeg } from "../lib/env.mjs"; + +export default { + id: "ffmpeg-baseline", + displayName: "ffmpeg (re-encode floor)", + vendor: "reference", + kind: "reference", + automation: "cli", + processName: null, + appPath: null, + bundleId: null, + install: null, + + detect() { + try { + const { ffmpeg, source } = resolveFfmpeg(); + return { installed: true, version: source, path: ffmpeg }; + } catch (e) { + return { installed: false, version: null, path: null, error: e.message }; + } + }, + + async prepare() { + return { + // The floor deliberately applies nothing. Listing the two output features it *does* + // honour keeps the fidelity score honest rather than showing a bare zero. + appliedFeatures: ["targetResolution", "targetFps"], + notes: ["No compositing: this row is the encode-only reference, not a product."], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const { ffmpeg } = resolveFfmpeg(); + const out = this.outputPath(ctx); + const t = ctx.scenario.output; + + const args = [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + ctx.source.path, + "-vf", + `scale=${t.width}:${t.height}:flags=bicubic,format=yuv420p`, + "-r", + String(t.fps), + "-c:v", + "h264_videotoolbox", + "-b:v", + "20M", + "-profile:v", + "high", + "-c:a", + "aac", + "-b:a", + "128k", + "-movflags", + "+faststart", + out, + ]; + + return new Promise((resolve, reject) => { + const child = spawn(ffmpeg, args, { stdio: ["ignore", "ignore", "pipe"] }); + let stderr = ""; + child.stderr.on("data", (d) => { + stderr += d.toString(); + }); + // The process is the export: commit the instant it is live. + ctx.commit(); + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) resolve(); + else reject(new Error(`ffmpeg exited ${code}: ${stderr.trim().slice(0, 500)}`)); + }); + }); + }, + + async cleanup() { + // Nothing to tear down: the floor spawns one ffmpeg and it exits. + }, +}; diff --git a/benchmark/drivers/focusee.mjs b/benchmark/drivers/focusee.mjs new file mode 100644 index 00000000..ce97d01b --- /dev/null +++ b/benchmark/drivers/focusee.mjs @@ -0,0 +1,176 @@ +/** + * FocuSee — the closest pitch-for-pitch rival, currently unmeasurable. + * + * **FocuSee 2.4.1 rejects every MP4 it is given.** Its own import panel and `open -a` both end + * at *"The source file is damaged and cannot be opened."* — for the benchmark fixture and for a + * real 2560×1440 H.264 screen recording alike. The app is not sandboxed (no + * `com.apple.security.app-sandbox` entitlement), so this is not a file-access grant that could + * be fixed by choosing the file through a picker. Verified on macOS 26.5 with the direct + * download from imobie; the Mac App Store build may differ. + * + * The rest of the driver is written and works: FocuSee is a native Cocoa app, so unlike the + * Electron entrants its whole interface is published to the accessibility API — the canvas-size + * buttons, the Padding / Inset / Roundness / Shadow values and the Export button are all + * addressable by name. If a later build fixes the import, this driver should measure it as-is. + * + * Install note: the vendor ships a ~5 MB downloader stub rather than the app. It is notarised + * (iMobie Inc., team 2QJGLWL8Y6) and installs FocuSee.app into /Applications on launch, but it + * is a GUI installer, so `bench.mjs install` cannot fetch this one unattended. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { sleep } from "../lib/measure.mjs"; +import { activateApp, appIsRunning, jxa, launchApp, osa, quitApp } from "../lib/uiScript.mjs"; + +const APP = "/Applications/FocuSee.app"; +const PROC = "FocuSee"; + +/** Read every static-text value in FocuSee's edit window — how its state is inspected. */ +function editorText() { + return JSON.parse( + jxa(` + const se = Application("System Events"); + const p = se.processes["${PROC}"]; + const win = p.windows().find(w => { try { return w.name() === "edit"; } catch (e) { return false; } }); + function txt(el, d) { + if (d > 10) return []; + let out = []; + try { + const r = el.role(); + if (r === "AXStaticText" || r === "AXButton") { + const v = el.value() || el.name(); + if (v && String(v) !== "button") out.push(String(v).slice(0, 60)); + } + } catch (e) {} + try { for (const k of el.uiElements()) out = out.concat(txt(k, d + 1)); } catch (e) {} + return out; + } + JSON.stringify(win ? txt(win, 0) : []); + `), + ); +} + +export default { + id: "focusee", + displayName: "FocuSee", + vendor: "iMobie", + kind: "gui", + automation: "ax+menu", + processName: PROC, + appPath: APP, + bundleId: "com.imobie.FocuSee", + install: { + method: "manual", + url: "https://focusee.imobie.com/go/download.php?product=fs", + appName: "FocuSee.app", + approxMB: 5, + licence: "commercial — trial exports are watermarked", + notes: [ + "The download is a GUI installer stub, not the app, so this one cannot be installed unattended.", + "Run the stub once during preflight; it places FocuSee.app in /Applications itself.", + ], + }, + + detect() { + if (!existsSync(APP)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* keep null */ + } + return { installed: true, version, path: APP }; + }, + + async prepare(ctx) { + if (appIsRunning(PROC)) await quitApp(PROC, { force: true }); + await sleep(2000); + // FocuSee registers as an MP4 handler; opening the file this way is what creates a + // project without having to drive its drag-and-drop drop zone. + execFileSync("/usr/bin/open", ["-a", APP, ctx.source.path]); + await sleep(18000); + activateApp(PROC); + + const text = editorText(); + const damaged = text.some((t) => /damaged and cannot be opened/i.test(t)); + if (damaged) { + throw new Error( + "FocuSee refused the source: “The source file is damaged and cannot be opened.” " + + "Reproduced with a real 1440p H.264 recording too, so it is not specific to the benchmark " + + "fixture. The app is not sandboxed, so this is not a file-access grant.", + ); + } + if (!text.length) throw new Error("FocuSee did not open an edit window for the source clip"); + + // The composition controls are AX static texts paired with sliders; FocuSee exposes their + // values but not setters, so what the scenario can reach here is limited to the canvas + // aspect ratio. Whatever is applied is reported, never assumed. + const applied = ["targetResolution", "targetFps"]; + return { + appliedFeatures: applied, + notes: [`editor state: ${text.slice(0, 20).join(" · ")}`], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + activateApp(PROC); + await sleep(600); + + const clicked = JSON.parse( + jxa(` + const se = Application("System Events"); + const p = se.processes["${PROC}"]; + p.frontmost = true; + function findAll(el, d, out) { + if (d > 10) return out; + try { if (el.role() === "AXButton") { const n = el.name() || ""; if (n) out.push([String(n), el]); } } catch (e) {} + try { for (const k of el.uiElements()) findAll(k, d + 1, out); } catch (e) {} + return out; + } + let all = []; + for (const w of p.windows()) findAll(w, 0, all); + const hit = all.find(([n]) => /^export$/i.test(n)); + if (!hit) JSON.stringify({ ok: false, seen: all.map(a => a[0]).slice(0, 20) }); + else { hit[1].click(); JSON.stringify({ ok: true }); } + `), + ); + if (!clicked.ok) + throw new Error(`FocuSee: no Export button. Present: ${(clicked.seen ?? []).join(", ")}`); + await sleep(3000); + + const dir = out.replace(/\/[^/]+$/, ""); + const stem = out + .split("/") + .pop() + .replace(/\.mp4$/i, ""); + osa(`tell application "System Events" to tell process "${PROC}" + set frontmost to true + keystroke "g" using {command down, shift down} + delay 0.8 + keystroke "${dir}" + delay 0.5 + key code 36 + delay 1.0 + keystroke "a" using {command down} + keystroke "${stem}" + delay 0.4 + key code 36 + end tell`); + ctx.commit(); + }, + + async cleanup() { + if (appIsRunning(PROC)) await quitApp(PROC, { force: true }); + }, +}; diff --git a/benchmark/drivers/kap.mjs b/benchmark/drivers/kap.mjs new file mode 100644 index 00000000..e52189a7 --- /dev/null +++ b/benchmark/drivers/kap.mjs @@ -0,0 +1,234 @@ +/** + * Kap — the open-source minimum. + * + * Kap has no background, no padding, no corner radius, no shadow and no zooms: it trims and + * re-encodes, and that is all it claims to do. It cannot express the full-demo scenario and is + * not a peer of the other apps here. It is kept because it answers a question the synthetic + * ffmpeg floor cannot — what a *real, shipping app* costs to get a frame from disk to disk on + * this machine, Electron shell and all. Its row is always marked partial. + * + * Automation: Kap is Electron with no accessibility tree, so `System Events` sees an empty + * window. Launched with `--remote-debugging-port` its renderer is reachable, and the export + * button, the settings fields and the progress text are all plain DOM. + * + * Output: the export destination is a native popup menu that neither CDP nor the accessibility + * API can open, so the driver uses Kap's clipboard destination — the same render, writing to a + * temp directory — and adopts the file it produces. Kap's own "Export complete" is used as the + * stop signal, so the adoption copy is never counted. + */ +import { execFileSync } from "node:child_process"; +import { + copyFileSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { homedir, tmpdir } from "node:os"; +import { join } from "node:path"; +import { CdpSession, DOM_HELPERS, listTargets } from "../lib/cdp.mjs"; +import { now, sleep } from "../lib/measure.mjs"; +import { appIsRunning, quitApp } from "../lib/uiScript.mjs"; + +const APP = "/Applications/Kap.app"; +const BIN = `${APP}/Contents/MacOS/Kap`; +const PORT = 9334; +const HISTORY = join( + homedir(), + "Library", + "Application Support", + "Kap", + "export-usage-history.json", +); + +/** Every `//*.mp4` Kap could have written. */ +function tempExports() { + const base = process.env.TMPDIR || tmpdir(); + const out = new Map(); + let dirs = []; + try { + dirs = readdirSync(base); + } catch { + return out; + } + for (const d of dirs) { + const dir = join(base, d); + let entries = []; + try { + if (!statSync(dir).isDirectory()) continue; + entries = readdirSync(dir); + } catch { + continue; + } + for (const f of entries) { + if (!f.endsWith(".mp4")) continue; + const p = join(dir, f); + try { + out.set(p, statSync(p).mtimeMs); + } catch { + /* vanished */ + } + } + } + return out; +} + +export default { + id: "kap", + displayName: "Kap", + vendor: "Wulkano", + kind: "gui", + automation: "cdp", + processName: "Kap", + appPath: APP, + bundleId: "com.wulkano.kap", + install: { + method: "dmg", + url: "https://github.com/wulkano/Kap/releases/download/v3.6.0/Kap-3.6.0-arm64.dmg", + version: "3.6.0", + appName: "Kap.app", + approxMB: 119, + licence: "MIT — free", + }, + + detect() { + if (!existsSync(BIN)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* keep null */ + } + return { installed: true, version, path: BIN }; + }, + + async prepare(ctx) { + // Kap picks its default format from a usage ledger rather than a setting. Promoting mp4 + // there is how you make the editor open on MP4 instead of GIF without touching the UI. + if (existsSync(HISTORY)) { + try { + const h = JSON.parse(readFileSync(HISTORY, "utf8")); + for (const k of Object.keys(h)) h[k].lastUsed = 1; + h.mp4 = { lastUsed: 99, plugins: { default: 99 } }; + writeFileSync(HISTORY, JSON.stringify(h, null, 1)); + } catch { + /* a fresh install has no ledger; the default is fine */ + } + } + + if (appIsRunning(this.processName)) await quitApp(this.processName, { force: true }); + await sleep(1500); + execFileSync("/bin/sh", [ + "-c", + `nohup ${JSON.stringify(BIN)} --remote-debugging-port=${PORT} >/dev/null 2>&1 &`, + ]); + await sleep(9000); + + execFileSync("/usr/bin/open", ["-a", APP, ctx.source.path]); + await sleep(8000); + + const target = (await listTargets(PORT)).find((t) => t.url.includes("editor.html")); + if (!target) throw new Error("Kap did not open an editor window for the source clip"); + const s = new CdpSession(target.webSocketDebuggerUrl); + await s.open(); + await s.eval(DOM_HELPERS); + ctx.state.cdp = s; + + const t = ctx.scenario.output; + const applied = []; + const state = JSON.parse( + await s.eval(`(() => { + const ins = [...document.querySelectorAll("input")]; + const setNative = (el, v) => { + const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, "value").set; + setter.call(el, String(v)); + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + }; + const w = ins.find(i => i.value === "1920") || ins[ins.length - 3]; + const h = ins.find(i => i.value === "1080") || ins[ins.length - 2]; + const f = ins[ins.length - 1]; + if (w) setNative(w, ${t.width}); + if (h) setNative(h, ${t.height}); + if (f) setNative(f, ${t.fps}); + return JSON.stringify({ + format: (document.querySelector(".format") || {}).innerText, + plugin: (document.querySelector(".plugin") || {}).innerText, + values: [...document.querySelectorAll("input")].map(i => i.value), + }); + })()`), + ); + if (/mp4/i.test(state.format ?? "")) applied.push("targetResolution", "targetFps"); + + return { + appliedFeatures: applied, + notes: [ + `format selector reads "${state.format}", destination "${state.plugin}"`, + "Kap has no background, padding, corner-radius, shadow or zoom features — the full-demo scenario cannot be expressed, so this row is a re-encode reference rather than a competitor.", + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const s = ctx.state.cdp; + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + const before = tempExports(); + + const clicked = await s.eval(`(() => { + const b = document.querySelector("button.start-export") + || [...document.querySelectorAll("button")].find(x => /convert/i.test(x.innerText)); + if (!b) return "no-button"; + b.click(); + return "clicked"; + })()`); + if (clicked !== "clicked") + throw new Error(`Kap: could not find the Convert button (${clicked})`); + ctx.commit(); + + // Kap reports its own progress and completion in the DOM. That is a better stop signal + // than the temp file, which is written before Kap has finished with it. + const deadline = now() + 30 * 60 * 1000; + let done = false; + while (now() < deadline) { + await sleep(500); + const txt = await s.eval(`document.body.innerText.slice(0, 400)`); + if (/export complete|drag and drop to copy/i.test(txt)) { + ctx.markComplete(); + done = true; + break; + } + const m = /Converting\s*—\s*(\d+)s remaining/i.exec(txt); + if (m) ctx.progress?.(null, `${m[1]}s remaining`); + } + if (!done) throw new Error("Kap never reported the export as complete"); + + // Adopt whatever appeared in the temp tree. The copy happens after markComplete, so it + // is outside the measured interval. + const after = tempExports(); + const fresh = [...after.entries()] + .filter(([p, m]) => !before.has(p) || before.get(p) !== m) + .sort((a, b) => b[1] - a[1]); + if (!fresh.length) + throw new Error("Kap reported completion but wrote no file into the temp tree"); + mkdirSync(ctx.outDir, { recursive: true }); + copyFileSync(fresh[0][0], out); + ctx.state.kapTempPath = fresh[0][0]; + }, + + async cleanup(ctx) { + ctx.state?.cdp?.close(); + if (appIsRunning(this.processName)) await quitApp(this.processName, { force: true }); + }, +}; diff --git a/benchmark/drivers/openscreen-cli.mjs b/benchmark/drivers/openscreen-cli.mjs new file mode 100644 index 00000000..eb774a30 --- /dev/null +++ b/benchmark/drivers/openscreen-cli.mjs @@ -0,0 +1,152 @@ +/** + * OpenScreen, headless. + * + * The subject of the benchmark, driven through its own `openscreen export` command. The + * project is written as JSON rather than built in the editor, so the scenario is exact and + * byte-reproducible — see lib/openscreenProject.mjs. + * + * Because this path skips the UI entirely it is *not* directly comparable to Screen Studio's + * or Camtasia's numbers; `openscreen-gui` exists for that comparison, and the report keeps the + * two rows apart. + */ +import { execFileSync, spawn } from "node:child_process"; +import { existsSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { buildProject } from "../lib/openscreenProject.mjs"; + +const APP = "/Applications/Openscreen.app"; +const BIN = `${APP}/Contents/MacOS/Openscreen`; + +export default { + id: "openscreen-cli", + displayName: "OpenScreen (CLI)", + vendor: "OpenScreen", + kind: "cli", + automation: "cli", + processName: "Openscreen", + appPath: APP, + bundleId: "com.etiennelescot.openscreen", + install: { + method: "dmg", + // Resolved at install time from the GitHub release feed; see lib/install.mjs. + url: "github:getopenscreen/openscreen", + appName: "Openscreen.app", + approxMB: 250, + licence: "MIT — free, no account, no watermark", + notes: ["The CLI ships inside the normal app bundle; there is nothing extra to install."], + }, + + detect() { + if (!existsSync(BIN)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* unreadable plist — report installed without a version */ + } + return { installed: true, version, path: BIN }; + }, + + /** + * OpenScreen's `padding` is 0-100 on its own scale, not a percentage of the frame. The + * default below is a starting point; `bench.mjs calibrate` measures the inset it actually + * produces and solves for the value that matches every other app. + */ + defaultPaddingControl(scenario) { + return Math.round(Math.min(100, Math.max(0, scenario.effects.paddingPercent * 10))); + }, + + async prepare(ctx) { + const outDir = join(ctx.workDir, "projects", "openscreen-cli"); + const { projectPath } = buildProject({ + sourcePath: ctx.source.path, + scenario: ctx.scenario, + outDir, + title: ctx.scenario.id, + paddingControl: ctx.paddingControl ?? this.defaultPaddingControl(ctx.scenario), + }); + ctx.state.projectPath = projectPath; + + return { + appliedFeatures: [ + "background", + "padding", + "cornerRadius", + "shadow", + "zooms", + "targetResolution", + "targetFps", + ], + notes: [ + `project: ${projectPath}`, + "MP4 export is fixed at 60 fps (MP4_EXPORT_FPS, src/cli/CliExportRunner.tsx) — which is why the pinned target is 60.", + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + + const args = ["export", ctx.state.projectPath, "-o", out, "--quality", "good", "--json"]; + return new Promise((resolve, reject) => { + const child = spawn(BIN, args, { stdio: ["ignore", "pipe", "pipe"] }); + let committed = false; + let stderrTail = ""; + let buf = ""; + + child.stdout.on("data", (d) => { + buf += d.toString(); + const lines = buf.split("\n"); + buf = lines.pop() ?? ""; + for (const line of lines) { + if (!line.trim()) continue; + let ev; + try { + ev = JSON.parse(line); + } catch { + continue; + } + // `started` fires once the hidden renderer is up and the render begins. Taking + // t0 here rather than at spawn keeps Electron's cold boot out of the export + // number — the GUI apps are warm when their clock starts, so this one is too. + // The boot cost is still recorded, as launchToCommitMs. + if (!committed && (ev.event === "started" || ev.event === "progress")) { + committed = true; + ctx.commit(); + } + if (ev.event === "progress") ctx.progress?.(ev.percentage); + if (ev.event === "done") ctx.state.reportedOutput = ev.outputPath; + } + }); + child.stderr.on("data", (d) => { + stderrTail = (stderrTail + d.toString()).slice(-2000); + }); + child.on("error", reject); + child.on("close", (code) => { + if (!committed) ctx.commit(); // never leave the run without a t0 + if (code === 0) resolve(); + else + reject(new Error(`openscreen export exited ${code}: ${stderrTail.trim().slice(0, 600)}`)); + }); + }); + }, + + async cleanup() { + // CLI exports leave the on-device STT server running; on an 8 GB machine those orphans + // distort the next run's memory figures and the process sampler's totals. + try { + execFileSync("/usr/bin/pkill", ["-f", "whisper-stt-server"], { stdio: "ignore" }); + } catch { + /* none running */ + } + }, +}; diff --git a/benchmark/drivers/openscreen-gui.mjs b/benchmark/drivers/openscreen-gui.mjs new file mode 100644 index 00000000..1c9724f3 --- /dev/null +++ b/benchmark/drivers/openscreen-gui.mjs @@ -0,0 +1,246 @@ +/** + * OpenScreen, through its own editor. + * + * The CLI leg (`openscreen-cli`) measures the render engine with no interface in the way. That + * is the right number to compare against Cap's CLI, and the wrong one to compare against an app + * that can only be driven by clicking — a UI leg carries the editor's own overhead, and the + * subject of a benchmark should not be the only entrant excused from it. + * + * So this driver does what a person does: opens the project in the editor, opens the export + * dialog, picks MP4 / 1080p / 60 / H.264, presses Export and answers the save panel. + * + * OpenScreen is Electron, so the editor is reached over CDP and every control is found by its + * visible text. The two native surfaces on the path — the File menu and the save panel — are + * driven through System Events. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { CdpSession, DOM_HELPERS, listTargets } from "../lib/cdp.mjs"; +import { sleep } from "../lib/measure.mjs"; +import { buildProject } from "../lib/openscreenProject.mjs"; +import { + activateApp, + appIsRunning, + clickMenuItem, + listWindows, + osa, + quitApp, +} from "../lib/uiScript.mjs"; + +const APP = "/Applications/Openscreen.app"; +const BIN = `${APP}/Contents/MacOS/Openscreen`; +const PORT = 9335; + +const editorTarget = async () => + (await listTargets(PORT)).find((t) => t.url.includes("windowType=editor")); + +export default { + id: "openscreen-gui", + displayName: "OpenScreen (GUI)", + vendor: "OpenScreen", + kind: "gui", + automation: "cdp+menu", + processName: "Openscreen", + appPath: APP, + bundleId: "com.etiennelescot.openscreen", + install: null, // shares the install with openscreen-cli + + detect() { + if (!existsSync(BIN)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* keep null */ + } + return { installed: true, version, path: BIN }; + }, + + defaultPaddingControl(scenario) { + return Math.round(Math.min(100, Math.max(0, scenario.effects.paddingPercent * 10))); + }, + + async prepare(ctx) { + const outDir = join(ctx.workDir, "projects", "openscreen-gui"); + const { projectPath } = buildProject({ + sourcePath: ctx.source.path, + scenario: ctx.scenario, + outDir, + title: ctx.scenario.id, + paddingControl: ctx.paddingControl ?? this.defaultPaddingControl(ctx.scenario), + }); + ctx.state.projectPath = projectPath; + + // A single-instance lock keys on the userData path, so a stale instance must go before + // the debugging port can be opened on a fresh one. + if (appIsRunning(this.processName)) await quitApp(this.processName, { force: true }); + try { + execFileSync("/usr/bin/pkill", ["-f", "whisper-stt-server"], { stdio: "ignore" }); + } catch { + /* none running */ + } + await sleep(2000); + execFileSync("/bin/sh", [ + "-c", + `nohup ${JSON.stringify(BIN)} --remote-debugging-port=${PORT} >/dev/null 2>&1 &`, + ]); + await sleep(12000); + + // The launcher opens on the HUD; the editor is a separate window. switchToEditor never + // resolves its promise, so it is fired and then waited for by polling the target list. + const hud = (await listTargets(PORT)).find((t) => t.url.includes("hud-overlay")); + if (hud) { + const h = new CdpSession(hud.webSocketDebuggerUrl); + await h.open(); + // switchToEditor tears the HUD renderer down, so the CDP reply for this evaluate may + // never arrive. Fire it, give it a moment, move on. + try { + await h.send( + "Runtime.evaluate", + { expression: "window.electronAPI.switchToEditor()", awaitPromise: false }, + { timeoutMs: 5000 }, + ); + } catch { + /* expected when the page goes away mid-call */ + } + h.close(); + } + let ed = null; + for (let i = 0; i < 30 && !ed; i++) { + await sleep(1000); + ed = await editorTarget(); + } + if (!ed) throw new Error("the OpenScreen editor window never appeared"); + + const s = new CdpSession(ed.webSocketDebuggerUrl); + await s.open(); + await s.eval(DOM_HELPERS); + ctx.state.cdp = s; + + // File → Load Project… raises an in-app picker whose "Browse files…" button is what + // opens the real panel. Both steps are needed; the in-app list ignores ⇧⌘G. + activateApp(this.processName); + await sleep(700); + clickMenuItem(this.processName, "File", ["Load Project"]); + await sleep(2000); + await s.eval(`JSON.stringify(window.__osbench.click("Browse files"))`); + await sleep(2000); + osa(`tell application "System Events" to tell process "${this.processName}" + set frontmost to true + keystroke "g" using {command down, shift down} + delay 0.8 + keystroke "${projectPath}" + delay 0.6 + key code 36 + delay 1.4 + key code 36 + end tell`); + await sleep(10000); + + // Read the composition panel back: this is the app telling us what it thinks it loaded. + const panel = await s.eval("document.body.innerText.slice(0, 500)"); + const applied = ["targetResolution", "targetFps"]; + const e = ctx.scenario.effects; + if (panel.includes(e.background.color)) applied.push("background"); + if (/Padding/.test(panel)) applied.push("padding"); + if (new RegExp(`Roundness\\s*\\|?\\s*${e.cornerRadiusPx}`).test(panel.replace(/\n/g, " "))) { + applied.push("cornerRadius"); + } + if (/Shadow\s*\|?\s*(?!0%)\d+%/.test(panel.replace(/\n/g, " "))) applied.push("shadow"); + // Zooms live on the timeline rather than the composition panel; the project was written + // with them and the pixel verifier is what confirms they rendered. + if (e.zooms?.length) applied.push("zooms"); + + return { + appliedFeatures: applied, + notes: [ + `project: ${projectPath}`, + `composition panel after load: ${panel.replace(/\n+/g, " | ").slice(0, 220)}`, + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const s = ctx.state.cdp; + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + const t = ctx.scenario.output; + + await s.eval(`JSON.stringify(window.__osbench.click("Export", { exact: true }))`); + await sleep(2000); + + // The dialog's controls are plain buttons labelled with their value. + for (const label of ["MP4", `${t.height}p`, String(t.fps), "H.264"]) { + const r = JSON.parse( + await s.eval( + `JSON.stringify(window.__osbench.click(${JSON.stringify(label)}, { exact: true }))`, + ), + ); + if (!r.ok) throw new Error(`OpenScreen export dialog: no control labelled "${label}"`); + await sleep(350); + } + + const go = JSON.parse(await s.eval(`JSON.stringify(window.__osbench.click("Export MP4"))`)); + if (!go.ok) throw new Error("OpenScreen export dialog: no “Export MP4” button"); + + // Pressing Export raises the system save panel; the render starts when it is answered. + await sleep(2500); + const dir = out.replace(/\/[^/]+$/, ""); + // The save panel appends the format's extension itself, so a name that already carries + // one comes back as "…run0.mp4.mp4" and the watcher waits forever on a path that will + // never exist. Type the stem only. + const file = out + .split("/") + .pop() + .replace(/\.mp4$/i, ""); + osa(`tell application "System Events" to tell process "${this.processName}" + set frontmost to true + keystroke "g" using {command down, shift down} + delay 0.7 + keystroke "${dir}" + delay 0.5 + key code 36 + delay 1.0 + keystroke "a" using {command down} + keystroke "${file}" + delay 0.4 + key code 36 + delay 0.8 + end tell`); + ctx.commit(); + + // Answer a replace-confirmation if one appears, then let the runner's file watcher decide + // when the render is done. + await sleep(1200); + try { + osa(`tell application "System Events" to tell process "${this.processName}" + repeat with w in windows + try + if exists (button "Replace" of sheet 1 of w) then click button "Replace" of sheet 1 of w + end try + end repeat + end tell`); + } catch { + /* the common case: no alert */ + } + }, + + async cleanup(ctx) { + ctx.state?.cdp?.close(); + if (appIsRunning(this.processName)) await quitApp(this.processName, { force: true }); + try { + execFileSync("/usr/bin/pkill", ["-f", "whisper-stt-server"], { stdio: "ignore" }); + } catch { + /* none running */ + } + }, +}; diff --git a/benchmark/drivers/screen-studio.mjs b/benchmark/drivers/screen-studio.mjs new file mode 100644 index 00000000..563c7c51 --- /dev/null +++ b/benchmark/drivers/screen-studio.mjs @@ -0,0 +1,237 @@ +/** + * Screen Studio — the app that defined this category. + * + * Fully automated up to the point where it stops being possible: **export is gated behind + * account activation.** Pressing Export on an unactivated install opens an activation wall + * asking for an email or licence key. There is no trial export and no watermark path — the + * bundle contains no "free trial" strings at all. With a licence activated once (during + * preflight), every step below runs unattended and the app becomes a full peer in the table. + * + * Getting to that point took the most work of any app here, and the findings are worth stating + * because they shape the driver: + * + * 1. **It cannot be screenshotted.** The editor window is marked `kCGWindowSharingNone`, so + * macOS excludes it from every capture API. It is plainly visible to the person at the + * machine and invisible to `screencapture`, ScreenCaptureKit, and any agent driving pixels. + * 2. **It publishes no accessibility tree.** `System Events` sees a window containing three + * traffic-light buttons and nothing else. + * 3. Its only documented automation is three `screen-studio://record-*` deeplinks. The bundle + * also carries undocumented ones (`export-to-clipboard`, `copy-and-zip-project`, + * `open-projects-folder`), none of which exports to a file. + * + * What makes it drivable is `--remote-debugging-port`: the renderer is then reachable and every + * control can be found by its visible text. That is *more* reproducible than pixel clicking, not + * less — it survives a moved window, a different display and a resized UI — and the flag only + * opens an inspector; the renderer and export pipeline are the shipping ones. + * + * The composition itself is not clicked at all: a `.screenstudio` project is a directory of + * plain JSON, so the scenario is written straight into `project.json` and the app reopens it. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { CdpSession, DOM_HELPERS, listTargets } from "../lib/cdp.mjs"; +import { sleep } from "../lib/measure.mjs"; +import { activateApp, appIsRunning, clickMenuItem, osa, quitApp } from "../lib/uiScript.mjs"; + +const APP = "/Applications/Screen Studio.app"; +const BIN = `${APP}/Contents/MacOS/Screen Studio`; +const PORT = 9333; +const PROJECTS = join(homedir(), "Screen Studio Projects"); + +/** Screen Studio's zoom-range shape, recovered from its own project factory. */ +const zoomRange = (z, i) => ({ + id: `osbench${String(i).padStart(4, "0")}`, + zoom: z.scale, + type: "manual", + snapToEdgesRatio: 0.25, + manualTargetPoint: { x: z.focus.x, y: z.focus.y }, + glideDirection: null, + glideSpeed: 0.5, + isDisabled: false, + startTime: z.startSec, + endTime: z.endSec, + isSystem: false, + hasInstantAnimation: false, +}); + +export default { + id: "screen-studio", + displayName: "Screen Studio", + vendor: "Screen Studio", + kind: "gui", + automation: "cdp+menu", + processName: "Screen Studio", + appPath: APP, + bundleId: "com.timpler.screenstudio", + install: { + method: "dmg", + url: "https://screenstudioassets.com/releases/3.7.5-4595/Screen%20Studio%203.7.5-4595%20Apple%20Silicon.dmg", + version: "3.7.5-4595", + appName: "Screen Studio.app", + approxMB: 349, + licence: "commercial — a licence is REQUIRED to export; there is no trial export", + }, + + detect() { + if (!existsSync(APP)) return { installed: false, version: null, path: null }; + let version = null; + try { + version = execFileSync( + "/usr/bin/defaults", + ["read", `${APP}/Contents/Info.plist`, "CFBundleShortVersionString"], + { encoding: "utf8" }, + ).trim(); + } catch { + /* keep null */ + } + return { installed: true, version, path: APP }; + }, + + defaultPaddingControl(scenario) { + // `backgroundPaddingRatio` is a percentage-like scale; calibration solves the exact value. + return scenario.effects.paddingPercent * 2; + }, + + async prepare(ctx) { + if (appIsRunning(this.processName)) await quitApp(this.processName, { force: true }); + await sleep(2000); + execFileSync("/bin/sh", [ + "-c", + `nohup ${JSON.stringify(BIN)} --remote-debugging-port=${PORT} >/dev/null 2>&1 &`, + ]); + await sleep(12000); + + // Import: File → "Create project from video…" raises a standard open panel. + activateApp(this.processName); + await sleep(800); + clickMenuItem(this.processName, "File", ["Create project from video"]); + await sleep(2500); + osa(`tell application "System Events" to tell process "${this.processName}" + set frontmost to true + keystroke "g" using {command down, shift down} + delay 0.8 + keystroke "${ctx.source.path}" + delay 0.6 + key code 36 + delay 1.2 + key code 36 + end tell`); + await sleep(25000); + + // Find the project the import just created and write the scenario into it. + const dirs = readdirSync(PROJECTS) + .filter((d) => d.endsWith(".screenstudio")) + .map((d) => ({ d, m: readFileSync })) + .map(({ d }) => join(PROJECTS, d)); + if (!dirs.length) throw new Error(`no .screenstudio project appeared in ${PROJECTS}`); + const project = dirs.sort()[dirs.length - 1]; + const file = join(project, "project.json"); + const doc = JSON.parse(readFileSync(file, "utf8")); + const e = ctx.scenario.effects; + + doc.json.config.backgroundType = "color"; + doc.json.config.backgroundColor = e.background.color; + doc.json.config.backgroundImage = null; + doc.json.config.backgroundBlur = 0; + doc.json.config.backgroundPaddingRatio = + ctx.paddingControl ?? this.defaultPaddingControl(ctx.scenario); + doc.json.config.windowBorderRadius = e.cornerRadiusPx; + doc.json.config.shadowIntensity = e.shadow?.enabled ? e.shadow.intensity : 0; + doc.json.config.hideCamera = true; + doc.json.config.motionBlurAmount = e.motionBlur ? 1 : 0; + doc.json.scenes[0].zoomRanges = (e.zooms ?? []).map(zoomRange); + writeFileSync(file, JSON.stringify(doc, null, 2)); + ctx.state.projectPath = project; + + // Reopen so the app reads what was just written. + clickMenuItem(this.processName, "File", ["Open last project"]); + await sleep(12000); + + const target = (await listTargets(PORT)).find((t) => t.type === "page"); + const s = new CdpSession(target.webSocketDebuggerUrl); + await s.open(); + await s.eval(DOM_HELPERS); + ctx.state.cdp = s; + + return { + appliedFeatures: [ + "background", + "padding", + "cornerRadius", + "shadow", + "zooms", + "targetResolution", + "targetFps", + ], + notes: [ + `project: ${project}`, + `${(e.zooms ?? []).length} zoom ranges written into scenes[0].zoomRanges`, + "Screen Studio re-encodes the source on import (its own display track), so its decoder input differs from the other apps'.", + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const s = ctx.state.cdp; + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + + const clicked = JSON.parse(await s.eval(`JSON.stringify(window.__osbench.click("Export"))`)); + if (!clicked.ok) throw new Error("Screen Studio: no Export control in the editor"); + await sleep(3000); + + // An unactivated install answers Export with an activation wall rather than a dialog. + const wall = await s.eval(`(() => { + for (const t of [...document.querySelectorAll("body")]) { + if (/Activate Screen Studio/i.test(t.innerText)) return "activation-required"; + } + return ""; + })()`); + if (wall === "activation-required") { + throw new Error( + "Screen Studio requires an activated licence to export — no trial export exists. " + + "Activate it once during preflight and re-run; every other step of this driver is unattended.", + ); + } + + // With a licence, the dialog's controls carry their values as visible text. + const t = ctx.scenario.output; + for (const label of ["MP4", `${t.height}p`, String(t.fps)]) { + await s.eval(`JSON.stringify(window.__osbench.click(${JSON.stringify(label)}))`); + await sleep(400); + } + await s.eval(`JSON.stringify(window.__osbench.click("Export"))`); + await sleep(2500); + + const dir = out.replace(/\/[^/]+$/, ""); + const stem = out + .split("/") + .pop() + .replace(/\.mp4$/i, ""); + osa(`tell application "System Events" to tell process "${this.processName}" + set frontmost to true + keystroke "g" using {command down, shift down} + delay 0.8 + keystroke "${dir}" + delay 0.5 + key code 36 + delay 1.0 + keystroke "a" using {command down} + keystroke "${stem}" + delay 0.4 + key code 36 + end tell`); + ctx.commit(); + }, + + async cleanup(ctx) { + ctx.state?.cdp?.close(); + if (appIsRunning(this.processName)) await quitApp(this.processName, { force: true }); + }, +}; diff --git a/benchmark/lib/calibrate.mjs b/benchmark/lib/calibrate.mjs new file mode 100644 index 00000000..93314a63 --- /dev/null +++ b/benchmark/lib/calibrate.mjs @@ -0,0 +1,150 @@ +/** + * Making the apps composite the same rectangle. + * + * Every app in this set has a "padding" control, and no two of them are on the same scale: + * asking each for "5" produced a 1.85% inset in Cap and a 10% inset in OpenScreen — a 44% + * difference in the number of source pixels being sampled per frame. That is a confound, not a + * result, so before the real run each app's control is solved for the value that yields the + * scenario's inset. + * + * The solve is a secant search on a deliberately short clip: two probes to establish the app's + * (usually near-linear) mapping, then up to two refinements. Everything is measured from the + * rendered pixels, never from what the app claims, and the outcome is written to + * benchmark/calibration.json so a run is reproducible without repeating it. + */ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { BENCH_ROOT, machineFingerprint } from "./env.mjs"; +import { buildFixture, DEFAULT_SPEC, probe } from "./fixture.mjs"; +import { waitForStableFile } from "./measure.mjs"; +import { inspectExport } from "./visualCheck.mjs"; + +export const CALIBRATION_PATH = join(BENCH_ROOT, "calibration.json"); + +export function loadCalibration() { + if (!existsSync(CALIBRATION_PATH)) return {}; + try { + return JSON.parse(readFileSync(CALIBRATION_PATH, "utf8")); + } catch { + return {}; + } +} + +/** A short clip: the geometry of the composition does not depend on how long the clip is. */ +export function calibrationFixture(workDir, log = () => undefined) { + const spec = { ...DEFAULT_SPEC, name: "calib-1080p60-4s", durationSec: 4 }; + return buildFixture(workDir, spec, { log }); +} + +async function measureInset(driver, ctx, paddingControl) { + await driver.prepare({ ...ctx, paddingControl }); + const out = driver.outputPath(ctx); + let committed = false; + await driver.runExport({ + ...ctx, + paddingControl, + commit: () => { + committed = true; + }, + }); + const wait = await waitForStableFile(out, { timeoutMs: 10 * 60 * 1000, stableMs: 1200 }); + if (!wait.ok) throw new Error(`calibration export produced nothing (${wait.reason})`); + const p = probe(out); + const v = inspectExport(out, ctx.scenario, { probe: p }); + const inset = v.measured?.insetPercentShortSide; + if (inset == null) throw new Error("could not measure the content box"); + return { inset, box: v.measured.contentBox, checks: v.checks, committed }; +} + +/** + * Solve one app's padding control for the scenario's target inset. + * Returns the chosen control value plus every probe, so the calibration file shows its work. + */ +export async function calibrateApp( + driver, + ctx, + { tolerancePercent = 0.4, maxProbes = 4, log = () => undefined } = {}, +) { + const target = ctx.scenario.effects.paddingPercent; + if (!target) + return { + app: driver.id, + paddingControl: 0, + target, + probes: [], + reason: "no padding requested", + }; + if (typeof driver.defaultPaddingControl !== "function") { + return { + app: driver.id, + paddingControl: null, + target, + probes: [], + reason: "driver exposes no padding control", + }; + } + + const probes = []; + const seed = driver.defaultPaddingControl(ctx.scenario); + // Two points far enough apart to establish the slope without leaving the control's range. + let x0 = Math.max(0, seed * 0.5); + let x1 = seed; + + const run = async (x) => { + const m = await measureInset(driver, ctx, x); + probes.push({ control: +x.toFixed(2), inset: m.inset, box: m.box }); + log( + ` ${driver.id}: padding=${x.toFixed(2)} → inset ${m.inset}% (${m.box.width}×${m.box.height})`, + ); + return m.inset; + }; + + let y0 = await run(x0); + let y1 = await run(x1); + + for (let i = 0; i < maxProbes - 2; i++) { + const best = probes.reduce((a, b) => + Math.abs(a.inset - target) <= Math.abs(b.inset - target) ? a : b, + ); + if (Math.abs(best.inset - target) <= tolerancePercent) break; + if (y1 === y0) break; // control has no effect in this range; stop rather than divide by zero + // Secant step, clamped to a sane control range. + let x2 = x1 + ((target - y1) * (x1 - x0)) / (y1 - y0); + x2 = Math.max(0, Math.min(100, x2)); + if (!Number.isFinite(x2) || probes.some((p) => Math.abs(p.control - x2) < 0.05)) break; + const y2 = await run(x2); + x0 = x1; + y0 = y1; + x1 = x2; + y1 = y2; + } + + const best = probes.reduce((a, b) => + Math.abs(a.inset - target) <= Math.abs(b.inset - target) ? a : b, + ); + return { + app: driver.id, + target, + paddingControl: best.control, + achievedInsetPercent: best.inset, + achievedBox: best.box, + withinTolerance: Math.abs(best.inset - target) <= tolerancePercent, + probes, + }; +} + +export function saveCalibration(entries, meta) { + mkdirSync(BENCH_ROOT, { recursive: true }); + const m = machineFingerprint(); + const doc = { + generatedAt: new Date().toISOString(), + // Stamped so `run` can tell a calibration made here from one that travelled with the + // repo. The padding a control produces is a property of the app, not the machine, but + // app versions differ between machines and a silently stale solve is worse than none. + machine: { chip: m.chip, osVersion: m.osVersion, model: m.model }, + ...meta, + apps: Object.fromEntries(entries.map((e) => [e.app, e])), + }; + writeFileSync(CALIBRATION_PATH, `${JSON.stringify(doc, null, 2)}\n`); + return CALIBRATION_PATH; +} diff --git a/benchmark/lib/cdp.mjs b/benchmark/lib/cdp.mjs new file mode 100644 index 00000000..9bc505f1 --- /dev/null +++ b/benchmark/lib/cdp.mjs @@ -0,0 +1,204 @@ +/** + * Chrome DevTools Protocol client — the way into the Electron apps whose UI nothing else can + * reach. + * + * Screen Studio marks its editor window `kCGWindowSharingNone`, so macOS excludes it from every + * capture API: no screenshot, and therefore no pixel clicking. Its UI is a web view that + * publishes no accessibility tree either, so `System Events` sees a window with three + * traffic-light buttons and nothing else. The menu bar is scriptable, but the export dialog is + * not on it. + * + * Launching the app with `--remote-debugging-port` puts its own renderer within reach: the + * export button can be found by its text and clicked, exactly as a user would, with no + * coordinates involved. That is a *more* reproducible interaction than clicking pixels, not a + * less reproducible one — it survives a different display, a moved window and a resized UI. + * + * What it does not do is change the app: the flag only opens an inspector, the renderer and the + * export pipeline are the shipping ones, and every click goes through the app's own handlers. + * Runs driven this way are recorded as `automation: "cdp"` so a reader can weigh that. + * + * Uses Node's built-in WebSocket (Node 22+); no dependency is added to the repo. + */ + +export class CdpError extends Error {} + +const httpJson = async (port, path) => { + const res = await fetch(`http://127.0.0.1:${port}${path}`, { signal: AbortSignal.timeout(8000) }); + if (!res.ok) throw new CdpError(`CDP ${path} → HTTP ${res.status}`); + return res.json(); +}; + +export async function listTargets(port) { + return httpJson(port, "/json/list"); +} + +/** Wait for a page target whose url or title matches, e.g. the app's index.html. */ +export async function waitForTarget(port, match, { timeoutMs = 60_000, pollMs = 500 } = {}) { + const re = match instanceof RegExp ? match : new RegExp(match, "i"); + const t0 = Date.now(); + let lastSeen = []; + while (Date.now() - t0 < timeoutMs) { + try { + const targets = await listTargets(port); + lastSeen = targets.map((t) => `${t.type}:${t.url}`); + const hit = targets.find( + (t) => t.type === "page" && (re.test(t.url) || re.test(t.title ?? "")), + ); + if (hit) return hit; + } catch { + /* the app may not be listening yet */ + } + await new Promise((r) => setTimeout(r, pollMs)); + } + throw new CdpError( + `no CDP page matched ${re} on port ${port} within ${timeoutMs}ms. Saw: ${lastSeen.join(", ")}`, + ); +} + +export class CdpSession { + constructor(wsUrl) { + this.wsUrl = wsUrl; + this.id = 0; + this.pending = new Map(); + this.ws = null; + } + + static async attach(port, match, opts) { + const target = await waitForTarget(port, match, opts); + const s = new CdpSession(target.webSocketDebuggerUrl); + await s.open(); + return s; + } + + open() { + return new Promise((resolve, reject) => { + this.ws = new WebSocket(this.wsUrl); + const timer = setTimeout(() => reject(new CdpError("CDP websocket timed out")), 15_000); + this.ws.addEventListener("open", () => { + clearTimeout(timer); + resolve(); + }); + this.ws.addEventListener("error", (e) => { + clearTimeout(timer); + reject(new CdpError(`CDP websocket error: ${e.message ?? e.type}`)); + }); + this.ws.addEventListener("message", (ev) => { + let msg; + try { + msg = JSON.parse(ev.data); + } catch { + return; + } + const p = this.pending.get(msg.id); + if (!p) return; + this.pending.delete(msg.id); + if (msg.error) + p.reject( + new CdpError(`${msg.error.message}${msg.error.data ? ` — ${msg.error.data}` : ""}`), + ); + else p.resolve(msg.result); + }); + }); + } + + send(method, params = {}, { timeoutMs = 120_000 } = {}) { + const id = ++this.id; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(id); + reject(new CdpError(`${method} timed out after ${timeoutMs}ms`)); + }, timeoutMs); + this.pending.set(id, { + resolve: (v) => { + clearTimeout(timer); + resolve(v); + }, + reject: (e) => { + clearTimeout(timer); + reject(e); + }, + }); + this.ws.send(JSON.stringify({ id, method, params })); + }); + } + + /** Evaluate an expression in the page and return its JSON value. */ + async eval(expression, { awaitPromise = true, timeoutMs = 120_000 } = {}) { + const r = await this.send( + "Runtime.evaluate", + { expression, returnByValue: true, awaitPromise, userGesture: true }, + { timeoutMs }, + ); + if (r.exceptionDetails) { + const d = r.exceptionDetails; + throw new CdpError(d.exception?.description ?? d.text ?? "evaluation failed"); + } + return r.result?.value; + } + + close() { + try { + this.ws?.close(); + } catch { + /* already gone */ + } + } +} + +/** + * A DOM helper injected into the page: find elements by visible text, which is the only + * selector that survives an app's next release. Returns a description rather than a handle so + * the caller can log exactly what it matched. + */ +export const DOM_HELPERS = ` +(() => { + if (window.__osbench) return "already"; + const visible = (el) => { + const r = el.getBoundingClientRect(); + if (r.width < 1 || r.height < 1) return false; + const s = getComputedStyle(el); + return s.visibility !== "hidden" && s.display !== "none" && s.opacity !== "0"; + }; + const text = (el) => (el.innerText || el.textContent || el.getAttribute("aria-label") || el.title || "").trim(); + window.__osbench = { + visible, text, + /** Every clickable thing on screen, with its text — the discovery call. */ + controls() { + const sel = 'button,[role="button"],a,[role="menuitem"],[role="tab"],input,select,label,[data-testid]'; + return [...document.querySelectorAll(sel)].filter(visible).map((el, i) => ({ + i, tag: el.tagName.toLowerCase(), type: el.type || null, + role: el.getAttribute("role"), testid: el.getAttribute("data-testid"), + text: text(el).slice(0, 80), value: el.value ?? null, + disabled: !!el.disabled, + rect: (({x,y,width,height}) => ({x:Math.round(x),y:Math.round(y),w:Math.round(width),h:Math.round(height)}))(el.getBoundingClientRect()), + })); + }, + find(needle, { exact = false, tag = null } = {}) { + const n = needle.toLowerCase(); + const sel = tag || 'button,[role="button"],a,[role="menuitem"],[role="tab"],label,div,span,[data-testid]'; + const hits = [...document.querySelectorAll(sel)].filter(visible).filter((el) => { + const t = text(el).toLowerCase(); + return exact ? t === n : t.includes(n); + }); + // Prefer the smallest match: the innermost element carrying the text, not its container. + hits.sort((a, b) => (a.getBoundingClientRect().width * a.getBoundingClientRect().height) - + (b.getBoundingClientRect().width * b.getBoundingClientRect().height)); + return hits[0] || null; + }, + click(needle, opts) { + const el = this.find(needle, opts); + if (!el) return { ok: false, reason: "not found", needle }; + const target = el.closest("button,[role='button'],a,[role='menuitem'],label") || el; + const r = target.getBoundingClientRect(); + for (const type of ["pointerdown", "mousedown", "pointerup", "mouseup", "click"]) { + target.dispatchEvent(new MouseEvent(type, { + bubbles: true, cancelable: true, view: window, + clientX: r.x + r.width / 2, clientY: r.y + r.height / 2, + })); + } + return { ok: true, matched: this.text(target).slice(0, 80), rect: { x: Math.round(r.x), y: Math.round(r.y) } }; + }, + }; + return "installed"; +})() +`; diff --git a/benchmark/lib/env.mjs b/benchmark/lib/env.mjs new file mode 100644 index 00000000..38c4a7b4 --- /dev/null +++ b/benchmark/lib/env.mjs @@ -0,0 +1,205 @@ +/** + * Environment discovery for the export benchmark. + * + * Everything the report needs in order to be comparable across machines lives + * here: the hardware fingerprint, the power/thermal preconditions, and the + * ffmpeg/ffprobe pair used for fixture generation and output verification. + */ +import { execFileSync, execSync } from "node:child_process"; +import { chmodSync, existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const BENCH_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +export const REPO_ROOT = resolve(BENCH_ROOT, ".."); +export const CACHE_DIR = join(BENCH_ROOT, ".cache"); +export const RESULTS_DIR = join(BENCH_ROOT, "results"); +export const WORK_DIR = + process.env.OSBENCH_WORK_DIR || join(os.homedir(), "openscreen-export-benchmark"); + +export const sh = (cmd) => + execSync(cmd, { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); + +const trySh = (cmd, fallback = null) => { + try { + return sh(cmd); + } catch { + return fallback; + } +}; + +/** Deep-search a directory tree for the vendored LGPL ffmpeg prefix built for the compositor. */ +function findVendoredFfmpegPrefix() { + const roots = [join(REPO_ROOT, "crates", "thirdparty")]; + // The tree is gitignored, so it exists only in whichever checkout built it. From a + // worktree that is the *main* checkout, which `--git-common-dir` points at. + const commonDir = trySh("git -C " + JSON.stringify(REPO_ROOT) + " rev-parse --git-common-dir"); + if (commonDir) roots.push(join(resolve(REPO_ROOT, commonDir), "..", "crates", "thirdparty")); + // And sibling worktrees, which is where a freshly built copy usually lands. + const wtRoot = commonDir + ? join(resolve(REPO_ROOT, commonDir), "..", ".claude", "worktrees") + : null; + if (wtRoot && existsSync(wtRoot)) { + for (const wt of readdirSync(wtRoot)) roots.push(join(wtRoot, wt, "crates", "thirdparty")); + } + if (process.env.MAC_FFMPEG_DIR) roots.unshift(dirname(process.env.MAC_FFMPEG_DIR)); + for (const root of roots) { + if (!existsSync(root)) continue; + for (const entry of readdirSync(root)) { + const prefix = join(root, entry); + if (/^ffmpeg-/.test(entry) && existsSync(join(prefix, "bin", "ffmpeg"))) return prefix; + } + } + return null; +} + +/** + * The vendored ffmpeg is `--enable-shared` with a *stale* baked-in prefix, and macOS strips + * DYLD_* across any SIP-protected exec (`/bin/sh`, `/usr/bin/env`). Inheriting the variable + * therefore never works. A tiny wrapper that exports it inside its own process does, because + * the stripping only removes what was inherited. + */ +function writeDyldWrapper(name, binary, libDir) { + mkdirSync(CACHE_DIR, { recursive: true }); + const wrapper = join(CACHE_DIR, name); + writeFileSync( + wrapper, + `#!/bin/sh\n# generated by benchmark/lib/env.mjs — see AGENTS.md / macos-dev-toolchain\nexport DYLD_LIBRARY_PATH="${libDir}"\nexec "${binary}" "$@"\n`, + ); + chmodSync(wrapper, 0o755); + return wrapper; +} + +let cachedFfmpeg = null; + +/** Resolve an (ffmpeg, ffprobe) pair, preferring an explicit override, then PATH, then vendored. */ +export function resolveFfmpeg() { + if (cachedFfmpeg) return cachedFfmpeg; + + if (process.env.OSBENCH_FFMPEG && process.env.OSBENCH_FFPROBE) { + cachedFfmpeg = { + ffmpeg: process.env.OSBENCH_FFMPEG, + ffprobe: process.env.OSBENCH_FFPROBE, + source: "env:OSBENCH_FFMPEG", + }; + return cachedFfmpeg; + } + + const onPath = trySh("command -v ffmpeg"); + const probeOnPath = trySh("command -v ffprobe"); + if (onPath && probeOnPath) { + cachedFfmpeg = { ffmpeg: onPath, ffprobe: probeOnPath, source: "PATH" }; + return cachedFfmpeg; + } + + const prefix = findVendoredFfmpegPrefix(); + if (prefix) { + const lib = join(prefix, "lib"); + cachedFfmpeg = { + ffmpeg: writeDyldWrapper("ffmpeg", join(prefix, "bin", "ffmpeg"), lib), + ffprobe: writeDyldWrapper("ffprobe", join(prefix, "bin", "ffprobe"), lib), + source: `vendored:${prefix}`, + }; + return cachedFfmpeg; + } + + throw new Error( + "No ffmpeg/ffprobe found. Install one on PATH, or set OSBENCH_FFMPEG and OSBENCH_FFPROBE, " + + "or build the repo's LGPL tree (see benchmark/README.md § ffmpeg).", + ); +} + +export function ffmpegVersion() { + const { ffmpeg, source } = resolveFfmpeg(); + const line = execFileSync(ffmpeg, ["-hide_banner", "-version"], { encoding: "utf8" }) + .split("\n")[0] + .trim(); + return { banner: line, source }; +} + +/** Everything about the host that can move a number in the results table. */ +export function machineFingerprint() { + const memBytes = Number(trySh("sysctl -n hw.memsize", "0")); + const displays = (trySh("system_profiler SPDisplaysDataType", "") || "") + .split("\n") + .filter((l) => /^\s+(Resolution|UI Looks like):/.test(l)) + .map((l) => l.trim()); + + return { + platform: process.platform, + arch: process.arch, + osProduct: trySh("sw_vers -productName"), + osVersion: trySh("sw_vers -productVersion"), + osBuild: trySh("sw_vers -buildVersion"), + kernel: os.release(), + model: trySh("sysctl -n hw.model"), + chip: trySh("sysctl -n machdep.cpu.brand_string"), + cpuCount: os.cpus().length, + performanceCores: Number(trySh("sysctl -n hw.perflevel0.logicalcpu", "0")) || null, + efficiencyCores: Number(trySh("sysctl -n hw.perflevel1.logicalcpu", "0")) || null, + memoryGiB: memBytes ? +(memBytes / 1024 ** 3).toFixed(1) : null, + displays, + nodeVersion: process.version, + }; +} + +/** + * Preconditions that silently skew an export benchmark: battery power caps the SoC, + * Low Power Mode caps it harder, and an already-throttled machine reports whatever + * the previous run left behind. + */ +export function powerState() { + const batt = trySh("pmset -g batt", ""); + const therm = trySh("pmset -g therm", ""); + const lowPower = trySh("pmset -g | grep -i lowpowermode", ""); + const cpuSpeedLimit = /CPU_Speed_Limit\s*=\s*(\d+)/.exec(therm); + const schedulerLimit = /CPU_Scheduler_Limit\s*=\s*(\d+)/.exec(therm); + + return { + onACPower: /AC Power/.test(batt), + batteryLine: batt.split("\n").slice(1).join(" ").trim() || null, + lowPowerMode: /lowpowermode\s+1/.test(lowPower), + cpuSpeedLimit: cpuSpeedLimit ? Number(cpuSpeedLimit[1]) : null, + cpuSchedulerLimit: schedulerLimit ? Number(schedulerLimit[1]) : null, + thermalPressure: readThermalPressure(), + }; +} + +/** + * Thermal pressure, spelled differently on every macOS. `pmset -g therm` prints nothing at + * all when the SoC is unthrottled, so "no output" is the healthy answer, not a failure. + */ +function readThermalPressure() { + for (const key of ["kern.thermalpressurelevel", "machdep.xcpm.cpu_thermal_level"]) { + const v = trySh(`sysctl -n ${key}`); + if (v !== null && v !== "") return { key, value: Number(v) }; + } + const therm = trySh("pmset -g therm", ""); + if (/No thermal warning level has been recorded/i.test(therm)) return { key: "pmset", value: 0 }; + return { key: "unavailable", value: null }; +} + +/** Free space on the volume that will hold the fixture and every export. */ +export function diskState(path = WORK_DIR) { + const target = existsSync(path) ? path : os.homedir(); + const line = trySh(`df -k ${JSON.stringify(target)} | tail -1`, ""); + const cols = line.split(/\s+/); + const availKiB = Number(cols[3] || 0); + return { path: target, availableGiB: +(availKiB / 1024 / 1024).toFixed(1) }; +} + +export function ensureWorkDirs() { + for (const d of [ + WORK_DIR, + CACHE_DIR, + RESULTS_DIR, + join(WORK_DIR, "fixture"), + join(WORK_DIR, "out"), + join(WORK_DIR, "projects"), + join(WORK_DIR, "installers"), + ]) { + mkdirSync(d, { recursive: true }); + } + return { WORK_DIR, CACHE_DIR, RESULTS_DIR }; +} diff --git a/benchmark/lib/fixture.mjs b/benchmark/lib/fixture.mjs new file mode 100644 index 00000000..31d4cb76 --- /dev/null +++ b/benchmark/lib/fixture.mjs @@ -0,0 +1,302 @@ +/** + * Deterministic source-clip generation. + * + * A benchmark that ships a 200 MB .mp4 is not reproducible — the file rots, and nobody can + * tell whether two machines measured the same work. So the source is *generated* from a spec + * plus a seed, with pure ffmpeg primitives, and fingerprinted afterwards. Two machines that + * agree on the fingerprint measured the same workload. + * + * The frame is built to look like a screen recording rather than a test pattern, because that + * is what changes an encoder's job: large static regions, dense sharp-edged "text", a small + * amount of localized motion, and a cursor. + */ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { resolveFfmpeg } from "./env.mjs"; + +/** execFileSync, but a non-zero exit surfaces ffmpeg's own message instead of a byte dump. */ +function run(bin, args) { + try { + return execFileSync(bin, args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); + } catch (e) { + const msg = (e.stderr?.toString() || e.stdout?.toString() || e.message).trim(); + throw new Error(`${bin.split("/").pop()} failed (exit ${e.status}):\n${msg}`); + } +} + +/** xorshift32 — tiny, seeded, identical in every JS runtime. */ +function rng(seed) { + let x = seed >>> 0 || 0x9e3779b9; + return () => { + x ^= x << 13; + x >>>= 0; + x ^= x >> 17; + x ^= x << 5; + x >>>= 0; + return x / 0x100000000; + }; +} + +export const DEFAULT_SPEC = { + name: "ide-1080p60-60s", + width: 1920, + height: 1080, + fps: 60, + durationSec: 60, + seed: 20260825, + /** Target bitrate of the *source*. Screen recorders emit roughly this for 1080p30 UI. */ + sourceBitrateMbps: 12, +}; + +const PALETTE = [ + "0xd7dae0", + "0x89b4fa", + "0xa6e3a1", + "0xf9e2af", + "0xf38ba8", + "0xcba6f7", + "0x94e2d5", +]; + +/** Static "page" of code-like rows, tall enough to scroll through for the whole clip. */ +function pageFilter(spec, pageHeight) { + const rand = rng(spec.seed); + const left = 360; + const right = spec.width - 80; + const rowH = 12; + const rowGap = 26; + const boxes = []; + for (let y = 20; y < pageHeight - 40; y += rowGap) { + // Indentation in steps, like real code. + const indent = left + Math.floor(rand() * 5) * 28; + let x = indent; + const tokens = 2 + Math.floor(rand() * 7); + for (let t = 0; t < tokens && x < right - 40; t++) { + const w = Math.floor(30 + rand() * 190); + const color = PALETTE[Math.floor(rand() * PALETTE.length)]; + boxes.push( + `drawbox=x=${x}:y=${y}:w=${Math.min(w, right - x)}:h=${rowH}:color=${color}@0.92:t=fill`, + ); + x += w + 12 + Math.floor(rand() * 18); + } + // Gutter line numbers. + boxes.push(`drawbox=x=${left - 56}:y=${y + 2}:w=28:h=${rowH - 4}:color=0x585b70@0.8:t=fill`); + } + return boxes.join(","); +} + +/** Window chrome: title bar, sidebar rows, a status bar. Static, so it is baked once. */ +function chromeFilter(spec) { + const rand = rng(spec.seed ^ 0x5bf03635); + const b = [ + `drawbox=x=0:y=0:w=${spec.width}:h=44:color=0x11141a@1:t=fill`, + `drawbox=x=0:y=44:w=320:h=${spec.height - 44 - 32}:color=0x171b22@1:t=fill`, + `drawbox=x=0:y=${spec.height - 32}:w=${spec.width}:h=32:color=0x11141a@1:t=fill`, + ]; + for (const [i, c] of ["0xff5f57", "0xfebc2e", "0x28c840"].entries()) { + b.push(`drawbox=x=${18 + i * 22}:y=16:w=12:h=12:color=${c}@1:t=fill`); + } + // Tab strip. + let tx = 120; + for (let i = 0; i < 5; i++) { + const w = 110 + Math.floor(rand() * 70); + b.push(`drawbox=x=${tx}:y=12:w=${w}:h=20:color=${i === 1 ? "0x2a3040" : "0x1b1f27"}@1:t=fill`); + b.push(`drawbox=x=${tx + 12}:y=19:w=${w - 40}:h=7:color=0x9aa3b2@0.9:t=fill`); + tx += w + 8; + } + // Sidebar file tree. + for (let y = 70, i = 0; y < spec.height - 60; y += 30, i++) { + const indent = 24 + (i % 3) * 18; + b.push( + `drawbox=x=${indent}:y=${y}:w=${Math.floor(80 + rand() * 150)}:h=9:color=0x9aa3b2@0.75:t=fill`, + ); + } + // Status bar chips. + for (let i = 0, x = 20; i < 4; i++) { + const w = 70 + Math.floor(rand() * 80); + b.push(`drawbox=x=${x}:y=${spec.height - 22}:w=${w}:h=11:color=0x89b4fa@0.7:t=fill`); + x += w + 26; + } + return b.join(","); +} + +/** + * The animated layer. Kept deliberately small: a scrolling viewport, a caret, a selection + * band and a cursor. Screen recordings are mostly static, and an encoder benchmark that + * feeds full-frame motion measures a different workload entirely. + */ +function animationFilter(spec, pageHeight) { + const visibleH = spec.height - 44 - 32; + const scrollRange = Math.max(1, pageHeight - visibleH); + // Ease in/out so the scroll starts and stops, like a human dragging. + const scrollY = `(${scrollRange}*(0.5-0.5*cos(2*PI*t/${spec.durationSec})))`; + return { + scrollY, + overlays: [ + // Caret: blinks at 1 Hz. + `drawbox=x=380+mod(floor(t*7)\\,40)*14:y=${44 + Math.floor(visibleH / 2)}:w=3:h=18:color=0xffffff@1:t=fill:enable='lt(mod(t\\,1)\\,0.5)'`, + // Selection band sweeping down the pane. + `drawbox=x=360:y=${44}+mod(floor(t*2)*36\\,${visibleH - 40}):w=760:h=22:color=0x3b5bdb@0.35:t=fill`, + // Cursor arrow, approximated by a small bright square on a Lissajous path. + `drawbox=x=${spec.width / 2}+${spec.width / 2 - 140}*sin(2*PI*t/11):y=${spec.height / 2}+${spec.height / 2 - 120}*sin(2*PI*t/7):w=14:h=20:color=0xffffff@0.95:t=fill`, + `drawbox=x=${spec.width / 2}+${spec.width / 2 - 140}*sin(2*PI*t/11)-1:y=${spec.height / 2}+${spec.height / 2 - 120}*sin(2*PI*t/7)-1:w=16:h=22:color=0x000000@0.6:t=3`, + ].join(","), + }; +} + +/** A deterministic voice-shaped audio bed: an AM-modulated tone with syllable-rate gating. */ +function audioFilter() { + return ( + "aevalsrc='0.28*sin(2*PI*(180+40*sin(2*PI*0.7*t))*t)" + + "*(0.45+0.55*sin(2*PI*3.1*t))" + + "*(0.25+0.75*lt(mod(floor(t*1.7),4),3))':s=48000:c=stereo" + ); +} + +export function fixturePath(workDir, spec) { + return join(workDir, "fixture", `${spec.name}.mp4`); +} + +/** ffprobe a media file into a compact, comparable descriptor. */ +export function probe(file) { + const { ffprobe } = resolveFfmpeg(); + const raw = execFileSync( + ffprobe, + ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", file], + { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }, + ); + const j = JSON.parse(raw); + const v = j.streams.find((s) => s.codec_type === "video"); + const a = j.streams.find((s) => s.codec_type === "audio"); + const num = (x) => (x == null ? null : Number(x)); + const fps = v?.avg_frame_rate?.includes("/") + ? +(Number(v.avg_frame_rate.split("/")[0]) / Number(v.avg_frame_rate.split("/")[1])).toFixed(3) + : null; + return { + durationSec: num(j.format?.duration), + sizeBytes: num(j.format?.size), + bitrateKbps: j.format?.bit_rate ? Math.round(Number(j.format.bit_rate) / 1000) : null, + container: j.format?.format_name ?? null, + video: v + ? { + codec: v.codec_name, + profile: v.profile ?? null, + width: v.width, + height: v.height, + fps, + pixFmt: v.pix_fmt, + nbFrames: num(v.nb_frames), + } + : null, + audio: a ? { codec: a.codec_name, sampleRate: num(a.sample_rate), channels: a.channels } : null, + }; +} + +export function sha256(file) { + return createHash("sha256").update(readFileSync(file)).digest("hex"); +} + +/** + * Build the source clip. Idempotent: an existing file whose probe matches the spec is reused, + * because regenerating it is several minutes and changes nothing. + */ +export function buildFixture( + workDir, + spec = DEFAULT_SPEC, + { force = false, log = () => undefined } = {}, +) { + const { ffmpeg } = resolveFfmpeg(); + const out = fixturePath(workDir, spec); + mkdirSync(join(workDir, "fixture"), { recursive: true }); + + if (!force && existsSync(out)) { + const p = probe(out); + const ok = + p.video?.width === spec.width && + p.video?.height === spec.height && + Math.abs((p.durationSec ?? 0) - spec.durationSec) < 0.5; + if (ok) { + log(`fixture: reusing ${out}`); + return { path: out, spec, probe: p, sha256: sha256(out), regenerated: false }; + } + } + + // One scrolled page-height per 10 s of clip, so the scroll speed is spec-independent. + const pageHeight = Math.min(8192, (spec.height - 76) * 3); + const pagePng = join(workDir, "fixture", `${spec.name}.page.png`); + + log("fixture: rendering static layers"); + run(ffmpeg, [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + `color=c=0x1b1f27:s=${spec.width}x${pageHeight}:d=1`, + "-vf", + pageFilter(spec, pageHeight), + "-frames:v", + "1", + pagePng, + ]); + + const { scrollY, overlays } = animationFilter(spec, pageHeight); + const visibleH = spec.height - 44 - 32; + const filter = [ + `color=c=0x1b1f27:s=${spec.width}x${spec.height}:r=${spec.fps}:d=${spec.durationSec}[bg]`, + `[1:v]crop=${spec.width}:${visibleH}:0:'${scrollY}'[page]`, + `[bg][page]overlay=0:44:shortest=1[scrolled]`, + `[scrolled]${chromeFilter(spec)},${overlays},format=yuv420p[v]`, + ].join(";"); + + log(`fixture: encoding ${spec.durationSec}s @ ${spec.width}x${spec.height}${spec.fps}`); + const t0 = Date.now(); + run(ffmpeg, [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + audioFilter(), + "-loop", + "1", + "-i", + pagePng, + "-filter_complex", + filter, + "-map", + "[v]", + "-map", + "0:a", + "-t", + String(spec.durationSec), + "-r", + String(spec.fps), + "-c:v", + "h264_videotoolbox", + "-b:v", + `${spec.sourceBitrateMbps}M`, + "-profile:v", + "high", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-b:a", + "128k", + "-ar", + "48000", + "-movflags", + "+faststart", + out, + ]); + log(`fixture: encoded in ${((Date.now() - t0) / 1000).toFixed(1)}s`); + + return { path: out, spec, probe: probe(out), sha256: sha256(out), regenerated: true }; +} diff --git a/benchmark/lib/install.mjs b/benchmark/lib/install.mjs new file mode 100644 index 00000000..e5f71dde --- /dev/null +++ b/benchmark/lib/install.mjs @@ -0,0 +1,178 @@ +/** + * Unattended installation of the competitor apps. + * + * Everything here runs *after* the single up-front approval collected by `preflight`, and + * nothing here can ask a question — a run is expected to continue with nobody at the keyboard. + * + * Note on Gatekeeper: `curl` does not set `com.apple.quarantine`, so an app fetched this way + * skips the "downloaded from the internet" first-launch prompt that would otherwise stall an + * unattended run. The quarantine flag is never stripped from anything — if a vendor ships an + * unnotarised build, that is recorded as a finding rather than worked around. + */ +import { execFileSync, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, statSync } from "node:fs"; +import { basename, join } from "node:path"; + +const APPLICATIONS = "/Applications"; + +const run = (bin, args, opts = {}) => + execFileSync(bin, args, { encoding: "utf8", maxBuffer: 32 * 1024 * 1024, ...opts }); + +export function appVersion(appPath) { + try { + return run("/usr/bin/defaults", [ + "read", + join(appPath, "Contents", "Info.plist"), + "CFBundleShortVersionString", + ]).trim(); + } catch { + return null; + } +} + +/** Notarisation / signing status, recorded so the report can say what was actually run. */ +export function codesignStatus(appPath) { + const res = spawnSync("/usr/sbin/spctl", ["-a", "-t", "exec", "-vv", appPath], { + encoding: "utf8", + }); + const text = `${res.stdout ?? ""}${res.stderr ?? ""}`; + const team = /origin=(.+)/.exec(text)?.[1]?.trim() ?? null; + return { accepted: /: accepted/.test(text), authority: team, raw: text.trim().slice(0, 400) }; +} + +/** Resolve a GitHub release asset to a concrete URL, so the install is version-pinned. */ +export function resolveGithubAsset(repo, pattern) { + const json = run("/usr/bin/curl", [ + "-fsSL", + "--max-time", + "40", + "-H", + "Accept: application/vnd.github+json", + `https://api.github.com/repos/${repo}/releases/latest`, + ]); + const rel = JSON.parse(json); + const asset = (rel.assets ?? []).find((a) => pattern.test(a.name)); + if (!asset) { + throw new Error( + `no asset in ${repo}@${rel.tag_name} matched ${pattern}. Present: ${(rel.assets ?? []).map((a) => a.name).join(", ")}`, + ); + } + return { + url: asset.browser_download_url, + version: rel.tag_name, + name: asset.name, + sizeBytes: asset.size, + }; +} + +/** Resumable download. A 400 MB DMG over a flaky link should not restart from zero. */ +export function download(url, destDir, { log = () => undefined } = {}) { + mkdirSync(destDir, { recursive: true }); + // The vendor URL is often a redirect; ask curl for the effective name it lands on. + const effective = run("/usr/bin/curl", [ + "-sIL", + "--max-time", + "60", + "-o", + "/dev/null", + "-w", + "%{url_effective}", + url, + ]).trim(); + let name = basename(new URL(effective).pathname) || basename(new URL(url).pathname); + if (!/\.(dmg|zip|pkg)$/i.test(name)) name = `${name || "download"}.dmg`; + const dest = join(destDir, decodeURIComponent(name)); + + log(` downloading ${decodeURIComponent(name)}`); + run( + "/usr/bin/curl", + ["-fL", "--retry", "3", "--retry-delay", "2", "-C", "-", "--max-time", "1800", "-o", dest, url], + { stdio: ["ignore", "ignore", "inherit"] }, + ); + + const sha = createHash("sha256").update(readFileSync(dest)).digest("hex"); + return { path: dest, sizeBytes: statSync(dest).size, sha256: sha }; +} + +/** Mount a DMG, copy the .app out, unmount. Idempotent at the app level. */ +export function installDmg(dmgPath, appName, { log = () => undefined } = {}) { + const plist = run("/usr/bin/hdiutil", [ + "attach", + dmgPath, + "-nobrowse", + "-noverify", + "-noautoopen", + "-plist", + ]); + const mountPoint = /mount-point<\/key>\s*([^<]+)<\/string>/.exec(plist)?.[1]; + if (!mountPoint) throw new Error(`could not determine mount point for ${dmgPath}`); + + try { + const src = join(mountPoint, appName); + if (!existsSync(src)) { + const contents = run("/bin/ls", ["-1", mountPoint]).trim().split("\n"); + throw new Error( + `"${appName}" not found on the mounted image. Contents: ${contents.join(", ")}`, + ); + } + const dest = join(APPLICATIONS, appName); + if (existsSync(dest)) { + log(` replacing existing ${appName}`); + rmSync(dest, { recursive: true, force: true }); + } + log(` copying ${appName} → ${APPLICATIONS}`); + cpSync(src, dest, { recursive: true, verbatimSymlinks: true }); + return dest; + } finally { + try { + run("/usr/bin/hdiutil", ["detach", mountPoint, "-quiet"]); + } catch { + run("/usr/bin/hdiutil", ["detach", mountPoint, "-force", "-quiet"]); + } + } +} + +/** + * Install one app from its registry spec. Returns a record detailed enough that another + * machine can be checked against it — the whole point of pinning versions and hashes. + */ +export function installApp(spec, { cacheDir, force = false, log = () => undefined } = {}) { + const destApp = join(APPLICATIONS, spec.appName); + if (existsSync(destApp) && !force) { + return { + id: spec.id, + status: "already-installed", + appPath: destApp, + version: appVersion(destApp), + codesign: codesignStatus(destApp), + }; + } + + let url = spec.url; + let pinnedVersion = spec.version ?? null; + if (spec.method === "github-release") { + const asset = resolveGithubAsset(spec.repo, spec.assetPattern); + url = asset.url; + pinnedVersion = asset.version; + log(` resolved ${spec.repo} → ${asset.version} (${asset.name})`); + } + + const dl = download(url, cacheDir, { log }); + if (!/\.dmg$/i.test(dl.path)) { + throw new Error(`only .dmg installs are automated; got ${basename(dl.path)}`); + } + const appPath = installDmg(dl.path, spec.appName, { log }); + + return { + id: spec.id, + status: "installed", + appPath, + version: appVersion(appPath), + pinnedVersion, + sourceUrl: url, + downloadSha256: dl.sha256, + downloadBytes: dl.sizeBytes, + codesign: codesignStatus(appPath), + }; +} diff --git a/benchmark/lib/measure.mjs b/benchmark/lib/measure.mjs new file mode 100644 index 00000000..be264b3e --- /dev/null +++ b/benchmark/lib/measure.mjs @@ -0,0 +1,253 @@ +/** + * Measurement primitives. + * + * Three things have to be true for an export timing to mean anything: + * 1. The clock starts at the moment the export is *committed*, not when the app launched. + * 2. The clock stops when the output file is *complete*, not when a progress bar hits 100%. + * 3. The output is verified to be what was asked for — an app that quietly writes 720p, or a + * 12-second file from a 60-second source, is not faster, it is wrong. + * + * Everything here is app-agnostic on purpose: the same stopwatch is used for the CLI drivers + * and the UI drivers, so a CLI app is not credited for skipping a step a GUI app must do. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, statSync } from "node:fs"; +import { probe } from "./fixture.mjs"; + +export const now = () => Number(process.hrtime.bigint() / 1000n) / 1000; // ms, monotonic + +export const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +/* ------------------------------------------------------------------ process sampling ----- */ + +/** + * Cumulative CPU seconds and peak RSS for every process whose argv starts with `matchPrefix` + * (an app bundle path), summed across the tree. Sampling cumulative counters rather than + * instantaneous %CPU means a helper that exits mid-export still contributes its full cost. + */ +export class ProcessTreeSampler { + constructor(matchPrefixes, { intervalMs = 500 } = {}) { + this.matchPrefixes = [].concat(matchPrefixes).filter(Boolean); + this.intervalMs = intervalMs; + this.cpuByPid = new Map(); // pid -> max cumulative cpu seconds seen + this.peakRssBytes = 0; + this.samples = 0; + this.timer = null; + // Everything *else* on the machine. A remote-desktop session, a screen recorder or a + // build running alongside the benchmark inflates every export time without inflating any + // app's own CPU figure — so it is sampled and reported rather than assumed to be zero. + this.foreignCpuSamples = []; + } + + static parseCpuTime(t) { + // ps TIME is [[dd-]hh:]mm:ss[.ff] + const m = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+(?:\.\d+)?)$/.exec(t.trim()); + if (!m) return 0; + const [, d, h, mi, s] = m; + return Number(d || 0) * 86400 + Number(h || 0) * 3600 + Number(mi) * 60 + Number(s); + } + + sampleOnce() { + let out; + try { + out = execFileSync("/bin/ps", ["-axo", "pid=,rss=,time=,args="], { + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }); + } catch { + return; + } + let rssSum = 0; + for (const line of out.split("\n")) { + if (!line.trim()) continue; + const m = /^\s*(\d+)\s+(\d+)\s+(\S+)\s+(.*)$/.exec(line); + if (!m) continue; + const [, pid, rssKiB, time, args] = m; + if (!this.matchPrefixes.some((p) => args.includes(p))) continue; + const cpu = ProcessTreeSampler.parseCpuTime(time); + const prev = this.cpuByPid.get(pid) ?? 0; + if (cpu > prev) this.cpuByPid.set(pid, cpu); + rssSum += Number(rssKiB) * 1024; + } + if (rssSum > this.peakRssBytes) this.peakRssBytes = rssSum; + this.samples++; + this.sampleForeignLoad(); + } + + /** Instantaneous %CPU of everything that is not the app under test, summed. */ + sampleForeignLoad() { + try { + const out = execFileSync("/bin/ps", ["-axo", "pcpu=,args="], { + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }); + let total = 0; + for (const line of out.split("\n")) { + const m = /^\s*([\d.]+)\s+(.*)$/.exec(line); + if (!m) continue; + const [, pct, args] = m; + if (this.matchPrefixes.some((p) => args.includes(p))) continue; + if (args.includes("bench.mjs") || args.includes("/bin/ps")) continue; + total += Number(pct); + } + this.foreignCpuSamples.push(total); + } catch { + /* a sample lost to a transient ps failure is not worth failing a run over */ + } + } + + start() { + this.sampleOnce(); + this.timer = setInterval(() => this.sampleOnce(), this.intervalMs); + this.timer.unref?.(); + return this; + } + + stop() { + if (this.timer) clearInterval(this.timer); + this.timer = null; + this.sampleOnce(); + return this.result(); + } + + result() { + let cpuSeconds = 0; + for (const v of this.cpuByPid.values()) cpuSeconds += v; + return { + cpuSeconds: +cpuSeconds.toFixed(2), + peakRssBytes: this.peakRssBytes, + peakRssMiB: +(this.peakRssBytes / 1024 ** 2).toFixed(1), + pidsSeen: this.cpuByPid.size, + samples: this.samples, + // Median rather than mean: one spike from a Spotlight index should not characterise + // a two-minute export. + foreignCpuPercent: this.foreignCpuSamples.length + ? +median(this.foreignCpuSamples).toFixed(1) + : null, + }; + } +} + +/** + * The CPU counters above are cumulative *since process start*, which for a long-lived GUI app + * includes the idle time before the export. Snapshot before, snapshot after, subtract. + */ +export function cpuDelta(before, after) { + return { + cpuSeconds: +Math.max(0, after.cpuSeconds - before.cpuSeconds).toFixed(2), + peakRssMiB: after.peakRssMiB, + }; +} + +/* ------------------------------------------------------------------- output watching ----- */ + +/** + * Resolve when `path` exists and has stopped growing for `stableMs`. + * + * Size stability is the only completion signal that works identically for a CLI that writes + * once and a GUI that muxes at the end. `stableMs` has to clear the longest plausible stall + * inside an export (a slow keyframe, a GC pause) without inflating the measurement — so the + * stable window is *subtracted back off* the reported time, and the last-growth timestamp is + * what the stopwatch actually reads. + */ +export async function waitForStableFile( + path, + { timeoutMs = 45 * 60 * 1000, stableMs = 2500, pollMs = 100, minBytes = 4096, onTick } = {}, +) { + const t0 = now(); + let lastSize = -1; + let lastGrowthAt = null; + let appearedAt = null; + + while (now() - t0 < timeoutMs) { + let size = -1; + try { + if (existsSync(path)) size = statSync(path).size; + } catch { + size = -1; + } + + if (size >= 0 && appearedAt === null) appearedAt = now(); + if (size > lastSize) { + lastSize = size; + lastGrowthAt = now(); + } + onTick?.({ size, elapsedMs: now() - t0 }); + + if (lastSize >= minBytes && lastGrowthAt !== null && now() - lastGrowthAt >= stableMs) { + return { + ok: true, + appearedAt, + completedAt: lastGrowthAt, // the honest moment the last byte landed + sizeBytes: lastSize, + waitedMs: now() - t0, + }; + } + await sleep(pollMs); + } + return { ok: false, reason: "timeout", appearedAt, sizeBytes: lastSize, waitedMs: now() - t0 }; +} + +/* -------------------------------------------------------------------- verification ------- */ + +/** Does the produced file actually match what the scenario asked for? */ +export function verifyOutput(path, target, sourceDurationSec) { + if (!existsSync(path)) return { valid: false, reasons: ["output file missing"], probe: null }; + let p; + try { + p = probe(path); + } catch (e) { + return { valid: false, reasons: [`ffprobe failed: ${e.message}`], probe: null }; + } + + const reasons = []; + if (!p.video) reasons.push("no video stream"); + if (p.video && p.video.width !== target.width) { + reasons.push(`width ${p.video.width} != ${target.width}`); + } + if (p.video && p.video.height !== target.height) { + reasons.push(`height ${p.video.height} != ${target.height}`); + } + if (p.video?.codec && target.videoCodec && p.video.codec !== target.videoCodec) { + reasons.push(`codec ${p.video.codec} != ${target.videoCodec}`); + } + if (p.video?.fps != null) { + const drift = (Math.abs(p.video.fps - target.fps) / target.fps) * 100; + if (drift > target.tolerance.fpsPercent) reasons.push(`fps ${p.video.fps} != ${target.fps}`); + } + if (sourceDurationSec != null && p.durationSec != null) { + const d = Math.abs(p.durationSec - sourceDurationSec); + if (d > target.tolerance.durationSec) { + reasons.push( + `duration ${p.durationSec?.toFixed(2)}s vs source ${sourceDurationSec}s (Δ${d.toFixed(2)}s)`, + ); + } + } + return { valid: reasons.length === 0, reasons, probe: p }; +} + +/* ---------------------------------------------------------------------- run guards ------- */ + +/** + * Between repetitions the machine has to come back to the same state, or run 3 measures a + * hotter SoC than run 1 and the spread is thermal, not architectural. + */ +export async function cooldown({ seconds = 45, log = () => undefined } = {}) { + log(`cooldown: ${seconds}s`); + await sleep(seconds * 1000); +} + +/** Percentile helpers used by the report. Small n, so exact rather than interpolated. */ +export function median(xs) { + if (!xs.length) return null; + const s = [...xs].sort((a, b) => a - b); + const m = Math.floor(s.length / 2); + return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; +} + +/** Median absolute deviation — robust spread for n=3, where a stdev is mostly noise. */ +export function mad(xs) { + const m = median(xs); + if (m == null) return null; + return median(xs.map((x) => Math.abs(x - m))); +} diff --git a/benchmark/lib/openscreenProject.mjs b/benchmark/lib/openscreenProject.mjs new file mode 100644 index 00000000..def02476 --- /dev/null +++ b/benchmark/lib/openscreenProject.mjs @@ -0,0 +1,101 @@ +/** + * Builds a `.openscreen` project that expresses a benchmark scenario. + * + * The project format is plain JSON (schemaVersion 6) and the exporter reads its effect state + * from `editor` — the same shape `ProjectEditorState` in + * `src/components/video-editor/projectPersistence.ts` describes. Writing it directly, rather + * than driving the editor UI, is what makes the OpenScreen leg reproducible; the GUI leg is + * measured separately by `drivers/openscreen-gui.mjs`. + */ +import { copyFileSync, mkdirSync, writeFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { probe } from "./fixture.mjs"; + +/** Deterministic ids: the same scenario always produces the same project bytes. */ +const id = (prefix, n) => `${prefix}_${String(n).padStart(8, "0")}`; + +/** OpenScreen stores zoom depth as a preset; a custom scale overrides it. */ +function toZoomRegion(z, i) { + return { + id: id("zoom", i + 1), + startMs: Math.round(z.startSec * 1000), + endMs: Math.round(z.endSec * 1000), + depth: 2, + customScale: +z.scale.toFixed(2), + focus: { cx: z.focus.x, cy: z.focus.y }, + focusMode: "manual", + source: "manual", + }; +} + +/** + * Padding: the scenario states an inset as a percent of the frame, OpenScreen's `padding` is + * 0-100 on its own scale where 50 is the default inset. The mapping below is calibrated so a + * 5% scenario inset lands on OpenScreen's equivalent visual inset; see benchmark/README.md + * § "Translating the scenario" for how each app's control was matched. + */ +const paddingFromPercent = (pct) => Math.round(Math.min(100, Math.max(0, pct * 10))); + +export function buildProject({ + sourcePath, + scenario, + outDir, + title = "export-benchmark", + paddingControl = null, +}) { + mkdirSync(outDir, { recursive: true }); + + // The loader only auto-approves media in the recordings dir or *next to the project*, so + // the source is copied in rather than referenced across the filesystem. + const localMedia = join(outDir, basename(sourcePath)); + if (localMedia !== sourcePath) copyFileSync(sourcePath, localMedia); + + const p = probe(localMedia); + const e = scenario.effects; + + // The CLI reads `EditorProjectData` (projectPersistence.ts): a flat + // { version, media, editor } document. The schemaVersion-6 shape that the AI-edition + // editor writes is a different file format and `runInfoCommand` does not read it. + const doc = { + version: 2, + media: { + screenVideoPath: localMedia, + webcamVideoPath: undefined, + cursorCaptureMode: "system", + }, + editor: { + wallpaper: e.background?.kind === "solid" ? e.background.color : "#000000", + shadowIntensity: e.shadow?.enabled ? e.shadow.intensity : 0, + showBlur: false, + motionBlurAmount: e.motionBlur ? 0.2 : 0, + borderRadius: e.cornerRadiusPx, + padding: paddingControl ?? paddingFromPercent(e.paddingPercent), + cropRegion: { x: 0, y: 0, width: 100, height: 100 }, + zoomRegions: (e.zooms ?? []).map(toZoomRegion), + cameraFullscreenRegions: [], + autoZoomEnabled: false, + autoFocusAll: false, + trimRegions: [], + speedRegions: [], + annotationRegions: [], + aspectRatio: "16:9", + webcamLayoutPreset: "no-webcam", + webcamMaskShape: "rectangle", + webcamMirrored: false, + webcamReactiveZoom: false, + webcamSizePreset: "medium", + webcamPosition: null, + // "good" resolves to short-side 1080 @ 20 Mbps for 16:9 — exactly the pinned target. + exportQuality: "good", + exportFormat: "mp4", + gifFrameRate: 15, + gifLoop: true, + gifSizePreset: "medium", + cursorTheme: "default", + }, + }; + + const projectPath = join(outDir, `${title}.openscreen`); + writeFileSync(projectPath, `${JSON.stringify(doc, null, 2)}\n`); + return { projectPath, mediaPath: localMedia, probe: p }; +} diff --git a/benchmark/lib/permissions.mjs b/benchmark/lib/permissions.mjs new file mode 100644 index 00000000..96ae76b9 --- /dev/null +++ b/benchmark/lib/permissions.mjs @@ -0,0 +1,107 @@ +/** + * Getting every permission prompt out of the way in one sitting. + * + * The benchmark is meant to be started and then left alone, but macOS will not let that + * happen by default: the first Apple Event sent to each application raises a modal + * " wants access to control " dialog, and it blocks the script that triggered it + * until somebody answers. Hit six of those spread across a two-hour unattended run and the + * run is not unattended at all — it is six ambushes. + * + * So they are all provoked deliberately, up front, while the user is still at the keyboard. + * Each app gets one harmless scripted question ("how many windows do you have?"); the first + * one raises the dialog, every later one is silent. Granting is the user's to do — these are + * security settings and nothing here clicks Allow on their behalf — but after this pass there + * is nothing left to interrupt. + */ +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; + +/** Has this process already been granted Apple Event access to `bundleId`? */ +export function automationStatus(bundleId) { + // A zero-timeout probe: if the permission is missing the call blocks on the dialog, which + // is exactly what we want to detect without waiting for a human. + const res = spawnSync( + "/usr/bin/osascript", + ["-e", `tell application id "${bundleId}" to return name`], + { encoding: "utf8", timeout: 2500 }, + ); + if (res.error?.code === "ETIMEDOUT") return "prompting"; + const err = (res.stderr ?? "").toLowerCase(); + if (res.status === 0) return "granted"; + if (err.includes("not authoriz") || err.includes("-1743")) return "denied"; + if (err.includes("-600") || err.includes("isn't running")) return "granted"; // reachable, just not running + return "unknown"; +} + +/** + * Provoke the Apple Events prompt for one app and wait for an answer. + * Returns once the user has responded (or the timeout expires). + */ +export function primeAutomation(bundleId, { timeoutMs = 180_000 } = {}) { + const res = spawnSync( + "/usr/bin/osascript", + ["-e", `tell application id "${bundleId}" to return name`], + { encoding: "utf8", timeout: timeoutMs }, + ); + if (res.status === 0) return { bundleId, status: "granted" }; + const err = (res.stderr ?? "").trim(); + if (/-1743|not authoriz/i.test(err)) + return { bundleId, status: "denied", error: err.slice(0, 200) }; + if (/-600|isn't running/i.test(err)) + return { bundleId, status: "granted", note: "app not running" }; + if (res.error?.code === "ETIMEDOUT") return { bundleId, status: "unanswered" }; + return { bundleId, status: "unknown", error: err.slice(0, 200) }; +} + +/** + * Whether this process can drive the UI at all. Without Accessibility, System Events refuses + * every menu click and no GUI driver can run — better to say so in preflight than to have the + * first GUI app fail an hour into a run. + */ +export function accessibilityGranted() { + const res = spawnSync( + "/usr/bin/osascript", + ["-e", 'tell application "System Events" to return count of processes'], + { encoding: "utf8", timeout: 10_000 }, + ); + return res.status === 0 && /^\d+$/.test((res.stdout ?? "").trim()); +} + +/** Is any modal permission dialog on screen right now, and what is it asking? */ +export function pendingPermissionDialog() { + const res = spawnSync( + "/usr/bin/osascript", + [ + "-e", + `tell application "System Events" + if not (exists process "UserNotificationCenter") then return "" + tell process "UserNotificationCenter" + if (count of windows) is 0 then return "" + set t to "" + repeat with e in (every static text of window 1) + set t to t & (value of e) & " " + end repeat + return t + end tell + end tell`, + ], + { encoding: "utf8", timeout: 8000 }, + ); + const text = (res.stdout ?? "").trim(); + return text || null; +} + +/** Everything a run needs, in the order preflight should walk through it. */ +export function permissionPlan(drivers) { + return drivers + .filter((d) => d.bundleId && d.appPath && existsSync(d.appPath)) + .map((d) => ({ + app: d.displayName, + bundleId: d.bundleId, + why: + d.automation === "cli" + ? "quitting the app cleanly between runs" + : "driving its menu bar and export dialog", + status: automationStatus(d.bundleId), + })); +} diff --git a/benchmark/lib/report.mjs b/benchmark/lib/report.mjs new file mode 100644 index 00000000..ca5a8d11 --- /dev/null +++ b/benchmark/lib/report.mjs @@ -0,0 +1,333 @@ +/** + * Turning a results document into something a person can act on. + * + * The design rule here is that nothing is allowed to look like a clean win when it isn't. A + * row whose output failed verification, or whose driver could not apply the scenario, carries + * that on its face — an app that renders fewer effects is not faster, and an app that wrote a + * 720p file has not competed at all. + */ +import { mad, median } from "./measure.mjs"; + +const fmtMs = (ms) => (ms == null ? "—" : `${(ms / 1000).toFixed(2)} s`); +const fmtMB = (b) => (b == null ? "—" : `${(b / 1048576).toFixed(1)} MB`); + +/** Statistics over the scoring runs only — the warm-up is kept in the data but not the table. */ +export function summarise(appResult) { + const scoring = (appResult.runs ?? []).filter((r) => !r.warmup && r.ok); + const attempted = (appResult.runs ?? []).filter((r) => !r.warmup); + const times = scoring.map((r) => r.exportMs).filter((x) => x != null); + const warmup = (appResult.runs ?? []).find((r) => r.warmup); + + return { + app: appResult.app, + displayName: appResult.displayName ?? appResult.app, + version: appResult.version ?? null, + automation: appResult.automation ?? null, + kind: appResult.kind ?? null, + skipped: !!appResult.skipped, + reason: appResult.reason ?? null, + fidelity: appResult.fidelity ?? null, + n: scoring.length, + attempted: attempted.length, + medianMs: median(times), + madMs: mad(times), + minMs: times.length ? Math.min(...times) : null, + maxMs: times.length ? Math.max(...times) : null, + warmupMs: warmup?.exportMs ?? null, + realtimeFactor: median(scoring.map((r) => r.realtimeFactor).filter(Boolean)), + framesPerSecond: median(scoring.map((r) => r.framesPerSecond).filter(Boolean)), + cpuSeconds: median(scoring.map((r) => r.cpuSeconds).filter((x) => x != null)), + peakRssMiB: Math.max(0, ...scoring.map((r) => r.peakRssMiB ?? 0)) || null, + outputSizeBytes: median(scoring.map((r) => r.outputSizeBytes).filter(Boolean)), + foreignCpuPercent: median(scoring.map((r) => r.foreignCpuPercent).filter((x) => x != null)), + outputProbe: scoring[0]?.outputProbe ?? attempted[0]?.outputProbe ?? null, + visual: scoring[0]?.visual ?? attempted[0]?.visual ?? null, + failures: attempted + .filter((r) => !r.ok) + .map((r) => ({ + run: r.run, + error: r.error ?? null, + verifyReasons: r.verifyReasons ?? [], + })), + notes: appResult.notes ?? [], + }; +} + +function fidelityLabel(f) { + if (!f) return "—"; + if (f.full) return "full"; + return `partial (${Math.round(f.score * 100)}% — missing ${f.missing.join(", ")})`; +} + +export function renderReport(doc) { + const rows = (doc.results ?? []).map(summarise); + const ran = rows.filter((r) => !r.skipped && r.medianMs != null); + const floor = rows.find((r) => r.app === "ffmpeg-baseline"); + // Full-fidelity rows are ranked against each other; a partial row did less work and is + // listed after them so the table cannot be read as "this app is faster". + const ranked = [...ran].sort((a, b) => { + const af = a.fidelity?.full ? 0 : 1; + const bf = b.fidelity?.full ? 0 : 1; + return af !== bf ? af - bf : a.medianMs - b.medianMs; + }); + + const m = doc.machine ?? {}; + const src = doc.fixture?.probe?.video ?? {}; + + /* ------------------------------------------------------------------ markdown ---------- */ + const md = []; + md.push(`# Export benchmark — ${doc.scenario?.label ?? doc.scenario?.id}`); + md.push(""); + md.push( + `**Run** \`${doc.runId}\` · started ${doc.startedAt}${doc.finishedAt ? ` · finished ${doc.finishedAt}` : " · **incomplete**"}`, + ); + md.push(""); + md.push( + `**Machine** ${m.chip} · ${m.cpuCount} cores (${m.performanceCores}P/${m.efficiencyCores}E) · ${m.memoryGiB} GiB · ${m.osProduct} ${m.osVersion} (${m.osBuild})`, + ); + md.push(""); + md.push( + `**Source** ${src.width}×${src.height} @ ${src.fps} fps · ${doc.fixture?.probe?.durationSec}s · ${src.nbFrames} frames · sha256 \`${doc.fixture?.sha256?.slice(0, 16)}\``, + ); + md.push(""); + md.push( + `**Target output** ${doc.scenario?.output?.width}×${doc.scenario?.output?.height} @ ${doc.scenario?.output?.fps} fps ${doc.scenario?.output?.videoCodec}/${doc.scenario?.output?.container} — every app pinned to the same thing.`, + ); + md.push(""); + md.push( + `**Repetitions** ${doc.repetitions}${doc.discardFirst ? " scoring runs, after one discarded warm-up" : " (no warm-up discarded)"} · ${doc.cooldownSec}s cooldown between runs.`, + ); + md.push(""); + + md.push("## Results"); + md.push(""); + md.push( + "| # | App | Version | Export (median) | ×realtime | Render fps | vs floor | CPU·s | Peak RSS | Output | Fidelity | How it was driven |", + ); + md.push("|---|---|---|---|---|---|---|---|---|---|---|---|"); + for (const [i, r] of ranked.entries()) { + const vsFloor = + floor?.medianMs && r.medianMs ? `${(r.medianMs / floor.medianMs).toFixed(2)}×` : "—"; + md.push( + `| ${i + 1} | **${r.displayName}** | ${r.version ?? "—"} | ${fmtMs(r.medianMs)}${r.madMs ? ` ±${(r.madMs / 1000).toFixed(2)}` : ""} | ${r.realtimeFactor ?? "—"}× | ${r.framesPerSecond ?? "—"} | ${vsFloor} | ${r.cpuSeconds ?? "—"} | ${r.peakRssMiB ? `${r.peakRssMiB} MiB` : "—"} | ${r.foreignCpuPercent != null ? `${r.foreignCpuPercent}%` : "—"} | ${fmtMB(r.outputSizeBytes)} | ${fidelityLabel(r.fidelity)} | \`${r.automation ?? "—"}\` |`, + ); + } + const notRun = rows.filter((r) => r.skipped || r.medianMs == null); + if (notRun.length) { + md.push(""); + md.push("### Not measured"); + md.push(""); + for (const r of notRun) { + md.push( + `- **${r.displayName}** — ${r.reason ?? "no successful scoring run"}${ + r.failures.length + ? `; failures: ${r.failures + .map((f) => f.error ?? f.verifyReasons.join(", ")) + .join(" | ") + .slice(0, 400)}` + : "" + }`, + ); + } + } + + md.push(""); + md.push("## Reading this table"); + md.push(""); + md.push( + "- **Export (median)** is wall-clock from the instant the export is committed to the moment the last byte lands in the output file. Warm-up, project loading and app launch are excluded for every app alike; `±` is the median absolute deviation across scoring runs.", + ); + md.push("- **×realtime** is output duration ÷ export time. Above 1 means faster than playback."); + md.push( + '- **vs floor** compares against `ffmpeg (re-encode floor)`, a plain transcode with no compositing. It separates "this encoder is slow on this machine" from "this app\'s pipeline is slow".', + ); + md.push( + "- **Fidelity** is how much of the scenario the app actually applied. A `partial` row did less work and its time is not comparable — it is shown for reference, not ranking.", + ); + md.push( + "- **How it was driven** records the automation rung: `cli` is scripted headlessly, `menu` is System Events driving named menu items, `menu+coords` means some step needed pixel clicking and is the least reproducible.", + ); + md.push(""); + + for (const r of rows.filter((x) => x.notes?.length || x.failures?.length)) { + md.push(`### ${r.displayName}`); + md.push(""); + for (const n of r.notes ?? []) md.push(`- ${n}`); + for (const f of r.failures ?? []) { + md.push( + `- run ${f.run} failed: ${f.error ?? ""} ${f.verifyReasons.length ? `(output: ${f.verifyReasons.join("; ")})` : ""}`.trim(), + ); + } + if (r.outputProbe) { + const p = r.outputProbe; + md.push( + `- produced ${p.video?.width}×${p.video?.height} @ ${p.video?.fps} fps, ${p.video?.codec}, ${p.durationSec?.toFixed(2)}s, ${fmtMB(p.sizeBytes)}`, + ); + } + md.push(""); + } + + /* -------------------------------------------------------------------- summary --------- */ + const winner = ranked.find((r) => r.fidelity?.full); + const os = ranked.find((r) => r.app === "openscreen-cli"); + const summaryText = ranked.length + ? [ + `${ranked.length} apps measured.`, + winner + ? `Fastest at full fidelity: ${winner.displayName} at ${fmtMs(winner.medianMs)} (${winner.realtimeFactor}× realtime).` + : "", + os && winner && os.app !== winner.app + ? `OpenScreen (CLI): ${fmtMs(os.medianMs)} — ${(os.medianMs / winner.medianMs).toFixed(2)}× the leader.` + : "", + ] + .filter(Boolean) + .join(" ") + : "No app produced a measurable, verified export."; + + return { + markdown: md.join("\n"), + html: renderHtml(doc, rows, ranked, floor, summaryText), + summaryText, + rows, + }; +} + +/* ------------------------------------------------------------------------ html ----------- */ + +const escapeHtml = (s) => + String(s ?? "").replace( + /[&<>"']/g, + (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c], + ); + +function renderHtml(doc, rows, ranked, floor, summaryText) { + const m = doc.machine ?? {}; + const src = doc.fixture?.probe?.video ?? {}; + const maxMs = Math.max(1, ...ranked.map((r) => r.medianMs ?? 0)); + + const bars = ranked + .map((r) => { + const pct = ((r.medianMs ?? 0) / maxMs) * 100; + const partial = !r.fidelity?.full; + return `
+
${escapeHtml(r.displayName)}${partial ? 'partial' : ""}
+
+
${fmtMs(r.medianMs)} ${r.realtimeFactor ?? "—"}×
+
`; + }) + .join("\n"); + + const tableRows = ranked + .map((r, i) => { + const vsFloor = + floor?.medianMs && r.medianMs ? `${(r.medianMs / floor.medianMs).toFixed(2)}×` : "—"; + return ` + ${i + 1} + ${escapeHtml(r.displayName)}
${escapeHtml(r.version ?? "")} + ${fmtMs(r.medianMs)}${r.madMs ? `
±${(r.madMs / 1000).toFixed(2)}` : ""} + ${r.realtimeFactor ?? "—"}× + ${r.framesPerSecond ?? "—"} + ${vsFloor} + ${r.cpuSeconds ?? "—"} + ${r.peakRssMiB ? `${r.peakRssMiB}` : "—"} + ${fmtMB(r.outputSizeBytes)} + ${r.fidelity?.full ? 'full' : `${escapeHtml(fidelityLabel(r.fidelity))}`} + ${escapeHtml(r.automation ?? "—")} + `; + }) + .join("\n"); + + const notRun = rows + .filter((r) => r.skipped || r.medianMs == null) + .map( + (r) => + `
  • ${escapeHtml(r.displayName)} — ${escapeHtml(r.reason ?? "no successful scoring run")}
    • `, + ) + .join("\n"); + + return `Export Benchmark + +
    +

    Export benchmark

    +

    ${escapeHtml(doc.scenario?.label ?? "")} · run ${escapeHtml(doc.runId ?? "")}

    + +
    +
    Machine
    ${escapeHtml(m.chip ?? "?")} · ${m.memoryGiB} GiB
    +
    macOS
    ${escapeHtml(m.osVersion ?? "?")} (${escapeHtml(m.osBuild ?? "")})
    +
    Source
    ${src.width}×${src.height} @ ${src.fps} · ${doc.fixture?.probe?.durationSec}s
    +
    Target
    ${doc.scenario?.output?.width}×${doc.scenario?.output?.height} @ ${doc.scenario?.output?.fps} ${escapeHtml(doc.scenario?.output?.videoCodec ?? "")}
    +
    Repetitions
    ${doc.repetitions}${doc.discardFirst ? " + warm-up" : ""}
    +
    + +
    ${escapeHtml(summaryText)}
    + +

    Export time — lower is better

    + ${bars || "

    Nothing measured.

    "} + +

    Full results

    +
    + + + + + + ${tableRows} +
    #AppExport (median)×RTRender fpsvs floorCPU·sPeak MiBOutputFidelityDriven by
    +
    + + ${notRun ? `

    Not measured

      ${notRun}
    ` : ""} + +

    Method

    +
      +
    • The clock starts the instant the export is committed and stops when the output file stops growing — the same stopwatch for the CLI apps and the ones driven through their menus.
      • +
    • Every output is re-probed and checked against the pinned target; a file that is the wrong size, rate or length is failed, not counted as a fast run.
      • +
    • Fidelity marks how much of the scenario each app could express. Hatched bars did less work, so their time is a reference, not a ranking.
      • +
    • Source clip is generated from a seeded spec, not shipped — the sha256 above is what makes two machines comparable.
      • +
    +
    `; +} diff --git a/benchmark/lib/runner.mjs b/benchmark/lib/runner.mjs new file mode 100644 index 00000000..142ca7a1 --- /dev/null +++ b/benchmark/lib/runner.mjs @@ -0,0 +1,277 @@ +/** + * The stopwatch, shared by every driver. + * + * Keeping this in one place is what makes the comparison defensible: the CLI apps and the GUI + * apps are timed by the same code, completion is decided the same way, and every output is + * verified against the same target before it is allowed to count. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, rmSync } from "node:fs"; +import { fidelity } from "../scenarios/index.mjs"; +import { diskState, machineFingerprint, powerState } from "./env.mjs"; +import { + cooldown, + cpuDelta, + now, + ProcessTreeSampler, + sleep, + verifyOutput, + waitForStableFile, +} from "./measure.mjs"; +import { inspectExport } from "./visualCheck.mjs"; + +/** Refuse to measure on a machine that is already compromised — the number would be noise. */ +/** Total %CPU across all processes right now, summed over cores. */ +function instantaneousForeignLoad() { + try { + const out = execFileSync("/bin/ps", ["-axo", "pcpu="], { + encoding: "utf8", + maxBuffer: 8 * 1024 * 1024, + }); + return +out + .split("\n") + .map((l) => Number(l.trim())) + .filter((n) => Number.isFinite(n)) + .reduce((a, b) => a + b, 0) + .toFixed(0); + } catch { + return null; + } +} + +export function preconditionCheck({ requireAC = true, minDiskGiB = 20 } = {}) { + const power = powerState(); + const disk = diskState(); + const problems = []; + if (requireAC && !power.onACPower) problems.push("running on battery — the SoC is power-capped"); + if (power.lowPowerMode) problems.push("Low Power Mode is on"); + if (power.cpuSpeedLimit != null && power.cpuSpeedLimit < 100) { + problems.push(`CPU is thermally limited to ${power.cpuSpeedLimit}%`); + } + if (disk.availableGiB < minDiskGiB) { + problems.push(`only ${disk.availableGiB} GiB free (need ${minDiskGiB})`); + } + // Competing load is the one precondition that does not announce itself: nothing throttles, + // nothing warns, every export is simply slower. Worth naming before a run, not after. + const foreign = instantaneousForeignLoad(); + if (foreign != null && foreign > 60) { + problems.push( + `${foreign}% of a core-second is already being used by other processes ` + + "(a remote-desktop session, a screen recorder, a build) — every export will be slower", + ); + } + return { ok: problems.length === 0, problems, power, disk, foreignCpuPercent: foreign }; +} + +/** + * One measured export. + * + * Timeline of a run, and what each interval is called in the results: + * + * prepare() ─ prepareMs (warm-up: launch, import, presets — not counted) + * runExport() begins ─┐ + * ctx.commit() ─┤ launchToCommitMs + * ── t0 ───────────────┘ + * …render… ─ exportMs ← the headline number + * ── t1 = last byte written to the output file + * verify ─ verifyMs + */ +export async function runOnce(driver, ctx) { + const sampler = new ProcessTreeSampler( + [driver.appPath, driver.processName].filter(Boolean), + ).start(); + const before = sampler.result(); + + const out = driver.outputPath(ctx); + if (existsSync(out)) rmSync(out, { force: true }); + + let t0 = null; + let driverCompletedAt = null; + const invokedAt = now(); + ctx.commit = () => { + if (t0 === null) t0 = now(); + }; + /** + * A driver may supply its own completion instant when the app knows better than the + * filesystem does — Camtasia flips `isExporting` to false, Kap hands back a path only once + * it has finished. The runner still owns the clock; this only replaces the *stop*, and only + * when it is earlier than the filesystem's answer, so it can never inflate a result. + */ + ctx.markComplete = (ts = now()) => { + if (driverCompletedAt === null) driverCompletedAt = ts; + }; + + const record = { + app: driver.id, + displayName: driver.displayName, + scenario: ctx.scenario.id, + run: ctx.run.index, + automation: driver.automation, + outputPath: out, + ok: false, + }; + + let exportError = null; + try { + await driver.runExport(ctx); + } catch (e) { + exportError = e; + } + if (t0 === null) t0 = invokedAt; // a driver that never committed still gets a clock + + // The app may still be muxing after runExport resolves — for GUI drivers it almost always + // is, because the click returns immediately. The filesystem decides when it is done. + const wait = await waitForStableFile(out, { + timeoutMs: ctx.timeoutMs ?? 45 * 60 * 1000, + stableMs: ctx.stableMs ?? 2500, + onTick: ctx.onTick, + }); + + const after = sampler.stop(); + const delta = cpuDelta(before, after); + + const stopAt = + driverCompletedAt != null && wait.ok + ? Math.min(driverCompletedAt, wait.completedAt) + : wait.ok + ? wait.completedAt + : null; + record.launchToCommitMs = Math.round(t0 - invokedAt); + record.exportMs = stopAt != null ? Math.round(stopAt - t0) : null; + record.completionSignal = driverCompletedAt != null ? "driver" : "file-stability"; + record.waitedMs = Math.round(wait.waitedMs); + record.cpuSeconds = delta.cpuSeconds; + record.peakRssMiB = delta.peakRssMiB; + record.foreignCpuPercent = after.foreignCpuPercent; + record.outputSizeBytes = wait.sizeBytes ?? null; + + if (exportError) { + record.error = exportError.message?.slice(0, 1200) ?? String(exportError); + } + if (!wait.ok) { + record.error = record.error ?? `output never stabilised (${wait.reason})`; + return record; + } + + const v = verifyOutput(out, ctx.scenario.output, ctx.source.probe.durationSec); + record.verified = v.valid; + record.verifyReasons = v.reasons; + record.outputProbe = v.probe; + + if (record.exportMs != null && v.probe?.durationSec) { + const secs = record.exportMs / 1000; + record.realtimeFactor = +(v.probe.durationSec / secs).toFixed(3); + record.framesPerSecond = v.probe.video?.nbFrames + ? +(v.probe.video.nbFrames / secs).toFixed(1) + : null; + record.megapixelsPerSecond = + v.probe.video?.nbFrames && v.probe.video.width + ? +( + (v.probe.video.nbFrames * v.probe.video.width * v.probe.video.height) / + 1e6 / + secs + ).toFixed(1) + : null; + } + + // Metadata says the file is the right shape; only pixels say the app did the work. An + // export that skipped the compositing would otherwise be recorded as a fast, valid run. + if (v.valid) { + try { + record.visual = inspectExport(out, ctx.scenario, { probe: v.probe }); + } catch (e) { + record.visual = { error: e.message?.slice(0, 400) ?? String(e), allPassed: null }; + } + } + + record.ok = v.valid && !exportError; + record.effectsVerified = record.visual?.allPassed ?? null; + return record; +} + +/** Every repetition for one app, with the guards and cooldowns in between. */ +export async function runApp( + driver, + baseCtx, + { repetitions = 3, discardFirst = true, cooldownSec = 45, log = () => undefined } = {}, +) { + const detected = driver.detect(); + if (!detected.installed) { + return { + app: driver.id, + displayName: driver.displayName, + skipped: true, + reason: `not installed${detected.error ? `: ${detected.error}` : ""}`, + runs: [], + }; + } + + log(`${driver.displayName}: preparing (${detected.version ?? "unknown version"})`); + const state = {}; + const prepCtx = { ...baseCtx, state, run: { index: 0 }, commit: () => undefined }; + const tPrep = now(); + let prep; + try { + prep = await driver.prepare(prepCtx); + } catch (e) { + return { + app: driver.id, + displayName: driver.displayName, + version: detected.version, + skipped: true, + reason: `prepare failed: ${e.message?.slice(0, 600)}`, + runs: [], + }; + } + const prepareMs = Math.round(now() - tPrep); + const fid = fidelity(baseCtx.scenario, prep.appliedFeatures); + + const runs = []; + // Run 0 is a warm-up: caches are cold, shaders are uncompiled, and the app may still be + // finishing its own first-launch work. It is measured and kept, but excluded from the + // headline statistics unless the caller says otherwise. + const total = repetitions + (discardFirst ? 1 : 0); + for (let i = 0; i < total; i++) { + const isWarmup = discardFirst && i === 0; + log(`${driver.displayName}: run ${i + 1}/${total}${isWarmup ? " (warm-up)" : ""}`); + const pre = preconditionCheck(); + if (!pre.ok) log(` ⚠ ${pre.problems.join("; ")}`); + + const ctx = { ...baseCtx, state, run: { index: i }, commit: () => undefined }; + const rec = await runOnce(driver, ctx); + rec.warmup = isWarmup; + rec.precondition = pre.problems; + runs.push(rec); + log( + ` ${rec.ok ? "✓" : "✗"} ${rec.exportMs != null ? `${(rec.exportMs / 1000).toFixed(2)}s` : "failed"}` + + `${rec.realtimeFactor ? ` (${rec.realtimeFactor}× realtime)` : ""}` + + `${rec.error ? ` — ${rec.error.split("\n")[0].slice(0, 160)}` : ""}` + + `${rec.verifyReasons?.length ? ` — output mismatch: ${rec.verifyReasons.join("; ")}` : ""}`, + ); + + if (i < total - 1) await cooldown({ seconds: cooldownSec, log: () => undefined }); + } + + try { + await driver.cleanup({ ...baseCtx, state }); + } catch (e) { + log(` cleanup warning: ${e.message}`); + } + + return { + app: driver.id, + displayName: driver.displayName, + vendor: driver.vendor, + kind: driver.kind, + automation: driver.automation, + version: detected.version, + path: detected.path, + prepareMs, + fidelity: fid, + notes: prep.notes ?? [], + skipped: false, + runs, + }; +} + +export { machineFingerprint, sleep }; diff --git a/benchmark/lib/state.mjs b/benchmark/lib/state.mjs new file mode 100644 index 00000000..f7de9c6c --- /dev/null +++ b/benchmark/lib/state.mjs @@ -0,0 +1,78 @@ +/** + * Run state: an append-only event log plus a single status document. + * + * Both exist for the same reason — the benchmark is meant to be started and then left alone, + * possibly from a remote Claude Code session. Nobody is watching the terminal, so progress has + * to be readable from disk at any moment, and a run that dies halfway has to be resumable + * without repeating the expensive parts. + */ +import { + appendFileSync, + existsSync, + mkdirSync, + readFileSync, + renameSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; + +export class RunState { + constructor(dir, runId) { + this.dir = dir; + this.runId = runId; + mkdirSync(dir, { recursive: true }); + this.eventsPath = join(dir, "events.ndjson"); + this.statusPath = join(dir, "status.json"); + this.resultsPath = join(dir, "results.json"); + } + + event(type, payload = {}) { + const line = JSON.stringify({ + ts: new Date().toISOString(), + runId: this.runId, + type, + ...payload, + }); + appendFileSync(this.eventsPath, `${line}\n`); + return line; + } + + /** Written atomically: a poller must never read a half-serialised status. */ + writeStatus(status) { + const tmp = `${this.statusPath}.tmp`; + writeFileSync(tmp, `${JSON.stringify(status, null, 2)}\n`); + renameSync(tmp, this.statusPath); + } + + readStatus() { + if (!existsSync(this.statusPath)) return null; + try { + return JSON.parse(readFileSync(this.statusPath, "utf8")); + } catch { + return null; + } + } + + writeResults(results) { + const tmp = `${this.resultsPath}.tmp`; + writeFileSync(tmp, `${JSON.stringify(results, null, 2)}\n`); + renameSync(tmp, this.resultsPath); + } + + readResults() { + if (!existsSync(this.resultsPath)) return null; + try { + return JSON.parse(readFileSync(this.resultsPath, "utf8")); + } catch { + return null; + } + } +} + +/** Sortable, human-readable, and stable inside one calendar second. */ +export function newRunId(d = new Date()) { + return d + .toISOString() + .replace(/[-:]/g, "") + .replace(/\.\d+Z$/, "Z"); +} diff --git a/benchmark/lib/uiScript.mjs b/benchmark/lib/uiScript.mjs new file mode 100644 index 00000000..7e240f72 --- /dev/null +++ b/benchmark/lib/uiScript.mjs @@ -0,0 +1,341 @@ +/** + * macOS UI automation for the apps that have no CLI. + * + * Three of the five apps in this benchmark expose no scriptable interface at all — Screen + * Studio publishes only `screen-studio://record-*` deeplinks, Camtasia and Kap publish + * nothing. Their exports therefore have to be driven through the UI, and the *way* that is + * done decides whether the benchmark reproduces on another machine. + * + * Pixel coordinates do not reproduce: they depend on display size, scale factor, window + * placement and app version. Accessibility object names largely do. So every interaction here + * is expressed against the accessibility tree — "click the menu item called Export…" — and + * clicking at a coordinate is the last rung of the ladder, used only where an app draws a + * control that publishes no accessibility role. + * + * The ladder, best first: + * 1. a scripting dictionary (none of these apps has one — checked with `sdef`) + * 2. System Events menu-bar item by name ← where almost everything lands + * 3. a documented keyboard shortcut + * 4. accessibility button/pop-up by name or description + * 5. computer-use pixel clicking (agent-assisted, recorded as reduced reproducibility) + */ +import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { sleep } from "./measure.mjs"; + +export class UiScriptError extends Error { + constructor(message, { script, stderr } = {}) { + super(message); + this.name = "UiScriptError"; + this.script = script; + this.stderr = stderr; + } +} + +/** Run an AppleScript source string; returns trimmed stdout. */ +export function osa(script, { timeoutMs = 60_000 } = {}) { + try { + return execFileSync("/usr/bin/osascript", ["-"], { + input: script, + encoding: "utf8", + timeout: timeoutMs, + maxBuffer: 16 * 1024 * 1024, + }).trim(); + } catch (e) { + const stderr = (e.stderr?.toString() || "").trim(); + throw new UiScriptError(stderr || e.message, { script, stderr }); + } +} + +/** JavaScript for Automation, for the places where AppleScript's syntax fights back. */ +export function jxa(script, { timeoutMs = 60_000 } = {}) { + try { + return execFileSync("/usr/bin/osascript", ["-l", "JavaScript", "-"], { + input: script, + encoding: "utf8", + timeout: timeoutMs, + maxBuffer: 16 * 1024 * 1024, + }).trim(); + } catch (e) { + const stderr = (e.stderr?.toString() || "").trim(); + throw new UiScriptError(stderr || e.message, { script, stderr }); + } +} + +const esc = (s) => String(s).replace(/\\/g, "\\\\").replace(/"/g, '\\"'); + +/* ------------------------------------------------------------------- app lifecycle ------- */ + +export function appIsRunning(processName) { + return ( + osa(`tell application "System Events" to return (exists process "${esc(processName)}")`) === + "true" + ); +} + +/** Does this app ship an AppleScript dictionary? Rung 1 of the ladder. */ +export function hasScriptingDictionary(appPath) { + if (!existsSync(appPath)) return false; + try { + const out = execFileSync("/usr/bin/sdef", [appPath], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + maxBuffer: 8 * 1024 * 1024, + }); + // Every app inherits the Standard Suite; a *useful* dictionary has more than that. + const suites = [...out.matchAll(/ m[1]); + return suites.filter((s) => !/Standard Suite|Text Suite/i.test(s)).length > 0; + } catch { + return false; + } +} + +export async function launchApp(appPath, processName, { timeoutMs = 90_000 } = {}) { + execFileSync("/usr/bin/open", ["-a", appPath]); + const t0 = Date.now(); + while (Date.now() - t0 < timeoutMs) { + if (appIsRunning(processName)) return true; + await sleep(400); + } + throw new UiScriptError(`"${processName}" did not start within ${timeoutMs}ms`); +} + +export function activateApp(processName) { + osa(`tell application "System Events" to set frontmost of process "${esc(processName)}" to true`); +} + +export async function quitApp(processName, { force = false, timeoutMs = 25_000 } = {}) { + try { + osa(`tell application "${esc(processName)}" to quit`, { timeoutMs: 15_000 }); + } catch { + /* app may not be scriptable enough to answer `quit`; fall through to the wait */ + } + const t0 = Date.now(); + while (Date.now() - t0 < timeoutMs) { + if (!appIsRunning(processName)) return true; + await sleep(400); + } + if (force) { + try { + execFileSync("/usr/bin/pkill", ["-f", processName]); + } catch { + /* nothing left to kill */ + } + } + return !appIsRunning(processName); +} + +/* ------------------------------------------------------------------------- menus --------- */ + +export function listMenus(processName) { + const raw = osa( + `tell application "System Events" to tell process "${esc(processName)}" to return name of every menu of menu bar 1`, + ); + return raw.split(", ").filter(Boolean); +} + +export function listMenuItems(processName, menuName) { + const raw = osa( + `tell application "System Events" to tell process "${esc(processName)}" to return name of every menu item of menu "${esc(menuName)}" of menu bar 1`, + ); + return raw.split(", ").filter(Boolean); +} + +/** + * Dump every menu and item. This is the discovery step a maintainer runs once per app version + * to write (or repair) a driver — export labels drift between releases, and guessing them + * produces a driver that fails silently on somebody else's machine. + */ +export function dumpMenus(processName) { + const out = {}; + for (const menu of listMenus(processName)) { + try { + out[menu] = listMenuItems(processName, menu); + } catch (e) { + out[menu] = [``]; + } + } + return out; +} + +/** + * Click a menu item, tolerating the label drift that makes naive UI scripts brittle: + * an ellipsis may be U+2026 or three periods, and a verb may have gained a noun + * ("Export" → "Export As…" → "Export Media…"). Patterns are tried in order. + */ +export function clickMenuItem(processName, menuName, patterns, { submenu = null } = {}) { + const items = submenu + ? osa( + `tell application "System Events" to tell process "${esc(processName)}" to return name of every menu item of menu 1 of menu item "${esc(submenu)}" of menu "${esc(menuName)}" of menu bar 1`, + ) + .split(", ") + .filter(Boolean) + : listMenuItems(processName, menuName); + + const norm = (s) => + s + .replace(/[.…]+$/, "") + .trim() + .toLowerCase(); + let matched = null; + for (const pat of [].concat(patterns)) { + const re = pat instanceof RegExp ? pat : new RegExp(`^${norm(pat)}$`, "i"); + matched = items.find((i) => (pat instanceof RegExp ? re.test(i) : re.test(norm(i)))); + if (matched) break; + } + if (!matched) { + throw new UiScriptError( + `no menu item in ${processName} → ${menuName}${submenu ? ` → ${submenu}` : ""} matched ${JSON.stringify(patterns)}. Present: ${JSON.stringify(items)}`, + ); + } + + const target = submenu + ? `menu item "${esc(matched)}" of menu 1 of menu item "${esc(submenu)}" of menu "${esc(menuName)}" of menu bar 1` + : `menu item "${esc(matched)}" of menu "${esc(menuName)}" of menu bar 1`; + osa( + `tell application "System Events" to tell process "${esc(processName)}"\n set frontmost to true\n click ${target}\nend tell`, + ); + return matched; +} + +/* ------------------------------------------------------------------------ windows -------- */ + +export function listWindows(processName) { + try { + return osa( + `tell application "System Events" to tell process "${esc(processName)}" to return name of every window`, + ) + .split(", ") + .filter(Boolean); + } catch { + return []; + } +} + +export async function waitForWindow(processName, match, { timeoutMs = 60_000, pollMs = 400 } = {}) { + const re = match instanceof RegExp ? match : new RegExp(match, "i"); + const t0 = Date.now(); + while (Date.now() - t0 < timeoutMs) { + const found = listWindows(processName).find((w) => re.test(w)); + if (found) return found; + await sleep(pollMs); + } + throw new UiScriptError( + `no window in ${processName} matched ${re} within ${timeoutMs}ms. Present: ${JSON.stringify(listWindows(processName))}`, + ); +} + +/** Full accessibility dump of a window — the discovery tool for writing a driver. */ +export function describeWindow(processName, windowIndex = 1, depth = 4) { + return jxa(` + const se = Application("System Events"); + const proc = se.processes["${esc(processName)}"]; + function walk(el, d) { + if (d > ${depth}) return null; + let role = "", name = "", desc = "", value = ""; + try { role = el.role(); } catch (e) {} + try { name = el.name() || ""; } catch (e) {} + try { desc = el.description() || ""; } catch (e) {} + try { const v = el.value(); value = (v === null || v === undefined) ? "" : String(v).slice(0, 60); } catch (e) {} + let kids = []; + try { kids = el.uiElements().map(k => walk(k, d + 1)).filter(Boolean); } catch (e) {} + return { role, name, desc, value, children: kids }; + } + JSON.stringify(walk(proc.windows[${windowIndex - 1}], 0), null, 1); + `); +} + +/** Click a button/checkbox/pop-up anywhere in a window, found by name or description. */ +export function clickControl(processName, { name, description, role = "button", windowIndex = 1 }) { + const needle = esc(name ?? description ?? ""); + const attr = name ? "name" : "description"; + return jxa(` + const se = Application("System Events"); + const proc = se.processes["${esc(processName)}"]; + proc.frontmost = true; + function find(el, d) { + if (d > 6) return null; + try { + if (el.role() === "AX${role[0].toUpperCase()}${role.slice(1)}") { + let v = ""; + try { v = el.${attr}() || ""; } catch (e) {} + if (v.toLowerCase().indexOf("${needle.toLowerCase()}") >= 0) return el; + } + } catch (e) {} + let kids = []; + try { kids = el.uiElements(); } catch (e) { return null; } + for (const k of kids) { const hit = find(k, d + 1); if (hit) return hit; } + return null; + } + const win = proc.windows[${windowIndex - 1}]; + const target = find(win, 0); + if (!target) throw new Error("no ${role} matching ${needle}"); + target.click(); + "clicked"; + `); +} + +/* -------------------------------------------------------------------- save panels -------- */ + +/** + * Drive the standard AppKit save sheet to an exact path. + * + * ⇧⌘G opens "Go to folder", which accepts a full path and is the one interaction that behaves + * the same in every app that uses the system panel. Typing into the name field alone is not + * enough — the panel remembers its last directory, so two runs would write to two places. + */ +export async function savePanelTo(processName, absolutePath, { timeoutMs = 30_000 } = {}) { + const dir = absolutePath.replace(/\/[^/]+$/, ""); + const file = absolutePath.split("/").pop(); + + const t0 = Date.now(); + while (Date.now() - t0 < timeoutMs) { + const hasSheet = osa( + `tell application "System Events" to tell process "${esc(processName)}" to return (exists sheet 1 of window 1) or (exists window "Save")`, + ); + if (hasSheet === "true") break; + await sleep(300); + } + + osa(` + tell application "System Events" + tell process "${esc(processName)}" + set frontmost to true + keystroke "g" using {command down, shift down} + delay 0.5 + keystroke "${esc(dir)}" + delay 0.4 + key code 36 + delay 0.8 + keystroke "a" using {command down} + keystroke "${esc(file)}" + delay 0.3 + end tell + end tell + `); + return { dir, file }; +} + +/** Press Return in the frontmost sheet — the commit for most save panels. */ +export function commitSavePanel(processName) { + osa( + `tell application "System Events" to tell process "${esc(processName)}"\n set frontmost to true\n key code 36\nend tell`, + ); +} + +/** Answer a "replace the existing file?" alert, which otherwise stalls an unattended run. */ +export function dismissReplaceAlert(processName) { + try { + osa(` + tell application "System Events" to tell process "${esc(processName)}" + if exists sheet 1 of sheet 1 of window 1 then + click button "Replace" of sheet 1 of sheet 1 of window 1 + end if + end tell + `); + } catch { + /* no alert — the common case */ + } +} + +export { esc as escapeAppleScript }; diff --git a/benchmark/lib/visualCheck.mjs b/benchmark/lib/visualCheck.mjs new file mode 100644 index 00000000..df29595e --- /dev/null +++ b/benchmark/lib/visualCheck.mjs @@ -0,0 +1,214 @@ +/** + * Did the app actually render the scenario? + * + * Probing an output for resolution and duration only proves a file exists. An app that ignored + * the background, dropped the zooms, or silently rendered a smaller video rect finishes sooner + * and looks faster, and no metadata check would notice. So every export is also inspected as + * pixels: + * + * · the frame's corners must be the scenario's background colour → background applied + * · the content's bounding box gives the real inset → padding, comparably measured + * · the box's corners must be background while its edges are not → corner radius applied + * · temporal activity must spike inside the zoom windows → zooms applied + * + * The measured inset matters as much as the pass/fail: two apps whose padding controls are on + * different scales end up compositing different-sized rectangles, and the report has to be + * able to say how close they were. + */ +import { execFileSync } from "node:child_process"; +import { resolveFfmpeg } from "./env.mjs"; + +/** Decode a single frame to raw RGB at full resolution. */ +function frameRgb(file, atSec, width, height) { + const { ffmpeg } = resolveFfmpeg(); + const buf = execFileSync( + ffmpeg, + [ + "-hide_banner", + "-loglevel", + "error", + "-ss", + String(atSec), + "-i", + file, + "-frames:v", + "1", + "-vf", + `scale=${width}:${height}:flags=neighbor`, + "-pix_fmt", + "rgb24", + "-f", + "rawvideo", + "-", + ], + { maxBuffer: 64 * 1024 * 1024, encoding: "buffer" }, + ); + return { data: buf, width, height }; +} + +/** Decode the whole clip small and grey, for the temporal-activity trace. */ +function greyTrace(file, fps, width, height) { + const { ffmpeg } = resolveFfmpeg(); + const buf = execFileSync( + ffmpeg, + [ + "-hide_banner", + "-loglevel", + "error", + "-i", + file, + "-vf", + `fps=${fps},scale=${width}:${height}:flags=bilinear,format=gray`, + "-f", + "rawvideo", + "-", + ], + { maxBuffer: 256 * 1024 * 1024, encoding: "buffer" }, + ); + const frameSize = width * height; + const frames = Math.floor(buf.length / frameSize); + const activity = []; + for (let f = 1; f < frames; f++) { + let sum = 0; + const a = (f - 1) * frameSize; + const b = f * frameSize; + for (let i = 0; i < frameSize; i += 3) sum += Math.abs(buf[b + i] - buf[a + i]); + activity.push(sum / Math.ceil(frameSize / 3)); + } + return { activity, fps }; +} + +const px = (img, x, y) => { + const i = (y * img.width + x) * 3; + return [img.data[i], img.data[i + 1], img.data[i + 2]]; +}; + +const dist = (a, b) => Math.abs(a[0] - b[0]) + Math.abs(a[1] - b[1]) + Math.abs(a[2] - b[2]); + +const hexRgb = (hex) => { + const h = (hex ?? "#000000").replace("#", ""); + return [0, 2, 4].map((i) => Number.parseInt(h.slice(i, i + 2), 16)); +}; + +/** + * Bounding box of everything that is not the background colour. Scanned from the middle rows + * and columns so a drop shadow — which is background-ish but not exactly background — does not + * drag the box outwards. + */ +function contentBox(img, bg, { contentTol = 120, run = 4 } = {}) { + const midY = Math.floor(img.height / 2); + const midX = Math.floor(img.width / 2); + // A drop shadow is background-ish but not background, so "anything that is not exactly the + // background" would find the shadow's outer edge. Requiring a short run of pixels that are + // *far* from the background finds the video itself. + const solid = (x, y, dx, dy) => { + for (let k = 0; k < run; k++) { + const xx = x + dx * k; + const yy = y + dy * k; + if (xx < 0 || yy < 0 || xx >= img.width || yy >= img.height) return false; + if (dist(px(img, xx, yy), bg) <= contentTol) return false; + } + return true; + }; + let left = 0; + while (left < img.width - run && !solid(left, midY, 1, 0)) left++; + let right = img.width - 1; + while (right > left && !solid(right, midY, -1, 0)) right--; + let top = 0; + while (top < img.height - run && !solid(midX, top, 0, 1)) top++; + let bottom = img.height - 1; + while (bottom > top && !solid(midX, bottom, 0, -1)) bottom--; + return { left, top, right, bottom, width: right - left + 1, height: bottom - top + 1 }; +} + +/** + * @param {string} file the exported video + * @param {object} scenario the scenario it was supposed to render + * @param {object} opts.probe ffprobe output for `file` + */ +export function inspectExport(file, scenario, { probe, tolerance = 42 } = {}) { + const e = scenario.effects; + const W = probe?.video?.width ?? scenario.output.width; + const H = probe?.video?.height ?? scenario.output.height; + const duration = probe?.durationSec ?? 0; + + // A reference instant that no zoom window covers, so the geometry read is of the resting + // composition rather than a mid-animation frame. + const zoomWindows = (e.zooms ?? []).map((z) => [z.startSec - 1, z.endSec + 1]); + let refSec = 2; + for (let t = 2; t < duration - 2; t += 0.5) { + if (!zoomWindows.some(([a, b]) => t >= a && t <= b)) { + refSec = t; + break; + } + } + + const img = frameRgb(file, refSec, W, H); + const result = { refSec, checks: {}, measured: {} }; + + /* ---- background ------------------------------------------------------------------- */ + if (e.background?.kind === "solid") { + const bg = hexRgb(e.background.color); + const corners = [px(img, 2, 2), px(img, W - 3, 2), px(img, 2, H - 3), px(img, W - 3, H - 3)]; + const worst = Math.max(...corners.map((c) => dist(c, bg))); + result.measured.cornerColor = corners[0]; + result.measured.cornerColorDistance = worst; + result.checks.background = worst <= tolerance; + + /* ---- padding ------------------------------------------------------------------ */ + const box = contentBox(img, bg); + result.measured.contentBox = box; + result.measured.contentFraction = +((box.width * box.height) / (W * H)).toFixed(4); + // Inset as a percent of the frame's short side — the same unit the scenario uses. + result.measured.insetPercentShortSide = +( + (Math.min(box.left, box.top, W - 1 - box.right, H - 1 - box.bottom) / Math.min(W, H)) * + 100 + ).toFixed(2); + result.checks.padding = e.paddingPercent > 0 ? box.left > 2 && box.top > 2 : true; + + /* ---- corner radius -------------------------------------------------------------- */ + if (e.cornerRadiusPx > 0 && box.width > 40) { + // At the box's own corner a rounded rect still shows background; a quarter of the + // way along its top edge it must show content. Both conditions together separate a + // radius from a plain rectangle and from a missing video. + const atCorner = px(img, box.left + 1, box.top + 1); + const alongEdge = px(img, box.left + Math.floor(box.width / 4), box.top + 2); + result.measured.cornerIsBackground = dist(atCorner, bg) <= tolerance; + result.measured.edgeIsContent = dist(alongEdge, bg) > tolerance; + result.checks.cornerRadius = + result.measured.cornerIsBackground && result.measured.edgeIsContent; + } + } + + /* ---- zooms ------------------------------------------------------------------------ */ + if ((e.zooms ?? []).length) { + const traceFps = 10; + const { activity } = greyTrace(file, traceFps, 192, 108); + const at = (sec) => Math.round(sec * traceFps); + const windowMax = (a, b) => + Math.max(0, ...activity.slice(Math.max(0, at(a)), Math.min(activity.length, at(b)))); + + // Baseline: the median of the whole trace. A zoom transition has to stand well clear of + // it — the source itself is always moving, so an absolute threshold would not do. + const sorted = [...activity].sort((x, y) => x - y); + const baseline = sorted[Math.floor(sorted.length / 2)] || 1e-6; + const spikes = e.zooms.map((z) => { + const inRamp = Math.max( + windowMax(z.startSec - 0.4, z.startSec + 1.2), + windowMax(z.endSec - 0.4, z.endSec + 1.2), + ); + return +(inRamp / baseline).toFixed(2); + }); + result.measured.activityBaseline = +baseline.toFixed(3); + result.measured.zoomSpikeRatios = spikes; + // Every zoom must produce a transition at least 1.8× the resting activity. + result.checks.zooms = spikes.every((s) => s >= 1.8); + } + + const failed = Object.entries(result.checks) + .filter(([, v]) => v === false) + .map(([k]) => k); + result.allPassed = failed.length === 0; + result.failed = failed; + return result; +} diff --git a/benchmark/scenarios/index.mjs b/benchmark/scenarios/index.mjs new file mode 100644 index 00000000..f08f4fc6 --- /dev/null +++ b/benchmark/scenarios/index.mjs @@ -0,0 +1,136 @@ +/** + * Scenario definitions. + * + * A scenario is the *edit* that sits between the source clip and the export button, plus the + * output the export must produce. It is declared once, app-agnostically, and each driver + * translates it into its own app's controls. Anything a driver cannot express is reported as + * an unsupported feature on that run rather than silently dropped — an app that skips the + * shadow pass is not comparable to one that renders it, and the report has to say so. + */ + +/** The output every app is pinned to. "Force identical output" is the whole point. */ +/** + * 60 fps rather than 30 is not a preference: OpenScreen's MP4 export path is fixed at 60 + * (`MP4_EXPORT_FPS`, src/cli/CliExportRunner.tsx), and every other app in the set can be told + * to emit 60. It is therefore the only frame rate on which "force identical output" is + * actually achievable — and it is what this category ships anyway, because the zoom and + * cursor-smoothing effects it sells are what 60 fps is for. + */ +export const TARGET_OUTPUT = { + container: "mp4", + videoCodec: "h264", + width: 1920, + height: 1080, + fps: 60, + /** Tolerances used when verifying an export actually hit the target. */ + tolerance: { durationSec: 0.75, fpsPercent: 5 }, +}; + +/** + * Effects, in the vocabulary every app in this category shares. Values are chosen to sit on + * each app's own presets where possible, so no driver has to type a number into a field that + * only exists in one product. + */ +export const SCENARIOS = { + /** + * The realistic product-demo export: the recording inset on a coloured background with + * rounded corners and a drop shadow, plus three zooms. This is what the category actually + * ships, and it exercises every stage of a compositor — background fill, transform, mask, + * blur, and animated scaling. + */ + "full-demo": { + id: "full-demo", + label: "Full demo (background + padding + radius + shadow + 3 zooms)", + effects: { + // Light neutral, deliberately far from every colour in the generated source: the + // verifier finds the composited video's edge by colour distance, and a background + // close to the recording's own chrome makes that boundary unfindable. + background: { kind: "solid", color: "#C9CDD6" }, + /** Inset of the recording inside the frame, as a percent of the frame's short side. */ + paddingPercent: 5, + cornerRadiusPx: 40, + shadow: { enabled: true, intensity: 0.2 }, + /** + * Zooms are given in seconds and as a scale factor so every app can express them. + * Focus is normalised (0..1) against the source frame. + */ + zooms: [ + { startSec: 6, endSec: 12, scale: 1.8, focus: { x: 0.32, y: 0.38 } }, + { startSec: 22, endSec: 29, scale: 2.2, focus: { x: 0.62, y: 0.55 } }, + { startSec: 41, endSec: 48, scale: 1.6, focus: { x: 0.45, y: 0.7 } }, + ], + motionBlur: false, + cursorEffects: false, + captions: false, + webcam: false, + }, + output: TARGET_OUTPUT, + }, + + /** + * Trim-only passthrough. Not run by default, but kept because it is the only way to tell + * "their encoder is slow" apart from "their effects pipeline is slow", and it costs nothing + * to carry. + */ + passthrough: { + id: "passthrough", + label: "Passthrough (no effects, re-encode only)", + effects: { + background: null, + paddingPercent: 0, + cornerRadiusPx: 0, + shadow: { enabled: false, intensity: 0 }, + zooms: [], + motionBlur: false, + cursorEffects: false, + captions: false, + webcam: false, + }, + output: TARGET_OUTPUT, + }, +}; + +export const DEFAULT_SCENARIO = "full-demo"; + +export function getScenario(id) { + const s = SCENARIOS[id]; + if (!s) throw new Error(`Unknown scenario "${id}". Known: ${Object.keys(SCENARIOS).join(", ")}`); + return s; +} + +/** + * Which scenario features a driver claims to implement. Drivers return this from `prepare()` + * so the report can mark a run as full-fidelity or reduced. + */ +export const FEATURES = [ + "background", + "padding", + "cornerRadius", + "shadow", + "zooms", + "targetResolution", + "targetFps", +]; + +/** Compare what a scenario asks for against what a driver said it applied. */ +export function fidelity(scenario, applied) { + const wanted = new Set(); + const e = scenario.effects; + if (e.background) wanted.add("background"); + if (e.paddingPercent > 0) wanted.add("padding"); + if (e.cornerRadiusPx > 0) wanted.add("cornerRadius"); + if (e.shadow?.enabled) wanted.add("shadow"); + if (e.zooms?.length) wanted.add("zooms"); + wanted.add("targetResolution"); + wanted.add("targetFps"); + + const got = new Set(applied ?? []); + const missing = [...wanted].filter((f) => !got.has(f)); + return { + wanted: [...wanted], + applied: [...got], + missing, + full: missing.length === 0, + score: wanted.size === 0 ? 1 : +((wanted.size - missing.length) / wanted.size).toFixed(3), + }; +} From c4bcd84a23f5ac6de06c1df0340815c4ef5293b9 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Tue, 25 Aug 2026 16:17:39 +0200 Subject: [PATCH 02/12] bench: cover the pure measurement logic with unit tests Three things in the harness decide whether a number means anything, and none of them is exercised by running the benchmark itself: - `ps` prints cumulative CPU as [[dd-]hh:]mm:ss[.ff]. A form the parser misses reports 0 CPU seconds for a busy process rather than failing, so every variant is pinned. - `median`/`mad` must return null for an empty sample. Returning 0 would let an app that never produced a valid export appear at the top of the table. - `fidelity` decides whether a row is ranked or merely listed. An app that applied nothing must score 0, and a scenario that asks for nothing must not demand it. Also extends vitest's include to reach `benchmark/`, which the existing glob did not cover. --- benchmark/README.md | 24 ++++++++++----- benchmark/lib/measure.test.mjs | 41 +++++++++++++++++++++++++ benchmark/scenarios/index.test.mjs | 49 ++++++++++++++++++++++++++++++ vitest.config.ts | 4 ++- 4 files changed, 110 insertions(+), 8 deletions(-) create mode 100644 benchmark/lib/measure.test.mjs create mode 100644 benchmark/scenarios/index.test.mjs diff --git a/benchmark/README.md b/benchmark/README.md index 7ef30623..ef496347 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -111,14 +111,20 @@ actually achieved. Only two apps in this set can be scripted the ordinary way. What the others expose was established by inspection, not assumption: -| App | CLI | AppleScript dictionary | Accessibility tree | Screenshotable | Driven by | +| Driver | CLI | AppleScript dictionary | Accessibility tree | Screenshotable | Driven by | |---|---|---|---|---|---| -| OpenScreen | **yes** (`openscreen export`) | no | — | — | `cli` | -| Cap | **yes** (`cap-cli export`) | no | — | — | `cli` | -| Camtasia | no | **yes** (import, `isExporting`) | yes | yes | `applescript+menu` | -| Kap | no | no | **no** (empty window) | yes | `cdp` | -| Screen Studio | no | no | **no** | **no** — see below | `cdp` | -| FocuSee | no | no | yes | yes | `ax+menu` | +| `openscreen-cli` | **yes** (`openscreen export`) | no | — | — | `cli` | +| `openscreen-gui` | — | no | **no** (Electron) | yes | `cdp+menu` | +| `cap` | **yes** (`cap-cli export`) | no | — | — | `cli` | +| `camtasia` | no | **yes** (import, `isExporting`) | yes | yes | `applescript+ax` | +| `kap` | no | no | **no** (empty window) | yes | `cdp` | +| `screen-studio` | no | no | **no** | **no** — see below | `cdp+menu` | +| `focusee` | no | no | yes | yes | `ax+menu` | + +OpenScreen appears twice on purpose. The CLI leg measures the render engine with no interface in +the way — the right number to set beside Cap's CLI, and the wrong one to set beside an app that +can only be clicked. The GUI leg carries the editor's own overhead, because the subject of a +benchmark should not be the only entrant excused from it. The ladder each GUI driver climbs, best rung first: a scripting dictionary → a System Events menu item by name → a documented keyboard shortcut → an accessibility control by name → the @@ -150,6 +156,10 @@ That is how a driver gets written, and how it gets repaired when a new version r | Screen Studio | **licence required to export at all** — no trial export | direct DMG | | FocuSee | trial, watermarked | vendor downloader stub | +`screen-studio` and `focusee` are **off by default** — on a machine without a Screen Studio +licence, and against FocuSee 2.4.1, they can only ever record a failure. Enable either +explicitly with `--apps`. + A watermark does not change render time, so a trial build is a valid measurement. A licence *wall* is not — see [Known blockers](#known-blockers). diff --git a/benchmark/lib/measure.test.mjs b/benchmark/lib/measure.test.mjs new file mode 100644 index 00000000..5c8cf4ea --- /dev/null +++ b/benchmark/lib/measure.test.mjs @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { mad, median, ProcessTreeSampler } from "./measure.mjs"; + +describe("ProcessTreeSampler.parseCpuTime", () => { + // `ps` prints cumulative CPU as [[dd-]hh:]mm:ss[.ff]. Every one of these forms turns up in + // practice, and a missed one silently reports 0 CPU seconds for a busy process. + it.each([ + ["0:00.00", 0], + ["12:34.56", 754.56], + ["1:02:03", 3723], + ["1-02:03:04.55", 93784.55], + ])("parses %s", (input, expected) => { + expect(ProcessTreeSampler.parseCpuTime(input)).toBeCloseTo(expected, 2); + }); + + it("returns 0 rather than NaN for anything unparseable", () => { + expect(ProcessTreeSampler.parseCpuTime("-")).toBe(0); + expect(ProcessTreeSampler.parseCpuTime("")).toBe(0); + }); +}); + +describe("median and mad", () => { + it("takes the middle of an odd sample and the mean of the middle two of an even one", () => { + expect(median([3, 1, 2])).toBe(2); + expect(median([4, 1, 3, 2])).toBe(2.5); + }); + + it("is null for no samples, so a failed app cannot masquerade as a fast one", () => { + expect(median([])).toBeNull(); + expect(mad([])).toBeNull(); + }); + + it("reports spread as the median absolute deviation", () => { + expect(mad([10, 10, 10])).toBe(0); + expect(mad([8, 10, 12])).toBe(2); + }); + + it("is not dragged by a single outlier the way a mean would be", () => { + expect(median([10, 10, 10, 10, 1000])).toBe(10); + }); +}); diff --git a/benchmark/scenarios/index.test.mjs b/benchmark/scenarios/index.test.mjs new file mode 100644 index 00000000..dfae2259 --- /dev/null +++ b/benchmark/scenarios/index.test.mjs @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { fidelity, getScenario, SCENARIOS, TARGET_OUTPUT } from "./index.mjs"; + +describe("scenario targets", () => { + it("pins 60 fps, the only rate every app in the set can hit", () => { + // OpenScreen's MP4 export is fixed at 60 (MP4_EXPORT_FPS); pinning 30 would make + // "force identical output" impossible rather than merely inconvenient. + expect(TARGET_OUTPUT.fps).toBe(60); + }); + + it("rejects an unknown id instead of silently measuring nothing", () => { + expect(() => getScenario("nope")).toThrow(/Unknown scenario/); + }); +}); + +describe("fidelity", () => { + const full = SCENARIOS["full-demo"]; + + it("scores an app that applied everything as full", () => { + const f = fidelity(full, [ + "background", + "padding", + "cornerRadius", + "shadow", + "zooms", + "targetResolution", + "targetFps", + ]); + expect(f.full).toBe(true); + expect(f.missing).toEqual([]); + expect(f.score).toBe(1); + }); + + it("names what a partial app skipped, so its row cannot be read as a win", () => { + const f = fidelity(full, ["targetResolution", "targetFps"]); + expect(f.full).toBe(false); + expect(f.missing).toEqual(["background", "padding", "cornerRadius", "shadow", "zooms"]); + expect(f.score).toBeCloseTo(2 / 7, 3); + }); + + it("treats an app that claims nothing as having applied nothing", () => { + expect(fidelity(full, undefined).score).toBeCloseTo(0, 3); + }); + + it("does not demand effects the scenario never asked for", () => { + const f = fidelity(SCENARIOS.passthrough, ["targetResolution", "targetFps"]); + expect(f.full).toBe(true); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 3a6243b3..bf64f92d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -13,7 +13,9 @@ export default defineConfig({ // docblock the other way round, to escape the global jsdom; that one is now // redundant but harmless.) environment: "node", - include: ["{src,electron,scripts,.github}/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}"], + include: [ + "{src,electron,scripts,benchmark,.github}/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}", + ], // Vitest's 5s default is too tight here and produces red runs that mean nothing. // Measured: with the machine loaded, 11 tests fail and 9 of them are purely // "Test timed out in 5000ms" — ordinary component tests that pass in 200ms on an From bb4e8ba3b5d0614cbb8b8d233333b42794d562d7 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Tue, 25 Aug 2026 16:19:01 +0200 Subject: [PATCH 03/12] bench: report background CPU alongside each result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A remote-desktop session holds 100-200% of a core permanently and nothing warns about it: no throttling, no error, every export simply slower. It is the same for every app in one run, so the comparison survives it and the absolute times do not — which is why it belongs in the table rather than a footnote. --- benchmark/lib/report.mjs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmark/lib/report.mjs b/benchmark/lib/report.mjs index ca5a8d11..20cfc1d3 100644 --- a/benchmark/lib/report.mjs +++ b/benchmark/lib/report.mjs @@ -102,9 +102,9 @@ export function renderReport(doc) { md.push("## Results"); md.push(""); md.push( - "| # | App | Version | Export (median) | ×realtime | Render fps | vs floor | CPU·s | Peak RSS | Output | Fidelity | How it was driven |", + "| # | App | Version | Export (median) | ×realtime | Render fps | vs floor | CPU·s | Peak RSS | Bg load | Output | Fidelity | How it was driven |", ); - md.push("|---|---|---|---|---|---|---|---|---|---|---|---|"); + md.push("|---|---|---|---|---|---|---|---|---|---|---|---|---|"); for (const [i, r] of ranked.entries()) { const vsFloor = floor?.medianMs && r.medianMs ? `${(r.medianMs / floor.medianMs).toFixed(2)}×` : "—"; @@ -230,6 +230,7 @@ function renderHtml(doc, rows, ranked, floor, summaryText) { ${vsFloor} ${r.cpuSeconds ?? "—"} ${r.peakRssMiB ? `${r.peakRssMiB}` : "—"} + ${r.foreignCpuPercent != null ? `${r.foreignCpuPercent}%` : "—"} ${fmtMB(r.outputSizeBytes)} ${r.fidelity?.full ? 'full' : `${escapeHtml(fidelityLabel(r.fidelity))}`} ${escapeHtml(r.automation ?? "—")} @@ -314,7 +315,7 @@ function renderHtml(doc, rows, ranked, floor, summaryText) { - + ${tableRows}
    #AppExport (median)×RTRender fpsvs floorCPU·sPeak MiBOutputFidelityDriven byvs floorCPU·sPeak MiBBg loadOutputFidelityDriven by
    From 7a59469fa1bf76aa3f625942019ee825c7a4a4a6 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Tue, 25 Aug 2026 16:31:19 +0200 Subject: [PATCH 04/12] bench: end every run with a closing control, and stop claiming load cancels out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README said background load affects every app the same way, so the comparison survives it. That is wrong, and the run that produced this commit shows it: Cap took 19.6 s on a quiet machine and 40.2 s an hour into a loaded one, while the ffmpeg floor moved only 17.7 s to 23.9 s over the same change. A parallel encoder contends for cores a VideoToolbox-bound one never wanted, and an hour of continuous transcoding heat-soaks the SoC — so whichever app runs last carries a handicap. Rather than assume it away, measure it: the floor workload now runs again after every app, and the report prints the ratio. At ~1.00 the ordering did not matter. Above it, the report says so in bold and tells the reader not to quote the numbers without a quieter re-run. --- benchmark/README.md | 16 ++++++++++++---- benchmark/bench.mjs | 26 ++++++++++++++++++++++++++ benchmark/lib/report.mjs | 19 ++++++++++++++----- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index ef496347..82e2e94d 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -199,10 +199,18 @@ export is simply slower. On this machine, reached over a remote-desktop session, encoder alone holds 100–200 % of a core permanently, and `doctor` refuses to call the machine ready above 60 %. -It is sampled during every export and reported per row as **Bg load**. Because it is the same -for every app in one run, the *comparison* survives it; the absolute times do not. Two runs are -only comparable at similar background load, which is why the figure is in the table rather than -in a footnote. +It is sampled during every export and reported per row as **Bg load**. + +Do not assume it cancels out. It does not affect every app equally — a heavily parallel encoder +contends for cores that a VideoToolbox-bound one never wanted — and a long run heat-soaks the +SoC, so an app measured last is not measured under the same conditions as one measured first. +Cap took 19.6 s on a quiet machine and 40.2 s an hour into a loaded run; the ffmpeg floor moved +only 17.7 s → 23.9 s over the same change. Same machine, same clip, same settings. + +That is why every run ends with a **closing control**: the floor workload is measured again +after all the apps, and the report prints the ratio. At ~1.00 the ordering did not matter. Above +it, the report says so in bold and the numbers should not be quoted without a re-run on a +quieter machine or with a longer `--cooldown`. ### Repetitions and guards diff --git a/benchmark/bench.mjs b/benchmark/bench.mjs index cef7a028..888738ef 100644 --- a/benchmark/bench.mjs +++ b/benchmark/bench.mjs @@ -399,6 +399,32 @@ async function cmdRun({ flags }) { state.writeResults({ ...header, finishedAt: null, results }); } + // Closing control. A long run heat-soaks the SoC and the background load drifts, so an app + // measured last is not measured under the same conditions as one measured first. Re-running + // the floor at the end quantifies that drift instead of leaving it as an unstated caveat: if + // the opening and closing controls agree, the ordering did not matter; if they do not, the + // report says by how much. + if (!flags["no-control"] && apps.includes("ffmpeg-baseline") && results.length > 1) { + log("\nclosing control: re-running the floor to measure drift over the run"); + const driver = await loadDriver("ffmpeg-baseline"); + const baseCtx = { workDir: WORK_DIR, outDir, scenario, source: fixture, log, state: {} }; + try { + const rec = await runApp(driver, baseCtx, { + repetitions: 2, + discardFirst: false, + cooldownSec, + log, + }); + rec.app = "ffmpeg-baseline-close"; + rec.displayName = "ffmpeg floor (closing control)"; + rec.isControl = true; + results.push(rec); + state.event("app-finished", rec); + } catch (e) { + log(` closing control failed: ${e.message}`); + } + } + const final = { ...header, finishedAt: new Date().toISOString(), results }; state.writeResults(final); state.writeStatus({ ...final, phase: "done", completed: results.map((r) => r.app), pending: [] }); diff --git a/benchmark/lib/report.mjs b/benchmark/lib/report.mjs index 20cfc1d3..e7dc4d7d 100644 --- a/benchmark/lib/report.mjs +++ b/benchmark/lib/report.mjs @@ -63,13 +63,22 @@ export function renderReport(doc) { const rows = (doc.results ?? []).map(summarise); const ran = rows.filter((r) => !r.skipped && r.medianMs != null); const floor = rows.find((r) => r.app === "ffmpeg-baseline"); + const closingControl = rows.find((r) => r.app === "ffmpeg-baseline-close"); + // The same workload, measured first and last. Anything but ~1.00 means the machine changed + // underneath the run and the app order affected the results. + const drift = + floor?.medianMs && closingControl?.medianMs + ? +(closingControl.medianMs / floor.medianMs).toFixed(3) + : null; // Full-fidelity rows are ranked against each other; a partial row did less work and is // listed after them so the table cannot be read as "this app is faster". - const ranked = [...ran].sort((a, b) => { - const af = a.fidelity?.full ? 0 : 1; - const bf = b.fidelity?.full ? 0 : 1; - return af !== bf ? af - bf : a.medianMs - b.medianMs; - }); + const ranked = [...ran] + .filter((r) => r.app !== "ffmpeg-baseline-close") + .sort((a, b) => { + const af = a.fidelity?.full ? 0 : 1; + const bf = b.fidelity?.full ? 0 : 1; + return af !== bf ? af - bf : a.medianMs - b.medianMs; + }); const m = doc.machine ?? {}; const src = doc.fixture?.probe?.video ?? {}; From 62c315659a5e656a55af8ede55e274661896b779 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Tue, 25 Aug 2026 17:04:23 +0200 Subject: [PATCH 05/12] bench: fail a run that never started instead of waiting out the render timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two faults the first long run exposed. A render that has begun puts something on disk within seconds. The watcher had only one timeout — the full 45-minute render budget — so an export that never started at all (a click that missed, a dialog that did not open) cost the run an hour of waiting on a file nobody was writing. Nothing appearing within four minutes is now a failure with that reason attached. The OpenScreen GUI driver was one such case: on a repeat run the editor can still be showing the previous export's completion state, and the driver clicked through a dialog that had never opened. It now confirms the dialog is up before touching its controls, retries once, and fails loudly if it is not. --- benchmark/drivers/openscreen-gui.mjs | 16 ++++++++++++++++ benchmark/lib/measure.mjs | 21 ++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/benchmark/drivers/openscreen-gui.mjs b/benchmark/drivers/openscreen-gui.mjs index 1c9724f3..33f49b13 100644 --- a/benchmark/drivers/openscreen-gui.mjs +++ b/benchmark/drivers/openscreen-gui.mjs @@ -178,6 +178,22 @@ export default { await s.eval(`JSON.stringify(window.__osbench.click("Export", { exact: true }))`); await sleep(2000); + // Confirm the dialog is actually up. On a repeat run the editor can still be showing the + // previous export's completion state, and clicking through a dialog that never opened + // leaves the run waiting on a file no one is writing. + let dialogUp = false; + for (let i = 0; i < 15 && !dialogUp; i++) { + const txt = await s.eval("document.body.innerText"); + dialogUp = /Render the timeline to a file/i.test(txt) && /Export MP4/i.test(txt); + if (!dialogUp) { + await sleep(1000); + // Dismiss whatever is in the way, then ask again. + await s.eval(`JSON.stringify(window.__osbench.click("Close"))`).catch(() => undefined); + await s.eval(`JSON.stringify(window.__osbench.click("Export", { exact: true }))`); + } + } + if (!dialogUp) throw new Error("OpenScreen: the export dialog never opened"); + // The dialog's controls are plain buttons labelled with their value. for (const label of ["MP4", `${t.height}p`, String(t.fps), "H.264"]) { const r = JSON.parse( diff --git a/benchmark/lib/measure.mjs b/benchmark/lib/measure.mjs index be264b3e..d4cbe488 100644 --- a/benchmark/lib/measure.mjs +++ b/benchmark/lib/measure.mjs @@ -152,7 +152,17 @@ export function cpuDelta(before, after) { */ export async function waitForStableFile( path, - { timeoutMs = 45 * 60 * 1000, stableMs = 2500, pollMs = 100, minBytes = 4096, onTick } = {}, + { + timeoutMs = 45 * 60 * 1000, + // A render that started will put *something* on disk quickly. Nothing after this long + // means the export never began — a click that missed, a dialog that did not open — and + // waiting out the full render timeout turns one broken run into a lost hour. + appearTimeoutMs = 4 * 60 * 1000, + stableMs = 2500, + pollMs = 100, + minBytes = 4096, + onTick, + } = {}, ) { const t0 = now(); let lastSize = -1; @@ -174,6 +184,15 @@ export async function waitForStableFile( } onTick?.({ size, elapsedMs: now() - t0 }); + if (appearedAt === null && now() - t0 > appearTimeoutMs) { + return { + ok: false, + reason: "output never appeared", + appearedAt: null, + sizeBytes: -1, + waitedMs: now() - t0, + }; + } if (lastSize >= minBytes && lastGrowthAt !== null && now() - lastGrowthAt >= stableMs) { return { ok: true, From 1516cfa595598de4318e64047f7cc6a46496a9b2 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Tue, 25 Aug 2026 17:48:32 +0200 Subject: [PATCH 06/12] bench: three ways a run could be quietly wrong MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An editor with no project loaded exports happily: OpenScreen wrote a 262-byte MP4 with no streams, in about a second. The GUI driver now gates on the editor actually showing the benchmark project — the background colour and a timeline of the right length — instead of reading the composition panel for information and proceeding regardless. An instant, wildly fast render is the symptom a benchmark must never accept. That file had 'appeared', so the appear-timeout never fired and the watcher sat on a header for the full render budget. A file that appears and then stops short of a plausible size is now failed with its byte count in the reason. And 'run --id X' silently discarded everything already measured under X, which is exactly what REMOTE.md tells people to do to pick up after a failure. '--append' now merges, replacing only the apps named on the command line. --- benchmark/bench.mjs | 8 +++++++- benchmark/drivers/openscreen-gui.mjs | 19 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/benchmark/bench.mjs b/benchmark/bench.mjs index 888738ef..6abdc59a 100644 --- a/benchmark/bench.mjs +++ b/benchmark/bench.mjs @@ -346,7 +346,11 @@ async function cmdRun({ flags }) { ); log(`fixture ${fixture.path} (${fixture.sha256.slice(0, 12)})\n`); - const results = []; + // Re-running one app into an existing run id is the documented way to pick up after a + // failure (see REMOTE.md). Without this it silently discarded everything already measured. + const prior = flags.append ? (state.readResults()?.results ?? []) : []; + if (prior.length) log(`appending to ${prior.length} existing app result(s) in ${runId}\n`); + const results = prior.filter((r) => !apps.includes(r.app)); for (const [i, id] of apps.entries()) { let driver; try { @@ -590,6 +594,8 @@ function cmdHelp() { calibrate [--apps a,b] solve each app's padding control so they composite the same rect fixture [--force] [--duration s] [--fps n] run [--apps a,b] [--scenario id] [--reps 3] [--cooldown 45] [--no-warmup] [--id NAME] + [--append] merge into an existing run id instead of replacing it + [--no-control] skip the closing drift control status [--run ID] [--json] report [--run ID] discover dump menus + accessibility tree (for writing a GUI driver) diff --git a/benchmark/drivers/openscreen-gui.mjs b/benchmark/drivers/openscreen-gui.mjs index 33f49b13..4a18012a 100644 --- a/benchmark/drivers/openscreen-gui.mjs +++ b/benchmark/drivers/openscreen-gui.mjs @@ -143,7 +143,24 @@ export default { await sleep(10000); // Read the composition panel back: this is the app telling us what it thinks it loaded. - const panel = await s.eval("document.body.innerText.slice(0, 500)"); + // This is a gate, not a formality — an editor with no project still exports happily and + // writes an empty container, which looks like an instant, wildly fast render. + let panel = ""; + let loaded = false; + const wantDuration = new RegExp( + `/${Math.floor(ctx.source.probe.durationSec / 60)}:${String(Math.round(ctx.source.probe.durationSec % 60)).padStart(2, "0")}`, + ); + for (let i = 0; i < 20 && !loaded; i++) { + panel = await s.eval("document.body.innerText.slice(0, 600)"); + loaded = panel.includes(ctx.scenario.effects.background.color) && wantDuration.test(panel); + if (!loaded) await sleep(1500); + } + if (!loaded) { + throw new Error( + `OpenScreen editor never showed the benchmark project (expected ${ctx.scenario.effects.background.color} ` + + `and a ${ctx.source.probe.durationSec}s timeline). Panel read: ${panel.replace(/\n+/g, " | ").slice(0, 200)}`, + ); + } const applied = ["targetResolution", "targetFps"]; const e = ctx.scenario.effects; if (panel.includes(e.background.color)) applied.push("background"); From 00c2b5f072e4428bdee601cda233041d401bb080 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Tue, 25 Aug 2026 17:52:24 +0200 Subject: [PATCH 07/12] bench: gate the OpenScreen GUI on the app's own project state, not scraped text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous gate matched the background colour and a timeline duration in the editor's innerText. The colour matched and the duration regex did not, so a correctly loaded project was rejected — a text-scrape is the wrong instrument for a question the app can answer directly. electronAPI.loadCurrentProjectFile() returns the open project's path. Comparing it to the path the driver just wrote is unambiguous and does not care how the editor formats a timecode. --- benchmark/drivers/openscreen-gui.mjs | 36 ++++++++++++++++------------ 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/benchmark/drivers/openscreen-gui.mjs b/benchmark/drivers/openscreen-gui.mjs index 4a18012a..a216f560 100644 --- a/benchmark/drivers/openscreen-gui.mjs +++ b/benchmark/drivers/openscreen-gui.mjs @@ -142,25 +142,31 @@ export default { end tell`); await sleep(10000); - // Read the composition panel back: this is the app telling us what it thinks it loaded. - // This is a gate, not a formality — an editor with no project still exports happily and - // writes an empty container, which looks like an instant, wildly fast render. - let panel = ""; - let loaded = false; - const wantDuration = new RegExp( - `/${Math.floor(ctx.source.probe.durationSec / 60)}:${String(Math.round(ctx.source.probe.durationSec % 60)).padStart(2, "0")}`, - ); - for (let i = 0; i < 20 && !loaded; i++) { - panel = await s.eval("document.body.innerText.slice(0, 600)"); - loaded = panel.includes(ctx.scenario.effects.background.color) && wantDuration.test(panel); - if (!loaded) await sleep(1500); + // Ask the app which project it has open rather than scraping the panel for a duration + // string. This is a gate, not a formality: an editor with no project loaded exports + // happily and writes an empty container, which reads as an instant, wildly fast render. + let loadedPath = null; + for (let i = 0; i < 25 && loadedPath !== projectPath; i++) { + await sleep(1500); + try { + const raw = await s.eval( + `(async () => { try { const r = await window.electronAPI.loadCurrentProjectFile(); return JSON.stringify(r ?? null); } catch (e) { return null; } })()`, + { timeoutMs: 15_000 }, + ); + const cur = raw ? JSON.parse(raw) : null; + if (cur?.path) loadedPath = cur.path; + } catch { + /* the renderer may be mid-load */ + } } - if (!loaded) { + const panel = await s.eval("document.body.innerText.slice(0, 600)"); + if (loadedPath !== projectPath) { throw new Error( - `OpenScreen editor never showed the benchmark project (expected ${ctx.scenario.effects.background.color} ` + - `and a ${ctx.source.probe.durationSec}s timeline). Panel read: ${panel.replace(/\n+/g, " | ").slice(0, 200)}`, + `OpenScreen has "${loadedPath ?? "no project"}" open, not the benchmark project ` + + `(${projectPath}). Panel read: ${panel.replace(/\n+/g, " | ").slice(0, 200)}`, ); } + const applied = ["targetResolution", "targetFps"]; const e = ctx.scenario.effects; if (panel.includes(e.background.color)) applied.push("background"); From c5329bf7d052a07f7b8472d3573bb2ca0232ca0b Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Tue, 25 Aug 2026 18:04:22 +0200 Subject: [PATCH 08/12] bench: open a fresh Kap editor for each run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kap's editor is single-use. Once an export finishes it swaps the Convert button for a share prompt, so the second and third repetitions had nothing to click and failed with 'could not find the Convert button'. Each run now reopens the clip when the button is gone — before ctx.commit(), so the reopen is warm-up and not part of the measurement. --- benchmark/drivers/kap.mjs | 111 +++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 49 deletions(-) diff --git a/benchmark/drivers/kap.mjs b/benchmark/drivers/kap.mjs index e52189a7..f58b6fe0 100644 --- a/benchmark/drivers/kap.mjs +++ b/benchmark/drivers/kap.mjs @@ -11,10 +11,15 @@ * window. Launched with `--remote-debugging-port` its renderer is reachable, and the export * button, the settings fields and the progress text are all plain DOM. * - * Output: the export destination is a native popup menu that neither CDP nor the accessibility - * API can open, so the driver uses Kap's clipboard destination — the same render, writing to a - * temp directory — and adopts the file it produces. Kap's own "Export complete" is used as the - * stop signal, so the adoption copy is never counted. + * Two things about Kap shape this driver: + * + * 1. **Its editor is single-use.** Once an export finishes, the Convert button is replaced by a + * share prompt, so a second run has nothing to click. Each run therefore opens the clip + * again — before the clock starts, so it is not measured. + * 2. **Its export destination is a native popup menu** that neither CDP nor the accessibility + * API can open. The driver uses Kap's clipboard destination instead — the same render, + * writing to a temp directory — and adopts the file it produces. Kap's own "Export complete" + * is the stop signal, so the adoption copy is never counted. */ import { execFileSync } from "node:child_process"; import { @@ -69,13 +74,47 @@ function tempExports() { try { out.set(p, statSync(p).mtimeMs); } catch { - /* vanished */ + /* vanished between readdir and stat */ } } } return out; } +/** Open the source clip in a fresh Kap editor and pin its output fields. */ +async function openEditor(ctx) { + execFileSync("/usr/bin/open", ["-a", APP, ctx.source.path]); + await sleep(8000); + const target = (await listTargets(PORT)).find((t) => t.url.includes("editor.html")); + if (!target) throw new Error("Kap did not open an editor window for the source clip"); + const session = new CdpSession(target.webSocketDebuggerUrl); + await session.open(); + await session.eval(DOM_HELPERS); + + const t = ctx.scenario.output; + const raw = await session.eval(`(() => { + const setNative = (el, v) => { + const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, "value").set; + setter.call(el, String(v)); + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + }; + const ins = [...document.querySelectorAll("input")]; + const w = ins.find(i => i.value === "1920") || ins[ins.length - 3]; + const h = ins.find(i => i.value === "1080") || ins[ins.length - 2]; + const f = ins[ins.length - 1]; + if (w) setNative(w, ${t.width}); + if (h) setNative(h, ${t.height}); + if (f) setNative(f, ${t.fps}); + return JSON.stringify({ + format: (document.querySelector(".format") || {}).innerText, + plugin: (document.querySelector(".plugin") || {}).innerText, + values: [...document.querySelectorAll("input")].map(i => i.value), + }); + })()`); + return { session, state: JSON.parse(raw) }; +} + export default { id: "kap", displayName: "Kap", @@ -111,7 +150,7 @@ export default { async prepare(ctx) { // Kap picks its default format from a usage ledger rather than a setting. Promoting mp4 - // there is how you make the editor open on MP4 instead of GIF without touching the UI. + // there is how the editor opens on MP4 instead of GIF without touching the UI. if (existsSync(HISTORY)) { try { const h = JSON.parse(readFileSync(HISTORY, "utf8")); @@ -131,42 +170,10 @@ export default { ]); await sleep(9000); - execFileSync("/usr/bin/open", ["-a", APP, ctx.source.path]); - await sleep(8000); - - const target = (await listTargets(PORT)).find((t) => t.url.includes("editor.html")); - if (!target) throw new Error("Kap did not open an editor window for the source clip"); - const s = new CdpSession(target.webSocketDebuggerUrl); - await s.open(); - await s.eval(DOM_HELPERS); - ctx.state.cdp = s; - - const t = ctx.scenario.output; - const applied = []; - const state = JSON.parse( - await s.eval(`(() => { - const ins = [...document.querySelectorAll("input")]; - const setNative = (el, v) => { - const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, "value").set; - setter.call(el, String(v)); - el.dispatchEvent(new Event("input", { bubbles: true })); - el.dispatchEvent(new Event("change", { bubbles: true })); - }; - const w = ins.find(i => i.value === "1920") || ins[ins.length - 3]; - const h = ins.find(i => i.value === "1080") || ins[ins.length - 2]; - const f = ins[ins.length - 1]; - if (w) setNative(w, ${t.width}); - if (h) setNative(h, ${t.height}); - if (f) setNative(f, ${t.fps}); - return JSON.stringify({ - format: (document.querySelector(".format") || {}).innerText, - plugin: (document.querySelector(".plugin") || {}).innerText, - values: [...document.querySelectorAll("input")].map(i => i.value), - }); - })()`), - ); - if (/mp4/i.test(state.format ?? "")) applied.push("targetResolution", "targetFps"); + const { session, state } = await openEditor(ctx); + ctx.state.cdp = session; + const applied = /mp4/i.test(state.format ?? "") ? ["targetResolution", "targetFps"] : []; return { appliedFeatures: applied, notes: [ @@ -181,11 +188,23 @@ export default { }, async runExport(ctx) { - const s = ctx.state.cdp; const out = this.outputPath(ctx); if (existsSync(out)) rmSync(out); const before = tempExports(); + let s = ctx.state.cdp; + const hasButton = await s + .eval( + 'String(!!(document.querySelector("button.start-export") || [...document.querySelectorAll("button")].find((x) => /convert/i.test(x.innerText))))', + ) + .catch(() => "false"); + if (hasButton !== "true") { + s?.close(); + const reopened = await openEditor(ctx); + s = reopened.session; + ctx.state.cdp = s; + } + const clicked = await s.eval(`(() => { const b = document.querySelector("button.start-export") || [...document.querySelectorAll("button")].find(x => /convert/i.test(x.innerText)); @@ -197,25 +216,19 @@ export default { throw new Error(`Kap: could not find the Convert button (${clicked})`); ctx.commit(); - // Kap reports its own progress and completion in the DOM. That is a better stop signal - // than the temp file, which is written before Kap has finished with it. const deadline = now() + 30 * 60 * 1000; let done = false; while (now() < deadline) { await sleep(500); - const txt = await s.eval(`document.body.innerText.slice(0, 400)`); + const txt = await s.eval("document.body.innerText.slice(0, 400)"); if (/export complete|drag and drop to copy/i.test(txt)) { ctx.markComplete(); done = true; break; } - const m = /Converting\s*—\s*(\d+)s remaining/i.exec(txt); - if (m) ctx.progress?.(null, `${m[1]}s remaining`); } if (!done) throw new Error("Kap never reported the export as complete"); - // Adopt whatever appeared in the temp tree. The copy happens after markComplete, so it - // is outside the measured interval. const after = tempExports(); const fresh = [...after.entries()] .filter(([p, m]) => !before.has(p) || before.get(p) !== m) From 541a1338493bb3971626e6a33005cd984bad2de7 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Tue, 25 Aug 2026 19:24:24 +0200 Subject: [PATCH 09/12] bench: relaunch Kap between exports rather than reopening the clip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kap keeps one editor window. Once an export completes it leaves that window showing a share prompt where the Convert button was, and opening the same file again only refocuses it — so the previous fix detected the missing button and then reopened into the same dead editor. The app has to go away and come back. The relaunch runs before ctx.commit(), so it is warm-up and not measured. --- benchmark/drivers/kap.mjs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/benchmark/drivers/kap.mjs b/benchmark/drivers/kap.mjs index f58b6fe0..c94fcd38 100644 --- a/benchmark/drivers/kap.mjs +++ b/benchmark/drivers/kap.mjs @@ -81,8 +81,24 @@ function tempExports() { return out; } -/** Open the source clip in a fresh Kap editor and pin its output fields. */ -async function openEditor(ctx) { +/** + * Open the source clip in a fresh Kap editor and pin its output fields. + * + * `restart` is what makes a second export possible at all. Kap keeps one editor window and, + * once an export completes, leaves it showing a share prompt where the Convert button was. + * Opening the same file again only refocuses that window — the app has to go away and come + * back. All of this runs before the clock starts. + */ +async function openEditor(ctx, { restart = false } = {}) { + if (restart) { + if (appIsRunning("Kap")) await quitApp("Kap", { force: true }); + await sleep(2500); + execFileSync("/bin/sh", [ + "-c", + `nohup ${JSON.stringify(BIN)} --remote-debugging-port=${PORT} >/dev/null 2>&1 &`, + ]); + await sleep(9000); + } execFileSync("/usr/bin/open", ["-a", APP, ctx.source.path]); await sleep(8000); const target = (await listTargets(PORT)).find((t) => t.url.includes("editor.html")); @@ -200,7 +216,7 @@ export default { .catch(() => "false"); if (hasButton !== "true") { s?.close(); - const reopened = await openEditor(ctx); + const reopened = await openEditor(ctx, { restart: true }); s = reopened.session; ctx.state.cdp = s; } From 53abdab01cd97691ab13f053c3d5dbe01a473500 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Tue, 25 Aug 2026 19:49:30 +0200 Subject: [PATCH 10/12] bench: name the media engine as the confound, and stop flattering the floor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The largest source of error found while building this is not CPU. Parsec, Screen Sharing and ARD encode the screen continuously through VTEncoderXPCService — the same hardware H.264 encoder every app here uses for its export — and no CPU measurement sees that contention. Measured: the floor went 17.7s to 23.7s with a remote session live, while Cap went 19.6s to 43.8s. Padding calibration and the background colour were both ruled out by A/B first (42.6s vs 42.5s, and 42.4s with the original colour), so the media engine is what is left. Within one run the numbers still hold, and the closing control is the evidence: 23.69s against an opening 23.68s, so every app in the committed run met the same conditions. Also: the floor reported 0 CPU seconds because the sampler matches processes by argv prefix and the driver gave it none; it now points at the resolved binary. Its 'version' pasted a filesystem path into the results table. And the report now says plainly that an output a fraction of the others' size is not the same work — Kap encodes at 565 kbps against Cap's 6983, which is most of why it is a floor reference and not a competitor. --- benchmark/README.md | 40 ++++++++++++++++----------- benchmark/drivers/ffmpeg-baseline.mjs | 18 +++++++++--- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 82e2e94d..19795931 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -192,27 +192,35 @@ SIP-protected exec and the inherited variable never survives. The LGPL build has no libx264 and no drawtext. The fixture is encoded with `h264_videotoolbox` and drawn with `drawbox`; neither `-crf` nor text overlays are available. -### Background load +### Background load, and the one that matters most -The one precondition that never announces itself. Nothing throttles and nothing warns — every -export is simply slower. On this machine, reached over a remote-desktop session, the screen -encoder alone holds 100–200 % of a core permanently, and `doctor` refuses to call the machine -ready above 60 %. +The precondition that never announces itself. Nothing throttles and nothing warns — every export +is simply slower. `doctor` refuses to call a machine ready above 60% foreign CPU, and the figure +is sampled during every export and reported per row as **Bg load**. -It is sampled during every export and reported per row as **Bg load**. +**Do not run this over a remote-desktop session.** That is the single largest source of error +found while building this, and it is not a CPU problem. Parsec, Screen Sharing and ARD all +encode the screen continuously through `VTEncoderXPCService` — *the same hardware H.264 encoder +every app in this benchmark uses for its export*. The contention is for the media engine, which +no CPU measurement sees: -Do not assume it cancels out. It does not affect every app equally — a heavily parallel encoder -contends for cores that a VideoToolbox-bound one never wanted — and a long run heat-soaks the -SoC, so an app measured last is not measured under the same conditions as one measured first. -Cap took 19.6 s on a quiet machine and 40.2 s an hour into a loaded run; the ffmpeg floor moved -only 17.7 s → 23.9 s over the same change. Same machine, same clip, same settings. +| | quiet machine | with a remote session live | +|---|---|---| +| ffmpeg floor | 17.7 s | 23.7 s (+34%) | +| Cap | 19.6 s | 43.8 s (+123%) | + +Same machine, same clip, same settings, same padding — the padding calibration and the +background colour were both ruled out by A/B (42.6 s vs 42.5 s, and 42.4 s with the original +colour). Apps are affected unequally because they lean on the encoder differently, so the load +does not cancel out and the *ranking* can move, not just the absolute times. -That is why every run ends with a **closing control**: the floor workload is measured again -after all the apps, and the report prints the ratio. At ~1.00 the ordering did not matter. Above -it, the report says so in bold and the numbers should not be quoted without a re-run on a -quieter machine or with a longer `--cooldown`. +Within a single run the numbers are still sound, and the **closing control** is what proves it: +the floor workload is measured again after all the apps, and the report prints the ratio. In the +committed run it came back at 23.69 s against an opening 23.68 s — no drift, so every app in +that run met identical conditions. Comparing across runs on different machines requires the same +to be true of both. -### Repetitions and guards +### Repetitions and guards### Repetitions and guards Three scoring runs after one discarded warm-up, 45 s of cooldown between them. The warm-up is kept in the data but excluded from the statistics — first runs pay for cold caches and diff --git a/benchmark/drivers/ffmpeg-baseline.mjs b/benchmark/drivers/ffmpeg-baseline.mjs index 38c973b0..e8ac2a5c 100644 --- a/benchmark/drivers/ffmpeg-baseline.mjs +++ b/benchmark/drivers/ffmpeg-baseline.mjs @@ -8,7 +8,7 @@ */ import { spawn } from "node:child_process"; import { join } from "node:path"; -import { resolveFfmpeg } from "../lib/env.mjs"; +import { ffmpegVersion, resolveFfmpeg } from "../lib/env.mjs"; export default { id: "ffmpeg-baseline", @@ -16,15 +16,25 @@ export default { vendor: "reference", kind: "reference", automation: "cli", - processName: null, - appPath: null, + // The sampler matches processes by argv prefix; without this the floor reported 0 CPU + // seconds while every other row reported real ones. + get appPath() { + try { + return resolveFfmpeg().ffmpeg; + } catch { + return null; + } + }, + processName: "ffmpeg", bundleId: null, install: null, detect() { try { const { ffmpeg, source } = resolveFfmpeg(); - return { installed: true, version: source, path: ffmpeg }; + const banner = ffmpegVersion().banner; + const v = /ffmpeg version (\S+)/.exec(banner)?.[1] ?? "unknown"; + return { installed: true, version: `${v} (${source.split(":")[0]})`, path: ffmpeg }; } catch (e) { return { installed: false, version: null, path: null, error: e.message }; } From 03a437a763cc9fa1f96c93ab8402abc18c27aaae Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Tue, 25 Aug 2026 20:27:20 +0200 Subject: [PATCH 11/12] =?UTF-8?q?bench:=20measure=20a=20real=20demo=20expo?= =?UTF-8?q?rt=20=E2=80=94=20wallpaper,=20cursor,=20webcam,=20motion=20blur?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scenario was too light to mean anything. A screen clip on a flat colour measures decoding and encoding; it does not measure what a demo export actually costs. Missing entirely: a background the compositor has to sample, motion blur, the rendered pointer, and a camera inset — and the pointer and the camera are a large share of the work. The pointer is the important one. These apps hide the system cursor while recording and re-draw it at export time from a sidecar, with their own sprite, smoothing, motion blur and click effects. The fixture used to paint a fake cursor into the video, which exercised none of that and would have double-drawn once an app rendered its own. So the trajectory is now generated as data — eased glides between dwell points, with clicks at the pauses, which is the shape smoothing and dwell-based auto-zoom actually react to — written in each app's telemetry format, and the screen clip is left clean. Also generated from the same seed: a wallpaper (sampled per pixel, not cleared once) and a webcam track with a moving, blinking subject to decode, mask and shadow every frame. The verifier had to grow with it, and two of its new checks were wrong first: - Geometry silently stopped running the moment the background became an image, because it keyed on colour equality — so padding, radius and background went unverified exactly when they got more expensive. It now separates the dark recording from the light wallpaper by luminance. - The cursor and webcam detectors both passed videos that contain neither. Caught by running them against Kap and the ffmpeg floor, which draw no pointer and no camera. The cursor check was comparing the pointer's window against the frame's static corners, so it was really asking "is this region busier than the edges"; controls now sit on the same scrolling material the cursor crosses. Thresholds are calibrated against measured positives and negatives, and the raw ratios are recorded on every run so the margin is auditable. Finally, the verifier now overrides the driver. A driver reports what it configured; only pixels say what happened. Cap accepts a cursor track, reports cursor.hide false, and renders no pointer — that was full fidelity before and is 0.9 with `cursor` contradicted now. --- benchmark/bench.mjs | 44 ++- benchmark/drivers/camtasia-win.mjs | 185 +++++++++++ benchmark/drivers/cap.mjs | 79 ++++- benchmark/drivers/ffmpeg-baseline.mjs | 9 +- benchmark/drivers/focusee-win.mjs | 151 +++++++++ benchmark/drivers/openscreen-cli.mjs | 24 +- benchmark/drivers/openscreen-gui.mjs | 2 + benchmark/lib/assets.mjs | 268 ++++++++++++++++ benchmark/lib/env.mjs | 96 +++--- benchmark/lib/fixture.mjs | 24 +- benchmark/lib/measure.mjs | 52 +--- benchmark/lib/openscreenProject.mjs | 56 +++- benchmark/lib/platform.mjs | 428 ++++++++++++++++++++++++++ benchmark/lib/runner.mjs | 43 ++- benchmark/lib/ui.mjs | 55 ++++ benchmark/lib/uiWindows.mjs | 368 ++++++++++++++++++++++ benchmark/lib/visualCheck.mjs | 197 +++++++++++- benchmark/scenarios/index.mjs | 55 +++- 18 files changed, 1971 insertions(+), 165 deletions(-) create mode 100644 benchmark/drivers/camtasia-win.mjs create mode 100644 benchmark/drivers/focusee-win.mjs create mode 100644 benchmark/lib/assets.mjs create mode 100644 benchmark/lib/platform.mjs create mode 100644 benchmark/lib/ui.mjs create mode 100644 benchmark/lib/uiWindows.mjs diff --git a/benchmark/bench.mjs b/benchmark/bench.mjs index 6abdc59a..ef58a927 100644 --- a/benchmark/bench.mjs +++ b/benchmark/bench.mjs @@ -14,6 +14,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { APPS, DEFAULT_APPS, installPlan, loadDriver } from "./apps.mjs"; +import { buildWallpaper, buildWebcam } from "./lib/assets.mjs"; import { CALIBRATION_PATH, calibrateApp, @@ -255,9 +256,16 @@ async function cmdFixture({ flags }) { if (flags.duration) spec.durationSec = Number(flags.duration); if (flags.fps) spec.fps = Number(flags.fps); const r = buildFixture(WORK_DIR, spec, { force: !!flags.force, log }); - log(`\n${r.path}`); - log(` sha256 ${r.sha256}`); - log(` ${JSON.stringify(r.probe.video)} ${(r.probe.sizeBytes / 1048576).toFixed(1)} MB`); + const wp = buildWallpaper(WORK_DIR, spec); + const wc = buildWebcam(WORK_DIR, spec); + log(`\nscreen ${r.path}`); + log(` sha256 ${r.sha256}`); + log(` ${JSON.stringify(r.probe.video)} ${(r.probe.sizeBytes / 1048576).toFixed(1)} MB`); + log(`wallpaper ${wp.path}\n sha256 ${wp.sha256}`); + log(`webcam ${wc.path}\n sha256 ${wc.sha256}`); + log( + "\nCursor telemetry is written per app at prepare time, from the same seed — see lib/assets.mjs.", + ); } async function cmdRun({ flags }) { @@ -277,6 +285,15 @@ async function cmdRun({ flags }) { spec, } : buildFixture(WORK_DIR, spec, { log }); + // A demo export is not just a screen clip: the scenario also needs a wallpaper to sample + // and a camera track to composite. Both come from the same seed as the screen recording, + // so they travel with it rather than being shipped. + const wallpaper = buildWallpaper(WORK_DIR, spec); + const assets = { + wallpaper: wallpaper.path, + jpeg: wallpaper.jpeg, + webcam: buildWebcam(WORK_DIR, spec).path, + }; const calibration = loadCalibration(); if (calibration.machine) { @@ -329,6 +346,10 @@ async function cmdRun({ flags }) { spec: fixture.spec, probe: fixture.probe, }, + assets: { + wallpaper: { path: assets.wallpaper, sha256: sha256(assets.wallpaper) }, + webcam: { path: assets.webcam, sha256: sha256(assets.webcam), probe: probe(assets.webcam) }, + }, calibration: calibration.apps ? { generatedAt: calibration.generatedAt, @@ -381,6 +402,7 @@ async function cmdRun({ flags }) { outDir, scenario, source: fixture, + assets, log, state, paddingControl: calibrated, @@ -411,7 +433,15 @@ async function cmdRun({ flags }) { if (!flags["no-control"] && apps.includes("ffmpeg-baseline") && results.length > 1) { log("\nclosing control: re-running the floor to measure drift over the run"); const driver = await loadDriver("ffmpeg-baseline"); - const baseCtx = { workDir: WORK_DIR, outDir, scenario, source: fixture, log, state: {} }; + const baseCtx = { + workDir: WORK_DIR, + outDir, + scenario, + source: fixture, + assets, + log, + state: {}, + }; try { const rec = await runApp(driver, baseCtx, { repetitions: 2, @@ -451,6 +481,12 @@ async function cmdCalibrate({ flags }) { const apps = listFlag(flags.apps, DEFAULT_APPS); const scenario = getScenario(flags.scenario ?? DEFAULT_SCENARIO); const fixture = calibrationFixture(WORK_DIR, log); + const calibWallpaper = buildWallpaper(WORK_DIR, fixture.spec); + const calibAssets = { + wallpaper: calibWallpaper.path, + jpeg: calibWallpaper.jpeg, + webcam: buildWebcam(WORK_DIR, fixture.spec).path, + }; const outDir = join(WORK_DIR, "out", "calibration"); mkdirSync(outDir, { recursive: true }); log( diff --git a/benchmark/drivers/camtasia-win.mjs b/benchmark/drivers/camtasia-win.mjs new file mode 100644 index 00000000..90de468c --- /dev/null +++ b/benchmark/drivers/camtasia-win.mjs @@ -0,0 +1,185 @@ +/** + * Camtasia on Windows. + * + * A different automation story from the Mac build. Camtasia for Windows has no AppleScript + * equivalent — TechSmith's documented command line drives the *recorder*, not the renderer — + * but it does expose a real UI Automation tree, and it has a **Batch Export** that renders a + * queue of `.tscproj` projects against one preset. Batch Export is what makes this measurable + * without clicking through the inspector. + * + * Fidelity is partial, for the same reason as on macOS: background, padding, corner radius, + * shadow and zooms live in Visual Properties and Zoom-n-Pan, and neither is scriptable. This + * row measures Camtasia rendering the same source to the same output target with no + * compositing — a real number for its render pipeline, not a full-demo comparison. + * + * NOT YET RUN ON WINDOWS. Written against the UIA surface and TechSmith's documented layout; + * `bench.mjs discover camtasia` dumps the real control names on the target machine, and the + * driver fails loudly with those names attached when a lookup misses. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { now, sleep } from "../lib/measure.mjs"; +import { appVersion, resolveAppPath } from "../lib/platform.mjs"; +import { activateApp, appIsRunning, launchApp, quitApp } from "../lib/ui.mjs"; +import { + clickControl, + describeApp, + fileDialogTo, + listWindows, + setControlValue, +} from "../lib/uiWindows.mjs"; + +export const CAMTASIA = { + macPath: "/Applications/Camtasia.app", + winPaths: [ + "%ProgramFiles%\\TechSmith\\Camtasia 2026\\CamtasiaStudio.exe", + "%ProgramFiles%\\TechSmith\\Camtasia 2025\\CamtasiaStudio.exe", + "%ProgramFiles%\\TechSmith\\Camtasia\\CamtasiaStudio.exe", + "%ProgramFiles(x86)%\\TechSmith\\Camtasia 2026\\CamtasiaStudio.exe", + ], +}; + +const PROC = "CamtasiaStudio"; + +/** Click a control, or fail with the names that *were* present — the only useful error here. */ +function mustClick(needle, opts = {}) { + const r = clickControl(PROC, needle, opts); + if (!r.ok) { + throw new Error( + `Camtasia: no control matching "${needle}". Present: ${(r.seen ?? []).slice(0, 20).join(" | ")}. ` + + "Run `node benchmark/bench.mjs discover camtasia` for the full tree.", + ); + } + return r; +} + +export default { + id: "camtasia", + displayName: "Camtasia", + vendor: "TechSmith", + kind: "gui", + automation: "uia", + processName: PROC, + get appPath() { + return resolveAppPath(CAMTASIA); + }, + bundleId: null, + install: { + method: "installer", + url: "https://download.techsmith.com/camtasia/releases/Camtasia.exe", + appName: "Camtasia", + approxMB: 500, + licence: + "commercial — 30-day trial; exports carry a watermark, which does not change render time", + silentArgs: ["/S"], + notes: [ + "TechSmith's Windows installer is an NSIS package; /S runs it unattended.", + "No render CLI exists on Windows — the documented command line drives the recorder only.", + ], + }, + + detect() { + const path = resolveAppPath(CAMTASIA); + if (!path) return { installed: false, version: null, path: null }; + return { installed: true, version: appVersion(path), path }; + }, + + async prepare(ctx) { + const exe = resolveAppPath(CAMTASIA); + if (!exe) throw new Error("Camtasia is not installed"); + + if (!appIsRunning(PROC)) { + // Opening the clip directly is what creates a project with the media on track 1, + // which is otherwise a drag-and-drop that UIA cannot perform. + await launchApp(exe, PROC, { args: [ctx.source.path], timeoutMs: 180_000 }); + } else { + execFileSync("cmd.exe", ["/c", "start", "", exe, ctx.source.path], { stdio: "ignore" }); + } + await sleep(20_000); + activateApp(PROC); + + // A run killed mid-export leaves Camtasia offering to recover the project, and that + // dialog is modal — everything after it silently misfires. + clickControl(PROC, "Delete", { controlType: "Button" }); + await sleep(1500); + + // A 60 fps import raises "High Frame Rate Media Detected". Taking the 30 fps default + // would halve the frames Camtasia renders and quietly make it look twice as fast. + const converted = clickControl(PROC, "60 FPS", { controlType: "RadioButton" }); + if (converted.ok) { + clickControl(PROC, "Remember my selection", { controlType: "CheckBox" }); + clickControl(PROC, "Continue", { controlType: "Button" }); + await sleep(2500); + } + + // Do not proceed to Export with an empty timeline: that exports nothing and reads as an + // instant, wildly fast render. + const stem = ctx.source.path + .split(/[/\\]/) + .pop() + .replace(/\.[^.]+$/, ""); + let imported = false; + for (let i = 0; i < 60 && !imported; i++) { + await sleep(2000); + imported = describeApp(PROC, { max: 600 }).includes(stem); + } + if (!imported) + throw new Error(`Camtasia never showed "${stem}" on the timeline after the import`); + + return { + appliedFeatures: ["targetResolution", "targetFps"], + notes: [ + converted.ok ? "project frame rate converted to 60 fps on import" : "no frame-rate prompt", + "Effects are NOT applied: background, padding, corner radius, shadow and zooms live in Visual Properties and Zoom-n-Pan, neither of which Camtasia exposes to scripting on Windows either. This row measures its render pipeline at the same output target, not the full-demo composition.", + ], + }; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + mkdirSync(ctx.outDir, { recursive: true }); + + activateApp(PROC); + await sleep(600); + mustClick("Export", { controlType: "Button" }); + await sleep(2500); + // The Export menu offers Local File / Screencast / YouTube … + clickControl(PROC, "Local File", { controlType: "MenuItem" }); + await sleep(3000); + + // The trial offers watermarked export or a licence key. A watermark is a cheap overlay + // and does not change render time, so the trial path is a valid measurement. + clickControl(PROC, "Export with Watermark", { controlType: "Button" }); + await sleep(2500); + + // Windows' file dialog takes a full path in its name field — no ⇧⌘G equivalent needed. + await fileDialogTo(PROC, out); + ctx.commit(); + + // Camtasia shows a render progress dialog; its disappearance is the completion signal. + const deadline = now() + 30 * 60 * 1000; + let sawProgress = false; + while (now() < deadline) { + await sleep(1000); + const wins = listWindows(PROC).join(" | "); + const rendering = /render|produc|export/i.test(wins); + if (rendering) sawProgress = true; + if (sawProgress && !rendering) { + ctx.markComplete(); + return; + } + // If the progress window was never observable, let the file watcher decide. + if (!sawProgress && existsSync(out)) return; + } + }, + + async cleanup() { + if (appIsRunning(PROC)) await quitApp(PROC, { force: true }); + }, +}; diff --git a/benchmark/drivers/cap.mjs b/benchmark/drivers/cap.mjs index 7c355c07..162e952e 100644 --- a/benchmark/drivers/cap.mjs +++ b/benchmark/drivers/cap.mjs @@ -12,9 +12,25 @@ import { execFileSync, spawn } from "node:child_process"; import { copyFileSync, existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import { writeCapCursor } from "../lib/assets.mjs"; +import { appVersion, IS_WIN, resolveAppPath } from "../lib/platform.mjs"; -const APP = "/Applications/Cap.app"; -const CLI = `${APP}/Contents/MacOS/cap-cli`; +export const CAP = { + macPath: "/Applications/Cap.app", + winPaths: [ + "%LOCALAPPDATA%\\Programs\\Cap\\Cap.exe", + "%ProgramFiles%\\Cap\\Cap.exe", + "%LOCALAPPDATA%\\Cap\\Cap.exe", + ], +}; + +const APP = IS_WIN ? resolveAppPath(CAP) : "/Applications/Cap.app"; +// The CLI sits beside the desktop binary on Windows and inside the bundle on macOS. +const CLI = APP + ? IS_WIN + ? APP.replace(/Cap\.exe$/i, "cap-cli.exe") + : `${APP}/Contents/MacOS/cap-cli` + : null; /** #RRGGBB → the [r,g,b] triple Cap's colour background expects. */ const rgb = (hex) => { @@ -67,15 +83,29 @@ export default { mkdirSync(join(project, "content"), { recursive: true }); copyFileSync(ctx.source.path, join(project, "content", "display.mp4")); + // A demo export composites more than the screen: a camera track to mask and shadow, and + // a pointer rendered from telemetry rather than baked into the recording. + const wantsCamera = e.webcam?.enabled && ctx.assets?.webcam; + if (wantsCamera) copyFileSync(ctx.assets.webcam, join(project, "content", "camera.mp4")); + const wantsCursor = e.cursor?.enabled; + if (wantsCursor) writeCapCursor(project, ctx.source.spec); + let wallpaperPath = null; + if (e.background?.kind === "image" && ctx.assets?.wallpaper) { + wallpaperPath = join(project, "content", "wallpaper.png"); + copyFileSync(ctx.assets.wallpaper, wallpaperPath); + } + // A single-segment studio recording: the smallest shape `cap project validate` accepts, // and the one an import would produce. writeFileSync( join(project, "recording-meta.json"), `${JSON.stringify( { - platform: "MacOS", + platform: IS_WIN ? "Windows" : "MacOS", pretty_name: `openscreen-benchmark-${ctx.scenario.id}`, display: { path: "content/display.mp4", fps: ctx.source.probe.video.fps }, + ...(wantsCamera ? { camera: { path: "content/camera.mp4", fps: 30 } } : {}), + ...(wantsCursor ? { cursor: "content/cursor.json" } : {}), }, null, 2, @@ -89,19 +119,36 @@ export default { ); const duration = ctx.source.probe.durationSec; - base.background.source = { - type: "color", - value: rgb(e.background?.color ?? "#000000"), - alpha: 255, - }; + // An image the compositor samples per pixel, not a fill it clears once — which is what + // these apps' own wallpapers cost, and what the first version of this scenario missed. + base.background.source = wallpaperPath + ? { type: "image", path: wallpaperPath } + : { type: "color", value: rgb(e.background?.color ?? "#000000"), alpha: 255 }; base.background.blur = 0; base.background.padding = ctx.paddingControl ?? this.defaultPaddingControl(ctx.scenario); base.background.rounding = e.cornerRadiusPx; // Cap's `shadow` is 0-100; the scenario's intensity is 0-1. base.background.shadow = e.shadow?.enabled ? Math.round(e.shadow.intensity * 100) : 0; - base.camera.hide = true; - base.cursor.hide = !e.cursorEffects; - base.screenMotionBlur = e.motionBlur ? 1 : 0; + // Camera inset: masked, rounded and shadowed, in the same corner as every other app. + base.camera.hide = !wantsCamera; + if (wantsCamera) { + base.camera.size = e.webcam.sizePercent ?? 25; + base.camera.rounding = e.webcam.shape === "rounded" ? 25 : 0; + base.camera.shadow = e.webcam.shadow ? 60 : 0; + base.camera.position = { x: "right", y: "bottom" }; + } + + // Cursor: rendered from the telemetry written above, with the smoothing, size and + // motion blur the scenario asks for. + base.cursor.hide = !wantsCursor; + if (wantsCursor) { + base.cursor.size = e.cursor.sizePercent ?? 100; + base.cursor.motionBlur = e.cursor.motionBlur ? 1 : 0; + base.cursor.animationStyle = e.cursor.smoothing >= 0.5 ? "mellow" : "regular"; + base.cursor.raw = false; + } + + base.screenMotionBlur = e.motionBlur?.enabled ? e.motionBlur.amount * 2 : 0; base.timeline = { segments: [{ recordingSegment: 0, timescale: 1, start: 0, end: duration }], @@ -139,11 +186,18 @@ export default { const verify = JSON.parse( execFileSync(CLI, ["project", "config", "get", project], { encoding: "utf8" }), ); + // Read back what Cap kept, not what was sent: `config set` silently resets anything it + // will not accept, and claiming a feature the app dropped is how a benchmark lies. const applied = ["targetResolution", "targetFps"]; - if (verify.background?.source?.type === "color") applied.push("background"); + if (["color", "image", "wallpaper", "gradient"].includes(verify.background?.source?.type)) { + applied.push("background"); + } if (verify.background?.padding > 0) applied.push("padding"); if (verify.background?.rounding > 0) applied.push("cornerRadius"); if (verify.background?.shadow > 0) applied.push("shadow"); + if (verify.screenMotionBlur > 0) applied.push("motionBlur"); + if (wantsCursor && verify.cursor?.hide === false) applied.push("cursor"); + if (wantsCamera && verify.camera?.hide === false) applied.push("webcam"); if ( (verify.timeline?.zoomSegments ?? []).length === (e.zooms ?? []).length && e.zooms?.length @@ -157,6 +211,7 @@ export default { notes: [ `project: ${project}`, `zoom segments written: ${(verify.timeline?.zoomSegments ?? []).length}`, + `camera track: ${wantsCamera ? "content/camera.mp4" : "none"}; cursor telemetry: ${wantsCursor ? "content/cursor.json" : "none"}`, ], }; }, diff --git a/benchmark/drivers/ffmpeg-baseline.mjs b/benchmark/drivers/ffmpeg-baseline.mjs index e8ac2a5c..05ba23e0 100644 --- a/benchmark/drivers/ffmpeg-baseline.mjs +++ b/benchmark/drivers/ffmpeg-baseline.mjs @@ -9,6 +9,7 @@ import { spawn } from "node:child_process"; import { join } from "node:path"; import { ffmpegVersion, resolveFfmpeg } from "../lib/env.mjs"; +import { pickH264Encoder } from "../lib/platform.mjs"; export default { id: "ffmpeg-baseline", @@ -41,11 +42,16 @@ export default { }, async prepare() { + const enc = pickH264Encoder(resolveFfmpeg().ffmpeg); return { // The floor deliberately applies nothing. Listing the two output features it *does* // honour keeps the fidelity score honest rather than showing a bare zero. appliedFeatures: ["targetResolution", "targetFps"], - notes: ["No compositing: this row is the encode-only reference, not a product."], + notes: [ + `encoder: ${enc.encoder}${enc.hardware ? " (hardware)" : " (SOFTWARE)"}`, + "No compositing: this row is the encode-only reference, not a product.", + ...(enc.note ? [enc.note] : []), + ], }; }, @@ -55,6 +61,7 @@ export default { async runExport(ctx) { const { ffmpeg } = resolveFfmpeg(); + const enc = pickH264Encoder(ffmpeg); const out = this.outputPath(ctx); const t = ctx.scenario.output; diff --git a/benchmark/drivers/focusee-win.mjs b/benchmark/drivers/focusee-win.mjs new file mode 100644 index 00000000..5f3c931b --- /dev/null +++ b/benchmark/drivers/focusee-win.mjs @@ -0,0 +1,151 @@ +/** + * FocuSee on Windows. + * + * The closest pitch-for-pitch rival to OpenScreen, and on Windows it is a first-class entrant: + * the vendor's download *is* the Windows application (`focusee-en-v2-setup.exe`), where the Mac + * side ships only a downloader stub, and the macOS build refused every MP4 it was handed with + * "The source file is damaged and cannot be opened." That failure is specific to the Mac build + * and there is a fair chance this one imports normally. + * + * FocuSee is a native app on both platforms, so unlike the Electron entrants its whole + * interface is published to the automation API — canvas size, Padding / Inset / Roundness / + * Shadow, and the Export button are all addressable by name. + * + * NOT YET RUN ON WINDOWS. `bench.mjs discover focusee` dumps the real control names; every + * lookup here fails loudly with what it did find. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { sleep } from "../lib/measure.mjs"; +import { appVersion, resolveAppPath } from "../lib/platform.mjs"; +import { activateApp, appIsRunning, launchApp, quitApp } from "../lib/ui.mjs"; +import { clickControl, describeApp, fileDialogTo, setControlValue } from "../lib/uiWindows.mjs"; + +export const FOCUSEE = { + macPath: "/Applications/FocuSee.app", + winPaths: [ + "%ProgramFiles%\\Gemoo\\FocuSee\\FocuSee.exe", + "%ProgramFiles(x86)%\\Gemoo\\FocuSee\\FocuSee.exe", + "%LOCALAPPDATA%\\Programs\\FocuSee\\FocuSee.exe", + "%ProgramFiles%\\FocuSee\\FocuSee.exe", + ], +}; + +const PROC = "FocuSee"; + +export default { + id: "focusee", + displayName: "FocuSee", + vendor: "iMobie / Gemoo", + kind: "gui", + automation: "uia", + processName: PROC, + get appPath() { + return resolveAppPath(FOCUSEE); + }, + bundleId: null, + install: { + method: "installer", + // The link the vendor's download button resolves to on Windows. + url: "https://focusee.imobie-resource.com/product/focusee-en-v2-setup.exe", + appName: "FocuSee", + approxMB: 120, + licence: "commercial — trial exports are watermarked", + silentArgs: ["/S"], + notes: [ + "On Windows the vendor serves the real application, not the downloader stub the Mac side gets.", + "If /S is rejected the installer is not NSIS; run it once by hand during preflight.", + ], + }, + + detect() { + const path = resolveAppPath(FOCUSEE); + if (!path) return { installed: false, version: null, path: null }; + return { installed: true, version: appVersion(path), path }; + }, + + async prepare(ctx) { + const exe = resolveAppPath(FOCUSEE); + if (!exe) throw new Error("FocuSee is not installed"); + + if (appIsRunning(PROC)) await quitApp(PROC, { force: true }); + await sleep(2000); + // Opening the clip through the shell creates a project without having to drive the + // drag-and-drop drop zone, which no automation API can perform. + await launchApp(exe, PROC, { args: [ctx.source.path], timeoutMs: 180_000 }); + await sleep(20_000); + activateApp(PROC); + + const tree = describeApp(PROC, { max: 600 }); + if (/damaged and cannot be opened/i.test(tree)) { + throw new Error( + "FocuSee refused the source: “The source file is damaged and cannot be opened.” " + + "This is the same failure the macOS build showed; if it reproduces here, the app cannot be benchmarked.", + ); + } + + // FocuSee's composition controls are sliders with a readable numeric label. Where a + // ValuePattern exists the scenario is applied; where it does not, the feature is simply + // not claimed — the pixel verifier is what decides, not this list. + const e = ctx.scenario.effects; + const applied = ["targetResolution", "targetFps"]; + if (clickControl(PROC, "16:9", { controlType: "Button" }).ok) applied.push("targetResolution"); + if (setControlValue(PROC, "Padding", String(ctx.paddingControl ?? e.paddingPercent))) + applied.push("padding"); + if (setControlValue(PROC, "Roundness", String(e.cornerRadiusPx))) applied.push("cornerRadius"); + if ( + e.shadow?.enabled && + setControlValue(PROC, "Shadow", String(Math.round(e.shadow.intensity * 100))) + ) { + applied.push("shadow"); + } + + return { + appliedFeatures: [...new Set(applied)], + notes: [ + "Zooms are not applied: FocuSee generates them from its own cursor telemetry, which a file import has none of, and its manual zoom editor is not addressable.", + `controls reached: ${[...new Set(applied)].join(", ")}`, + ], + }; + }, + + /** FocuSee's padding control is 0-100 on its own scale; `bench.mjs calibrate` solves it. */ + defaultPaddingControl(scenario) { + return scenario.effects.paddingPercent; + }, + + outputPath(ctx) { + return join(ctx.outDir, `${this.id}-${ctx.scenario.id}-run${ctx.run.index}.mp4`); + }, + + async runExport(ctx) { + const out = this.outputPath(ctx); + if (existsSync(out)) rmSync(out); + mkdirSync(ctx.outDir, { recursive: true }); + + activateApp(PROC); + await sleep(600); + const r = clickControl(PROC, "Export", { controlType: "Button" }); + if (!r.ok) { + throw new Error( + `FocuSee: no Export button. Present: ${(r.seen ?? []).slice(0, 20).join(" | ")}. ` + + "Run `node benchmark/bench.mjs discover focusee`.", + ); + } + await sleep(3000); + + // The export sheet offers format and resolution before the file dialog. + clickControl(PROC, "MP4", { controlType: "Button" }); + clickControl(PROC, "1080", { controlType: "Button" }); + await sleep(800); + clickControl(PROC, "Export", { controlType: "Button" }); + + await fileDialogTo(PROC, out); + ctx.commit(); + }, + + async cleanup() { + if (appIsRunning(PROC)) await quitApp(PROC, { force: true }); + }, +}; diff --git a/benchmark/drivers/openscreen-cli.mjs b/benchmark/drivers/openscreen-cli.mjs index eb774a30..b443e242 100644 --- a/benchmark/drivers/openscreen-cli.mjs +++ b/benchmark/drivers/openscreen-cli.mjs @@ -13,9 +13,23 @@ import { execFileSync, spawn } from "node:child_process"; import { existsSync, rmSync } from "node:fs"; import { join } from "node:path"; import { buildProject } from "../lib/openscreenProject.mjs"; +import { appVersion, IS_WIN, resolveAppPath } from "../lib/platform.mjs"; -const APP = "/Applications/Openscreen.app"; -const BIN = `${APP}/Contents/MacOS/Openscreen`; +export const OPENSCREEN = { + macPath: "/Applications/Openscreen.app", + winPaths: [ + "%ProgramFiles%\\Openscreen\\Openscreen.exe", + "%LOCALAPPDATA%\\Programs\\Openscreen\\Openscreen.exe", + "%LOCALAPPDATA%\\openscreen\\Openscreen.exe", + ], +}; + +/** + * The CLI ships inside the normal application bundle on both platforms, so there is nothing + * extra to install — only a different place to look. + */ +const APP = IS_WIN ? resolveAppPath(OPENSCREEN) : "/Applications/Openscreen.app"; +const BIN = IS_WIN ? APP : `${APP}/Contents/MacOS/Openscreen`; export default { id: "openscreen-cli", @@ -68,9 +82,12 @@ export default { outDir, title: ctx.scenario.id, paddingControl: ctx.paddingControl ?? this.defaultPaddingControl(ctx.scenario), + assets: ctx.assets ?? {}, + spec: ctx.source.spec, }); ctx.state.projectPath = projectPath; + const e = ctx.scenario.effects; return { appliedFeatures: [ "background", @@ -78,6 +95,9 @@ export default { "cornerRadius", "shadow", "zooms", + ...(e.motionBlur?.enabled ? ["motionBlur"] : []), + ...(e.cursor?.enabled ? ["cursor"] : []), + ...(e.webcam?.enabled && ctx.assets?.webcam ? ["webcam"] : []), "targetResolution", "targetFps", ], diff --git a/benchmark/drivers/openscreen-gui.mjs b/benchmark/drivers/openscreen-gui.mjs index a216f560..0e03db3c 100644 --- a/benchmark/drivers/openscreen-gui.mjs +++ b/benchmark/drivers/openscreen-gui.mjs @@ -73,6 +73,8 @@ export default { outDir, title: ctx.scenario.id, paddingControl: ctx.paddingControl ?? this.defaultPaddingControl(ctx.scenario), + assets: ctx.assets ?? {}, + spec: ctx.source.spec, }); ctx.state.projectPath = projectPath; diff --git a/benchmark/lib/assets.mjs b/benchmark/lib/assets.mjs new file mode 100644 index 00000000..62e7066a --- /dev/null +++ b/benchmark/lib/assets.mjs @@ -0,0 +1,268 @@ +/** + * The rest of a real recording: a wallpaper, a webcam track, and cursor telemetry. + * + * The first version of this benchmark fed the apps a screen clip and nothing else, and that + * measured the wrong thing. An export in this category is not a transcode with a coloured + * border — it is a compositor pass that samples a background image, transforms and masks the + * recording, renders a *synthetic* cursor from telemetry with smoothing and motion blur, draws + * a webcam inset with its own mask and shadow, and motion-blurs the whole thing. Leave the + * cursor and the camera out and the expensive half of the pipeline never runs. + * + * Two consequences shape this file: + * + * 1. **The cursor must not be drawn into the source.** These apps hide the system pointer while + * recording and re-render it at export time from a sidecar. Baking a cursor into the pixels + * would exercise nothing and would double-draw once an app rendered its own. So the + * trajectory is generated here, written in each app's telemetry format by its driver, and + * the screen clip is left clean. + * 2. **Everything is generated from the same seed**, so a second machine reproduces the whole + * bundle — wallpaper, webcam and cursor path included — and can prove it by hash. + */ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { resolveFfmpeg } from "./env.mjs"; +import { sha256 } from "./fixture.mjs"; +import { pickH264Encoder } from "./platform.mjs"; + +function run(bin, args) { + try { + return execFileSync(bin, args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); + } catch (e) { + const msg = (e.stderr?.toString() || e.stdout?.toString() || e.message).trim(); + throw new Error(`${bin.split(/[/\\]/).pop()} failed (exit ${e.status}):\n${msg}`); + } +} + +/* -------------------------------------------------------------------- wallpaper ---------- */ + +/** + * A background the compositor has to *sample*, not fill. + * + * A flat colour is a single clear; an image is a texture upload and a per-pixel fetch for the + * whole frame, every frame — which is what the apps' own wallpapers cost. Deliberately light, + * so the dark recording's edge stays findable by the geometry verifier. + */ +export function buildWallpaper(workDir, spec) { + const { ffmpeg } = resolveFfmpeg(); + const dir = join(workDir, "fixture"); + mkdirSync(dir, { recursive: true }); + const out = join(dir, `${spec.name}.wallpaper.png`); + const jpg = join(dir, `${spec.name}.wallpaper.jpg`); + if (existsSync(out) && existsSync(jpg)) { + return { path: out, jpeg: jpg, sha256: sha256(out), regenerated: false }; + } + + // A soft diagonal gradient with a few large translucent discs — visually plausible as a + // product-demo backdrop, and high-frequency enough that a sampler cannot shortcut it. + const w = spec.width; + const h = spec.height; + const discs = [ + [0.18, 0.24, 0.3, "0xffffff@0.16"], + [0.74, 0.18, 0.22, "0xf3d9c0@0.22"], + [0.62, 0.78, 0.34, "0xc9d7ee@0.20"], + [0.3, 0.86, 0.18, "0xffffff@0.12"], + ] + .map(([cx, cy, r, color]) => { + const rr = Math.round(r * Math.min(w, h)); + const x = Math.round(cx * w - rr); + const y = Math.round(cy * h - rr); + // drawbox has no ellipse; a stack of inset boxes reads as a soft blob once blurred. + return `drawbox=x=${x}:y=${y}:w=${rr * 2}:h=${rr * 2}:color=${color}:t=fill`; + }) + .join(","); + + run(ffmpeg, [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + `gradients=s=${w}x${h}:c0=0x2b3a55:c1=0xd9c9b4:x0=0:y0=0:x1=${w}:y1=${h}:n=2`, + "-vf", + `${discs},gblur=sigma=${Math.round(Math.min(w, h) / 12)},format=rgb24`, + "-frames:v", + "1", + out, + ]); + // A JPEG copy, for apps that take the wallpaper inline rather than by path. + run(ffmpeg, ["-hide_banner", "-loglevel", "error", "-y", "-i", out, "-q:v", "6", jpg]); + return { path: out, jpeg: jpg, sha256: sha256(out), regenerated: true }; +} + +/* ---------------------------------------------------------------------- webcam ----------- */ + +/** + * A webcam track: a person-shaped subject that moves, on a backdrop. + * + * The point is not realism, it is cost — a second video stream to decode, scale, mask into a + * rounded or circular inset, and drop a shadow behind, for every frame. Small and 30 fps + * because that is what webcams actually deliver, and a driver that scales it to the screen + * clip's 60 fps is doing the work a real project would. + */ +export function buildWebcam(workDir, spec) { + const { ffmpeg } = resolveFfmpeg(); + const enc = pickH264Encoder(ffmpeg); + const dir = join(workDir, "fixture"); + mkdirSync(dir, { recursive: true }); + const out = join(dir, `${spec.name}.webcam.mp4`); + if (existsSync(out)) return { path: out, sha256: sha256(out), regenerated: false }; + + const W = 1280; + const H = 720; + const fps = 30; + // Head and shoulders that drift and breathe, so no two frames are identical and the + // encoder cannot coast — a static webcam would cost almost nothing to composite. + const headX = `${W / 2}-170+40*sin(2*PI*t/9)`; + const headY = `${H / 2}-120+22*sin(2*PI*t/5)`; + const filter = [ + `color=c=0x1d2430:s=${W}x${H}:r=${fps}:d=${spec.durationSec}`, + `drawbox=x=0:y=0:w=${W}:h=${H}:color=0x243044@1:t=fill`, + // backdrop pool of light + `drawbox=x=${Math.round(W * 0.2)}:y=0:w=${Math.round(W * 0.6)}:h=${H}:color=0x2e3b52@0.8:t=fill`, + // shoulders + `drawbox=x='${W / 2}-300+40*sin(2*PI*t/9)':y=${Math.round(H * 0.72)}:w=600:h=${Math.round(H * 0.3)}:color=0x3a4a63@1:t=fill`, + // head + `drawbox=x='${headX}':y='${headY}':w=340:h=380:color=0xd8b49a@1:t=fill`, + // hair + `drawbox=x='${headX}':y='${headY}':w=340:h=90:color=0x3b2f2a@1:t=fill`, + // eyes, which blink on a 4 s cycle + `drawbox=x='${headX}+80':y='${headY}+170':w=42:h=22:color=0x2a2320@1:t=fill:enable='gt(mod(t\\,4)\\,0.18)'`, + `drawbox=x='${headX}+218':y='${headY}+170':w=42:h=22:color=0x2a2320@1:t=fill:enable='gt(mod(t\\,4)\\,0.18)'`, + // mouth, moving as if speaking + `drawbox=x='${headX}+130':y='${headY}+270':w=80:h='14+10*abs(sin(2*PI*2.7*t))':color=0x8c4a44@1:t=fill`, + "format=yuv420p", + ].join(","); + + run(ffmpeg, [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + filter, + "-t", + String(spec.durationSec), + "-r", + String(fps), + "-c:v", + enc.encoder, + ...enc.rateArgs(6), + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + out, + ]); + return { path: out, sha256: sha256(out), regenerated: true, width: W, height: H, fps }; +} + +/* ---------------------------------------------------------------------- cursor ----------- */ + +/** + * The cursor trajectory, as data. + * + * Deterministic from the spec's seed, sampled at a realistic rate, and shaped like real use: + * long smooth glides, short pauses, and clicks at the pauses — which is exactly the signal the + * apps' smoothing, click effects and dwell-based auto-zoom react to. A straight-line sweep + * would let a smoothing implementation do nothing and cost nothing. + * + * Positions are normalised (0-1) against the screen frame, matching every format that consumes + * them; each driver translates this into its app's own sidecar. + */ +export function cursorTrack(spec, { sampleHz = 60 } = {}) { + const samples = []; + const total = Math.round(spec.durationSec * sampleHz); + // Dwell points the pointer travels between — a plausible tour of a UI. + const stops = [ + [0.12, 0.18], + [0.46, 0.32], + [0.78, 0.24], + [0.62, 0.66], + [0.24, 0.74], + [0.52, 0.48], + [0.86, 0.62], + [0.3, 0.36], + ]; + const legMs = (spec.durationSec * 1000) / stops.length; + const glideFraction = 0.62; // the rest of each leg is a pause + + for (let i = 0; i < total; i++) { + const timeMs = Math.round((i / sampleHz) * 1000); + const leg = Math.min(stops.length - 1, Math.floor(timeMs / legMs)); + const within = (timeMs % legMs) / legMs; + const from = stops[leg]; + const to = stops[(leg + 1) % stops.length]; + + let cx; + let cy; + let interactionType = "move"; + if (within < glideFraction) { + // Ease-in-out along the leg: acceleration is what smoothing has to work on. + const u = within / glideFraction; + const e = u < 0.5 ? 2 * u * u : 1 - (-2 * u + 2) ** 2 / 2; + cx = from[0] + (to[0] - from[0]) * e; + cy = from[1] + (to[1] - from[1]) * e; + } else { + cx = to[0]; + cy = to[1]; + // One click just after arriving, then stillness — the shape click effects expect. + const sincePause = (within - glideFraction) * legMs; + if (sincePause >= 120 && sincePause < 120 + 1000 / sampleHz) interactionType = "click"; + else if (sincePause >= 220 && sincePause < 220 + 1000 / sampleHz) interactionType = "mouseup"; + } + samples.push({ + timeMs, + cx: +cx.toFixed(5), + cy: +cy.toFixed(5), + visible: true, + interactionType, + }); + } + return samples; +} + +/** OpenScreen reads `.cursor.json`; schema version 2, normalised coordinates. */ +export function writeOpenscreenCursor(screenVideoPath, spec) { + const path = `${screenVideoPath}.cursor.json`; + writeFileSync( + path, + `${JSON.stringify({ version: 2, provider: "native", samples: cursorTrack(spec) }, null, 0)}\n`, + ); + return path; +} + +/** + * Cap stores its pointer track as a JSON array of `{ process_time_ms, x, y, ... }` beside the + * segment, referenced by `cursor` in recording-meta.json. Coordinates are normalised, as in + * `cap-project`'s `CursorEvents`. + */ +export function writeCapCursor(projectDir, spec) { + const path = join(projectDir, "content", "cursor.json"); + const track = cursorTrack(spec); + // Field names come from cap-project's CursorMoveEvent / CursorClickEvent: `time_ms`, not + // the `process_time_ms` a recording's raw log uses. Getting this wrong is silent — Cap + // parses the file, finds no usable events, and exports with no pointer at all. + const moves = track.map((s) => ({ + active_modifiers: [], + cursor_id: "0", + time_ms: s.timeMs, + x: s.cx, + y: s.cy, + })); + const clicks = track + .filter((s) => s.interactionType === "click" || s.interactionType === "mouseup") + .map((s) => ({ + active_modifiers: [], + cursor_num: 0, + cursor_id: "0", + time_ms: s.timeMs, + down: s.interactionType === "click", + })); + writeFileSync(path, `${JSON.stringify({ clicks, moves }, null, 0)}\n`); + return path; +} diff --git a/benchmark/lib/env.mjs b/benchmark/lib/env.mjs index 38c4a7b4..e015ec8e 100644 --- a/benchmark/lib/env.mjs +++ b/benchmark/lib/env.mjs @@ -10,6 +10,12 @@ import { chmodSync, existsSync, mkdirSync, readdirSync, writeFileSync } from "no import os from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { + diskFreeGiB, + IS_WIN, + machineFingerprint as platformFingerprint, + powerState as platformPower, +} from "./platform.mjs"; export const BENCH_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); export const REPO_ROOT = resolve(BENCH_ROOT, ".."); @@ -48,7 +54,8 @@ function findVendoredFfmpegPrefix() { if (!existsSync(root)) continue; for (const entry of readdirSync(root)) { const prefix = join(root, entry); - if (/^ffmpeg-/.test(entry) && existsSync(join(prefix, "bin", "ffmpeg"))) return prefix; + const bin = join(prefix, "bin", IS_WIN ? "ffmpeg.exe" : "ffmpeg"); + if (/^ffmpeg-/.test(entry) && existsSync(bin)) return prefix; } } return null; @@ -86,13 +93,32 @@ export function resolveFfmpeg() { return cachedFfmpeg; } - const onPath = trySh("command -v ffmpeg"); - const probeOnPath = trySh("command -v ffprobe"); + const which = IS_WIN ? "where" : "command -v"; + const onPath = trySh(`${which} ffmpeg`)?.split("\n")[0]?.trim() || null; + const probeOnPath = trySh(`${which} ffprobe`)?.split("\n")[0]?.trim() || null; if (onPath && probeOnPath) { cachedFfmpeg = { ffmpeg: onPath, ffprobe: probeOnPath, source: "PATH" }; return cachedFfmpeg; } + if (IS_WIN) { + // The repo vendors ffmpeg for the compositor on Windows too; scripts/fetch-ffmpeg.mjs + // puts it under crates/thirdparty. No wrapper is needed — Windows has no DYLD stripping. + const prefix = findVendoredFfmpegPrefix(); + if (prefix) { + cachedFfmpeg = { + ffmpeg: join(prefix, "bin", "ffmpeg.exe"), + ffprobe: join(prefix, "bin", "ffprobe.exe"), + source: `vendored:${prefix}`, + }; + return cachedFfmpeg; + } + throw new Error( + "No ffmpeg/ffprobe found. Install one on PATH (winget install Gyan.FFmpeg), " + + "or set OSBENCH_FFMPEG and OSBENCH_FFPROBE.", + ); + } + const prefix = findVendoredFfmpegPrefix(); if (prefix) { const lib = join(prefix, "lib"); @@ -120,73 +146,21 @@ export function ffmpegVersion() { /** Everything about the host that can move a number in the results table. */ export function machineFingerprint() { - const memBytes = Number(trySh("sysctl -n hw.memsize", "0")); - const displays = (trySh("system_profiler SPDisplaysDataType", "") || "") - .split("\n") - .filter((l) => /^\s+(Resolution|UI Looks like):/.test(l)) - .map((l) => l.trim()); - - return { - platform: process.platform, - arch: process.arch, - osProduct: trySh("sw_vers -productName"), - osVersion: trySh("sw_vers -productVersion"), - osBuild: trySh("sw_vers -buildVersion"), - kernel: os.release(), - model: trySh("sysctl -n hw.model"), - chip: trySh("sysctl -n machdep.cpu.brand_string"), - cpuCount: os.cpus().length, - performanceCores: Number(trySh("sysctl -n hw.perflevel0.logicalcpu", "0")) || null, - efficiencyCores: Number(trySh("sysctl -n hw.perflevel1.logicalcpu", "0")) || null, - memoryGiB: memBytes ? +(memBytes / 1024 ** 3).toFixed(1) : null, - displays, - nodeVersion: process.version, - }; + return platformFingerprint(); } /** - * Preconditions that silently skew an export benchmark: battery power caps the SoC, - * Low Power Mode caps it harder, and an already-throttled machine reports whatever - * the previous run left behind. + * Preconditions that silently skew an export benchmark: battery power caps the SoC, a + * power-saver plan caps it harder, and an already-throttled machine reports whatever the + * previous run left behind. */ export function powerState() { - const batt = trySh("pmset -g batt", ""); - const therm = trySh("pmset -g therm", ""); - const lowPower = trySh("pmset -g | grep -i lowpowermode", ""); - const cpuSpeedLimit = /CPU_Speed_Limit\s*=\s*(\d+)/.exec(therm); - const schedulerLimit = /CPU_Scheduler_Limit\s*=\s*(\d+)/.exec(therm); - - return { - onACPower: /AC Power/.test(batt), - batteryLine: batt.split("\n").slice(1).join(" ").trim() || null, - lowPowerMode: /lowpowermode\s+1/.test(lowPower), - cpuSpeedLimit: cpuSpeedLimit ? Number(cpuSpeedLimit[1]) : null, - cpuSchedulerLimit: schedulerLimit ? Number(schedulerLimit[1]) : null, - thermalPressure: readThermalPressure(), - }; -} - -/** - * Thermal pressure, spelled differently on every macOS. `pmset -g therm` prints nothing at - * all when the SoC is unthrottled, so "no output" is the healthy answer, not a failure. - */ -function readThermalPressure() { - for (const key of ["kern.thermalpressurelevel", "machdep.xcpm.cpu_thermal_level"]) { - const v = trySh(`sysctl -n ${key}`); - if (v !== null && v !== "") return { key, value: Number(v) }; - } - const therm = trySh("pmset -g therm", ""); - if (/No thermal warning level has been recorded/i.test(therm)) return { key: "pmset", value: 0 }; - return { key: "unavailable", value: null }; + return platformPower(); } /** Free space on the volume that will hold the fixture and every export. */ export function diskState(path = WORK_DIR) { - const target = existsSync(path) ? path : os.homedir(); - const line = trySh(`df -k ${JSON.stringify(target)} | tail -1`, ""); - const cols = line.split(/\s+/); - const availKiB = Number(cols[3] || 0); - return { path: target, availableGiB: +(availKiB / 1024 / 1024).toFixed(1) }; + return diskFreeGiB(path); } export function ensureWorkDirs() { diff --git a/benchmark/lib/fixture.mjs b/benchmark/lib/fixture.mjs index 31d4cb76..094257a2 100644 --- a/benchmark/lib/fixture.mjs +++ b/benchmark/lib/fixture.mjs @@ -15,6 +15,7 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, statSync } from "node:fs"; import { join } from "node:path"; import { resolveFfmpeg } from "./env.mjs"; +import { pickH264Encoder } from "./platform.mjs"; /** execFileSync, but a non-zero exit surfaces ffmpeg's own message instead of a byte dump. */ function run(bin, args) { @@ -139,9 +140,11 @@ function animationFilter(spec, pageHeight) { `drawbox=x=380+mod(floor(t*7)\\,40)*14:y=${44 + Math.floor(visibleH / 2)}:w=3:h=18:color=0xffffff@1:t=fill:enable='lt(mod(t\\,1)\\,0.5)'`, // Selection band sweeping down the pane. `drawbox=x=360:y=${44}+mod(floor(t*2)*36\\,${visibleH - 40}):w=760:h=22:color=0x3b5bdb@0.35:t=fill`, - // Cursor arrow, approximated by a small bright square on a Lissajous path. - `drawbox=x=${spec.width / 2}+${spec.width / 2 - 140}*sin(2*PI*t/11):y=${spec.height / 2}+${spec.height / 2 - 120}*sin(2*PI*t/7):w=14:h=20:color=0xffffff@0.95:t=fill`, - `drawbox=x=${spec.width / 2}+${spec.width / 2 - 140}*sin(2*PI*t/11)-1:y=${spec.height / 2}+${spec.height / 2 - 120}*sin(2*PI*t/7)-1:w=16:h=22:color=0x000000@0.6:t=3`, + // No cursor is drawn here on purpose. Every app in this set hides the system pointer + // while recording and re-renders it at export time from a telemetry sidecar, with its + // own theme, smoothing and motion blur — which is a large part of what an export + // costs. Baking one in would exercise none of that and would double-draw once an app + // rendered its own. The trajectory lives in lib/assets.mjs → cursorTrack(). ].join(","), }; } @@ -208,6 +211,9 @@ export function buildFixture( { force = false, log = () => undefined } = {}, ) { const { ffmpeg } = resolveFfmpeg(); + // The encoder differs by platform and GPU vendor; picking it here keeps the fixture + // generatable everywhere while recording which one produced it. + const enc = pickH264Encoder(ffmpeg); const out = fixturePath(workDir, spec); mkdirSync(join(workDir, "fixture"), { recursive: true }); @@ -296,7 +302,15 @@ export function buildFixture( "+faststart", out, ]); - log(`fixture: encoded in ${((Date.now() - t0) / 1000).toFixed(1)}s`); + log(`fixture: encoded in ${((Date.now() - t0) / 1000).toFixed(1)}s using ${enc.encoder}`); + if (enc.note) log(`fixture: ${enc.note}`); - return { path: out, spec, probe: probe(out), sha256: sha256(out), regenerated: true }; + return { + path: out, + spec, + probe: probe(out), + sha256: sha256(out), + regenerated: true, + encoder: enc.encoder, + }; } diff --git a/benchmark/lib/measure.mjs b/benchmark/lib/measure.mjs index d4cbe488..7ca0a7f0 100644 --- a/benchmark/lib/measure.mjs +++ b/benchmark/lib/measure.mjs @@ -10,9 +10,10 @@ * Everything here is app-agnostic on purpose: the same stopwatch is used for the CLI drivers * and the UI drivers, so a CLI app is not credited for skipping a step a GUI app must do. */ -import { execFileSync } from "node:child_process"; + import { existsSync, statSync } from "node:fs"; import { probe } from "./fixture.mjs"; +import { instantaneousLoadPercent, listProcesses, parseMacCpuTime } from "./platform.mjs"; export const now = () => Number(process.hrtime.bigint() / 1000n) / 1000; // ms, monotonic @@ -48,52 +49,27 @@ export class ProcessTreeSampler { } sampleOnce() { - let out; - try { - out = execFileSync("/bin/ps", ["-axo", "pid=,rss=,time=,args="], { - encoding: "utf8", - maxBuffer: 16 * 1024 * 1024, - }); - } catch { - return; - } + const procs = listProcesses(); let rssSum = 0; - for (const line of out.split("\n")) { - if (!line.trim()) continue; - const m = /^\s*(\d+)\s+(\d+)\s+(\S+)\s+(.*)$/.exec(line); - if (!m) continue; - const [, pid, rssKiB, time, args] = m; + for (const { pid, rssBytes, cpuSeconds, args } of procs) { if (!this.matchPrefixes.some((p) => args.includes(p))) continue; - const cpu = ProcessTreeSampler.parseCpuTime(time); const prev = this.cpuByPid.get(pid) ?? 0; - if (cpu > prev) this.cpuByPid.set(pid, cpu); - rssSum += Number(rssKiB) * 1024; + if (cpuSeconds > prev) this.cpuByPid.set(pid, cpuSeconds); + rssSum += rssBytes; } if (rssSum > this.peakRssBytes) this.peakRssBytes = rssSum; this.samples++; - this.sampleForeignLoad(); + this.sampleForeignLoad(procs); } /** Instantaneous %CPU of everything that is not the app under test, summed. */ - sampleForeignLoad() { - try { - const out = execFileSync("/bin/ps", ["-axo", "pcpu=,args="], { - encoding: "utf8", - maxBuffer: 16 * 1024 * 1024, - }); - let total = 0; - for (const line of out.split("\n")) { - const m = /^\s*([\d.]+)\s+(.*)$/.exec(line); - if (!m) continue; - const [, pct, args] = m; - if (this.matchPrefixes.some((p) => args.includes(p))) continue; - if (args.includes("bench.mjs") || args.includes("/bin/ps")) continue; - total += Number(pct); - } - this.foreignCpuSamples.push(total); - } catch { - /* a sample lost to a transient ps failure is not worth failing a run over */ - } + sampleForeignLoad(procs) { + const total = instantaneousLoadPercent(); + if (total == null) return; + // Subtract nothing: the figure is "how busy is this machine besides the measurement", + // and the app under test is a small share of it during an export. Callers read it as + // context, not as an exact complement. + this.foreignCpuSamples.push(total); } start() { diff --git a/benchmark/lib/openscreenProject.mjs b/benchmark/lib/openscreenProject.mjs index def02476..ffbe8940 100644 --- a/benchmark/lib/openscreenProject.mjs +++ b/benchmark/lib/openscreenProject.mjs @@ -9,6 +9,7 @@ */ import { copyFileSync, mkdirSync, writeFileSync } from "node:fs"; import { basename, join } from "node:path"; +import { wallpaperDataUri, writeOpenscreenCursor } from "./assets.mjs"; import { probe } from "./fixture.mjs"; /** Deterministic ids: the same scenario always produces the same project bytes. */ @@ -42,6 +43,10 @@ export function buildProject({ outDir, title = "export-benchmark", paddingControl = null, + /** Generated wallpaper and camera track — see lib/assets.mjs. */ + assets = {}, + /** The fixture spec, needed to regenerate the cursor track deterministically. */ + spec = null, }) { mkdirSync(outDir, { recursive: true }); @@ -53,6 +58,32 @@ export function buildProject({ const p = probe(localMedia); const e = scenario.effects; + // Cursor telemetry rides beside the screen video as `