diff --git a/README.md b/README.md index fb18a14..d1a42f4 100644 --- a/README.md +++ b/README.md @@ -353,20 +353,25 @@ export default defineConfig({ ### Popups -Pages you never wrap (popups, `context.newPage()`) fall through to plain Playwright. To get plugin behavior in a popup — an OAuth window, say — wrap it with a second `addPlugins` call, using **fresh plugin instances**: +Popups are wrapped automatically. When a wrapped page opens one — an OAuth window, say — the popup gets the same plugin treatment, no wiring: ```ts const popupPromise = page.waitForEvent("popup"); await page.getByRole("button", { name: "Sign in" }).click(); -await using popup = await addPlugins({ - page: await popupPromise, - testInfo, - plugins: [spinnerWaiter(), videoMode()], -}); +const popup = await popupPromise; // already wrapped await popup.getByRole("button", { name: "Approve" }).click(); ``` -Playwright screencasts each page separately, so the popup's `videoMode` produces its own video; its artifacts get a `-2` suffix (`video-rendered-2.webm`, `video-mode-2.json`, …) so they sit next to the main page's in the same output dir. Reusing the main page's `videoMode` instance on the popup would wipe the main timeline, so it throws instead — one instance per page. See [spec/popup.spec.ts](spec/popup.spec.ts). +In video mode, the popup renders as an overlay **in the main page's video**: scaled to fit 90% of the frame over the dimmed page, faded in and out on open/close, with popup clicks pointer-annotated inside the overlay. One composed video per test, popups included. The popup's facts land in `video-mode.json` under `children`. + +Details and escape hatches: + +- Plugins can control what a popup gets via the `forPopup(ctx)` hook — return a plugin for the popup, or `null` to skip. Hookless plugins are re-registered as-is (fine for stateless ones). +- `addPlugins({ ..., popups: false })` turns auto-wrap off. You can then wrap the popup manually with **fresh plugin instances** — a fresh `videoMode()` gives the popup its own standalone video, with `-2`-suffixed artifacts (`video-rendered-2.webm`, `video-mode-2.json`, …). +- Wrapping an already-wrapped page throws, as does reusing an active `videoMode` instance on a second page — one instance per page. +- Popup dialogs (`alert`/`confirm`/`prompt` opened by the popup) aren't annotated in video mode yet. + +See [spec/popup.spec.ts](spec/popup.spec.ts) and [spec/popup-overlay-demo.spec.ts](spec/popup-overlay-demo.spec.ts). ## Writing your own plugin diff --git a/spec/auth-demo-app.ts b/spec/auth-demo-app.ts index 2ee91e8..596fc57 100644 --- a/spec/auth-demo-app.ts +++ b/spec/auth-demo-app.ts @@ -16,6 +16,75 @@ const demoStyle = ` `; +/** + * Demo-video variant of the auth flow: same app page, but the popup is a + * realistic sign-in form (inert username/password fields, a Sign in button + * that notifies the opener and closes) on a visibly different background so + * the popout reads clearly in the rendered overlay. + */ +export const routeSignInDemoApp = async (context: BrowserContext) => { + await routeAuthDemoApp(context); + // The app page gets a colored background so the dimmed page under the + // popup overlay reads clearly in the rendered video. + await context.route("https://app.middlewright.test/**", async (route) => { + await route.fulfill({ + body: ` + ${demoStyle} + +
+

middlewright dashboard

+ + + +
+ `, + contentType: "text/html", + }); + }); + await context.route("https://auth.middlewright.test/**", async (route) => { + await route.fulfill({ + body: ` + ${demoStyle} + +
+

Sign in to middlewright

+ + + + + + +
+ `, + contentType: "text/html", + }); + }); +}; + export const routeAuthDemoApp = async (context: BrowserContext) => { await context.route("https://app.middlewright.test/**", async (route) => { await route.fulfill({ @@ -50,6 +119,7 @@ export const routeAuthDemoApp = async (context: BrowserContext) => { diff --git a/spec/popup-overlay-demo.spec.ts b/spec/popup-overlay-demo.spec.ts new file mode 100644 index 0000000..c386301 --- /dev/null +++ b/spec/popup-overlay-demo.spec.ts @@ -0,0 +1,48 @@ +// Demo-grade popup flow with the full watchable treatment — pointer +// highlights, step captions, address bar, popup overlay composite. The +// rendered output doubles as the PR/README demo video. +import { stat } from "node:fs/promises"; +import { test, expect } from "@playwright/test"; +import { addPlugins, videoMode } from "../src/index.ts"; +import { routeSignInDemoApp } from "./auth-demo-app.ts"; + +test.use({ video: "on", viewport: { width: 960, height: 540 } }); + +test("auth popup demo", async ({ page: basePage, context }, testInfo) => { + await routeSignInDemoApp(context); + const video = videoMode(); + { + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video] }); + + const popupPromise = basePage.waitForEvent("popup"); + await test.step("Open the sign-in popup", async () => { + await page.goto("https://app.middlewright.test/"); + // Real frames on each side of the popup span keep the composite honest + // (and the demo watchable) — an instant flow would land before the + // screencast's first frame. + await page.waitForTimeout(500); + await page.getByRole("button", { name: "Sign in" }).click(); + }); + + const popup = await popupPromise; + await test.step("Sign in as mmkal", async () => { + await popup.waitForTimeout(500); + await popup.getByLabel("Username").fill("mmkal"); + await popup.getByLabel("Password").fill("hunter2"); + await popup.getByRole("button", { name: "Sign in" }).click(); + }); + + await test.step("Back on the app, signed in", async () => { + await page.getByText("Signed in as mmkal").waitFor(); + await page.waitForTimeout(500); + }); + } + + const metadata = await video.metadata(); + expect(metadata).toMatchObject({ + children: [{ closedAt: expect.any(Number), openedAt: expect.any(Number) }], + outputs: { raw: "video-raw.webm", rendered: "video-rendered.webm" }, + }); + expect(metadata.children[0].closedAt!).toBeGreaterThan(metadata.children[0].openedAt); + expect((await stat(video.outputPaths().rendered)).size).toBeGreaterThan(0); +}); diff --git a/spec/popup-video.spec.ts b/spec/popup-video.spec.ts index 6f81c06..c40fbc6 100644 --- a/spec/popup-video.spec.ts +++ b/spec/popup-video.spec.ts @@ -1,10 +1,92 @@ +import { execFile as execFileCallback } from "node:child_process"; import { stat } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; import { test, expect } from "@playwright/test"; import { addPlugins, videoMode } from "../src/index.ts"; import { routeAuthDemoApp } from "./auth-demo-app.ts"; test.use({ video: "on" }); +test("captures an auto-wrapped popup's raw screencast for the composite", async ({ + page: basePage, + context, +}, testInfo) => { + await routeAuthDemoApp(context); + const video = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); + { + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video] }); + await page.goto("https://app.middlewright.test/"); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + await (await popupPromise).getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + } + + const metadata = await video.metadata(); + expect(metadata.children).toMatchObject([ + { + // The demo popup closes itself after Approve, like a real OAuth popup — + // closedAt comes from the close event, and there is no settled + // recordingEndedAt (the screencast start approximates the timeline). + closedAt: expect.any(Number), + highlights: [{ method: "click" }], + openedAt: expect.any(Number), + raw: "video-raw-popup-1.webm", + viewport: { height: expect.any(Number), width: expect.any(Number) }, + }, + ]); + const [child] = metadata.children; + expect(child.closedAt!).toBeGreaterThan(child.openedAt); + expect((await stat(join(testInfo.outputDir, child.raw!))).size).toBeGreaterThan(0); +}); + +test("renders the popup as a dimmed overlay in one composed video", async ({ + page: basePage, + context, +}, testInfo) => { + await routeAuthDemoApp(context); + const video = videoMode({ + addressBar: false, + finalHold: 0, + highlight: { mode: "outline", duration: 500 }, + trimStart: "never", + }); + { + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video] }); + await page.goto("https://app.middlewright.test/"); + // Let the screencast capture real frames on each side of the popup span — + // an instant flow lands entirely before the recorder's first frame. + await page.waitForTimeout(500); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + const popup = await popupPromise; + await popup.waitForTimeout(500); + await popup.getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + await page.waitForTimeout(500); + } + + await expect(video.metadata()).resolves.toMatchObject({ + outputs: { rendered: "video-rendered.webm" }, + }); + const frames = await videoFrameSamples(video.outputPaths().rendered); + // The demo app's background is a light gray (~245) throughout, so a + // darkened corner marks a frame where the popup backdrop dim is active. + // The downscale blends the thin dim border with its bright neighbors, so + // dimmed corners read ~211 (overlay up) down to ~147 (exit fade), against + // ~245 when lit. + const dimmedFrames = frames.filter((frame) => frame.corner < 235); + const litFrames = frames.filter((frame) => frame.corner >= 235); + expect(dimmedFrames.length).toBeGreaterThan(0); + expect(litFrames.length).toBeGreaterThan(0); + // While dimmed, the popup's white card sits centered above the backdrop. + const overlayFrames = dimmedFrames.filter((frame) => frame.centerPeak > 220); + expect(overlayFrames.length).toBeGreaterThan(0); +}); + test("records separate videos for the main page and an auth popup", async ({ page: basePage, context, @@ -13,7 +95,7 @@ test("records separate videos for the main page and an auth popup", async ({ const video = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); let popupVideo!: ReturnType; { - await using page = await addPlugins({ page: basePage, testInfo, plugins: [video] }); + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video], popups: false }); await page.goto("https://app.middlewright.test/"); const popupPromise = basePage.waitForEvent("popup"); @@ -46,3 +128,49 @@ test("records separate videos for the main page and an auth popup", async ({ expect((await stat(path)).size).toBeGreaterThan(0); } }); + +const execFile = promisify(execFileCallback); + +/** + * Decode the video to small grayscale frames and sample each one: a pixel + * near the bottom-left corner (page background), and the brightest pixel of + * the central quarter (the popup card when the overlay is up). 0-255. + */ +const videoFrameSamples = async (path: string) => { + const size = 64; + const { stdout } = await execFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-i", + path, + "-vf", + `fps=10,scale=${size}:${size},format=gray`, + "-f", + "rawvideo", + "-pix_fmt", + "gray", + "pipe:1", + ], + { encoding: "buffer", maxBuffer: 64 * 1024 * 1024 }, + ); + const frameSize = size * size; + const frames: { centerPeak: number; corner: number }[] = []; + + for (let offset = 0; offset + frameSize <= stdout.length; offset += frameSize) { + let centerPeak = 0; + for (let y = Math.floor(size * 0.375); y < Math.floor(size * 0.625); y += 1) { + for (let x = Math.floor(size * 0.375); x < Math.floor(size * 0.625); x += 1) { + centerPeak = Math.max(centerPeak, stdout[offset + y * size + x]); + } + } + frames.push({ + centerPeak, + corner: stdout[offset + (size - 4) * size + 3], + }); + } + + return frames; +}; diff --git a/spec/popup.spec.ts b/spec/popup.spec.ts index 8f2fae3..11c1202 100644 --- a/spec/popup.spec.ts +++ b/spec/popup.spec.ts @@ -1,8 +1,122 @@ import { test, expect } from "@playwright/test"; import { addPlugins, videoMode } from "../src/index.ts"; +import type { Plugin } from "../src/index.ts"; import { routeAuthDemoApp } from "./auth-demo-app.ts"; -test("a popup wrapped with addPlugins runs actions through its own plugins", async ({ +test("popups are wrapped automatically", async ({ page: basePage, context }, testInfo) => { + await routeAuthDemoApp(context); + const actions: string[] = []; + // No forPopup hook: the plugin is re-registered as-is on the popup. + const recorder: Plugin = { + name: "action-recorder", + middleware: async (ctx, next) => { + actions.push(`${ctx.method} on ${new URL(ctx.page.url()).host}`); + return next(); + }, + }; + await using page = await addPlugins({ page: basePage, testInfo, plugins: [recorder] }); + await page.goto("https://app.middlewright.test/"); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + const popup = await popupPromise; + + // No addPlugins call for the popup — it's already wrapped. + await popup.getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + + expect(actions).toEqual([ + "click on app.middlewright.test", + "click on auth.middlewright.test", + "waitFor on app.middlewright.test", + ]); +}); + +test("forPopup controls what popups get", async ({ page: basePage, context }, testInfo) => { + await routeAuthDemoApp(context); + const actions: string[] = []; + const record = (label: string) => { + const middleware: Plugin["middleware"] = async (ctx, next) => { + actions.push(`${label}: ${ctx.method} on ${new URL(ctx.page.url()).host}`); + return next(); + }; + return middleware; + }; + const inherited: Plugin = { + name: "inherited", + middleware: record("parent"), + forPopup: () => ({ name: "inherited-child", middleware: record("child") }), + }; + const skipped: Plugin = { + name: "skipped-on-popups", + middleware: record("skipped"), + forPopup: () => null, + }; + await using page = await addPlugins({ page: basePage, testInfo, plugins: [inherited, skipped] }); + await page.goto("https://app.middlewright.test/"); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + await (await popupPromise).getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + + expect(actions).toEqual([ + "parent: click on app.middlewright.test", + "skipped: click on app.middlewright.test", + "child: click on auth.middlewright.test", + "parent: waitFor on app.middlewright.test", + "skipped: waitFor on app.middlewright.test", + ]); +}); + +test("wrapping an already-wrapped page throws", async ({ page: basePage, context }, testInfo) => { + await routeAuthDemoApp(context); + await using page = await addPlugins({ page: basePage, testInfo, plugins: [] }); + + await expect(addPlugins({ page: basePage, testInfo, plugins: [] })).rejects.toThrow( + "already has plugins", + ); + + // Popups are auto-wrapped, so wrapping one manually is also a double wrap. + await page.goto("https://app.middlewright.test/"); + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + const popup = await popupPromise; + await expect(addPlugins({ page: popup, testInfo, plugins: [] })).rejects.toThrow( + "popups: false", + ); +}); + +test("popups: false leaves popups unwrapped", async ({ page: basePage, context }, testInfo) => { + await routeAuthDemoApp(context); + const actions: string[] = []; + const recorder: Plugin = { + name: "action-recorder", + middleware: async (ctx, next) => { + actions.push(`${ctx.method} on ${new URL(ctx.page.url()).host}`); + return next(); + }, + }; + await using page = await addPlugins({ + page: basePage, + testInfo, + plugins: [recorder], + popups: false, + }); + await page.goto("https://app.middlewright.test/"); + + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + await (await popupPromise).getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + + expect(actions).toEqual([ + "click on app.middlewright.test", + "waitFor on app.middlewright.test", + ]); +}); + +test("videoMode records popup actions as a child timeline", async ({ page: basePage, context, }, testInfo) => { @@ -11,6 +125,37 @@ test("a popup wrapped with addPlugins runs actions through its own plugins", asy await using page = await addPlugins({ page: basePage, testInfo, plugins: [video] }); await page.goto("https://app.middlewright.test/"); + const popupPromise = basePage.waitForEvent("popup"); + await page.getByRole("button", { name: "Sign in" }).click(); + await (await popupPromise).getByRole("button", { name: "Approve" }).click(); + await page.getByText("Signed in as mmkal").waitFor(); + + const metadata = await video.metadata(); + expect(metadata).toMatchObject({ + // The main timeline has only the main page's actions... + highlights: [{ method: "click" }, { method: "waitFor" }], + // ...and the popup's actions land on a child timeline of the same clock. + children: [ + { + openedAt: expect.any(Number), + highlights: [{ method: "click" }], + }, + ], + }); + const [child] = metadata.children; + expect(child.openedAt).toBeGreaterThanOrEqual(metadata.highlights[0].start); + expect(child.openedAt).toBeLessThanOrEqual(child.highlights[0].start); +}); + +test("a manually wrapped popup runs actions through its own plugins", async ({ + page: basePage, + context, +}, testInfo) => { + await routeAuthDemoApp(context); + const video = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video], popups: false }); + await page.goto("https://app.middlewright.test/"); + const popupPromise = basePage.waitForEvent("popup"); await page.getByRole("button", { name: "Sign in" }).click(); const popupVideo = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); @@ -41,7 +186,7 @@ test("each videoMode instance still owns its artifacts after the test", async ({ const video = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); let popupVideo!: ReturnType; { - await using page = await addPlugins({ page: basePage, testInfo, plugins: [video] }); + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video], popups: false }); await page.goto("https://app.middlewright.test/"); const popupPromise = basePage.waitForEvent("popup"); @@ -74,7 +219,7 @@ test("reusing one videoMode instance on a popup fails with a clear error", async }, testInfo) => { await routeAuthDemoApp(context); const video = videoMode({ finalHold: 0, highlight: { mode: "outline", duration: 300 }, trimStart: "never" }); - await using page = await addPlugins({ page: basePage, testInfo, plugins: [video] }); + await using page = await addPlugins({ page: basePage, testInfo, plugins: [video], popups: false }); await page.goto("https://app.middlewright.test/"); const popupPromise = basePage.waitForEvent("popup"); diff --git a/spec/video-mode.spec.ts b/spec/video-mode.spec.ts index ba28fa1..b3c1977 100644 --- a/spec/video-mode.spec.ts +++ b/spec/video-mode.spec.ts @@ -927,7 +927,7 @@ test("deadAir runs actions without video highlighting and records metadata", asy await basePage.waitForSelector('#result:has-text("(no style)")'); await expect(page.videoMode.metadata()).resolves.toMatchObject({ outputs: {}, - schemaVersion: 1, + schemaVersion: 2, timebase: "ms", }); expect((await page.videoMode.metadata()).deadAir).toContainEqual( @@ -947,7 +947,7 @@ test("deadAir runs actions without video highlighting and records metadata", asy expect(metadata).toMatchObject({ highlights: [], outputs: {}, - schemaVersion: 1, + schemaVersion: 2, timebase: "ms", }); expect(metadata.deadAir).toContainEqual( diff --git a/src/plugin-system.ts b/src/plugin-system.ts index 2f14da7..fb05a2a 100644 --- a/src/plugin-system.ts +++ b/src/plugin-system.ts @@ -104,6 +104,14 @@ export type PageExtensionContext = { testInfo: TestInfo; }; +export type PopupPluginContext = { + /** The newly opened popup page, not yet wrapped. */ + page: Page; + /** The wrapped page that opened the popup. */ + parentPage: Page; + testInfo: TestInfo; +}; + export type Plugin = { name: string; /** Middleware to wrap locator actions. Called in registration order. */ @@ -112,6 +120,13 @@ export type Plugin = { testLifecycle?: (emitter: Emittery) => void | (() => void); /** Add explicit test controls to the page returned from addPlugins. */ pageExtension?: (ctx: PageExtensionContext) => PageExtension; + /** + * Called when a page wrapped with this plugin opens a popup, to produce the + * plugin registered on the popup — often a fresh instance tied to this one. + * Return null to skip this plugin on popups. Plugins without this hook are + * re-registered as-is (fine for stateless plugins). + */ + forPopup?: (ctx: PopupPluginContext) => Plugin | false | null | undefined; }; const PLUGIN_STATE = Symbol("playwrightPluginState"); @@ -184,9 +199,24 @@ export const addPlugins = async (p page: Page; testInfo: TestInfo; plugins: Plugins; + /** + * Automatically add plugins to popups this page opens (and to their popups, + * recursively). Plugins may define `forPopup` to control or skip what gets + * registered on the popup. Default: true. Pass false to leave popups + * unwrapped — they fall through to original Playwright behavior and can be + * wrapped manually with fresh plugin instances. + */ + popups?: boolean; boxedStackPrefixes?: (defaults: string[]) => string[]; }): Promise>> => { const { page, testInfo, plugins, boxedStackPrefixes } = params; + if (getPluginState(page)) { + throw new Error( + "this page already has plugins added. Popups are auto-wrapped by default - " + + "pass popups: false to the parent addPlugins call for manual control, " + + "and use fresh plugin instances for each page", + ); + } // Patch Locator prototype once globally patchLocatorPrototype(page, boxedStackPrefixes); @@ -228,11 +258,51 @@ export const addPlugins = async (p // Emit beforeTest await state.lifecycleEmitter.emitSerial("beforeTest", { page, testInfo }); + // Auto-wrap popups (default on). The child addPlugins call attaches plugin + // state synchronously in the tick the popup event fires — before test code + // awaiting waitForEvent("popup") gets to act on the popup — because this + // listener is registered ahead of the test's own. + const childWraps: Promise[] = []; + let onPopup: ((popup: Page) => void) | undefined; + if (params.popups !== false) { + onPopup = (popupPage) => { + const childPlugins = plugins + .filter((plugin): plugin is Plugin => !!plugin) + .map((plugin) => + plugin.forPopup + ? plugin.forPopup({ page: popupPage, parentPage: page, testInfo }) + : plugin, + ); + const wrap = addPlugins({ page: popupPage, testInfo, plugins: childPlugins }); + childWraps.push(wrap); + // Failures surface at dispose; avoid unhandled-rejection noise meanwhile. + wrap.catch(() => {}); + }; + page.on("popup", onPopup); + } + // Add async dispose pageWithPlugins[Symbol.asyncDispose] = async () => { + if (onPopup) { + page.off("popup", onPopup); + } + // Children dispose first (newest first) so their plugins can finalize -- + // and, later, feed facts to parent plugins -- before the parent's own + // lifecycle events run. A failed child wrap must not stop the parent + // finalizing; it rethrows below once cleanup is done. + const settledChildren = await Promise.allSettled(childWraps); + for (const result of [...settledChildren].reverse()) { + if (result.status === "fulfilled") { + await result.value[Symbol.asyncDispose](); + } + } await state.lifecycleEmitter.emitSerial("afterTest", { page, testInfo }); await state.lifecycleEmitter.emitSerial("afterTestFinalize", { page, testInfo }); state.lifecycleCleanups.forEach((cleanup) => cleanup()); + const failedChildWrap = settledChildren.find((result) => result.status === "rejected"); + if (failedChildWrap) { + throw failedChildWrap.reason; + } }; return pageWithPlugins; diff --git a/src/plugins/video-mode.ts b/src/plugins/video-mode.ts index 3f44357..cee6302 100644 --- a/src/plugins/video-mode.ts +++ b/src/plugins/video-mode.ts @@ -18,6 +18,7 @@ import { extname, join } from "node:path"; import { promisify } from "node:util"; import type { Dialog, Locator, Page, TestInfo } from "@playwright/test"; import type { + ActionMiddleware, ActionTiming, LocatorWithOriginal, Plugin, @@ -141,6 +142,17 @@ export type VideoModeHighlight = VideoModeSpan & { fillReveal?: VideoModeFillReveal; image?: string; method?: OverrideableMethod; + /** + * Set at render time on projected popup highlights: maps child-frame + * coordinates (which `fillReveal` and its screenshots use) into the + * composite frame — scale plus the overlay's top-left corner. + */ + overlayTransform?: { + scale: number; + viewport: VideoModeViewport; + x: number; + y: number; + }; pan?: VideoModePan; rect: VideoModeRect; sourceFrameAt?: number; @@ -160,11 +172,34 @@ export type VideoModeAddressBar = VideoModeSpan & { url: string; }; +/** + * Recorded facts for a popup opened by the recorded page. Timestamps share + * the parent timeline (ms since the parent instance's timebase); the child's + * raw screencast has its own clock, mapped via `recordingEndedAt`. + */ +export type VideoModeChild = { + /** Parent-timeline ms when the popup closed. Missing: open at render end. */ + closedAt?: number; + highlights: VideoModeHighlight[]; + /** Parent-timeline ms when the popup opened. */ + openedAt: number; + /** Raw screencast artifact for the popup, when video was recorded. */ + raw?: string; + /** + * Parent-timeline ms of the popup recorder's settled endpoint (the raw + * video's last frame). Missing when the popup closed itself — the raw + * video's own end approximates `closedAt` then. + */ + recordingEndedAt?: number; + viewport?: VideoModeViewport; +}; + export type VideoModeMetadata = { - schemaVersion: 1; + schemaVersion: 2; timebase: "ms"; addressBars: VideoModeAddressBar[]; captions: VideoModeCaption[]; + children: VideoModeChild[]; deadAir: VideoModeSpan[]; highlights: VideoModeHighlight[]; outputs: VideoModeOutputs; @@ -268,6 +303,7 @@ type VideoModeState = { /** Distinguishes artifact filenames when a test has several instances. */ artifactSuffix: string; captions: VideoModeCaption[]; + children: VideoModeChild[]; deadAirDepth: number; deadAirSpans: VideoModeSpan[]; highlights: VideoModeHighlight[]; @@ -688,7 +724,7 @@ const translateVideoTimeline = (options: { text: caption.text, })), deadAir: options.deadAir.map((span) => translateVideoSpan(span, options.offsetMs)), - highlights: options.highlights.map((highlight) => { + highlights: options.highlights.map((highlight): VideoModeHighlight => { const start = Math.max(0, Math.round(highlight.start + options.offsetMs)); return { ...highlight, @@ -731,10 +767,14 @@ const metadataFor = (state: VideoModeState): VideoModeMetadata => { .filter((addressBar) => addressBar.end > addressBar.start) .sort((left, right) => left.start - right.start || left.end - right.end), captions: normalizeVideoCaptions(state.captions), + children: state.children.map((child) => ({ + ...child, + highlights: normalizeVideoHighlights(child.highlights), + })), deadAir: mergeVideoSpans(state.deadAirSpans), highlights: normalizeVideoHighlights(state.highlights), outputs: state.outputs, - schemaVersion: 1, + schemaVersion: 2, sourceRange: normalizeSourceRange(state.sourceRange), timebase: "ms", }; @@ -1426,8 +1466,7 @@ const recordFillReveal = async (options: { value.length === 0 || value.length > captureOptions.maxCharacters || style.direction === "rtl" || - !["left", "start"].includes(style.textAlign) || - (element instanceof HTMLInputElement && element.type === "password") + !["left", "start"].includes(style.textAlign) ) { return { ...geometry, kind: "fallback" as const }; } @@ -1534,10 +1573,16 @@ const recordFillReveal = async (options: { return { ...geometry, kind: "fallback" as const }; } context.font = style.font; - const graphemes = Array.from( - new Intl.Segmenter(undefined, { granularity: "grapheme" }).segment(value), - ({ segment }) => segment, - ); + // A password input renders one bullet per character, so measure those + // glyphs — the reveal only ever shows the screenshot's dots, never the + // value. + const masked = element instanceof HTMLInputElement && element.type === "password"; + const graphemes = masked + ? Array.from(value, () => "•") + : Array.from( + new Intl.Segmenter(undefined, { granularity: "grapheme" }).segment(value), + ({ segment }) => segment, + ); const letterSpacing = pixels(style.letterSpacing); const textIndent = pixels(style.textIndent); const revealStops = graphemes.map((_, index) => { @@ -2286,6 +2331,12 @@ const videoPieces = (options: { addressBars: VideoModeAddressBar[]; frameDurationMs: number; highlights: VideoModeHighlight[]; + /** + * Source spans that must reach the output (popup enter/exit animations). + * Overlapping-hold skips normally jump the footage between two highlights; + * a skip is cancelled when it would leap across one of these. + */ + keepSpans: VideoModeSpan[]; preActionStabilizationMs: number; segments: RenderVideoSegment[]; }): VideoPiece[] => { @@ -2333,8 +2384,10 @@ const videoPieces = (options: { const nextHighlight = highlights[highlightIndex + 1]; if (highlight.start > cursor) { - const postAction = previousHighlight?.fillReveal ? previousHighlight : undefined; - const preAction = highlight.fillReveal ? highlight : undefined; + const stabilizable = (candidate: VideoModeHighlight | undefined) => + candidate?.fillReveal && !candidate.overlayTransform ? candidate : undefined; + const postAction = stabilizable(previousHighlight); + const preAction = stabilizable(highlight); // `trim` chooses whole source frames. Round down so the boundary frame // belongs to the stabilized piece instead of leaking from the raw gap. const preActionStart = preAction @@ -2388,7 +2441,13 @@ const videoPieces = (options: { let nextCursor = actionEnd; if (nextHighlight && highlight.end > nextHighlight.start) { - nextCursor = Math.max(nextCursor, nextHighlight.start); + const skipTo = Math.max(nextCursor, nextHighlight.start); + const skipCrossesKeptSpan = options.keepSpans.some( + (span) => span.start < skipTo && span.end > nextCursor, + ); + if (!skipCrossesKeptSpan) { + nextCursor = skipTo; + } } cursor = Math.min(segment.end, nextCursor); @@ -2398,7 +2457,10 @@ const videoPieces = (options: { if (segment.end > cursor) { pieces.push({ end: segment.end, - postAction: previousHighlight?.fillReveal ? previousHighlight : undefined, + postAction: + previousHighlight?.fillReveal && !previousHighlight.overlayTransform + ? previousHighlight + : undefined, speed: segment.speed, start: cursor, }); @@ -2627,8 +2689,20 @@ const highlightCursorPoint = ( highlight: VideoModeHighlight, video: { width: number; height: number }, ) => { - const rect = highlight.fillReveal - ? scaleVideoModeRect(highlight.fillReveal.initialRect, highlight.viewport, video) + // fillReveal rects live in child-frame coordinates on projected popup + // highlights — map them through the overlay transform first. + const transform = highlight.overlayTransform; + const projectedInitialRect = + highlight.fillReveal && transform + ? { + height: highlight.fillReveal.initialRect.height * transform.scale, + width: highlight.fillReveal.initialRect.width * transform.scale, + x: transform.x + highlight.fillReveal.initialRect.x * transform.scale, + y: transform.y + highlight.fillReveal.initialRect.y * transform.scale, + } + : highlight.fillReveal?.initialRect; + const rect = projectedInitialRect + ? scaleVideoModeRect(projectedInitialRect, highlight.viewport, video) : scaleHighlight(highlight, video); return { @@ -2930,6 +3004,7 @@ const renderedVideoFilter = (options: { highlightMode: "outline" | "pointer"; highlightInputs: HighlightInput[]; highlights: VideoModeHighlight[]; + keepSpans: VideoModeSpan[]; preActionStabilizationMs: number; segments: RenderVideoSegment[]; textPointerInput?: PointerInput; @@ -2942,10 +3017,23 @@ const renderedVideoFilter = (options: { addressBars: options.addressBars, frameDurationMs: options.video.frameDurationMs, highlights: options.highlights, + keepSpans: options.keepSpans, preActionStabilizationMs: options.preActionStabilizationMs, segments: options.segments, }); const renderedPieces = renderedVideoPieces(pieces); + if (process.env.MIDDLEWRIGHT_DEBUG_PIECES) { + for (const piece of renderedPieces) { + console.log( + `piece src[${piece.start}-${piece.end}] out[${Math.round(piece.outputStart)}-${Math.round(piece.outputEnd)}] speed=${piece.speed}` + + (piece.highlight ? ` highlight=${piece.highlight.method} hstart=${piece.highlight.start}` : "") + + (piece.addressBar ? " addressBar" : "") + + (piece.highlight?.overlayTransform ? " overlay" : "") + + (piece.highlight?.fillReveal ? " fillReveal" : "") + + (piece.highlight?.image ? ` image=${piece.highlight.image}` : ""), + ); + } + } const targets = cursorTargets({ highlights: options.highlights, pieces: renderedPieces, @@ -3097,6 +3185,188 @@ const renderedVideoFilter = (options: { continue; } + // Fill reveal inside a popup overlay: the base is a frozen composite + // frame from just before the fill (popup risen, field empty, dim and + // parent intact), and the typed reveal is the child screenshot's content + // rect scaled and positioned through the overlay transform. + if (piece.highlight && fillReveal && postFillInput && piece.highlight.overlayTransform) { + const transform = piece.highlight.overlayTransform; + const projectLength = (value: number) => Math.round(value * transform.scale); + const scaledImage = { + height: Math.max(2, projectLength(transform.viewport.height)), + width: Math.max(2, projectLength(transform.viewport.width)), + }; + const contentLocal = { + height: Math.max(1, Math.min(scaledImage.height, projectLength(fillReveal.contentRect.height))), + width: Math.max(1, Math.min(scaledImage.width, projectLength(fillReveal.contentRect.width))), + x: Math.max(0, projectLength(fillReveal.contentRect.x)), + y: Math.max(0, projectLength(fillReveal.contentRect.y)), + }; + const contentAbsolute = { + x: Math.round(transform.x + fillReveal.contentRect.x * transform.scale), + y: Math.round(transform.y + fillReveal.contentRect.y * transform.scale), + }; + const duration = renderedPieceDuration(piece); + const durationSeconds = formatSeconds(duration); + const revealStops = fillReveal.revealStops + .map((stop) => Math.max(1, Math.min(contentLocal.width, projectLength(stop)))) + .filter((stop, stopIndex, stops) => stopIndex === 0 || stop !== stops[stopIndex - 1]); + const revealSteps = fillReveal.revealBands.flatMap((band) => { + const y = Math.max(0, Math.min(contentLocal.height - 1, projectLength(band.y))); + const height = Math.max(1, Math.min(contentLocal.height - y, projectLength(band.height))); + return revealStops.map((width) => ({ height, width, y })); + }); + const target = plan.targets.find( + (candidate) => candidate.highlight === piece.highlight, + ); + const revealEnd = + options.highlightMode === "pointer" + ? Math.max(0, duration - TEXT_CURSOR_POINTER_TAIL_MS) + : duration; + const pointerArrival = target + ? Math.max(0, target.arriveAt - renderedPiece.outputStart) + : 0; + const availableAfterArrival = Math.max(0, revealEnd - pointerArrival); + const preRevealHold = Math.min( + TEXT_CURSOR_HOLD_IDEAL_MS, + availableAfterArrival / 2, + ); + const revealStart = Math.max( + 0, + Math.min(revealEnd, pointerArrival + preRevealHold), + ); + // The base predates the fill, so it shows the field unfocused. The + // post-fill screenshot has the focus ring: overlay the field's ring + // region from it at reveal start, immediately cover its text with the + // pre-fill screenshot's empty content box, and let the reveal bands + // type over that — the ring appears when the cursor lands and the + // letters arrive inside it, continuous with the live footage after. + const ringPaddingPx = 4; + const ringSource = { + height: fillReveal.initialRect.height + 2 * ringPaddingPx, + width: fillReveal.initialRect.width + 2 * ringPaddingPx, + x: fillReveal.initialRect.x - ringPaddingPx, + y: fillReveal.initialRect.y - ringPaddingPx, + }; + const ringLocal = { + height: Math.max(1, Math.min(scaledImage.height, projectLength(ringSource.height))), + width: Math.max(1, Math.min(scaledImage.width, projectLength(ringSource.width))), + x: Math.max(0, projectLength(ringSource.x)), + y: Math.max(0, projectLength(ringSource.y)), + }; + const ringAbsolute = { + x: Math.round(transform.x + ringSource.x * transform.scale), + y: Math.round(transform.y + ringSource.y * transform.scale), + }; + const baseLabel = `fillbase${index}`; + // One frame back only: rewinding further can cross the previous fill's + // completion (wiping its value from the frozen base). The content box + // is covered with the pre-fill empty state from t=0 below, so anchor + // imprecision inside this field can't leak early-typed text either. + const freezeStart = Math.max(0, piece.start - options.video.frameDurationMs); + filters.push( + [ + `[0:v]trim=start=${formatSeconds(freezeStart)}:end=${formatSeconds( + freezeStart + options.video.frameDurationMs, + )}`, + "setpts=PTS-STARTPTS", + `tpad=stop_mode=clone:stop_duration=${formatSeconds( + Math.max(0, duration - options.video.frameDurationMs), + )}`, + `trim=start=0:end=${durationSeconds}`, + `setpts=PTS-STARTPTS[${baseLabel}]`, + ].join(","), + ); + + if (revealSteps.length === 0) { + filters.push(`[${baseLabel}]null[${label}]`); + continue; + } + + const splitLabels = revealSteps.map((_, stepIndex) => `fillpost${index}x${stepIndex}`); + filters.push( + [ + `[${postFillInput.inputIndex}:v]scale=w=${scaledImage.width}:h=${scaledImage.height}`, + `crop=w=${contentLocal.width}:h=${contentLocal.height}:x=${contentLocal.x}:y=${contentLocal.y}`, + `trim=start=0:end=${durationSeconds}`, + "setpts=PTS-STARTPTS", + `split=${revealSteps.length}${splitLabels.map((splitLabel) => `[${splitLabel}]`).join("")}`, + ].join(","), + ); + + let composedLabel = baseLabel; + if (preFillInput) { + const ringLabel = `fillring${index}`; + const emptyLabel = `fillempty${index}`; + const revealStartEnable = `enable='gte(t\\,${formatSeconds(revealStart)})'`; + filters.push( + [ + `[${postFillInput.inputIndex}:v]scale=w=${scaledImage.width}:h=${scaledImage.height}`, + `crop=w=${ringLocal.width}:h=${ringLocal.height}:x=${ringLocal.x}:y=${ringLocal.y}`, + `trim=start=0:end=${durationSeconds}`, + `setpts=PTS-STARTPTS[${ringLabel}]`, + ].join(","), + ); + filters.push( + [ + `[${preFillInput.inputIndex}:v]scale=w=${scaledImage.width}:h=${scaledImage.height}`, + `crop=w=${contentLocal.width}:h=${contentLocal.height}:x=${contentLocal.x}:y=${contentLocal.y}`, + `trim=start=0:end=${durationSeconds}`, + `setpts=PTS-STARTPTS[${emptyLabel}]`, + ].join(","), + ); + filters.push( + [ + `[${composedLabel}][${ringLabel}]overlay=x=${ringAbsolute.x}`, + `y=${ringAbsolute.y}`, + revealStartEnable, + `shortest=1[fillringcomposed${index}]`, + ].join(":"), + ); + filters.push( + [ + `[fillringcomposed${index}][${emptyLabel}]overlay=x=${contentAbsolute.x}`, + `y=${contentAbsolute.y}`, + `shortest=1[fillemptycomposed${index}]`, + ].join(":"), + ); + composedLabel = `fillemptycomposed${index}`; + } + for (let stepIndex = 0; stepIndex < revealSteps.length; stepIndex += 1) { + const step = revealSteps[stepIndex]; + const cropLabel = `fillcrop${index}x${stepIndex}`; + const nextLabel = `fillcomposed${index}x${stepIndex}`; + const showAt = + revealStart + + ((revealEnd - revealStart) * (stepIndex + 1)) / + (revealSteps.length + 1); + filters.push( + `${[ + `[${splitLabels[stepIndex]}]crop=w=${step.width}`, + `h=${step.height}`, + "x=0", + `y=${step.y}`, + ].join(":")}[${cropLabel}]`, + ); + filters.push( + [ + `[${composedLabel}][${cropLabel}]overlay=x=${contentAbsolute.x}`, + `y=${contentAbsolute.y + step.y}`, + `enable='gte(t\\,${formatSeconds(showAt)})'`, + `shortest=1[${nextLabel}]`, + ].join(":"), + ); + composedLabel = nextLabel; + } + + filters.push( + options.highlightMode === "outline" + ? `[${composedLabel}]${drawboxFilter(piece.highlight, options.video)}[${label}]` + : `[${composedLabel}]null[${label}]`, + ); + continue; + } + if (piece.highlight && fillReveal && preFillInput && postFillInput) { const scaledViewport = scaledViewportSize(piece.highlight.viewport, options.video); const contentRect = scaleVideoModeRect( @@ -3873,6 +4143,8 @@ const renderVideo = async (options: { highlightMode: "outline" | "pointer"; highlights: VideoModeHighlight[]; inputPath: string; + /** Source spans that must not be skipped (popup enter/exit animations). */ + keepSpans: VideoModeSpan[]; outputDir: string; outputPath: string; deadAir: VideoModeSpan[]; @@ -3981,6 +4253,7 @@ const renderVideo = async (options: { addressBars: options.addressBars, frameDurationMs: info.frameDurationMs, highlights: options.highlights, + keepSpans: options.keepSpans, preActionStabilizationMs, segments, }), @@ -4034,6 +4307,7 @@ const renderVideo = async (options: { highlightMode: options.highlightMode, highlightInputs, highlights: options.highlights, + keepSpans: options.keepSpans, preActionStabilizationMs, segments, textPointerInput, @@ -4076,6 +4350,392 @@ const renderVideo = async (options: { return true; }; +const VIDEO_MODE_COMPOSITE_FILE = "video-composite.webm"; +// Playwright extends a closed page's final screencast frame by the time since +// that frame arrived, with this minimum (see VIDEO_MODE_RECORDER_SETTLE_MS). +const RECORDER_FINAL_FRAME_MIN_PADDING_MS = 1000; +const CHILD_OVERLAY_MAX_FRACTION = 0.9; +const CHILD_OVERLAY_BACKDROP_OPACITY = 0.4; +const CHILD_OVERLAY_FADE_MS = 300; + +/** A popup screencast placed on the parent timeline as a scaled overlay. */ +type VideoModeChildLayer = { + child: VideoModeChild; + /** Composite-time close (ms) — where the exit fade starts. */ + closeMs: number; + /** Shift applied to child-raw frames to land them in composite time (ms). */ + delayMs: number; + /** Overlay visibility window in composite time (ms), including exit fade. */ + enableFromMs: number; + enableToMs: number; + /** Scaled placement in composite pixels, centered and even-sized. */ + height: number; + width: number; + x: number; + y: number; + path: string; + rawInfo: VideoInfo; +}; + +const childCompositeLayers = async (options: { + children: VideoModeChild[]; + outputDir: string; + timelineOffsetMs: number; + video: VideoInfo; +}): Promise => { + const layers: VideoModeChildLayer[] = []; + + for (const child of options.children) { + if (!child.raw) continue; + const path = join(options.outputDir, child.raw); + const rawInfo = await videoInfo(path); + // A settled recorder maps the raw end to a known parent time. A popup + // that closed itself has no settled endpoint — and its screencast t=0 is + // the first *captured* frame, which lags the popup event by the initial + // paint. Playwright pads the final frame by >=1s on close, so the first + // frame lands near closedAt + padding - duration; openedAt is the floor. + const childOffsetMs = + child.recordingEndedAt === undefined + ? Math.max( + child.openedAt, + (child.closedAt === undefined ? child.openedAt : child.closedAt) + + RECORDER_FINAL_FRAME_MIN_PADDING_MS - + rawInfo.durationMs, + ) + : child.recordingEndedAt - rawInfo.durationMs; + const scale = Math.min( + 1, + (CHILD_OVERLAY_MAX_FRACTION * options.video.width) / rawInfo.width, + (CHILD_OVERLAY_MAX_FRACTION * options.video.height) / rawInfo.height, + ); + const width = Math.max(2, 2 * Math.round((rawInfo.width * scale) / 2)); + const height = Math.max(2, 2 * Math.round((rawInfo.height * scale) / 2)); + const enableFromMs = Math.max(0, child.openedAt + options.timelineOffsetMs); + const closeMs = Math.min( + options.video.durationMs, + (child.closedAt === undefined ? child.openedAt + rawInfo.durationMs : child.closedAt) + + options.timelineOffsetMs, + ); + // The exit fade runs AFTER close (the screencast's padded final frame + // supplies footage): a popup that closes itself right after a click would + // otherwise put that click — and its hold's freeze frame — mid-fade. + const enableToMs = Math.min(options.video.durationMs, closeMs + CHILD_OVERLAY_FADE_MS); + + if (closeMs <= enableFromMs) continue; + + layers.push({ + child, + closeMs, + delayMs: childOffsetMs + options.timelineOffsetMs, + enableFromMs, + enableToMs, + height, + path, + rawInfo, + width, + x: Math.round((options.video.width - width) / 2), + y: Math.round((options.video.height - height) / 2), + }); + } + + return layers; +}; + +/** + * Pass A of the popup composite: overlay each popup screencast onto the + * parent's raw footage — dimmed backdrop, scaled to fit, alpha-faded in and + * out, windowed to the popup's open/close span, newest stacked on top. The + * output shares the parent raw timeline exactly, so the annotation render + * (pass B) runs on it unchanged; holds there freeze the composite, so it + * never matters which source triggered them. + */ +const compositeChildOverlays = async (options: { + /** Continuous frame rate for the overlay chains (see fps note below). */ + fps: number; + inputPath: string; + layers: VideoModeChildLayer[]; + outputPath: string; +}) => { + const filters: string[] = []; + // The parent screencast is as sparse as the child ones (a static page emits + // no frames), and overlay only emits output at primary-input frame times — + // without resampling, the whole popup window can contain zero composite + // frames. A continuous base gives every enable window frames to land on. + filters.push(`[0:v]fps=${formatFilterNumber(options.fps)}[base]`); + let currentLabel = "base"; + + options.layers.forEach((layer, index) => { + const from = formatSeconds(layer.enableFromMs); + const to = formatSeconds(layer.enableToMs); + const enable = `enable='between(t\\,${from}\\,${to})'`; + const dimLabel = `dim${index}`; + const childLabel = `popup${index}`; + const outLabel = `composite${index}`; + const fadeOutStartMs = layer.closeMs; + + filters.push( + `[${currentLabel}]drawbox=x=0:y=0:w=iw:h=ih:color=black@${CHILD_OVERLAY_BACKDROP_OPACITY}:t=fill:${enable}[${dimLabel}]`, + ); + filters.push( + [ + `[${index + 1}:v]setpts=PTS+${formatSeconds(layer.delayMs)}/TB`, + // A mostly-static popup screencast has sparse frames; without + // resampling, the frame that happens to pass `fade` mid-ramp keeps + // its partial alpha while framesync repeats it for the whole window, + // ghosting the overlay. Continuous frames give fade real timestamps. + `fps=${formatFilterNumber(options.fps)}`, + `scale=w=${layer.width}:h=${layer.height}`, + "format=yuva420p", + `fade=t=in:st=${from}:d=${formatSeconds(CHILD_OVERLAY_FADE_MS)}:alpha=1`, + `fade=t=out:st=${formatSeconds(fadeOutStartMs)}:d=${formatSeconds(CHILD_OVERLAY_FADE_MS)}:alpha=1[${childLabel}]`, + ].join(","), + ); + // Slide up from the bottom edge on enter (cubic ease-out), slide back + // down after close (cubic ease-in), resting at the centered position + // between. Times are seconds; commas escaped for the filter graph. + const fadeSeconds = formatSeconds(CHILD_OVERLAY_FADE_MS); + const offscreenY = "H"; + const enterProgress = `min(max((t-${from})/${fadeSeconds}\\,0)\\,1)`; + const exitProgress = `min(max((t-${formatSeconds(layer.closeMs)})/${fadeSeconds}\\,0)\\,1)`; + const slideY = [ + `if(lt(t\\,${formatSeconds(layer.enableFromMs + CHILD_OVERLAY_FADE_MS)})`, + `\\,${layer.y}+(${offscreenY}-${layer.y})*pow(1-${enterProgress}\\,2)`, + `\\,if(gt(t\\,${formatSeconds(layer.closeMs)})`, + `\\,${layer.y}+(${offscreenY}-${layer.y})*pow(${exitProgress}\\,2)`, + `\\,${layer.y}))`, + ].join(""); + filters.push( + `[${dimLabel}][${childLabel}]overlay=x=${layer.x}:y='${slideY}':eval=frame:eof_action=pass:${enable}[${outLabel}]`, + ); + currentLabel = outLabel; + }); + + await execFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + options.inputPath, + ...options.layers.flatMap((layer) => ["-i", layer.path]), + "-filter_complex", + filters.join(";"), + "-map", + `[${currentLabel}]`, + "-an", + options.outputPath, + ], + { maxBuffer: 10 * 1024 * 1024 }, + ); +}; + +/** + * Project a popup highlight into composite coordinates. The child frame is + * scaled into an overlay box on the parent frame, so rects become + * parent-frame pixels; child-frame pixel treatments (pans, fill reveals, + * screenshot stills) drop away, leaving the plain box/pointer/freeze path. + */ +const projectChildHighlight = (options: { + highlight: VideoModeHighlight; + layer: VideoModeChildLayer; + video: VideoInfo; +}): VideoModeHighlight => { + const { highlight, layer } = options; + const viewport = options.layer.child.viewport || { + height: layer.rawInfo.height, + width: layer.rawInfo.width, + }; + const viewportToChildPixels = Math.min( + layer.rawInfo.width / viewport.width, + layer.rawInfo.height / viewport.height, + ); + const scale = viewportToChildPixels * (layer.width / layer.rawInfo.width); + // Holds clone the composite frame at actionEnd. The overlay stays fully + // visible until close (the exit animation runs after), so the freeze only + // needs two small guards: never sample past close, and keep the source + // slice wider than a frame tick so the trim can't come up empty (an + // instant click's slice is a few ms — often between ticks). Widening may + // nudge the highlight a few ms earlier; order-preserving in practice. + const closeFloorMs = Math.floor(layer.closeMs); + const minSliceMs = options.video.frameDurationMs + 10; + let { actionEnd, end, start } = highlight; + if (actionEnd !== undefined) { + actionEnd = Math.min(actionEnd, closeFloorMs); + if (actionEnd - start < minSliceMs) { + actionEnd = Math.min(closeFloorMs, start + minSliceMs); + } + if (actionEnd - start < minSliceMs) { + const shift = minSliceMs - (actionEnd - start); + start -= shift; + end -= shift; + } + } + + return { + ...highlight, + actionEnd, + end, + start, + dialog: undefined, + // Fill reveals keep their child-frame geometry and screenshots; the + // render projects them through overlayTransform. Screenshot stills + // without a reveal (e.g. password fallbacks) would render full-frame, so + // they drop away and the hold freezes the composite instead. + fillReveal: highlight.fillReveal, + image: highlight.fillReveal ? highlight.image : undefined, + overlayTransform: { + scale, + viewport, + x: layer.x, + y: layer.y, + }, + pan: undefined, + rect: { + height: highlight.rect.height * scale, + width: highlight.rect.width * scale, + x: layer.x + highlight.rect.x * scale, + y: layer.y + highlight.rect.y * scale, + }, + sourceFrameAt: undefined, + viewport: { height: options.video.height, width: options.video.width }, + }; +}; + +/** + * The action-recording middleware, shared between a videoMode instance (which + * records onto its own state) and its popup child recorders (which record onto + * a child state that shares the parent's clock and dead-air spans). + */ +const videoModeActionMiddleware = (options: { + /** Gates recording on the owning instance's deadAir() nesting. */ + deadAirState: VideoModeState; + highlight: ResolvedVideoModeHighlight; + /** Parent-only hook, used for first-locator trim start. */ + onBeforeRecording?: (timing: ActionTiming) => void; + /** Where highlights, dead air, and images are recorded. */ + recordingState: VideoModeState; + skipMethods: OverrideableMethod[]; + skipStackFrames: string[]; +}): ActionMiddleware => { + const { deadAirState, highlight, recordingState: state, skipMethods, skipStackFrames } = options; + + return async ({ args, locator, method, testInfo, timing }, next) => { + if (deadAirState.deadAirDepth > 0) return next(); + + // Skip if called from internal helpers (navigation, login flows etc) + if (skipStackFrames.length > 0) { + const stack = new Error().stack || ""; + if (skipStackFrames.some((frame) => stack.includes(frame))) return next(); + } + + options.onBeforeRecording?.(timing); + + if (method === "waitFor") { + let result: unknown; + try { + result = await next(); + } finally { + recordActionElapsedDeadAirFromTiming(state, timing, { minimumMs: 0 }); + } + + if (highlight.mode !== "off" && !skipMethods.includes(method)) { + await recordHighlight({ + pan: "return", + color: highlight.color, + durationMs: highlight.durationMs, + locator, + method, + requireVisible: true, + startAfterScreenshot: true, + state, + testInfo, + thickness: highlight.thickness, + }); + } + + return result; + } + + recordMiddlewareWaitBeforeVideoMode(state, timing); + + if (skipMethods.includes(method)) { + try { + return await next(); + } finally { + if (timing.attachedAtStart) { + recordActionElapsedDeadAirFromTiming(state, timing, { minimumMs: 50 }); + } + recordAttachedWaitFromTiming(state, timing); + } + } + + const recordedHighlight = + highlight.mode === "off" + ? undefined + : await recordHighlight({ + pan: method === "fill" ? "off" : "stay", + color: highlight.color, + durationMs: highlight.durationMs, + locator, + method, + requireVisible: false, + startAfterScreenshot: false, + state, + testInfo, + thickness: highlight.thickness, + }); + + try { + const result = await next(); + if ( + recordedHighlight && + method === "fill" && + typeof args[0] === "string" && + args[0].length > 0 + ) { + await recordFillReveal({ + highlight: recordedHighlight, + locator, + state, + testInfo, + }); + } + if (recordedHighlight && state.startedAt !== undefined) { + recordedHighlight.actionEnd = Math.max( + recordedHighlight.start, + Math.round(performance.now() - state.startedAt), + ); + } + if (recordedHighlight?.pan) { + await finalizePanHighlightAfterAction({ + highlight: recordedHighlight, + locator, + state, + }); + } + return result; + } finally { + if (recordedHighlight && state.startedAt !== undefined) { + recordedHighlight.actionEnd = + recordedHighlight.actionEnd || + Math.max( + recordedHighlight.start, + Math.round(performance.now() - state.startedAt), + ); + } + if ( + !recordedHighlight && + (timing.attachedAtStart || timing.attachedAt === undefined) + ) { + recordActionElapsedDeadAirFromTiming(state, timing, { minimumMs: 50 }); + } + recordAttachedWaitFromTiming(state, timing); + } + }; +}; + /** Records video-mode facts and renders annotations into the recorded video. */ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => { if (process.env.PWDEBUG) { @@ -4091,10 +4751,11 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => { metadata: async () => ({ addressBars: [], captions: [], + children: [], deadAir: [], highlights: [], outputs: {}, - schemaVersion: 1, + schemaVersion: 2, sourceRange: {}, timebase: "ms", }), @@ -4113,6 +4774,7 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => { return { ...controls, name: "video-mode", + forPopup: () => null, pageExtension: ({ testInfo }) => { testInfoForOutputPaths = testInfo; return { videoMode: controls }; @@ -4136,6 +4798,7 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => { addressBars: [], artifactSuffix: "", captions: [], + children: [], deadAirDepth: 0, deadAirSpans: [], highlightImageIndex: 0, @@ -4197,9 +4860,104 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => { }, }; + /** + * A popup recorder bound to this instance: it records facts (highlights on + * the parent clock, open/close spans, the popup's raw screencast) into + * `state.children` and renders nothing itself. Grandchild popups recurse — + * every popup lands flat in `state.children`. Popup dialogs are not + * annotated yet (tasks/popup-overlay-video.md). + */ + const createPopupRecorder = (popupPage: Page): Plugin | null => { + if (state.startedAt === undefined) { + return null; + } + + const child: VideoModeChild = { + highlights: [], + openedAt: getVideoTimestamp(), + viewport: popupPage.viewportSize() || undefined, + }; + state.children.push(child); + const childIndex = state.children.length; + // The child records onto the parent clock: highlight times land directly + // on the parent timeline, waits merge into the parent's dead air (a span + // is dead only if no source is active), and images get a child suffix. + const childRecordingState: VideoModeState = { + addressBars: [], + artifactSuffix: `${state.artifactSuffix}-popup-${childIndex}`, + captions: [], + children: [], + deadAirDepth: 0, + deadAirSpans: state.deadAirSpans, + highlightImageIndex: 0, + highlights: child.highlights, + outputs: {}, + sourceRange: {}, + startedAt: state.startedAt, + }; + + return { + // Named video-mode so middleware wait-timing lookups match. + name: "video-mode", + forPopup: ({ page: grandchildPage }) => createPopupRecorder(grandchildPage), + middleware: videoModeActionMiddleware({ + deadAirState: state, + highlight, + recordingState: childRecordingState, + skipMethods, + skipStackFrames, + }), + testLifecycle: (emitter) => { + const onClose = () => { + if (child.closedAt === undefined) { + child.closedAt = getVideoTimestamp(); + } + }; + popupPage.on("close", onClose); + const offAfterTestFinalize = emitter.on( + "afterTestFinalize", + async ({ page, testInfo }) => { + child.viewport = child.viewport || page.viewportSize() || undefined; + const video = page.video(); + if (!page.isClosed()) { + if (video) { + await settleVideoRecorder(page); + } + const closeStartedAt = performance.now(); + await page.close({ runBeforeUnload: false }); + const closeEndedAt = performance.now(); + if (video && state.startedAt !== undefined) { + child.recordingEndedAt = Math.round( + (closeStartedAt + closeEndedAt) / 2 - state.startedAt, + ); + } + } + onClose(); + if (video) { + const raw = suffixArtifactFileName( + VIDEO_MODE_RAW_FILE, + childRecordingState.artifactSuffix, + ); + await mkdir(testInfo.outputDir, { recursive: true }); + const recordedVideoPath = await video.path(); + await waitForNonEmptyFile(recordedVideoPath); + await copyFile(recordedVideoPath, join(testInfo.outputDir, raw)); + child.raw = raw; + } + }, + ); + return () => { + offAfterTestFinalize(); + popupPage.off("close", onClose); + }; + }, + }; + }; + return { ...controls, name: "video-mode", + forPopup: ({ page: popupPage }) => createPopupRecorder(popupPage), pageExtension: ({ testInfo }) => { if (activePage) { throw new Error( @@ -4214,125 +4972,24 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => { return { videoMode: controls }; }, - middleware: async ({ args, locator, method, testInfo, timing }, next) => { - if (state.deadAirDepth > 0) return next(); - - // Skip if called from internal helpers (navigation, login flows etc) - if (skipStackFrames.length > 0) { - const stack = new Error().stack || ""; - if (skipStackFrames.some((frame) => stack.includes(frame))) return next(); - } - - if ( - trimStart.firstLocator && - state.sourceRange.start === undefined && - state.startedAt !== undefined - ) { - controls.setStartTime(Math.max(0, Math.round(timing.actionStartedAt - state.startedAt))); - } - - if (method === "waitFor") { - let result: unknown; - try { - result = await next(); - } finally { - recordActionElapsedDeadAirFromTiming(state, timing, { minimumMs: 0 }); - } - - if (highlight.mode !== "off" && !skipMethods.includes(method)) { - await recordHighlight({ - pan: "return", - color: highlight.color, - durationMs: highlight.durationMs, - locator, - method, - requireVisible: true, - startAfterScreenshot: true, - state, - testInfo, - thickness: highlight.thickness, - }); - } - - return result; - } - - recordMiddlewareWaitBeforeVideoMode(state, timing); - - if (skipMethods.includes(method)) { - try { - return await next(); - } finally { - if (timing.attachedAtStart) { - recordActionElapsedDeadAirFromTiming(state, timing, { minimumMs: 50 }); - } - recordAttachedWaitFromTiming(state, timing); - } - } - - const recordedHighlight = - highlight.mode === "off" - ? undefined - : await recordHighlight({ - pan: method === "fill" ? "off" : "stay", - color: highlight.color, - durationMs: highlight.durationMs, - locator, - method, - requireVisible: false, - startAfterScreenshot: false, - state, - testInfo, - thickness: highlight.thickness, - }); - - try { - const result = await next(); + middleware: videoModeActionMiddleware({ + deadAirState: state, + highlight, + onBeforeRecording: (timing) => { if ( - recordedHighlight && - method === "fill" && - typeof args[0] === "string" && - args[0].length > 0 + trimStart.firstLocator && + state.sourceRange.start === undefined && + state.startedAt !== undefined ) { - await recordFillReveal({ - highlight: recordedHighlight, - locator, - state, - testInfo, - }); - } - if (recordedHighlight && state.startedAt !== undefined) { - recordedHighlight.actionEnd = Math.max( - recordedHighlight.start, - Math.round(performance.now() - state.startedAt), + controls.setStartTime( + Math.max(0, Math.round(timing.actionStartedAt - state.startedAt)), ); } - if (recordedHighlight?.pan) { - await finalizePanHighlightAfterAction({ - highlight: recordedHighlight, - locator, - state, - }); - } - return result; - } finally { - if (recordedHighlight && state.startedAt !== undefined) { - recordedHighlight.actionEnd = - recordedHighlight.actionEnd || - Math.max( - recordedHighlight.start, - Math.round(performance.now() - state.startedAt), - ); - } - if ( - !recordedHighlight && - (timing.attachedAtStart || timing.attachedAt === undefined) - ) { - recordActionElapsedDeadAirFromTiming(state, timing, { minimumMs: 50 }); - } - recordAttachedWaitFromTiming(state, timing); - } - }, + }, + recordingState: state, + skipMethods, + skipStackFrames, + }), testLifecycle: (emitter) => { let addressBarOriginalGoto: Page["goto"] | undefined; @@ -4346,6 +5003,7 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => { dialogHighlightQueue = Promise.resolve(); state.addressBars = []; state.captions = []; + state.children = []; state.deadAirDepth = 0; state.deadAirSpans = []; state.highlightImageIndex = 0; @@ -4597,6 +5255,77 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => { highlights, offsetMs: timelineOffset, }); + + // Popup composite (pass A): overlay each popup's screencast onto the + // raw footage, then annotate that composite instead of the raw. The + // composite shares the raw timeline, so nothing downstream changes. + let renderInputPath = paths.raw; + // Popup enter/exit animations must reach the output even when a + // hold's overlap-skip would jump across them. + const renderKeepSpans: VideoModeSpan[] = []; + const childLayers = await childCompositeLayers({ + children: metadataBeforeVideo.children, + outputDir: testInfo.outputDir, + timelineOffsetMs: timelineOffset, + video: rawVideoInfo, + }); + if (childLayers.length > 0) { + const compositePath = join( + testInfo.outputDir, + suffixArtifactFileName(VIDEO_MODE_COMPOSITE_FILE, state.artifactSuffix), + ); + await compositeChildOverlays({ + fps: 1000 / rawVideoInfo.frameDurationMs, + inputPath: paths.raw, + layers: childLayers, + outputPath: compositePath, + }); + renderInputPath = compositePath; + for (const layer of childLayers) { + renderKeepSpans.push( + { end: layer.enableFromMs + CHILD_OVERLAY_FADE_MS, start: layer.enableFromMs }, + { end: layer.enableToMs, start: layer.closeMs }, + ); + } + // A parent action right after a popup closes (waiting for the + // signed-in state, say) would hold a freeze frame from inside the + // overlay's exit animation — a ghost popup flashing back after it + // disappeared. Shift such highlights past the fade window. + for (const parentHighlight of renderTimeline.highlights) { + for (const layer of childLayers) { + const fadeEndMs = layer.enableToMs + rawVideoInfo.frameDurationMs; + if ( + parentHighlight.start >= layer.closeMs - rawVideoInfo.frameDurationMs && + parentHighlight.start < fadeEndMs + ) { + const shift = fadeEndMs - parentHighlight.start; + parentHighlight.start += shift; + parentHighlight.end += shift; + if (parentHighlight.actionEnd !== undefined) { + parentHighlight.actionEnd += shift; + } + if (parentHighlight.sourceFrameAt !== undefined) { + parentHighlight.sourceFrameAt += shift; + } + } + } + } + const projectedChildHighlights = childLayers.flatMap((layer) => + translateVideoTimeline({ + addressBars: [], + captions: [], + deadAir: [], + highlights: layer.child.highlights, + offsetMs: timelineOffset, + }).highlights.map((highlight) => + projectChildHighlight({ highlight, layer, video: rawVideoInfo }), + ), + ); + renderTimeline.highlights.push(...projectedChildHighlights); + renderTimeline.highlights.sort( + (left, right) => left.start - right.start || left.end - right.end, + ); + } // A selector-driven trim start resolves over the protocol and can // land a few milliseconds after a highlight recorded at effectively // the same moment. The race must not drop that highlight, so a start @@ -4678,7 +5407,8 @@ export const videoMode = (options: VideoModeOptions = {}): VideoModePlugin => { finalHoldMs: finalHold, highlightMode: highlight.mode === "pointer" ? "pointer" : "outline", highlights: renderTimeline.highlights, - inputPath: paths.raw, + inputPath: renderInputPath, + keepSpans: renderKeepSpans, outputDir: testInfo.outputDir, outputPath: paths.rendered, sourceRange, diff --git a/tasks/complete/2026-08-13-popup-overlay-video.md b/tasks/complete/2026-08-13-popup-overlay-video.md new file mode 100644 index 0000000..e759ec4 --- /dev/null +++ b/tasks/complete/2026-08-13-popup-overlay-video.md @@ -0,0 +1,60 @@ +# Popup overlay videos + auto-wrapped popup plugins + +--- +status: done +size: large +branch: popup-overlay +base: popup-plugins (PR #32) +pr: https://github.com/iterate/middlewright/pull/33 +--- + +**Status summary**: done. Popups auto-wrap by default (plugin-system `forPopup` hook, `popups: false` opt-out, double-wrap error). Video mode records popups as child timelines and renders them as a dimmed 90%-fit overlay in ONE composed video via a two-pass render (composite pass, then the untouched annotation pass). Permanent demo spec + README updated. Deferred: popup dialog annotations; unified-piece holds inside the overlay use plain freezes (child pans/fill-reveals degrade to box highlights). + +## Checklist + +- [x] Phase 1: plugin-system guard + auto-wrap _(`popups: false`, `forPopup`, double-wrap error; specs migrated)_ +- [x] Phase 2: videoMode child recorder — facts + metadata `children` schema _(v2; parent-clock child highlights, raw screencast copy, close calibration)_ +- [x] Phase 3: render integration _(two-pass: composite with dim+fade+fps-resample, then existing piece machinery; projected child highlights; hold slices anchored clear of close)_ +- [x] Phase 4: docs + permanent demo spec (`spec/popup-overlay-demo.spec.ts`) + refreshed PR video + +## Implementation notes (what differed from the plan) + +- Decision 6's "unified piece timeline" landed as a **two-pass render**: pass A composites popup screencasts onto the parent raw footage (same timeline), pass B is the existing single-source piece/hold/cursor machinery over the composite. Holds freeze the composite as decided; child highlights project into composite coordinates as plain box/pointer highlights (child pans/fill-reveals/stills degrade — upgrade path stays open). +- Both ffmpeg chains need `fps` resampling: static pages emit sparse screencast frames, which ghosted fades and left the popup window without composite frames. +- Exit fade runs AFTER close (screencast's padded final frame supplies footage) — a self-closing popup otherwise puts its own click mid-fade. +- Deferred: popup dialog annotations, scale-zoom enter animation (alpha fade only). + +## Context + +PR #32 (branch `popup-plugins`) made popups wrappable: per-instance videoMode artifact namespacing, so a popup's own instance renders a separate `-2` video. This design goes further: popups get wrapped *automatically*, and video mode renders the popup's screencast as a scaled overlay on the main page's video — one composed output, since rendering happens in post anyway. + +## Decisions + +1. **Per-instance artifact namespacing** — shipped in PR #32. Multiple `videoMode()` instances per test auto-index their artifacts; first instance keeps legacy names. +2. **Reuse guard** — shipped in PR #32. Wiring one active `videoMode()` instance to a second page throws; one instance per page. +3. **Separate videos per page is the current model** — shipped in PR #32; overlay composition builds on top (Playwright screencasts each page separately regardless). +4. **Parent-owned composition** — popup recording is a child sub-timeline of the main page's `videoMode`; only the parent renders, producing one video with the popup overlaid. Manual standalone instances (`popups: false` + fresh instance) keep separate `-2` videos. +5. **Auto-wrap is default-ON at plugin-system level** — wrapped pages listen for `"popup"` and auto-wrap it; plugins may declare `forPopup(ctx)` to produce their child (videoMode returns a parent-bound child recorder), stateless plugins re-register as-is. Opt out with `popups: false`. Parent dispose disposes children. +6. **Unified piece timeline** — child highlights/holds are first-class pieces in the parent's render plan. Complexity tamed: (i) ONE output stream — holds/freezes freeze the composite, so the triggering source never matters; (ii) pieces/highlights carry a source tag (parent | child N), child coordinates project through the overlay transform, planner stays single-timeline; (iii) the "parent quiet while popup visible" assumption is not load-bearing — simultaneous actions degrade gracefully. One cursor, planned globally, glides between page and overlay. +7. **Overlay presentation** — appears at popup creation, disappears at close (or render end); ~200ms scale+fade enter/exit (filter-level, no synthetic time; hard cut fallback); centered, fit to **90%** of the parent frame preserving popup aspect; parent footage live underneath, dimmed ~40%; multiple/nested popups stack newest-on-top. +8. **Double-wrap throws; no `.original()`** — `addPlugins` on an already-wrapped page errors (also guards accidental double-wrap of main pages). `popups: false` is the manual-control path. PR #32's manual-wrap specs become the `popups: false` coverage. +9. **Permanent demo spec** — commit the popup demo as `spec/popup-overlay-demo.spec.ts` with light assertions (rendered output exists; metadata contains a child with open/close span). Precedent: `scroll-pan-demo.spec.ts`. + +## Settled by recommendation + +- **Clock alignment**: every instance anchors to process-wide `performance.now()`; each source's raw video is calibrated to that clock at its close (existing `settleVideoRecorder` + close-midpoint technique, applied per source). Child→parent time mapping is arithmetic. +- **Dead-air**: unified — a span is dead air only if *no* source has activity; child recorders have no standalone compression (one timeline by construction). +- **Captions**: parent renders unified `test.step` captions full-frame; child recorders don't observe steps separately. +- **Edge cases**: popup closed mid-action → child sub-timeline ends (overlay exits); popup open at test end → child settled+closed during parent finalize, before the parent's own close; screencasts for late-created pages come free from context-level `video: "on"`. +- **Metadata**: `video-mode.json` gains `children: [{ openedAt, closedAt, viewport, highlights, deadAir, ... }]` mirroring the top-level shape, plus a `source` tag on rendered pieces. Schema version bump. + +## Implementation plan (phases, each a PR-able chunk on top of `popup-plugins`) + +1. **Plugin-system: guard + auto-wrap.** Throw on double-wrap. `popups: false` option. On `"popup"`: build child plugin list (`forPopup(ctx)` hook, else reuse), wrap the popup, register child disposal under the parent's dispose. Specs: auto-wrap applies middleware to popups; opt-out; double-wrap error; PR #32 specs migrate to `popups: false`. +2. **videoMode child recorder (facts only).** `forPopup` returns a parent-bound child: records highlights/open/close/viewport into the parent's state under a child entry; per-source clock calibration at close; metadata `children` schema. No rendering changes yet — specs assert metadata. +3. **Render integration.** Source-tagged pieces; overlay transform (90% fit, ~40% dim, enter/exit); child highlight/pointer coordinate projection; cross-source cursor planning; unified dead-air segments. ffmpeg-level specs sampling frames for the overlay + backdrop dim (same technique as `video-mode-ffmpeg.spec.ts`). +4. **Docs + demo.** Commit the permanent demo spec, README popup section update, refresh the PR demo video showing ONE composed video. + +Key files: `src/plugin-system.ts` (guard, auto-wrap, `forPopup`), `src/plugins/video-mode.ts` (child recorder, calibration, metadata, render), `spec/popup*.spec.ts`, `spec/auth-demo-app.ts`, README. + +Verification: full suite green at each phase; phase 3 verified by frame-sampling specs plus eyeballing the rendered demo video.