From f271bbb033f8cc6ccb21a3c2981d1c27b684e551 Mon Sep 17 00:00:00 2001 From: Yusuke Hirao Date: Wed, 22 Jul 2026 16:19:44 +0900 Subject: [PATCH] feat(beholder): capture console messages and page errors Collect console.log/warn/error/etc. and uncaught exceptions / unhandled promise rejections via page.on('console')/('pageerror'), for internal pages only. Returned as ScrapeResult.consoleLogs (ConsoleLogEntry[]), mirroring the existing resources array pattern rather than embedding into PageData. --- packages/@d-zero/beholder/README.md | 10 + packages/@d-zero/beholder/src/index.ts | 2 +- packages/@d-zero/beholder/src/scraper.ts | 426 ++++++++++-------- .../beholder/src/to-console-log-entry.spec.ts | 108 +++++ .../beholder/src/to-console-log-entry.ts | 40 ++ .../beholder/src/to-page-error-entry.spec.ts | 23 + .../beholder/src/to-page-error-entry.ts | 22 + packages/@d-zero/beholder/src/types.ts | 32 ++ 8 files changed, 478 insertions(+), 185 deletions(-) create mode 100644 packages/@d-zero/beholder/src/to-console-log-entry.spec.ts create mode 100644 packages/@d-zero/beholder/src/to-console-log-entry.ts create mode 100644 packages/@d-zero/beholder/src/to-page-error-entry.spec.ts create mode 100644 packages/@d-zero/beholder/src/to-page-error-entry.ts diff --git a/packages/@d-zero/beholder/README.md b/packages/@d-zero/beholder/README.md index c1cf77e1..05bbc17d 100644 --- a/packages/@d-zero/beholder/README.md +++ b/packages/@d-zero/beholder/README.md @@ -68,6 +68,16 @@ flowchart TD - `scrollHeight`: `{ desktop, mobile }`（各 `number | null`）。未計測時はフィールド全体が `null` - 配列要素のフィールド詳細は型（`MainContents*` / `ScrollHeightData`）を参照 +## console ログ・未捕捉例外の収集 + +`ScrapeResult.consoleLogs`（`ConsoleLogEntry[]`）に、内部ページ（`isExternal: false`）が出力した `console` メッセージ（全type）と、未捕捉例外・未処理の Promise rejection（`page.on('pageerror')`、`type: 'pageerror'` として区別、スタックトレース付き）が格納される。`resources` と同様に `pageUrl` を持つ戻り値配列で、イベント経由では提供されない。`result.type` が `"success"` / `"skipped"` / `"error"` のいずれであっても常に配列として存在する（該当ログがなければ空配列）ため、エラー発生時のデバッグにもそのまま使える。 + +```ts +for (const entry of result.consoleLogs) { + console.log(entry.type, entry.text, entry.args); +} +``` + ## DOM 文字列からメタ抽出（Puppeteer なし） HTML 文字列を jsdom などでパースしてから `Meta` を取り出したい場合、`extractMetaFromDocument` を使う。`Scraper` が内部で呼ぶ `collectHead → detectTags → classify` パイプラインと同じ実装を再利用するため、戻り値の `Meta` 形状は `scrapeStart` と同一。DOM ライブラリ（jsdom 等）はユーザランドの責務。 diff --git a/packages/@d-zero/beholder/src/index.ts b/packages/@d-zero/beholder/src/index.ts index 58394443..76c60aa9 100644 --- a/packages/@d-zero/beholder/src/index.ts +++ b/packages/@d-zero/beholder/src/index.ts @@ -18,7 +18,7 @@ export { detectCompress } from '@d-zero/shared/detect-compress'; export type { CompressType } from '@d-zero/shared/detect-compress'; export { detectCDN } from '@d-zero/shared/detect-cdn'; export type { CDNType } from '@d-zero/shared/detect-cdn'; -export type { ScrapeResult, ResourceEntry, PageData } from './types.js'; +export type { ScrapeResult, ResourceEntry, ConsoleLogEntry, PageData } from './types.js'; export type { ScraperOptions, ChangePhaseEvent, ScraperEventTypes } from './types.js'; export type { Resource, diff --git a/packages/@d-zero/beholder/src/scraper.ts b/packages/@d-zero/beholder/src/scraper.ts index e04fb658..bb25a864 100644 --- a/packages/@d-zero/beholder/src/scraper.ts +++ b/packages/@d-zero/beholder/src/scraper.ts @@ -1,5 +1,6 @@ import type { ChangePhaseEvent, + ConsoleLogEntry, ResourceEntry, ScraperEventTypes, ScraperOptions, @@ -14,7 +15,7 @@ import type { SkippedPageData, } from './types.js'; import type { PageScanPhase } from '@d-zero/puppeteer-page-scan'; -import type { Dialog, HTTPRequest, HTTPResponse, Page } from 'puppeteer'; +import type { ConsoleMessage, Dialog, HTTPRequest, HTTPResponse, Page } from 'puppeteer'; import { beforePageScan, devicePresets } from '@d-zero/puppeteer-page-scan'; import { detectCDN } from '@d-zero/shared/detect-cdn'; @@ -37,6 +38,8 @@ import { measureScrollHeight } from './measure-scroll-height.js'; import { emptyMeta } from './meta/classify.js'; import { findDisconnectionFailures } from './network-disconnection.js'; import { parseUrl } from './parse-url.js'; +import { toConsoleLogEntry } from './to-console-log-entry.js'; +import { toPageErrorEntry } from './to-page-error-entry.js'; const pid = `${process.pid}`; const log = scraperLog.extend(pid); @@ -86,7 +89,10 @@ export default class Scraper extends EventEmitter { * - `type: "error"` with `error` details when scraping fails * * Sub-resources are collected via the `resourceResponse` event and - * included in the returned `ScrapeResult.resources`. + * included in the returned `ScrapeResult.resources`. Console messages and + * uncaught page errors (internal pages only) are collected via + * Puppeteer's `console`/`pageerror` page events and included in + * `ScrapeResult.consoleLogs`. * @param page - The Puppeteer page instance to use for navigation and DOM evaluation. * @param url - The extended URL to scrape. * @param options - Optional scraper configuration overriding defaults. @@ -106,6 +112,7 @@ export default class Scraper extends EventEmitter { const metadataOnly = options?.metadataOnly ?? false; const imageLoadTimeout = options?.imageLoadTimeout ?? 5000; const resources: ResourceEntry[] = []; + const consoleLogs: ConsoleLogEntry[] = []; const failedRequests: Array<{ url: string; errorText: string }> = []; void this.emit('changePhase', { @@ -128,6 +135,7 @@ export default class Scraper extends EventEmitter { return { type: 'skipped', resources, + consoleLogs, ignored: { url, matchedText: url.pathname || '', @@ -164,7 +172,7 @@ export default class Scraper extends EventEmitter { isExternal, message: '', }); - return { type: 'success', pageData: result, resources }; + return { type: 'success', pageData: result, resources, consoleLogs }; } let headResult: PageData | SkippedPageData | null = options?.headCheckResult ?? null; @@ -186,6 +194,7 @@ export default class Scraper extends EventEmitter { scrollHeight: headResult.scrollHeight ?? null, }, resources, + consoleLogs, }; } @@ -197,6 +206,7 @@ export default class Scraper extends EventEmitter { captureImages, imageLoadTimeout, resources, + consoleLogs, failedRequests, options, ).catch((error) => { @@ -212,6 +222,7 @@ export default class Scraper extends EventEmitter { return { type: 'error', resources, + consoleLogs, failedRequests: failedRequests.length > 0 ? failedRequests : undefined, error: { name: fetchResult.name, @@ -246,6 +257,7 @@ export default class Scraper extends EventEmitter { return { type: 'skipped', resources, + consoleLogs, ignored: { url, matchedText: url.pathname || '', @@ -263,6 +275,7 @@ export default class Scraper extends EventEmitter { return { type: 'skipped', resources, + consoleLogs, ignored: { url, matchedText: headResult.matched.text, @@ -284,6 +297,7 @@ export default class Scraper extends EventEmitter { type: 'success', pageData: headResult, resources, + consoleLogs, failedRequests: failedRequests.length > 0 ? failedRequests : undefined, }; } @@ -342,7 +356,8 @@ export default class Scraper extends EventEmitter { * Frame" or "Session closed". * * Flow: - * 1. Register request/response/requestfailed listeners to capture sub-resources (internal pages only) + * 1. Register request/response/requestfailed/console/pageerror listeners to + * capture sub-resources and console output (internal pages only) * 2. Navigate to URL via `page.goto()` and track redirect chain * 3. Wait for DOM content and network idle * 4. Check for network disconnection errors and throw to trigger retry @@ -354,6 +369,7 @@ export default class Scraper extends EventEmitter { * @param captureImages - Whether to run the image extraction pipeline * @param imageLoadTimeout - Timeout (ms) for waiting lazy-loaded images to complete * @param resources - Mutable array to collect captured sub-resources into + * @param consoleLogs - Mutable array to collect captured console messages / page errors into * @param failedRequests - Mutable array to collect failed sub-resource requests into * @param options - Additional scraper options (e.g. `disableQueries`, `navigationTimeout`) * @returns Full page data or skipped page data if an exclusion rule matched @@ -386,6 +402,7 @@ export default class Scraper extends EventEmitter { captureImages: boolean, imageLoadTimeout: number, resources: ResourceEntry[], + consoleLogs: ConsoleLogEntry[], failedRequests: Array<{ url: string; errorText: string }>, options?: Partial, ): Promise { @@ -396,12 +413,16 @@ export default class Scraper extends EventEmitter { const domEvaluationTimeout = options?.domEvaluationTimeout ?? DEFAULT_DOM_EVALUATION_TIMEOUT; const networkLogs: Record = {}; + // Tracks in-flight `toConsoleLogEntry()` resolutions (async `jsonValue()` + // extraction) so callers can await them before reading `consoleLogs`. + const pendingConsoleWork: Promise[] = []; // Clear stale state from previous retries (@retryable may re-invoke this method // with the same page and mutable arrays, so we must reset to avoid accumulation) this.#cleanupPageListeners(); failedRequests.length = 0; resources.length = 0; + consoleLogs.length = 0; // Define named listeners so they can be individually removed on retry/cleanup const onDialog = async (dialog: Dialog) => { @@ -418,6 +439,8 @@ export default class Scraper extends EventEmitter { let onRequest: ((req: HTTPRequest) => void) | null = null; let onResponse: ((res: HTTPResponse) => void) | null = null; let onRequestFailed: ((req: HTTPRequest) => void) | null = null; + let onConsole: ((msg: ConsoleMessage) => void) | null = null; + let onPageError: ((error: unknown) => void) | null = null; if (!isExternal) { onRequest = (request: HTTPRequest) => { @@ -503,9 +526,30 @@ export default class Scraper extends EventEmitter { failedRequests.push({ url: request.url(), errorText }); }; + onConsole = (msg: ConsoleMessage) => { + pendingConsoleWork.push( + toConsoleLogEntry(msg, url.withoutHash).then( + (entry) => { + consoleLogs.push(entry); + }, + (error) => { + // A single malformed console message must not fail the whole + // scrape (e.g. mid-navigation execution-context teardown). + log('Error(CONSOLE_LOG): %s', error); + }, + ), + ); + }; + + onPageError = (error: unknown) => { + consoleLogs.push(toPageErrorEntry(error, url.withoutHash)); + }; + page.on('request', onRequest); page.on('response', onResponse); page.on('requestfailed', onRequestFailed); + page.on('console', onConsole); + page.on('pageerror', onPageError); } // Store cleanup function for retry/post-fetch removal @@ -514,221 +558,235 @@ export default class Scraper extends EventEmitter { if (onRequest) page.off('request', onRequest); if (onResponse) page.off('response', onResponse); if (onRequestFailed) page.off('requestfailed', onRequestFailed); + if (onConsole) page.off('console', onConsole); + if (onPageError) page.off('pageerror', onPageError); }; const navigationTimeout = options?.navigationTimeout ?? 60_000; - void this.emit('changePhase', { - pid: process.pid, - name: 'openPage', - url, - isExternal, - message: `%countdown(${navigationTimeout},openPage_${url.withoutHash},s)%s`, - }); - - if (url.username && url.password) { - await page.setExtraHTTPHeaders({ - Authorization: `Basic ${Buffer.from(`${url.username}:${url.password}`).toString('base64')}`, + // The whole navigation/extraction sequence below is wrapped in `finally` + // so that `pendingConsoleWork` is always awaited exactly once — on every + // return AND on every throw (e.g. `Page.goto returned null`, network + // disconnection). A per-return-statement `await` would miss thrown exits + // and let a still-resolving `toConsoleLogEntry()` push into `consoleLogs` + // after a `@retryable` retry has already cleared it for a new attempt. + try { + void this.emit('changePhase', { + pid: process.pid, + name: 'openPage', + url, + isExternal, + message: `%countdown(${navigationTimeout},openPage_${url.withoutHash},s)%s`, }); - } - const res = await page.goto(url.withoutHashAndAuth, { timeout: navigationTimeout }); - - if (!res) { - throw new Error('The method Page.goto returned null'); - } + if (url.username && url.password) { + await page.setExtraHTTPHeaders({ + Authorization: `Basic ${Buffer.from(`${url.username}:${url.password}`).toString('base64')}`, + }); + } - const destUrl = parseUrl(page.url(), parseOpts)!; - const redirectPaths = new Set(); + const res = await page.goto(url.withoutHashAndAuth, { timeout: navigationTimeout }); - if (url.withoutHash !== destUrl.withoutHash) { - const redirectChain = res - .request() - .redirectChain() - .map((req) => req.url()); - for (const redirectPath of redirectChain) { - redirectPaths.add(redirectPath); + if (!res) { + throw new Error('The method Page.goto returned null'); } - redirectPaths.add(destUrl.withoutHash); - } - if (destUrl.hostname !== url.hostname) { - isExternal = true; - } + const destUrl = parseUrl(page.url(), parseOpts)!; + const redirectPaths = new Set(); - const status = res.status(); - const statusText = res.statusText(); - const responseHeaders = res.headers(); - const contentType = responseHeaders['content-type']?.split(';')[0] || null; - const _contentLength = Number.parseInt(responseHeaders['content-length'] ?? ''); - const contentLength = Number.isFinite(_contentLength) ? _contentLength : null; + if (url.withoutHash !== destUrl.withoutHash) { + const redirectChain = res + .request() + .redirectChain() + .map((req) => req.url()); + for (const redirectPath of redirectChain) { + redirectPaths.add(redirectPath); + } + redirectPaths.add(destUrl.withoutHash); + } - if (!isHtmlContentType(contentType)) { - return { - url, - isTarget: false, - isExternal, - redirectPaths: [...redirectPaths], - status, - statusText, - contentType, - contentLength, - responseHeaders, - meta: emptyMeta(), - imageList: [], - anchorList: [], - html: '', - mainContents: null, - scrollHeight: null, - isSkipped: false, - }; - } + if (destUrl.hostname !== url.hostname) { + isExternal = true; + } - void this.emit('changePhase', { - pid: process.pid, - name: 'loadDOMContent', - url, - isExternal, - message: '', - }); + const status = res.status(); + const statusText = res.statusText(); + const responseHeaders = res.headers(); + const contentType = responseHeaders['content-type']?.split(';')[0] || null; + const _contentLength = Number.parseInt(responseHeaders['content-length'] ?? ''); + const contentLength = Number.isFinite(_contentLength) ? _contentLength : null; - await page - .waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 }) - .catch(() => {}); + if (!isHtmlContentType(contentType)) { + return { + url, + isTarget: false, + isExternal, + redirectPaths: [...redirectPaths], + status, + statusText, + contentType, + contentLength, + responseHeaders, + meta: emptyMeta(), + imageList: [], + anchorList: [], + html: '', + mainContents: null, + scrollHeight: null, + isSkipped: false, + }; + } - void this.emit('changePhase', { - pid: process.pid, - name: 'getHTML', - url, - isExternal, - message: '', - }); + void this.emit('changePhase', { + pid: process.pid, + name: 'loadDOMContent', + url, + isExternal, + message: '', + }); - const { title, html } = await page.evaluate(() => { - /* global document */ - return { - title: document.title, - html: document.documentElement.outerHTML, - }; - }); + await page + .waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 }) + .catch(() => {}); - if (isExternal) { - const externalMeta = emptyMeta(); - externalMeta.title = title; - return { + void this.emit('changePhase', { + pid: process.pid, + name: 'getHTML', url, - isTarget: false, isExternal, - redirectPaths: [...redirectPaths], - status, - statusText, - contentType, - contentLength, - responseHeaders, - meta: externalMeta, - imageList: [], - anchorList: [], - html, - mainContents: null, - scrollHeight: null, - isSkipped: false, - }; - } + message: '', + }); - void this.emit('changePhase', { - pid: process.pid, - name: 'waitNetworkIdle', - url, - isExternal, - message: '', - }); + const { title, html } = await page.evaluate(() => { + /* global document */ + return { + title: document.title, + html: document.documentElement.outerHTML, + }; + }); - await page - .waitForNavigation({ waitUntil: 'networkidle0', timeout: 5000 }) - .catch(() => {}); - - // Check for network disconnection errors in failed requests - const disconnectionFailures = findDisconnectionFailures(failedRequests); - if (disconnectionFailures.length > 0) { - const errorSummary = disconnectionFailures - .map((r) => `${r.url} (${r.errorText})`) - .join(', '); - throw new Error(`Network disconnection detected during page load: ${errorSummary}`); - } + if (isExternal) { + const externalMeta = emptyMeta(); + externalMeta.title = title; + return { + url, + isTarget: false, + isExternal, + redirectPaths: [...redirectPaths], + status, + statusText, + contentType, + contentLength, + responseHeaders, + meta: externalMeta, + imageList: [], + anchorList: [], + html, + mainContents: null, + scrollHeight: null, + isSkipped: false, + }; + } - const mainContents = await getMainContents(page, { - mainContentSelector: options?.mainContentSelector, - }); + void this.emit('changePhase', { + pid: process.pid, + name: 'waitNetworkIdle', + url, + isExternal, + message: '', + }); - void this.emit('changePhase', { - pid: process.pid, - name: 'getAnchors', - url, - isExternal, - message: `%countdown(${domEvaluationTimeout},getAnchors_${url.withoutHash},s)%s`, - }); - const anchorList = await getAnchorList(page, parseOpts, domEvaluationTimeout); + await page + .waitForNavigation({ waitUntil: 'networkidle0', timeout: 5000 }) + .catch(() => {}); + + // Check for network disconnection errors in failed requests + const disconnectionFailures = findDisconnectionFailures(failedRequests); + if (disconnectionFailures.length > 0) { + const errorSummary = disconnectionFailures + .map((r) => `${r.url} (${r.errorText})`) + .join(', '); + throw new Error( + `Network disconnection detected during page load: ${errorSummary}`, + ); + } - void this.emit('changePhase', { - pid: process.pid, - name: 'getMeta', - url, - isExternal, - message: `%countdown(${domEvaluationTimeout},getMeta_${url.withoutHash},s)%s`, - }); - const meta = await getMeta( - page, - { - url: url.withoutHashAndAuth, - html, - statusCode: status, - headers: responseHeaders ?? undefined, - }, - domEvaluationTimeout, - ); + const mainContents = await getMainContents(page, { + mainContentSelector: options?.mainContentSelector, + }); - let imageList: ImageElement[] = []; - let scrollHeight: ScrollHeightData | null = null; + void this.emit('changePhase', { + pid: process.pid, + name: 'getAnchors', + url, + isExternal, + message: `%countdown(${domEvaluationTimeout},getAnchors_${url.withoutHash},s)%s`, + }); + const anchorList = await getAnchorList(page, parseOpts, domEvaluationTimeout); - if (captureImages) { void this.emit('changePhase', { pid: process.pid, - name: 'extractImages', + name: 'getMeta', url, isExternal, - message: `%countdown(${domEvaluationTimeout},extractImages_${url.withoutHash},s)%s`, + message: `%countdown(${domEvaluationTimeout},getMeta_${url.withoutHash},s)%s`, }); - const fetched = await this.#fetchImages( + const meta = await getMeta( page, - url.withoutHashAndAuth, - isExternal, - imageLoadTimeout, + { + url: url.withoutHashAndAuth, + html, + statusCode: status, + headers: responseHeaders ?? undefined, + }, domEvaluationTimeout, ); - imageList = fetched.imageList; - scrollHeight = fetched.scrollHeight; - } else { - scrollHeight = await measureScrollHeight(page); - } - return { - url, - isTarget: true, - isExternal, - redirectPaths: [...redirectPaths], - status, - statusText, - contentType, - contentLength, - responseHeaders, - meta, - anchorList, - imageList, - html, - mainContents, - scrollHeight, - isSkipped: false, - }; + let imageList: ImageElement[] = []; + let scrollHeight: ScrollHeightData | null = null; + + if (captureImages) { + void this.emit('changePhase', { + pid: process.pid, + name: 'extractImages', + url, + isExternal, + message: `%countdown(${domEvaluationTimeout},extractImages_${url.withoutHash},s)%s`, + }); + const fetched = await this.#fetchImages( + page, + url.withoutHashAndAuth, + isExternal, + imageLoadTimeout, + domEvaluationTimeout, + ); + imageList = fetched.imageList; + scrollHeight = fetched.scrollHeight; + } else { + scrollHeight = await measureScrollHeight(page); + } + + return { + url, + isTarget: true, + isExternal, + redirectPaths: [...redirectPaths], + status, + statusText, + contentType, + contentLength, + responseHeaders, + meta, + anchorList, + imageList, + html, + mainContents, + scrollHeight, + isSkipped: false, + }; + } finally { + await Promise.all(pendingConsoleWork); + } } /** * Extracts image data from the page across multiple device presets. diff --git a/packages/@d-zero/beholder/src/to-console-log-entry.spec.ts b/packages/@d-zero/beholder/src/to-console-log-entry.spec.ts new file mode 100644 index 00000000..72222094 --- /dev/null +++ b/packages/@d-zero/beholder/src/to-console-log-entry.spec.ts @@ -0,0 +1,108 @@ +import type { ConsoleMessage, ConsoleMessageLocation, JSHandle } from 'puppeteer'; + +import { describe, expect, it, vi } from 'vitest'; + +import { toConsoleLogEntry } from './to-console-log-entry.js'; + +/** + * Builds a minimal `JSHandle` mock whose `jsonValue()` resolves with the given value. + * @param value + */ +function mockArg(value: unknown): JSHandle { + return { + jsonValue: () => Promise.resolve(value), + dispose: () => Promise.resolve(), + } as unknown as JSHandle; +} + +/** + * Builds a `JSHandle` mock whose `jsonValue()` rejects, simulating a handle + * that can no longer be resolved (e.g. a destroyed execution context). + * @param reason + */ +function mockRejectingArg(reason: unknown): JSHandle { + return { + jsonValue: () => Promise.reject(reason), + dispose: () => Promise.resolve(), + } as unknown as JSHandle; +} + +/** + * Builds a minimal `ConsoleMessage` mock. + * @param options + * @param options.type + * @param options.text + * @param options.args + * @param options.location + */ +function mockConsoleMessage(options: { + type?: string; + text?: string; + args?: JSHandle[]; + location?: ConsoleMessageLocation; +}): ConsoleMessage { + return { + type: () => options.type ?? 'log', + text: () => options.text ?? '', + args: () => options.args ?? [], + location: () => options.location ?? {}, + } as unknown as ConsoleMessage; +} + +describe('toConsoleLogEntry', () => { + it('resolves text, type, and pageUrl', async () => { + const msg = mockConsoleMessage({ type: 'warn', text: 'careful' }); + const entry = await toConsoleLogEntry(msg, 'https://example.com/'); + + expect(entry.pageUrl).toBe('https://example.com/'); + expect(entry.type).toBe('warn'); + expect(entry.text).toBe('careful'); + expect(entry.args).toEqual([]); + }); + + it('resolves each argument via jsonValue()', async () => { + const msg = mockConsoleMessage({ + args: [mockArg('a'), mockArg(1), mockArg({ b: 2 })], + }); + const entry = await toConsoleLogEntry(msg, 'https://example.com/'); + + expect(entry.args).toEqual(['a', 1, { b: 2 }]); + }); + + it('falls back to undefined for an argument whose jsonValue() rejects', async () => { + const msg = mockConsoleMessage({ + args: [mockArg('ok'), mockRejectingArg(new Error('destroyed context'))], + }); + const entry = await toConsoleLogEntry(msg, 'https://example.com/'); + + expect(entry.args).toEqual(['ok', undefined]); + }); + + it('disposes each argument handle after resolving', async () => { + const arg = mockArg('a'); + const disposeSpy = vi.spyOn(arg, 'dispose'); + const msg = mockConsoleMessage({ args: [arg] }); + await toConsoleLogEntry(msg, 'https://example.com/'); + + expect(disposeSpy).toHaveBeenCalledOnce(); + }); + + it('keeps location when the message has a source URL', async () => { + const location = { + url: 'https://example.com/app.js', + lineNumber: 3, + columnNumber: 7, + }; + const msg = mockConsoleMessage({ location }); + const entry = await toConsoleLogEntry(msg, 'https://example.com/'); + + expect(entry.location).toEqual(location); + }); + + it('omits location when the message has no source URL', async () => { + const msg = mockConsoleMessage({ location: {} }); + const entry = await toConsoleLogEntry(msg, 'https://example.com/'); + + expect(entry.location).toBeUndefined(); + }); +}); diff --git a/packages/@d-zero/beholder/src/to-console-log-entry.ts b/packages/@d-zero/beholder/src/to-console-log-entry.ts new file mode 100644 index 00000000..1f6ec759 --- /dev/null +++ b/packages/@d-zero/beholder/src/to-console-log-entry.ts @@ -0,0 +1,40 @@ +import type { ConsoleLogEntry } from './types.js'; +import type { ConsoleMessage } from 'puppeteer'; + +/** + * Converts a Puppeteer `ConsoleMessage` into a `ConsoleLogEntry`. + * + * WHY per-argument try/catch: `JSHandle.jsonValue()` can reject when the + * page's execution context has been destroyed (e.g. mid-navigation) or when + * the value cannot be serialized (e.g. a circular reference). A single + * failing argument must not drop the rest of the message's arguments. + * @param msg - The console message captured via `page.on('console')` + * @param pageUrl - The URL (without hash) of the page that produced the message + * @returns The resolved console log entry + */ +export async function toConsoleLogEntry( + msg: ConsoleMessage, + pageUrl: string, +): Promise { + const args = await Promise.all( + msg.args().map(async (arg) => { + try { + return await arg.jsonValue(); + } catch { + return; + } finally { + await arg.dispose().catch(() => {}); + } + }), + ); + const loc = msg.location(); + + return { + pageUrl, + type: msg.type(), + text: msg.text(), + args, + location: loc.url === undefined ? undefined : loc, + ts: Date.now(), + }; +} diff --git a/packages/@d-zero/beholder/src/to-page-error-entry.spec.ts b/packages/@d-zero/beholder/src/to-page-error-entry.spec.ts new file mode 100644 index 00000000..a6736de9 --- /dev/null +++ b/packages/@d-zero/beholder/src/to-page-error-entry.spec.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest'; + +import { toPageErrorEntry } from './to-page-error-entry.js'; + +describe('toPageErrorEntry', () => { + it('extracts message and stack from an Error', () => { + const error = new Error('boom'); + const entry = toPageErrorEntry(error, 'https://example.com/'); + + expect(entry.pageUrl).toBe('https://example.com/'); + expect(entry.type).toBe('pageerror'); + expect(entry.text).toBe('boom'); + expect(entry.stack).toBe(error.stack); + expect(entry.args).toEqual([]); + }); + + it('stringifies a non-Error throw value and omits stack', () => { + const entry = toPageErrorEntry('oops', 'https://example.com/'); + + expect(entry.text).toBe('oops'); + expect(entry.stack).toBeUndefined(); + }); +}); diff --git a/packages/@d-zero/beholder/src/to-page-error-entry.ts b/packages/@d-zero/beholder/src/to-page-error-entry.ts new file mode 100644 index 00000000..2bd04118 --- /dev/null +++ b/packages/@d-zero/beholder/src/to-page-error-entry.ts @@ -0,0 +1,22 @@ +import type { ConsoleLogEntry } from './types.js'; + +/** + * Converts an uncaught exception / unhandled Promise rejection (captured via + * `page.on('pageerror')`) into a `ConsoleLogEntry`. + * @param error - The value emitted by `page.on('pageerror')`; typed as `unknown` + * because Puppeteer allows non-`Error` throw values (e.g. `throw 'oops'`) + * @param pageUrl - The URL (without hash) of the page that produced the error + * @returns The console log entry, with `type: 'pageerror'` + */ +export function toPageErrorEntry(error: unknown, pageUrl: string): ConsoleLogEntry { + const isErr = error instanceof Error; + + return { + pageUrl, + type: 'pageerror', + text: isErr ? error.message : String(error), + args: [], + stack: isErr ? error.stack : undefined, + ts: Date.now(), + }; +} diff --git a/packages/@d-zero/beholder/src/types.ts b/packages/@d-zero/beholder/src/types.ts index 93bd7872..cf07cfe7 100644 --- a/packages/@d-zero/beholder/src/types.ts +++ b/packages/@d-zero/beholder/src/types.ts @@ -59,6 +59,7 @@ import type { Meta } from './meta/types.js'; import type { CDNType } from '@d-zero/shared/detect-cdn'; import type { CompressType } from '@d-zero/shared/detect-compress'; import type { ExURL } from '@d-zero/shared/parse-url'; +import type { ConsoleMessageType } from 'puppeteer'; /** * Scraped page data returned by the scraper after successfully processing a page. @@ -477,6 +478,8 @@ export type ScrapeResult = { pageData?: PageData; /** All sub-resources captured during the page load. */ resources: ResourceEntry[]; + /** All console messages and uncaught page errors captured during the page load. */ + consoleLogs: ConsoleLogEntry[]; /** Details about why the page was ignored, present when `type` is `"skipped"`. */ ignored?: { url: ExURL; matchedText: string; excludeKeywords: string[] }; /** Error details, present when `type` is `"error"`. */ @@ -498,6 +501,35 @@ export type ResourceEntry = { pageUrl: string; }; +/** + * A single console message or uncaught page error captured during page scraping. + * Captured via Puppeteer's `console` and `pageerror` page events; internal pages only. + */ +export type ConsoleLogEntry = { + /** The URL (without hash) of the page that produced this message. */ + pageUrl: string; + /** + * The console message type (Puppeteer's `ConsoleMessageType`, e.g. `"log"`, `"warn"`, `"error"`), + * or `"pageerror"` for an uncaught exception / unhandled Promise rejection. + */ + type: ConsoleMessageType | 'pageerror'; + /** The message text, or the error's `message` for `"pageerror"` entries. */ + text: string; + /** + * Arguments passed to the console call, resolved via `JSHandle.jsonValue()`. + * An individual argument is `undefined` when it could not be resolved (e.g. a + * destroyed execution context or a value `jsonValue()` cannot serialize). + * Always empty for `"pageerror"` entries. + */ + args: unknown[]; + /** Source location of the message, or `undefined` when unavailable (always `undefined` for `"pageerror"`). */ + location?: { url?: string; lineNumber?: number; columnNumber?: number }; + /** Stack trace text, present only for `"pageerror"` entries. */ + stack?: string; + /** Timestamp (ms since epoch) when the message was captured. */ + ts: number; +}; + /** * Event payload describing a phase transition in the scraping lifecycle. * Phases proceed roughly in order: scrapeStart -> openPage ->