diff --git a/package-lock.json b/package-lock.json index 72a350e..a9002f2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,11 +12,17 @@ "@earendil-works/pi-coding-agent": "0.84.1", "ajv": "8.20.0", "ajv-formats": "3.0.1", + "buffer-crc32": "0.2.13", "commander": "15.0.0", - "fastify": "5.11.3" + "fastify": "5.11.3", + "pdfjs-dist": "5.4.624", + "saxes": "6.0.0", + "yauzl": "3.1.3" }, "devDependencies": { + "@types/buffer-crc32": "0.2.4", "@types/node": "24.10.1", + "@types/yauzl": "2.10.3", "tsx": "4.23.12", "typescript": "5.9.3", "vitest": "4.1.10", @@ -2465,6 +2471,271 @@ "dev": true, "license": "MIT" }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.100.tgz", + "integrity": "sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA==", + "license": "MIT", + "optional": true, + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.100", + "@napi-rs/canvas-darwin-arm64": "0.1.100", + "@napi-rs/canvas-darwin-x64": "0.1.100", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.100", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.100", + "@napi-rs/canvas-linux-arm64-musl": "0.1.100", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.100", + "@napi-rs/canvas-linux-x64-gnu": "0.1.100", + "@napi-rs/canvas-linux-x64-musl": "0.1.100", + "@napi-rs/canvas-win32-arm64-msvc": "0.1.100", + "@napi-rs/canvas-win32-x64-msvc": "0.1.100" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.100.tgz", + "integrity": "sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.100.tgz", + "integrity": "sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.100.tgz", + "integrity": "sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.100.tgz", + "integrity": "sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.100.tgz", + "integrity": "sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.100.tgz", + "integrity": "sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.100.tgz", + "integrity": "sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.100.tgz", + "integrity": "sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.100.tgz", + "integrity": "sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-arm64-msvc": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.100.tgz", + "integrity": "sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.100.tgz", + "integrity": "sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, "node_modules/@oxc-project/types": { "version": "0.143.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.143.0.tgz", @@ -2751,6 +3022,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/buffer-crc32": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@types/buffer-crc32/-/buffer-crc32-0.2.4.tgz", + "integrity": "sha512-GSrhSZOK1/wazf2CjDp3CVJQKWzSc5Ugq3NyZ/RQqg1MWtmA9mAT6i6LzGKhzcRxDOl8aLB+AzvObDSlrMpvLw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -2786,6 +3067,16 @@ "undici-types": "~7.16.0" } }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@vitest/expect": { "version": "4.1.10", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.10.tgz", @@ -2977,6 +3268,15 @@ "fastq": "^1.17.1" } }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/chai": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", @@ -3643,6 +3943,13 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/node-readable-to-web-readable-stream": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/node-readable-to-web-readable-stream/-/node-readable-to-web-readable-stream-0.4.2.tgz", + "integrity": "sha512-/cMZNI34v//jUTrI+UIo4ieHAB5EZRY/+7OmXZgBxaWBMcW2tGdceIw06RFxWxrKZ5Jp3sI2i5TsRo+CBhtVLQ==", + "license": "MIT", + "optional": true + }, "node_modules/obug": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.4.tgz", @@ -3673,6 +3980,25 @@ "dev": true, "license": "MIT" }, + "node_modules/pdfjs-dist": { + "version": "5.4.624", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.624.tgz", + "integrity": "sha512-sm6TxKTtWv1Oh6n3C6J6a8odejb5uO4A4zo/2dgkHuC0iu8ZMAXOezEODkVaoVp8nX1Xzr+0WxFJJmUr45hQzg==", + "license": "Apache-2.0", + "engines": { + "node": ">=20.16.0 || >=22.3.0" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.88", + "node-readable-to-web-readable-stream": "^0.4.2" + } + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -3888,6 +4214,18 @@ "node": ">=10" } }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } + }, "node_modules/secure-json-parse": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", @@ -4267,6 +4605,12 @@ "node": ">=8" } }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "license": "MIT" + }, "node_modules/yaml": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", @@ -4282,6 +4626,19 @@ "funding": { "url": "https://github.com/sponsors/eemeli" } + }, + "node_modules/yauzl": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.1.3.tgz", + "integrity": "sha512-JCCdmlJJWv7L0q/KylOekyRaUrdEoUxWkWVcgorosTROCFWiS9p2NNPE9Yb91ak7b1N5SxAZEliWpspbZccivw==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "pend": "~1.2.0" + }, + "engines": { + "node": ">=12" + } } } } diff --git a/package.json b/package.json index 299fa93..4a2ce84 100644 --- a/package.json +++ b/package.json @@ -25,11 +25,17 @@ "@earendil-works/pi-coding-agent": "0.84.1", "ajv": "8.20.0", "ajv-formats": "3.0.1", + "buffer-crc32": "0.2.13", "commander": "15.0.0", - "fastify": "5.11.3" + "fastify": "5.11.3", + "pdfjs-dist": "5.4.624", + "saxes": "6.0.0", + "yauzl": "3.1.3" }, "devDependencies": { + "@types/buffer-crc32": "0.2.4", "@types/node": "24.10.1", + "@types/yauzl": "2.10.3", "tsx": "4.23.12", "typescript": "5.9.3", "vitest": "4.1.10", diff --git a/src/input/README.md b/src/input/README.md new file mode 100644 index 0000000..1643e9f --- /dev/null +++ b/src/input/README.md @@ -0,0 +1,24 @@ +# Safe package input + +`importPackage` inventories a local competition package without modifying source files or using the network. Existing callers keep the same required fields (`problemText`, the original `problemPath`, `dataAssets`, and `dataPaths`); extraction and inventory details are optional additions. + +## Problem selection + +Problem statements are selected deterministically by precedence group: + +1. `problem.md` or `problem.markdown` +2. `problem.txt` +3. `problem.pdf` +4. `problem.docx` + +Names are matched case-insensitively at any non-hidden package depth. The first non-empty precedence group wins. If that group contains more than one candidate (including duplicate names in separate directories, or both Markdown spellings), import fails with `PackageImportError.code === "ambiguous_problem"`; lower-priority candidates are data assets only when a higher-priority problem was selected. + +## Safety and limits + +All limits have conservative defaults in `DEFAULT_IMPORT_LIMITS` and may be reduced or raised through `importPackage(path, { limits })`. Invalid limits are rejected. Package roots and nested entries may not be symlinks, paths must remain beneath the package root, and total traversal is bounded by `maxPackageEntries` and `maxPackageDepth`. Every encountered child entry (file, directory, unsupported extension, or hidden name) consumes the entry budget; the package root itself does not. Hidden entries are then ignored and hidden directories are never recursed into. + +PDF text is extracted locally with the pinned `pdfjs-dist` Node build. Input bytes are passed directly to the parser with fetch, streaming, JavaScript evaluation, system fonts, and WebAssembly disabled. The importer records source SHA-256/size, PDF page count, extracted character count, and extractor version, while explicitly rejecting unavailable tooling, encryption, corruption, empty text, and byte/page/character limits. `maxPdfCharacters` applies to the exact returned text, including spaces inserted between text items, blank lines inserted between non-empty pages, and final normalization; construction is checked incrementally and asserted again before return. + +DOCX extraction performs a bounded magic-prefix check before opening ZIP data: normal ZIP signatures continue, OLE/CFB Office containers are classified as `docx_encrypted`, and other input is `docx_corrupt`. Extraction reads only required OOXML package parts and `word/document.xml` body text. It never decrypts content, executes payloads, follows non-internal relationships, or opens embedded objects. All relationship files reject duplicate IDs and external, escaping, or unknown normalized `TargetMode` values; package relationships require exactly one normalized main-document relationship; and content-type declarations reject duplicate/case-normalized part ambiguity. Any VBA/macro/ActiveX content type, relationship type, or package-part indicator is rejected before payload expansion; ZIP symlink entries are rejected at the archive boundary. + +Data assets retain SHA-256 and size computed from their original bytes. XLSX inspection is limited to workbook, relationship, and bounded worksheet-dimension XML; images are read only far enough to obtain bounded PNG/JPEG format and dimensions. CSV, JSON, legacy XLS, and Parquet are inventoried without parsing or sampling their records. Unreadable or over-limit optional asset metadata is surfaced in `warnings`, never silently treated as successful metadata. diff --git a/src/input/asset-metadata.ts b/src/input/asset-metadata.ts new file mode 100644 index 0000000..195e5b7 --- /dev/null +++ b/src/input/asset-metadata.ts @@ -0,0 +1,309 @@ +import { open as openFile } from "node:fs/promises"; +import { extname } from "node:path"; +import { + PackageImportError, + type AssetMetadata, + type ImageAssetMetadata, + type ImportLimits, + type XlsxSheetMetadata +} from "./types.js"; +import { readBoundedZip } from "./zip-reader.js"; +import { + attributeValue, + PACKAGE_RELATIONSHIPS_NAMESPACE, + parseXml, + SPREADSHEETML_NAMESPACE +} from "./xml.js"; + +const OFFICE_DOCUMENT_RELATIONSHIPS_NAMESPACE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; +const MAX_CAUSE_LENGTH = 240; +const XLSX_ZIP_CODES = { + corrupt: "metadata_unreadable", + encrypted: "metadata_unreadable", + zipSlip: "metadata_unreadable", + entryLimit: "metadata_limit", + uncompressedLimit: "metadata_limit" +} as const; + +function checkedImage(format: ImageAssetMetadata["format"], width: number, height: number, path: string, limits: ImportLimits): ImageAssetMetadata { + const pixels = width * height; + if (!Number.isSafeInteger(width) || !Number.isSafeInteger(height) || width <= 0 || height <= 0) { + throw new PackageImportError("metadata_unreadable", `Invalid ${format.toUpperCase()} dimensions.`, { path }); + } + if (!Number.isSafeInteger(pixels) || pixels > limits.maxImagePixels) { + throw new PackageImportError("metadata_limit", `Image has ${pixels} pixels; limit is ${limits.maxImagePixels}.`, { + path, + actual: pixels, + limit: limits.maxImagePixels + }); + } + return { kind: "image", format, width, height }; +} + +async function readPrefix(path: string, length: number): Promise { + const handle = await openFile(path, "r"); + try { + const buffer = Buffer.alloc(length); + const { bytesRead } = await handle.read(buffer, 0, length, 0); + return buffer.subarray(0, bytesRead); + } finally { + await handle.close(); + } +} + +function pngMetadata(bytes: Buffer, path: string, limits: ImportLimits): ImageAssetMetadata { + const signature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + if (bytes.length < 24 || !bytes.subarray(0, 8).equals(signature) || bytes.toString("ascii", 12, 16) !== "IHDR") { + throw new PackageImportError("metadata_unreadable", "Invalid PNG header.", { path }); + } + return checkedImage("png", bytes.readUInt32BE(16), bytes.readUInt32BE(20), path, limits); +} + +const JPEG_START_OF_FRAME_MARKERS = new Set([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf]); + +function jpegMetadata(bytes: Buffer, path: string, limits: ImportLimits, sourceBytes: number): ImageAssetMetadata { + if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) { + throw new PackageImportError("metadata_unreadable", "Invalid JPEG header.", { path }); + } + let offset = 2; + while (offset + 3 < bytes.length) { + while (bytes[offset] === 0xff) offset += 1; + const marker = bytes[offset]; + offset += 1; + if (marker === undefined || marker === 0xd9 || marker === 0xda) break; + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue; + if (offset + 2 > bytes.length) break; + const segmentLength = bytes.readUInt16BE(offset); + if (segmentLength < 2) throw new PackageImportError("metadata_unreadable", "Invalid JPEG segment length.", { path }); + if (JPEG_START_OF_FRAME_MARKERS.has(marker)) { + if (offset + 7 > bytes.length) break; + return checkedImage("jpeg", bytes.readUInt16BE(offset + 5), bytes.readUInt16BE(offset + 3), path, limits); + } + offset += segmentLength; + } + const code = bytes.length < sourceBytes ? "metadata_limit" : "metadata_unreadable"; + throw new PackageImportError(code, "JPEG dimensions were not found within the bounded header scan.", { + path, + actual: bytes.length, + limit: limits.maxImageHeaderBytes + }); +} + +async function imageMetadata(path: string, sourceBytes: number, extension: string, limits: ImportLimits): Promise { + const bytes = await readPrefix(path, Math.min(sourceBytes, limits.maxImageHeaderBytes)); + if (extension === ".png") return pngMetadata(bytes, path, limits); + return jpegMetadata(bytes, path, limits, sourceBytes); +} + +interface WorkbookSheet { + name: string; + relationshipId: string; +} + +function workbookSheets(xml: Buffer, path: string, sheetLimit: number, byteLimit: number): WorkbookSheet[] { + if (xml.length > byteLimit) { + throw new PackageImportError("metadata_limit", `XLSX workbook metadata exceeds ${byteLimit} bytes.`, { + path, + actual: xml.length, + limit: byteLimit + }); + } + const sheets: WorkbookSheet[] = []; + parseXml(xml, { + openTag(tag) { + if (tag.uri !== SPREADSHEETML_NAMESPACE || tag.local !== "sheet") return; + const name = attributeValue(tag, "name"); + const relationshipId = attributeValue(tag, "id", OFFICE_DOCUMENT_RELATIONSHIPS_NAMESPACE); + if (!name || !relationshipId) throw new PackageImportError("metadata_unreadable", "XLSX sheet is missing a name or relationship id.", { path }); + sheets.push({ name, relationshipId }); + if (sheets.length > sheetLimit) { + throw new PackageImportError("metadata_limit", `XLSX has more than ${sheetLimit} sheets.`, { + path, + actual: sheets.length, + limit: sheetLimit + }); + } + } + }); + return sheets; +} + +function decodePercentEscapes(value: string): string | undefined { + let decoded = value; + for (let depth = 0; depth < 4 && decoded.includes("%"); depth += 1) { + try { + const next = decodeURIComponent(decoded); + if (next === decoded) return undefined; + decoded = next; + } catch { + return undefined; + } + } + return decoded.includes("%") ? undefined : decoded; +} + +function normalizedOoxmlEntryKey(name: string): string | undefined { + const isDirectory = name.endsWith("/"); + const partName = isDirectory ? name.slice(0, -1) : name; + const decoded = decodePercentEscapes(partName); + if (decoded === undefined + || !decoded + || name.startsWith("/") + || partName.endsWith("/") + || decoded !== partName + || /[\u0000-\u001f\u007f]/.test(decoded) + || decoded.includes("\\") + || decoded.includes("?") + || decoded.includes("#")) return undefined; + const normalized = decoded.normalize("NFC"); + if (normalized !== decoded || normalized.split("/").some((segment) => !segment || segment === "." || segment === "..")) return undefined; + return normalized.toLowerCase(); +} + +function validateOoxmlEntryNames(entryNames: Set, path: string): void { + const keys = new Set(); + for (const name of entryNames) { + const key = normalizedOoxmlEntryKey(name); + if (key === undefined || keys.has(key)) { + throw new PackageImportError("metadata_unreadable", "XLSX contains unsafe or ambiguous package part names.", { path: name }); + } + keys.add(key); + } +} + +function worksheetTargetMode(value: string): string | undefined { + const decoded = decodePercentEscapes(value.trim()); + return decoded?.trim().toLowerCase(); +} + +function worksheetTarget(target: string): string | undefined { + const decoded = decodePercentEscapes(target.trim()); + if (decoded === undefined || !decoded + || decoded.includes("%") + || /[\u0000-\u001f\u007f]/.test(decoded) + || decoded.includes("\\") + || decoded.includes("?") + || decoded.includes("#") + || decoded.startsWith("/") + || decoded.startsWith("//") + || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(decoded)) return undefined; + const segments: string[] = []; + for (const segment of decoded.split("/")) { + if (!segment || segment === ".") return undefined; + if (segment === "..") return undefined; + segments.push(segment); + } + return `xl/${segments.join("/")}`; +} + +function worksheetRelationships(xml: Buffer, path: string, byteLimit: number): Map { + if (xml.length > byteLimit) { + throw new PackageImportError("metadata_limit", `XLSX relationships metadata exceeds ${byteLimit} bytes.`, { + path, + actual: xml.length, + limit: byteLimit + }); + } + const relationships = new Map(); + parseXml(xml, { + openTag(tag) { + if (tag.uri !== PACKAGE_RELATIONSHIPS_NAMESPACE || tag.local !== "Relationship") return; + const id = attributeValue(tag, "Id")?.trim(); + const target = attributeValue(tag, "Target")?.trim(); + const type = attributeValue(tag, "Type")?.trim(); + if (!id || !target || !type?.endsWith("/worksheet")) return; + const rawTargetMode = attributeValue(tag, "TargetMode"); + const targetMode = rawTargetMode === undefined ? undefined : worksheetTargetMode(rawTargetMode); + if (rawTargetMode !== undefined && targetMode !== "internal") { + throw new PackageImportError("metadata_unreadable", "External XLSX worksheet relationship is not followed.", { path }); + } + const resolvedTarget = worksheetTarget(target); + if (resolvedTarget === undefined) { + throw new PackageImportError("metadata_unreadable", "Unsafe XLSX worksheet target.", { path }); + } + if (relationships.has(id)) { + throw new PackageImportError("metadata_unreadable", "XLSX contains duplicate worksheet relationship IDs.", { path }); + } + relationships.set(id, resolvedTarget); + } + }); + return relationships; +} + +function worksheetDimension(xml: Buffer, path: string, limit: number): string | undefined { + if (xml.length > limit) { + throw new PackageImportError("metadata_limit", `XLSX worksheet metadata part exceeds ${limit} bytes.`, { + path, + actual: xml.length, + limit + }); + } + let dimension: string | undefined; + parseXml(xml, { + openTag(tag) { + if (dimension === undefined && tag.uri === SPREADSHEETML_NAMESPACE && tag.local === "dimension") { + dimension = attributeValue(tag, "ref"); + } + } + }); + return dimension; +} + +async function xlsxMetadata(path: string, sourceBytes: number, limits: ImportLimits): Promise { + if (sourceBytes > limits.maxMetadataBytes) { + throw new PackageImportError("metadata_limit", `XLSX is ${sourceBytes} bytes; metadata inspection limit is ${limits.maxMetadataBytes}.`, { + path, + actual: sourceBytes, + limit: limits.maxMetadataBytes + }); + } + try { + const archive = await readBoundedZip( + path, + { maxEntries: limits.maxMetadataZipEntries, maxUncompressedBytes: limits.maxMetadataUncompressedBytes }, + XLSX_ZIP_CODES, + (name) => name === "xl/workbook.xml" || name === "xl/_rels/workbook.xml.rels" || name.startsWith("xl/worksheets/") + ); + validateOoxmlEntryNames(archive.entryNames, path); + const workbook = archive.entries.get("xl/workbook.xml"); + const relationshipXml = archive.entries.get("xl/_rels/workbook.xml.rels"); + if (!workbook || !relationshipXml) throw new PackageImportError("metadata_unreadable", "XLSX is missing workbook metadata parts.", { path }); + const sheetRecords = workbookSheets(workbook, path, limits.maxXlsxSheets, limits.maxXlsxWorkbookBytes); + const relationships = worksheetRelationships(relationshipXml, path, limits.maxXlsxRelationshipsBytes); + const sheets: XlsxSheetMetadata[] = sheetRecords.map((sheet) => { + const target = relationships.get(sheet.relationshipId); + if (!target) throw new PackageImportError("metadata_unreadable", `XLSX relationship ${sheet.relationshipId} is missing.`, { path }); + const worksheet = archive.entries.get(target); + if (!worksheet) throw new PackageImportError("metadata_unreadable", `XLSX worksheet part is missing: ${target}`, { path }); + const dimension = worksheetDimension(worksheet, target, limits.maxXlsxWorksheetBytes); + return dimension === undefined ? { name: sheet.name } : { name: sheet.name, dimension }; + }); + return { + kind: "spreadsheet", + format: "xlsx", + sheets, + zipEntries: archive.entryCount, + uncompressedBytes: archive.uncompressedBytes + }; + } catch (error) { + if (error instanceof PackageImportError) throw error; + const value = error instanceof Error ? error.message : String(error); + const cause = value.replace(/[\r\n\t]+/g, " ").slice(0, MAX_CAUSE_LENGTH); + throw new PackageImportError("metadata_unreadable", "Could not read XLSX metadata.", { path, cause }); + } +} + +export async function inspectAssetMetadata( + path: string, + sourceBytes: number, + limits: ImportLimits +): Promise { + const extension = extname(path).toLowerCase(); + if (extension === ".xlsx") return xlsxMetadata(path, sourceBytes, limits); + if (extension === ".png" || extension === ".jpg" || extension === ".jpeg") return imageMetadata(path, sourceBytes, extension, limits); + if (extension === ".csv") return { kind: "bounded-inventory", format: "csv" }; + if (extension === ".json") return { kind: "bounded-inventory", format: "json" }; + if (extension === ".xls") return { kind: "bounded-inventory", format: "xls" }; + if (extension === ".parquet") return { kind: "bounded-inventory", format: "parquet" }; + return undefined; +} diff --git a/src/input/docx-extractor.ts b/src/input/docx-extractor.ts new file mode 100644 index 0000000..bc4de72 --- /dev/null +++ b/src/input/docx-extractor.ts @@ -0,0 +1,479 @@ +import { open as openFile } from "node:fs/promises"; +import { posix } from "node:path"; +import { PackageImportError, type ImportLimits, type ProblemExtractionMetadata } from "./types.js"; +import { readBoundedZip } from "./zip-reader.js"; +import { attributeValue, PACKAGE_RELATIONSHIPS_NAMESPACE, parseXml, WORDPROCESSINGML_NAMESPACE } from "./xml.js"; + +interface DocxExtraction { + text: string; + metadata: ProblemExtractionMetadata; +} + +const WORD_DOCUMENT_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"; +const OFFICE_DOCUMENT_RELATIONSHIP = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument"; +const OFFICE_DOCUMENT_RELATIONSHIP_KEY = OFFICE_DOCUMENT_RELATIONSHIP.toLowerCase(); +const CONTENT_TYPES_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/content-types"; +const MAIN_DOCUMENT_PART = "/word/document.xml"; +const PACKAGE_RELATIONSHIPS_PART = "_rels/.rels"; +const CFB_SIGNATURE = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]); +const ZIP_SIGNATURES = [Buffer.from("PK\u0003\u0004", "binary"), Buffer.from("PK\u0005\u0006", "binary"), Buffer.from("PK\u0007\u0008", "binary")]; +const MAX_CAUSE_LENGTH = 240; + +const DOCX_ZIP_CODES = { + corrupt: "docx_corrupt", + encrypted: "docx_encrypted", + zipSlip: "docx_zip_slip", + entryLimit: "docx_zip_entry_limit", + uncompressedLimit: "docx_uncompressed_limit" +} as const; + +function causeMessage(error: unknown): string { + const value = error instanceof Error ? error.message : String(error); + return value.replace(/[\r\n\t]+/g, " ").slice(0, MAX_CAUSE_LENGTH); +} + +async function readPrefix(path: string, length: number): Promise { + const handle = await openFile(path, "r"); + try { + const buffer = Buffer.alloc(length); + const { bytesRead } = await handle.read(buffer, 0, length, 0); + return buffer.subarray(0, bytesRead); + } finally { + await handle.close(); + } +} + +function hasSignature(prefix: Buffer, signature: Buffer): boolean { + return prefix.length >= signature.length && prefix.subarray(0, signature.length).equals(signature); +} + +async function classifyContainer(path: string): Promise { + const prefix = await readPrefix(path, CFB_SIGNATURE.length); + if (hasSignature(prefix, CFB_SIGNATURE)) { + throw new PackageImportError("docx_encrypted", "Encrypted Office containers are not allowed.", { path }); + } + if (!ZIP_SIGNATURES.some((signature) => hasSignature(prefix, signature))) { + throw new PackageImportError("docx_corrupt", "DOCX is neither a ZIP package nor an encrypted Office container.", { path }); + } +} + +function macroMarker(value: string | undefined): boolean { + if (value === undefined) return false; + if (/macro|vba|activex/i.test(value)) return true; + const decoded = decodePercentEscapes(value); + return decoded !== undefined && /macro|vba|activex/i.test(decoded.value); +} + +interface DecodedValue { + value: string; + changed: boolean; +} + +function decodePercentEscapes(value: string): DecodedValue | undefined { + if (!value.includes("%")) return { value, changed: false }; + let decoded = value; + let changed = false; + for (let depth = 0; depth < 4 && decoded.includes("%"); depth += 1) { + try { + const next = decodeURIComponent(decoded); + if (next === decoded) return undefined; + decoded = next; + changed = true; + } catch { + return undefined; + } + } + return decoded.includes("%") ? undefined : { value: decoded, changed }; +} + +interface CanonicalPartName { + value: string; + changed: boolean; +} + +function canonicalPartName(value: string): CanonicalPartName | undefined { + const decoded = decodePercentEscapes(value); + if (decoded === undefined + || !decoded.value + || /[\u0000-\u001f\u007f]/.test(decoded.value) + || decoded.value.includes("\\") + || decoded.value.includes("?") + || decoded.value.includes("#")) return undefined; + const normalized = decoded.value.normalize("NFC"); + const withLeadingSlash = normalized.startsWith("/") ? normalized : `/${normalized}`; + const segments = withLeadingSlash.split("/"); + const canonical: string[] = []; + let changed = decoded.changed || normalized !== decoded.value; + for (const [index, segment] of segments.entries()) { + if (segment === "") { + if (index !== 0) changed = true; + continue; + } + if (segment === ".") { + changed = true; + continue; + } + if (segment === "..") return undefined; + canonical.push(segment); + } + const result = `/${canonical.join("/")}`; + return { value: result, changed: changed || result !== withLeadingSlash }; +} + +interface CanonicalArchiveEntry { + key: string; + partName: string; +} + +function canonicalArchiveEntry(name: string): CanonicalArchiveEntry | undefined { + const isDirectory = name.endsWith("/"); + const partName = isDirectory ? name.slice(0, -1) : name; + if (!partName || name.startsWith("/") || partName.endsWith("/")) return undefined; + const canonical = canonicalPartName(partName); + if (canonical === undefined || canonical.changed || canonical.value !== `/${partName}`) return undefined; + const normalized = partName.normalize("NFC"); + return { key: `/${normalized}`.toLowerCase(), partName }; +} + +function entryIndicatesMacro(entry: CanonicalArchiveEntry): boolean { + const partName = entry.partName.normalize("NFC").toLowerCase(); + const baseName = posix.basename(partName); + return macroMarker(partName) || (baseName.endsWith(".bin") && baseName.includes("office")); +} + +function isContentTypesPart(name: string): boolean { + return !name.endsWith("/") && name.toLowerCase() === "[content_types].xml"; +} + +function isRelationshipPart(name: string): boolean { + return !name.endsWith("/") && name.toLowerCase().endsWith(".rels"); +} + +function validateContentTypes(xml: Buffer, path: string): void { + const defaults = new Set(); + const overrides = new Set(); + let documentContentType: string | undefined; + let documentOverrideCount = 0; + let depth = 0; + let rootSeen = false; + + parseXml(xml, { + openTag(tag) { + const parentDepth = depth; + depth += 1; + if (parentDepth === 0) { + rootSeen = true; + if (tag.uri !== CONTENT_TYPES_NAMESPACE || tag.local !== "Types") { + throw new PackageImportError("docx_corrupt", "DOCX content types XML has an invalid root element.", { path }); + } + return; + } + if (tag.uri !== CONTENT_TYPES_NAMESPACE || (tag.local !== "Default" && tag.local !== "Override")) return; + if (parentDepth !== 1) { + throw new PackageImportError("docx_corrupt", "DOCX content type declarations must be direct children of Types.", { path }); + } + const contentType = attributeValue(tag, "ContentType")?.trim(); + if (!contentType) { + throw new PackageImportError("docx_corrupt", "DOCX content type declaration is incomplete.", { path }); + } + if (macroMarker(contentType)) { + throw new PackageImportError("docx_macro_enabled", "Macro-enabled or VBA content types are not allowed.", { path }); + } + + if (tag.local === "Default") { + const extension = attributeValue(tag, "Extension")?.trim(); + if (!extension || !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(extension)) { + throw new PackageImportError("docx_corrupt", "DOCX default content type has an invalid extension.", { path }); + } + if (macroMarker(extension)) { + throw new PackageImportError("docx_macro_enabled", "Macro-enabled or VBA content types are not allowed.", { path }); + } + const key = extension.toLowerCase(); + if (defaults.has(key)) { + throw new PackageImportError("docx_corrupt", "DOCX contains ambiguous default content type declarations.", { path }); + } + defaults.add(key); + return; + } + + const partName = attributeValue(tag, "PartName")?.trim(); + if (!partName || !partName.startsWith("/")) { + throw new PackageImportError("docx_corrupt", "DOCX override content type has an invalid part name.", { path }); + } + if (macroMarker(partName)) { + throw new PackageImportError("docx_macro_enabled", "VBA package parts are not allowed.", { path }); + } + const canonical = canonicalPartName(partName); + if (canonical === undefined || canonical.changed) { + throw new PackageImportError("docx_corrupt", "DOCX override content type has an unsafe or non-canonical part name.", { path }); + } + const key = canonical.value.toLowerCase(); + if (overrides.has(key)) { + throw new PackageImportError("docx_corrupt", "DOCX contains ambiguous override content type declarations.", { path }); + } + overrides.add(key); + if (key === MAIN_DOCUMENT_PART) { + if (canonical.value !== MAIN_DOCUMENT_PART) { + throw new PackageImportError("docx_corrupt", "DOCX main document override has ambiguous casing.", { path }); + } + documentOverrideCount += 1; + documentContentType = contentType; + } + }, + closeTag() { + depth -= 1; + } + }); + + if (!rootSeen || depth !== 0 || documentOverrideCount !== 1 || documentContentType === undefined) { + throw new PackageImportError("docx_corrupt", "DOCX content types must declare word/document.xml exactly once.", { path }); + } + if (documentContentType !== WORD_DOCUMENT_CONTENT_TYPE) { + throw new PackageImportError("docx_corrupt", "Unexpected DOCX main document content type.", { path }); + } +} + +interface ResolvedRelationshipTarget { + partName: string; + changed: boolean; +} + +function relationshipSourceDirectory(path: string): string[] | undefined { + if (path === PACKAGE_RELATIONSHIPS_PART) return []; + const segments = path.split("/"); + const fileName = segments.at(-1); + if (fileName === undefined + || segments.length < 2 + || segments.at(-2) !== "_rels" + || !fileName.toLowerCase().endsWith(".rels") + || fileName.length === ".rels".length) return undefined; + return segments.slice(0, -2); +} + +function resolveInternalTarget(target: string, relationshipPath: string): ResolvedRelationshipTarget | undefined { + const sourceDirectory = relationshipSourceDirectory(relationshipPath); + const trimmed = target.trim(); + const decoded = decodePercentEscapes(trimmed); + if (sourceDirectory === undefined + || decoded === undefined + || !decoded.value + || /[\u0000-\u001f\u007f]/.test(decoded.value) + || decoded.value.includes("\\") + || decoded.value.includes("?") + || decoded.value.includes("#") + || decoded.value.startsWith("//") + || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(decoded.value)) return undefined; + + const segments = decoded.value.startsWith("/") ? [] : [...sourceDirectory]; + let changed = target !== trimmed || decoded.changed; + for (const [index, segment] of decoded.value.split("/").entries()) { + if (segment === "") { + if (index !== 0) changed = true; + continue; + } + if (segment === ".") { + changed = true; + continue; + } + if (segment === "..") { + changed = true; + if (segments.length === 0) return undefined; + segments.pop(); + continue; + } + segments.push(segment); + } + if (segments.length === 0) return undefined; + return { partName: `/${segments.join("/")}`, changed }; +} + +interface RelationshipSummary { + officeDocumentTargets: ResolvedRelationshipTarget[]; +} + +function validateRelationships(xml: Buffer, path: string): RelationshipSummary { + const ids = new Set(); + const officeDocumentTargets: ResolvedRelationshipTarget[] = []; + let depth = 0; + let rootSeen = false; + + parseXml(xml, { + openTag(tag) { + const parentDepth = depth; + depth += 1; + if (parentDepth === 0) { + rootSeen = true; + if (tag.uri !== PACKAGE_RELATIONSHIPS_NAMESPACE || tag.local !== "Relationships") { + throw new PackageImportError("docx_corrupt", "DOCX relationships XML has an invalid root element.", { path }); + } + return; + } + if (tag.uri !== PACKAGE_RELATIONSHIPS_NAMESPACE || tag.local !== "Relationship") return; + if (parentDepth !== 1) { + throw new PackageImportError("docx_corrupt", "DOCX relationships must be direct children of Relationships.", { path }); + } + const rawTargetMode = attributeValue(tag, "TargetMode"); + const targetMode = rawTargetMode === undefined ? undefined : decodePercentEscapes(rawTargetMode.trim())?.value.trim().toLowerCase(); + if (rawTargetMode !== undefined && (targetMode === undefined || targetMode !== "internal")) { + throw new PackageImportError("docx_external_relationship", "Non-internal DOCX relationship mode is not allowed.", { path }); + } + + const id = attributeValue(tag, "Id")?.trim(); + const type = attributeValue(tag, "Type")?.trim(); + const target = attributeValue(tag, "Target"); + if (!id || ids.has(id) || !target?.trim()) { + throw new PackageImportError("docx_corrupt", "DOCX relationships contain a missing or duplicate required value.", { path }); + } + const resolvedTarget = resolveInternalTarget(target, path); + if (resolvedTarget === undefined) { + throw new PackageImportError("docx_external_relationship", "DOCX relationship target is external or escapes the package.", { path }); + } + if (!type) { + throw new PackageImportError("docx_corrupt", "DOCX relationships contain a missing or duplicate required value.", { path }); + } + ids.add(id); + if (macroMarker(type) || macroMarker(resolvedTarget.partName)) { + throw new PackageImportError("docx_macro_enabled", "Macro or VBA relationships are not allowed.", { path }); + } + if (decodePercentEscapes(type)?.value.toLowerCase() === OFFICE_DOCUMENT_RELATIONSHIP_KEY) officeDocumentTargets.push(resolvedTarget); + }, + closeTag() { + depth -= 1; + } + }); + + if (!rootSeen || depth !== 0) { + throw new PackageImportError("docx_corrupt", "DOCX relationships XML is incomplete.", { path }); + } + return { officeDocumentTargets }; +} + +function validatePackageRelationships(summary: RelationshipSummary, path: string): void { + if (summary.officeDocumentTargets.length !== 1) { + throw new PackageImportError("docx_corrupt", "DOCX package must contain exactly one officeDocument relationship.", { path }); + } + const target = summary.officeDocumentTargets[0]; + if (target === undefined || target.changed || target.partName !== MAIN_DOCUMENT_PART) { + throw new PackageImportError("docx_corrupt", "DOCX package does not point to word/document.xml as its main document.", { path }); + } +} + +function parseBody(xml: Buffer, path: string, characterLimit: number): string { + const pieces: string[] = []; + let collectingText = false; + let characterCount = 0; + const append = (text: string): void => { + characterCount += text.length; + if (characterCount > characterLimit) { + throw new PackageImportError("docx_character_limit", `DOCX body exceeds ${characterLimit} characters.`, { + path, + actual: characterCount, + limit: characterLimit + }); + } + pieces.push(text); + }; + parseXml(xml, { + openTag(tag) { + if (tag.uri !== WORDPROCESSINGML_NAMESPACE) return; + if (tag.local === "t" || tag.local === "instrText") collectingText = true; + else if (tag.local === "tab") append("\t"); + else if (tag.local === "br" || tag.local === "cr") append("\n"); + }, + closeTag(tag) { + if (tag.uri !== WORDPROCESSINGML_NAMESPACE) return; + if (tag.local === "t" || tag.local === "instrText") collectingText = false; + else if (tag.local === "p") append("\n"); + else if (tag.local === "tc") append("\t"); + }, + text(text) { + if (collectingText) append(text); + } + }); + return pieces.join("").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim(); +} + +export async function extractDocx( + path: string, + identity: { sha256: string; sizeBytes: number }, + limits: ImportLimits +): Promise { + if (identity.sizeBytes > limits.maxProblemBytes) { + throw new PackageImportError("problem_file_limit", `DOCX is ${identity.sizeBytes} bytes; limit is ${limits.maxProblemBytes}.`, { + path, + actual: identity.sizeBytes, + limit: limits.maxProblemBytes + }); + } + try { + await classifyContainer(path); + const archive = await readBoundedZip( + path, + { maxEntries: limits.maxDocxZipEntries, maxUncompressedBytes: limits.maxDocxUncompressedBytes }, + DOCX_ZIP_CODES, + (name) => isContentTypesPart(name) || name.toLowerCase() === "word/document.xml" || isRelationshipPart(name) + ); + const contentTypes = archive.entries.get("[Content_Types].xml"); + const packageRelationships = archive.entries.get("_rels/.rels"); + const document = archive.entries.get("word/document.xml"); + if (!contentTypes || !packageRelationships || !document) { + throw new PackageImportError("docx_corrupt", "DOCX is missing required OOXML parts.", { path }); + } + for (const [name, xml] of archive.entries) { + if (xml.length > limits.maxDocxXmlPartBytes) { + throw new PackageImportError("docx_uncompressed_limit", `DOCX XML part ${name} exceeds ${limits.maxDocxXmlPartBytes} bytes.`, { + path: name, + actual: xml.length, + limit: limits.maxDocxXmlPartBytes + }); + } + } + const canonicalEntries = new Set(); + for (const name of archive.entryNames) { + if (macroMarker(name)) { + throw new PackageImportError("docx_macro_enabled", "Macro or VBA package parts are not allowed.", { path }); + } + const canonical = canonicalArchiveEntry(name); + if (canonical === undefined) { + throw new PackageImportError("docx_corrupt", "DOCX contains an unsafe or non-canonical package part name.", { path: name }); + } + if (canonicalEntries.has(canonical.key)) { + throw new PackageImportError("docx_corrupt", "DOCX contains case-ambiguous package part names.", { path: name }); + } + canonicalEntries.add(canonical.key); + if (entryIndicatesMacro(canonical)) { + throw new PackageImportError("docx_macro_enabled", "Macro or VBA package parts are not allowed.", { path }); + } + } + validateContentTypes(contentTypes, path); + const packageSummary = validateRelationships(packageRelationships, "_rels/.rels"); + validatePackageRelationships(packageSummary, "_rels/.rels"); + for (const [name, xml] of archive.entries) { + if (name.toLowerCase().endsWith(".rels") && name.toLowerCase() !== "_rels/.rels") validateRelationships(xml, name); + } + const text = parseBody(document, path, limits.maxDocxCharacters); + if (!text) throw new PackageImportError("docx_empty", "DOCX body contains no extractable text.", { path }); + if (text.length > limits.maxDocxCharacters) { + throw new PackageImportError("docx_character_limit", `DOCX body exceeds ${limits.maxDocxCharacters} characters after normalization.`, { + path, + actual: text.length, + limit: limits.maxDocxCharacters + }); + } + return { + text, + metadata: { + format: "docx", + sourceBytes: identity.sizeBytes, + sha256: identity.sha256, + extractedCharacters: text.length, + extractor: "ooxml-body-v2", + zipEntries: archive.entryCount, + uncompressedBytes: archive.uncompressedBytes + } + }; + } catch (error) { + if (error instanceof PackageImportError) throw error; + throw new PackageImportError("docx_corrupt", "Could not parse DOCX.", { path, cause: causeMessage(error) }); + } +} diff --git a/src/input/package-importer.ts b/src/input/package-importer.ts index 8649835..8c4bd1a 100644 --- a/src/input/package-importer.ts +++ b/src/input/package-importer.ts @@ -1,17 +1,83 @@ -import { lstat, readFile, realpath } from "node:fs/promises"; +import { lstat, readFile, readdir, realpath } from "node:fs/promises"; import { basename, extname, isAbsolute, relative, resolve, sep } from "node:path"; import { fileIdentity, sha256Text } from "../infrastructure/hash.js"; import type { DataAsset } from "../contracts/types.js"; +import { inspectAssetMetadata } from "./asset-metadata.js"; +import { extractDocx } from "./docx-extractor.js"; +import { extractPdf } from "./pdf-extractor.js"; +import { + PackageImportError, + type AssetMetadata, + type ImportedPackage, + type ImportLimits, + type ImportOptions, + type ImportWarning, + type ProblemExtractionMetadata +} from "./types.js"; -const SUPPORTED_EXTENSIONS = new Set([".md", ".markdown", ".txt", ".csv", ".json", ".xlsx", ".xls", ".parquet", ".png", ".jpg", ".jpeg"]); +export { + PackageImportError, + type AssetMetadata, + type ImportedPackage, + type ImportLimits, + type ImportOptions, + type ImportWarning, + type ProblemExtractionMetadata +} from "./types.js"; -export interface ImportedPackage { - rootPath: string; - problemPath: string; - problemText: string; - dataAssets: DataAsset[]; - dataPaths: Map; -} +/** + * Problem candidates are selected by these precedence groups, from highest to + * lowest. Markdown spellings intentionally share one precedence group; if more + * than one candidate exists in the selected group, import fails rather than + * depending on directory enumeration order. + */ +export const PROBLEM_CANDIDATE_PRIORITY = [ + ["problem.md", "problem.markdown"], + ["problem.txt"], + ["problem.pdf"], + ["problem.docx"] +] as const; + +export const DEFAULT_IMPORT_LIMITS: Readonly = Object.freeze({ + maxProblemBytes: 32 * 1024 * 1024, + maxTextCharacters: 2_000_000, + maxPackageEntries: 10_000, + maxPackageDepth: 64, + maxAssetBytes: 2 * 1024 * 1024 * 1024, + maxPdfPages: 500, + maxPdfCharacters: 2_000_000, + maxDocxZipEntries: 2_000, + maxDocxUncompressedBytes: 128 * 1024 * 1024, + maxDocxXmlPartBytes: 16 * 1024 * 1024, + maxDocxCharacters: 2_000_000, + maxMetadataBytes: 128 * 1024 * 1024, + maxMetadataZipEntries: 10_000, + maxMetadataUncompressedBytes: 256 * 1024 * 1024, + maxXlsxSheets: 1_000, + maxXlsxWorkbookBytes: 4 * 1024 * 1024, + maxXlsxRelationshipsBytes: 4 * 1024 * 1024, + maxXlsxWorksheetBytes: 1024 * 1024, + maxImageHeaderBytes: 1024 * 1024, + maxImagePixels: 250_000_000 +}); + +const MACRO_ENABLED_EXTENSIONS = new Set([".docm", ".dotm"]); + +const SUPPORTED_EXTENSIONS = new Set([ + ".md", + ".markdown", + ".txt", + ".pdf", + ".docx", + ".csv", + ".json", + ".xlsx", + ".xls", + ".parquet", + ".png", + ".jpg", + ".jpeg" +]); function mediaType(path: string): string { const extension = extname(path).toLowerCase(); @@ -19,6 +85,8 @@ function mediaType(path: string): string { ".md": "text/markdown", ".markdown": "text/markdown", ".txt": "text/plain", + ".pdf": "application/pdf", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".csv": "text/csv", ".json": "application/json", ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", @@ -31,67 +99,261 @@ function mediaType(path: string): string { return types[extension] ?? "application/octet-stream"; } +function normalizedLimits(overrides: Partial | undefined): ImportLimits { + const limits = { ...DEFAULT_IMPORT_LIMITS, ...overrides }; + for (const [name, value] of Object.entries(limits)) { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new TypeError(`Import limit ${name} must be a positive safe integer.`); + } + } + return limits; +} + function normalizeRelative(root: string, target: string): string { const value = relative(root, target).split(sep).join("/"); if (!value || value.startsWith("../") || value === ".." || isAbsolute(value)) { - throw new Error(`Unsafe path outside package: ${value || target}`); + throw new PackageImportError("unsafe_path", `Unsafe path outside package: ${value || target}`, { path: value || target }); } return value; } export async function resolveSafePath(rootPath: string, relativePath: string): Promise { - const root = await realpath(rootPath); - if (isAbsolute(relativePath)) throw new Error(`Unsafe absolute path outside package: ${relativePath}`); + const requestedRoot = resolve(rootPath); + const requestedStat = await lstat(requestedRoot); + if (requestedStat.isSymbolicLink()) throw new PackageImportError("symlink_input", `Symlink inputs are not allowed: ${rootPath}`, { path: rootPath }); + const root = await realpath(requestedRoot); + if (isAbsolute(relativePath)) { + throw new PackageImportError("unsafe_path", `Unsafe absolute path outside package: ${relativePath}`, { path: relativePath }); + } const candidate = resolve(root, relativePath); const normalized = normalizeRelative(root, candidate); const parentReal = await realpath(resolve(root, normalized, "..")); if (parentReal !== root && !parentReal.startsWith(`${root}${sep}`)) { - throw new Error(`Unsafe path outside package: ${relativePath}`); + throw new PackageImportError("unsafe_path", `Unsafe path outside package: ${relativePath}`, { path: relativePath }); } - let targetReal: string; try { - targetReal = await realpath(candidate); - } catch { - targetReal = candidate; - } - if (targetReal !== root && !targetReal.startsWith(`${root}${sep}`)) { - throw new Error(`Unsafe path outside package: ${relativePath}`); + const candidateStat = await lstat(candidate); + if (candidateStat.isSymbolicLink()) { + throw new PackageImportError("symlink_input", `Symlink inputs are not allowed: ${relativePath}`, { path: relativePath }); + } + const targetReal = await realpath(candidate); + if (targetReal !== root && !targetReal.startsWith(`${root}${sep}`)) { + throw new PackageImportError("unsafe_path", `Unsafe path outside package: ${relativePath}`, { path: relativePath }); + } + } catch (error) { + if (error instanceof PackageImportError) throw error; + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code !== "ENOENT") throw error; } return candidate; } -async function walk(root: string, directory: string, files: string[]): Promise { - const { readdir } = await import("node:fs/promises"); +interface WalkState { + entries: number; +} + +async function walk( + root: string, + directory: string, + files: string[], + limits: ImportLimits, + state: WalkState, + depth: number +): Promise { const entries = await readdir(directory, { withFileTypes: true }); - for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { - if (entry.name === ".git" || entry.name === "node_modules" || entry.name.startsWith(".")) continue; + for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name, "en"))) { const absolute = resolve(directory, entry.name); - if (entry.isDirectory()) { - await walk(root, absolute, files); - } else if (entry.isFile() && SUPPORTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) { + state.entries += 1; + if (state.entries > limits.maxPackageEntries) { + throw new PackageImportError("package_entry_limit", `Package has more than ${limits.maxPackageEntries} filesystem entries.`, { + actual: state.entries, + limit: limits.maxPackageEntries + }); + } + const relativePath = normalizeRelative(root, absolute); + const current = await lstat(absolute); + if (current.isSymbolicLink()) { + throw new PackageImportError("symlink_input", `Symlink inputs are not allowed: ${relativePath}`, { path: relativePath }); + } + if (current.isDirectory()) { + const childDepth = depth + 1; + if (childDepth > limits.maxPackageDepth) { + throw new PackageImportError("package_depth_limit", `Package directory depth exceeds ${limits.maxPackageDepth}.`, { + path: relativePath, + actual: childDepth, + limit: limits.maxPackageDepth + }); + } + // Hidden entries consume the traversal budget, but hidden directories are not recursed into. + if (entry.name.startsWith(".")) continue; + const resolvedDirectory = await realpath(absolute); + if (resolvedDirectory !== root && !resolvedDirectory.startsWith(`${root}${sep}`)) { + throw new PackageImportError("unsafe_path", "Directory changed outside package during traversal.", { path: relativePath }); + } + const checked = await lstat(absolute); + if (!checked.isDirectory() || checked.isSymbolicLink() || checked.dev !== current.dev || checked.ino !== current.ino) { + throw new PackageImportError("unsafe_path", "Directory changed during traversal.", { path: relativePath }); + } + await walk(root, resolvedDirectory, files, limits, state, childDepth); + continue; + } + // Hidden entries consume the traversal budget, but hidden files are not inventoried. + if (entry.name.startsWith(".")) continue; + const extension = extname(entry.name).toLowerCase(); + if (current.isFile() && MACRO_ENABLED_EXTENSIONS.has(extension)) { + throw new PackageImportError("docx_macro_enabled", `Macro-enabled Word input is not allowed: ${relativePath}`, { + path: relativePath + }); + } else if (current.isFile() && SUPPORTED_EXTENSIONS.has(extension)) { files.push(absolute); - } else if (entry.isSymbolicLink()) { - throw new Error(`Symlink inputs are not allowed: ${entry.name}`); } } } -export async function importPackage(packagePath: string): Promise { - const root = await realpath(resolve(packagePath)); - const rootStat = await lstat(root); - if (!rootStat.isDirectory()) throw new Error(`Package path is not a directory: ${packagePath}`); +function selectProblem(root: string, files: string[]): string { + const byName = new Map(); + for (const path of files) { + const name = basename(path).toLowerCase(); + const candidates = byName.get(name) ?? []; + candidates.push(path); + byName.set(name, candidates); + } + for (const group of PROBLEM_CANDIDATE_PRIORITY) { + const candidates = group.flatMap((name) => byName.get(name) ?? []).sort((left, right) => left.localeCompare(right, "en")); + if (candidates.length === 0) continue; + if (candidates.length > 1) { + const relativeCandidates = candidates.map((path) => normalizeRelative(root, path)); + throw new PackageImportError( + "ambiguous_problem", + `Multiple problem statements share the selected priority: ${relativeCandidates.join(", ")}`, + { candidates: relativeCandidates } + ); + } + const selected = candidates[0]; + if (selected === undefined) break; + return selected; + } + throw new PackageImportError( + "problem_missing", + "Package must contain problem.md, problem.markdown, problem.txt, problem.pdf, or problem.docx." + ); +} + +function decodeUtf8(bytes: Buffer, path: string): string { + try { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch (error) { + const value = error instanceof Error ? error.message : String(error); + const cause = value.replace(/[\r\n\t]+/g, " ").slice(0, 240); + throw new PackageImportError("problem_encoding", `Problem statement is not valid UTF-8: ${path}`, { path, cause }); + } +} + +async function extractProblem( + problemPath: string, + identity: { sha256: string; sizeBytes: number }, + limits: ImportLimits +): Promise<{ text: string; metadata: ProblemExtractionMetadata }> { + const extension = extname(problemPath).toLowerCase(); + if (extension === ".pdf") return extractPdf(problemPath, identity, limits); + if (extension === ".docx") return extractDocx(problemPath, identity, limits); + if (identity.sizeBytes > limits.maxProblemBytes) { + throw new PackageImportError("problem_file_limit", `Problem statement is ${identity.sizeBytes} bytes; limit is ${limits.maxProblemBytes}.`, { + path: problemPath, + actual: identity.sizeBytes, + limit: limits.maxProblemBytes + }); + } + const text = decodeUtf8(await readFile(problemPath), problemPath); + if (text.length > limits.maxTextCharacters) { + throw new PackageImportError("problem_file_limit", `Problem statement exceeds ${limits.maxTextCharacters} characters.`, { + path: problemPath, + actual: text.length, + limit: limits.maxTextCharacters + }); + } + if (!text.trim()) throw new PackageImportError("problem_empty", "Problem statement is empty.", { path: problemPath }); + return { + text, + metadata: { + format: extension === ".txt" ? "text" : "markdown", + sourceBytes: identity.sizeBytes, + sha256: identity.sha256, + extractedCharacters: text.length, + extractor: "utf8-v1" + } + }; +} + +export async function importPackage(packagePath: string, options: ImportOptions = {}): Promise { + const limits = normalizedLimits(options.limits); + const requestedRoot = resolve(packagePath); + const requestedStat = await lstat(requestedRoot); + if (requestedStat.isSymbolicLink()) { + throw new PackageImportError("symlink_input", `Symlink package roots are not allowed: ${packagePath}`, { path: packagePath }); + } + if (!requestedStat.isDirectory()) throw new Error(`Package path is not a directory: ${packagePath}`); + const root = await realpath(requestedRoot); + const checkedRoot = await lstat(requestedRoot); + if (!checkedRoot.isDirectory() + || checkedRoot.isSymbolicLink() + || checkedRoot.dev !== requestedStat.dev + || checkedRoot.ino !== requestedStat.ino) { + throw new PackageImportError("unsafe_path", "Package root changed during import.", { path: packagePath }); + } const files: string[] = []; - await walk(root, root, files); - const problemPath = files.find((path) => ["problem.md", "problem.markdown", "problem.txt"].includes(basename(path).toLowerCase())); - if (!problemPath) throw new Error("Package must contain problem.md, problem.markdown, or problem.txt."); - const problemText = await readFile(problemPath, "utf8"); - if (!problemText.trim()) throw new Error("Problem statement is empty."); + await walk(root, root, files, limits, { entries: 0 }, 0); + const problemPath = selectProblem(root, files); + const problemBeforeStat = await lstat(problemPath); + if (!problemBeforeStat.isFile() || problemBeforeStat.isSymbolicLink()) { + throw new PackageImportError("symlink_input", `Problem input changed or became a symlink during import: ${problemPath}`, { path: problemPath }); + } + if (problemBeforeStat.size > limits.maxProblemBytes) { + throw new PackageImportError("problem_file_limit", `Problem statement is ${problemBeforeStat.size} bytes; limit is ${limits.maxProblemBytes}.`, { + path: problemPath, + actual: problemBeforeStat.size, + limit: limits.maxProblemBytes + }); + } + const problemIdentity = await fileIdentity(problemPath); + const problem = await extractProblem(problemPath, problemIdentity, limits); + const problemAfterStat = await lstat(problemPath); + if (!problemAfterStat.isFile() + || problemAfterStat.isSymbolicLink() + || problemBeforeStat.dev !== problemAfterStat.dev + || problemBeforeStat.ino !== problemAfterStat.ino + || problemBeforeStat.size !== problemAfterStat.size + || problemBeforeStat.mtimeMs !== problemAfterStat.mtimeMs) { + throw new PackageImportError("unsafe_path", `Problem input changed while being extracted: ${problemPath}`, { path: problemPath }); + } const dataAssets: DataAsset[] = []; const dataPaths = new Map(); + const assetMetadata: Record = {}; + const warnings: ImportWarning[] = []; for (const path of files.filter((item) => item !== problemPath)) { const relativePath = normalizeRelative(root, path); + const beforeStat = await lstat(path); + if (!beforeStat.isFile() || beforeStat.isSymbolicLink()) { + throw new PackageImportError("symlink_input", `Input changed or became a symlink during import: ${relativePath}`, { path: relativePath }); + } + if (beforeStat.size > limits.maxAssetBytes) { + throw new PackageImportError("asset_file_limit", `Asset ${relativePath} is ${beforeStat.size} bytes; limit is ${limits.maxAssetBytes}.`, { + path: relativePath, + actual: beforeStat.size, + limit: limits.maxAssetBytes + }); + } const identity = await fileIdentity(path); + const afterStat = await lstat(path); + if (!afterStat.isFile() + || afterStat.isSymbolicLink() + || beforeStat.dev !== afterStat.dev + || beforeStat.ino !== afterStat.ino + || beforeStat.size !== afterStat.size + || beforeStat.mtimeMs !== afterStat.mtimeMs) { + throw new PackageImportError("unsafe_path", `Input changed while being inventoried: ${relativePath}`, { path: relativePath }); + } const asset: DataAsset = { artifact_id: `input-${sha256Text(`${relativePath}:${identity.sha256}`).slice(0, 12)}`, relative_path: relativePath, @@ -101,6 +363,22 @@ export async function importPackage(packagePath: string): Promise { + if (identity.sizeBytes > limits.maxProblemBytes) { + throw new PackageImportError("problem_file_limit", `PDF is ${identity.sizeBytes} bytes; limit is ${limits.maxProblemBytes}.`, { + path, + actual: identity.sizeBytes, + limit: limits.maxProblemBytes + }); + } + const bytes = await readFile(path); + let pdfjs: typeof import("pdfjs-dist/legacy/build/pdf.mjs"); + try { + pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); + } catch (error) { + throw new PackageImportError("pdf_tool_unavailable", "The local pdfjs extractor is unavailable.", { + path, + cause: causeMessage(error) + }); + } + + const loadingTask = pdfjs.getDocument({ + data: new Uint8Array(bytes), + disableAutoFetch: true, + disableFontFace: true, + disableRange: true, + disableStream: true, + isEvalSupported: false, + maxImageSize: limits.maxImagePixels, + stopAtErrors: true, + useSystemFonts: false, + useWasm: false, + useWorkerFetch: false, + verbosity: 0 + }); + try { + const document = await loadingTask.promise; + try { + if (document.numPages > limits.maxPdfPages) { + throw new PackageImportError("pdf_page_limit", `PDF has ${document.numPages} pages; limit is ${limits.maxPdfPages}.`, { + path, + actual: document.numPages, + limit: limits.maxPdfPages + }); + } + const pages: string[] = []; + let characters = 0; + const ensureFits = (base: number, additional: number): void => { + const actual = base + additional; + if (!Number.isSafeInteger(actual) || actual > limits.maxPdfCharacters) { + throw new PackageImportError("pdf_character_limit", `PDF extraction exceeds ${limits.maxPdfCharacters} characters.`, { + path, + actual, + limit: limits.maxPdfCharacters + }); + } + }; + for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber += 1) { + const page = await document.getPage(pageNumber); + const content = await page.getTextContent({ disableNormalization: false, includeMarkedContent: false }); + const textItems = content.items.filter(isTextItem); + const leadingPageSeparator = pages.length === 0 ? "" : "\n\n"; + const pagePieces: string[] = []; + let pageCharacters = 0; + for (const item of textItems) { + const itemText = item.str.trim(); + if (itemText.length === 0) continue; + const separator = pagePieces.length === 0 ? "" : " "; + const additional = separator.length + itemText.length; + if (!Number.isSafeInteger(pageCharacters + additional)) ensureFits(characters, Number.MAX_SAFE_INTEGER); + ensureFits(characters, leadingPageSeparator.length + pageCharacters + additional); + if (separator) pagePieces.push(separator); + pagePieces.push(itemText); + pageCharacters += additional; + } + if (pageCharacters === 0) continue; + pages.push(pagePieces.join("")); + characters += leadingPageSeparator.length + pageCharacters; + } + const text = pages.join("\n\n").trim(); + if (!text) throw new PackageImportError("pdf_empty", "PDF contains no extractable text.", { path }); + if (text.length > limits.maxPdfCharacters || text.length !== characters) { + throw new PackageImportError("pdf_character_limit", `PDF extraction exceeds ${limits.maxPdfCharacters} characters after normalization.`, { + path, + actual: text.length, + limit: limits.maxPdfCharacters + }); + } + return { + text, + metadata: { + format: "pdf", + sourceBytes: identity.sizeBytes, + sha256: identity.sha256, + extractedCharacters: text.length, + extractor: `pdfjs-dist/${pdfjs.version}`, + pages: document.numPages + } + }; + } finally { + await document.destroy(); + } + } catch (error) { + if (error instanceof PackageImportError) throw error; + const message = causeMessage(error); + const name = error instanceof Error ? error.name : ""; + if (name === "PasswordException" || /password|encrypted/i.test(message)) { + throw new PackageImportError("pdf_encrypted", "Encrypted PDF files are not allowed.", { path, cause: message }); + } + throw new PackageImportError("pdf_corrupt", "Could not extract PDF text.", { path, cause: message }); + } finally { + await loadingTask.destroy(); + } +} diff --git a/src/input/types.ts b/src/input/types.ts new file mode 100644 index 0000000..13a5fc7 --- /dev/null +++ b/src/input/types.ts @@ -0,0 +1,203 @@ +import { createHash } from "node:crypto"; +import type { DataAsset } from "../contracts/types.js"; + +const MAX_CAUSE_FINGERPRINT_INPUT = 4096; +const MAX_PUBLIC_PATH_LENGTH = 160; +const MAX_PUBLIC_CANDIDATES = 16; + +function diagnosticFingerprint(value: string): string { + return `sha256:${createHash("sha256").update(value.slice(0, MAX_CAUSE_FINGERPRINT_INPUT)).digest("hex").slice(0, 16)}`; +} + +function isPrivatePath(value: string): boolean { + return value.startsWith("/") + || value.startsWith("\\") + || /^[A-Za-z]:[\\/]/.test(value) + || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(value); +} + +function publicDiagnosticPath(value: string | undefined): string | undefined { + if (value === undefined || value.length === 0) return undefined; + if (/\r|\n|\t/.test(value) || isPrivatePath(value) || value === ".." || value.startsWith("../")) return undefined; + if (value.length > MAX_PUBLIC_PATH_LENGTH + || value.includes("%") + || value.includes("\\") + || value.includes("?") + || value.includes("#") + || value.split("/").some((segment) => segment === ".." || segment === ".")) { + return diagnosticFingerprint(value); + } + return value; +} + +export type PackageImportErrorCode = + | "ambiguous_problem" + | "problem_missing" + | "problem_empty" + | "problem_file_limit" + | "problem_encoding" + | "package_entry_limit" + | "package_depth_limit" + | "asset_file_limit" + | "symlink_input" + | "unsafe_path" + | "pdf_tool_unavailable" + | "pdf_corrupt" + | "pdf_encrypted" + | "pdf_empty" + | "pdf_page_limit" + | "pdf_character_limit" + | "docx_corrupt" + | "docx_encrypted" + | "docx_macro_enabled" + | "docx_external_relationship" + | "docx_zip_slip" + | "docx_zip_entry_limit" + | "docx_uncompressed_limit" + | "docx_character_limit" + | "docx_empty" + | "metadata_unreadable" + | "metadata_limit"; + +const PUBLIC_ERROR_MESSAGES: Readonly> = Object.freeze({ + ambiguous_problem: "Problem statement selection is ambiguous.", + problem_missing: "Package does not contain a supported problem statement.", + problem_empty: "Problem statement is empty.", + problem_file_limit: "Problem statement exceeds an import limit.", + problem_encoding: "Problem statement encoding is invalid.", + package_entry_limit: "Package exceeds the filesystem entry limit.", + package_depth_limit: "Package exceeds the filesystem depth limit.", + asset_file_limit: "An input asset exceeds an import limit.", + symlink_input: "Symlink inputs are not allowed.", + unsafe_path: "Unsafe path outside package.", + pdf_tool_unavailable: "The local PDF extractor is unavailable.", + pdf_corrupt: "PDF input could not be parsed.", + pdf_encrypted: "Encrypted PDF files are not allowed.", + pdf_empty: "PDF contains no extractable text.", + pdf_page_limit: "PDF exceeds the page limit.", + pdf_character_limit: "PDF extraction exceeds the character limit.", + docx_corrupt: "DOCX input could not be parsed.", + docx_encrypted: "Encrypted DOCX files are not allowed.", + docx_macro_enabled: "Macro-enabled DOCX files are not allowed.", + docx_external_relationship: "DOCX contains a non-internal relationship.", + docx_zip_slip: "DOCX contains an unsafe archive path.", + docx_zip_entry_limit: "DOCX exceeds the archive entry limit.", + docx_uncompressed_limit: "DOCX exceeds the uncompressed-byte limit.", + docx_character_limit: "DOCX extraction exceeds the character limit.", + docx_empty: "DOCX body contains no extractable text.", + metadata_unreadable: "Input metadata could not be read safely.", + metadata_limit: "Input metadata exceeds an inspection limit." +}); + +export interface PackageImportErrorDetails { + path?: string; + candidates?: string[]; + limit?: number; + actual?: number; + cause?: string; +} + +export class PackageImportError extends Error { + readonly code: PackageImportErrorCode; + readonly details: Readonly; + + constructor(code: PackageImportErrorCode, message: string, details: PackageImportErrorDetails = {}) { + void message; + const { cause, path, candidates, ...safeDetails } = details; + const normalizedCause = cause?.replace(/[\r\n\t]+/g, " ").slice(0, MAX_CAUSE_FINGERPRINT_INPUT); + const boundedCause = normalizedCause === undefined ? undefined : diagnosticFingerprint(normalizedCause); + const boundedDetails: PackageImportErrorDetails = { ...safeDetails }; + const publicPath = publicDiagnosticPath(path); + if (publicPath !== undefined) boundedDetails.path = publicPath; + if (candidates !== undefined) { + boundedDetails.candidates = candidates.slice(0, MAX_PUBLIC_CANDIDATES).map((candidate) => publicDiagnosticPath(candidate) ?? diagnosticFingerprint(candidate)); + } + if (boundedCause !== undefined) boundedDetails.cause = boundedCause; + super(PUBLIC_ERROR_MESSAGES[code], boundedCause === undefined ? undefined : { cause: boundedCause }); + this.name = "PackageImportError"; + this.code = code; + this.details = Object.freeze(boundedDetails); + } +} + +export interface ImportLimits { + maxProblemBytes: number; + maxTextCharacters: number; + maxPackageEntries: number; + maxPackageDepth: number; + maxAssetBytes: number; + maxPdfPages: number; + maxPdfCharacters: number; + maxDocxZipEntries: number; + maxDocxUncompressedBytes: number; + maxDocxXmlPartBytes: number; + maxDocxCharacters: number; + maxMetadataBytes: number; + maxMetadataZipEntries: number; + maxMetadataUncompressedBytes: number; + maxXlsxSheets: number; + maxXlsxWorkbookBytes: number; + maxXlsxRelationshipsBytes: number; + maxXlsxWorksheetBytes: number; + maxImageHeaderBytes: number; + maxImagePixels: number; +} + +export interface ImportOptions { + limits?: Partial; +} + +export interface ProblemExtractionMetadata { + format: "markdown" | "text" | "pdf" | "docx"; + sourceBytes: number; + sha256: string; + extractedCharacters: number; + extractor: string; + pages?: number; + zipEntries?: number; + uncompressedBytes?: number; +} + +export interface XlsxSheetMetadata { + name: string; + dimension?: string; +} + +export interface XlsxAssetMetadata { + kind: "spreadsheet"; + format: "xlsx"; + sheets: XlsxSheetMetadata[]; + zipEntries: number; + uncompressedBytes: number; +} + +export interface ImageAssetMetadata { + kind: "image"; + format: "png" | "jpeg"; + width: number; + height: number; +} + +export interface BoundedAssetMetadata { + kind: "bounded-inventory"; + format: "csv" | "json" | "xls" | "parquet"; +} + +export type AssetMetadata = XlsxAssetMetadata | ImageAssetMetadata | BoundedAssetMetadata; + +export interface ImportWarning { + code: "metadata_unreadable" | "metadata_limit"; + path: string; + message: string; +} + +export interface ImportedPackage { + rootPath: string; + problemPath: string; + problemText: string; + dataAssets: DataAsset[]; + dataPaths: Map; + problemMetadata?: ProblemExtractionMetadata; + assetMetadata?: Record; + warnings?: ImportWarning[]; +} diff --git a/src/input/xml.ts b/src/input/xml.ts new file mode 100644 index 0000000..d3c3f71 --- /dev/null +++ b/src/input/xml.ts @@ -0,0 +1,32 @@ +import { SaxesParser, type SaxesTagNS } from "saxes"; + +export const WORDPROCESSINGML_NAMESPACE = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"; +export const SPREADSHEETML_NAMESPACE = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; +export const PACKAGE_RELATIONSHIPS_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/relationships"; + +export function parseXml( + xml: Buffer | string, + handlers: { + openTag?: (tag: SaxesTagNS) => void; + closeTag?: (tag: SaxesTagNS) => void; + text?: (text: string) => void; + } +): void { + const parser = new SaxesParser({ xmlns: true, position: true }); + parser.on("opentag", (tag) => handlers.openTag?.(tag)); + parser.on("closetag", (tag) => handlers.closeTag?.(tag)); + parser.on("text", (text) => handlers.text?.(text)); + parser.on("cdata", (text) => handlers.text?.(text)); + parser.on("doctype", () => { + throw new Error("DOCTYPE declarations are not allowed."); + }); + parser.on("error", (error) => { + throw error; + }); + parser.write(xml.toString("utf8")); + parser.close(); +} + +export function attributeValue(tag: SaxesTagNS, local: string, uri?: string): string | undefined { + return Object.values(tag.attributes).find((attribute) => attribute.local === local && (uri === undefined || attribute.uri === uri))?.value; +} diff --git a/src/input/zip-reader.ts b/src/input/zip-reader.ts new file mode 100644 index 0000000..c6b609e --- /dev/null +++ b/src/input/zip-reader.ts @@ -0,0 +1,187 @@ +import crc32 from "buffer-crc32"; +import { open, type Entry, type ZipFile } from "yauzl"; +import { PackageImportError, type PackageImportErrorCode } from "./types.js"; + +export interface ZipLimits { + maxEntries: number; + maxUncompressedBytes: number; +} + +export interface ZipContents { + entries: Map; + entryNames: Set; + entryCount: number; + uncompressedBytes: number; +} + +export interface ZipErrorCodes { + corrupt: PackageImportErrorCode; + encrypted: PackageImportErrorCode; + zipSlip: PackageImportErrorCode; + entryLimit: PackageImportErrorCode; + uncompressedLimit: PackageImportErrorCode; +} + +const MAX_CAUSE_LENGTH = 240; + +function causeMessage(error: unknown): string { + const value = error instanceof Error ? error.message : String(error); + return value.replace(/[\r\n\t]+/g, " ").slice(0, MAX_CAUSE_LENGTH); +} + +function openArchive(path: string): Promise { + return new Promise((resolve, reject) => { + open(path, { + autoClose: false, + lazyEntries: true, + decodeStrings: true, + validateEntrySizes: true, + strictFileNames: true + }, (error, archive) => { + if (error) reject(error); + else resolve(archive); + }); + }); +} + +function readEntry(archive: ZipFile, entry: Entry, maximumBytes: number): Promise { + return new Promise((resolve, reject) => { + archive.openReadStream(entry, (error, stream) => { + if (error) { + reject(error); + return; + } + const chunks: Buffer[] = []; + let size = 0; + stream.on("data", (chunk: Buffer | string) => { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + size += buffer.length; + if (size > maximumBytes) stream.destroy(new Error(`Uncompressed entry exceeds ${maximumBytes} bytes.`)); + else chunks.push(buffer); + }); + stream.once("error", reject); + stream.once("end", () => { + const result = Buffer.concat(chunks); + if (result.length !== entry.uncompressedSize) { + reject(new Error(`ZIP entry size mismatch for ${entry.fileName}: expected ${entry.uncompressedSize}, got ${result.length}.`)); + return; + } + if (crc32.unsigned(result) !== entry.crc32) { + reject(new Error(`ZIP entry CRC mismatch for ${entry.fileName}.`)); + return; + } + resolve(result); + }); + }); + }); +} + +function unsafeEntryName(name: string): boolean { + return !name + || /[\u0000-\u001f\u007f]/.test(name) + || name.includes("\\") + || name.startsWith("/") + || /^[A-Za-z]:/.test(name) + || name.split("/").some((segment) => segment === ".." || segment === "."); +} + +function isUnixSymlink(entry: Entry): boolean { + const fileType = (entry.externalFileAttributes >>> 16) & 0o170000; + return fileType === 0o120000; +} + +function zipErrorCode(message: string, codes: ZipErrorCodes): PackageImportErrorCode { + if (/encrypt|password/i.test(message)) return codes.encrypted; + if (/invalid relative path|absolute path|invalid characters in fileName/i.test(message)) return codes.zipSlip; + return codes.corrupt; +} + +export async function readBoundedZip( + path: string, + limits: ZipLimits, + codes: ZipErrorCodes, + select: (entryName: string) => boolean +): Promise { + let archive: ZipFile; + try { + archive = await openArchive(path); + } catch (error) { + const message = causeMessage(error); + throw new PackageImportError(zipErrorCode(message, codes), "Could not open ZIP package.", { path, cause: message }); + } + + if (archive.entryCount > limits.maxEntries) { + archive.close(); + throw new PackageImportError(codes.entryLimit, `ZIP package has ${archive.entryCount} entries; limit is ${limits.maxEntries}.`, { + path, + actual: archive.entryCount, + limit: limits.maxEntries + }); + } + + return new Promise((resolve, reject) => { + const entries = new Map(); + const entryNames = new Set(); + let count = 0; + let total = 0; + let settled = false; + + const fail = (error: unknown): void => { + if (settled) return; + settled = true; + archive.close(); + if (error instanceof PackageImportError) reject(error); + else { + const message = causeMessage(error); + reject(new PackageImportError(zipErrorCode(message, codes), "Could not read ZIP package.", { path, cause: message })); + } + }; + + archive.on("error", fail); + archive.on("entry", (entry: Entry) => { + void (async () => { + count += 1; + if (count > limits.maxEntries) { + throw new PackageImportError(codes.entryLimit, `ZIP package exceeds ${limits.maxEntries} entries.`, { + path, + actual: count, + limit: limits.maxEntries + }); + } + if (unsafeEntryName(entry.fileName)) { + throw new PackageImportError(codes.zipSlip, `Unsafe ZIP entry path: ${entry.fileName}`, { path: entry.fileName }); + } + if (entryNames.has(entry.fileName)) { + throw new PackageImportError(codes.corrupt, `ZIP package contains a duplicate entry: ${entry.fileName}`, { path: entry.fileName }); + } + entryNames.add(entry.fileName); + if (entry.isEncrypted()) { + throw new PackageImportError(codes.encrypted, `Encrypted ZIP entry is not allowed: ${entry.fileName}`, { path: entry.fileName }); + } + if (isUnixSymlink(entry)) { + throw new PackageImportError(codes.zipSlip, `ZIP symlink entry is not allowed: ${entry.fileName}`, { path: entry.fileName }); + } + total += entry.uncompressedSize; + if (!Number.isSafeInteger(total) || total > limits.maxUncompressedBytes) { + throw new PackageImportError(codes.uncompressedLimit, `ZIP package expands to more than ${limits.maxUncompressedBytes} bytes.`, { + path, + actual: total, + limit: limits.maxUncompressedBytes + }); + } + if (!entry.fileName.endsWith("/") && select(entry.fileName)) { + const remaining = limits.maxUncompressedBytes - (total - entry.uncompressedSize); + entries.set(entry.fileName, await readEntry(archive, entry, remaining)); + } + archive.readEntry(); + })().catch(fail); + }); + archive.on("end", () => { + if (settled) return; + settled = true; + archive.close(); + resolve({ entries, entryNames, entryCount: count, uncompressedBytes: total }); + }); + archive.readEntry(); + }); +} diff --git a/tests/input-package-importer.test.ts b/tests/input-package-importer.test.ts new file mode 100644 index 0000000..7ac72e3 --- /dev/null +++ b/tests/input-package-importer.test.ts @@ -0,0 +1,727 @@ +import { describe, expect, it } from "vitest"; +import { deflateRawSync } from "node:zlib"; +import { mkdtemp, mkdir, readFile, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { + importPackage, + type ImportLimits +} from "../src/input/package-importer.js"; + +const tempPackage = async (name: string): Promise => mkdtemp(join(tmpdir(), `modeling-input-${name}-`)); + +function pdfDocument(pages: Array, encrypted = false): Buffer { + const objects: string[] = []; + const pageObjectNumbers: number[] = []; + const fontObject = 3; + const pagesObject = 2; + objects.push("<< /Type /Catalog /Pages 2 0 R >>"); + objects.push(`<< /Type /Pages /Kids [${pages.map((_, index) => `${4 + index * 2} 0 R`).join(" ")}] /Count ${pages.length} >>`); + objects.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"); + for (const [index, page] of pages.entries()) { + const pageObject = 4 + index * 2; + const contentObject = pageObject + 1; + pageObjectNumbers.push(pageObject); + const items = typeof page === "string" ? [page] : page; + const operations = items.map((text, itemIndex) => { + const escaped = text.replaceAll("\\", "\\\\").replaceAll("(", "\\(").replaceAll(")", "\\)"); + return `1 0 0 1 ${72 + itemIndex * 72} 720 Tm (${escaped}) Tj`; + }).join(" "); + const stream = `BT /F1 18 Tf ${operations} ET`; + objects.push(`<< /Type /Page /Parent ${pagesObject} 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 ${fontObject} 0 R >> >> /Contents ${contentObject} 0 R >>`); + objects.push(`<< /Length ${Buffer.byteLength(stream, "ascii")} >>\nstream\n${stream}\nendstream`); + } + const chunks: Buffer[] = [Buffer.from("%PDF-1.4\n", "ascii")]; + const offsets = [0]; + for (const [index, object] of objects.entries()) { + offsets.push(Buffer.concat(chunks).length); + chunks.push(Buffer.from(`${index + 1} 0 obj\n${object}\nendobj\n`, "ascii")); + } + if (encrypted) { + objects.push("<< /Filter /Standard /V 1 /R 2 /Length 40 /O <0000000000000000000000000000000000000000000000000000000000000000> /U <0000000000000000000000000000000000000000000000000000000000000000> /P -4 >>"); + const encryptedObject = objects.length; + offsets.push(Buffer.concat(chunks).length); + chunks.push(Buffer.from(`${encryptedObject} 0 obj\n${objects[encryptedObject - 1]}\nendobj\n`, "ascii")); + } + const xrefOffset = Buffer.concat(chunks).length; + const xref = [`xref`, `0 ${objects.length + 1}`, "0000000000 65535 f "]; + for (const offset of offsets.slice(1)) xref.push(`${String(offset).padStart(10, "0")} 00000 n `); + const encryptionTrailer = encrypted + ? ` /Encrypt ${objects.length} 0 R /ID [<00112233445566778899aabbccddeeff><00112233445566778899aabbccddeeff>]` + : ""; + xref.push(`trailer\n<< /Size ${objects.length + 1} /Root 1 0 R${encryptionTrailer} >>`, `startxref`, String(xrefOffset), "%%EOF\n"); + chunks.push(Buffer.from(`${xref.join("\n")}\n`, "ascii")); + void pageObjectNumbers; + return Buffer.concat(chunks); +} + +type ZipEntry = { name: string; data: Buffer; compression?: "store" | "deflate"; unixMode?: number }; + +function zip(entries: ZipEntry[]): Buffer { + const local: Buffer[] = []; + const central: Buffer[] = []; + let offset = 0; + for (const entry of entries) { + const name = Buffer.from(entry.name, "utf8"); + const compressed = entry.compression === "deflate" ? deflateRawSync(entry.data) : entry.data; + const method = entry.compression === "deflate" ? 8 : 0; + const crc32 = crc32Of(entry.data); + const header = Buffer.alloc(30 + name.length); + header.writeUInt32LE(0x04034b50, 0); + header.writeUInt16LE(20, 4); + header.writeUInt16LE(0, 6); + header.writeUInt16LE(method, 8); + header.writeUInt16LE(0, 10); + header.writeUInt16LE(0, 12); + header.writeUInt32LE(crc32, 14); + header.writeUInt32LE(compressed.length, 18); + header.writeUInt32LE(entry.data.length, 22); + header.writeUInt16LE(name.length, 26); + header.writeUInt16LE(0, 28); + name.copy(header, 30); + local.push(header, compressed); + const directory = Buffer.alloc(46 + name.length); + directory.writeUInt32LE(0x02014b50, 0); + directory.writeUInt16LE(entry.unixMode === undefined ? 20 : (3 << 8) | 20, 4); + directory.writeUInt16LE(20, 6); + directory.writeUInt16LE(0, 8); + directory.writeUInt16LE(method, 10); + directory.writeUInt16LE(0, 12); + directory.writeUInt16LE(0, 14); + directory.writeUInt32LE(crc32, 16); + directory.writeUInt32LE(compressed.length, 20); + directory.writeUInt32LE(entry.data.length, 24); + directory.writeUInt16LE(name.length, 28); + directory.writeUInt16LE(0, 30); + directory.writeUInt16LE(0, 32); + directory.writeUInt16LE(0, 34); + directory.writeUInt32LE(entry.unixMode === undefined ? 0 : (entry.unixMode << 16) >>> 0, 38); + directory.writeUInt32LE(offset, 42); + name.copy(directory, 46); + central.push(directory); + offset += header.length + compressed.length; + } + const localBytes = Buffer.concat(local); + const centralBytes = Buffer.concat(central); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(0, 4); + end.writeUInt16LE(0, 6); + end.writeUInt16LE(entries.length, 8); + end.writeUInt16LE(entries.length, 10); + end.writeUInt32LE(centralBytes.length, 12); + end.writeUInt32LE(localBytes.length, 16); + return Buffer.concat([localBytes, centralBytes, end]); +} + +function encryptedZipEntryFixture(name: string, data: Buffer): Buffer { + const archive = zip([{ name, data, compression: "deflate" }]); + archive.writeUInt16LE(archive.readUInt16LE(6) | 0x0001, 6); + const centralOffset = archive.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02])); + if (centralOffset < 0) throw new Error("Fixture ZIP has no central directory."); + archive.writeUInt16LE(archive.readUInt16LE(centralOffset + 8) | 0x0001, centralOffset + 8); + return archive; +} + +function crc32Of(data: Buffer): number { + let crc = 0xffffffff; + for (const byte of data) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + return (crc ^ 0xffffffff) >>> 0; +} + +const STANDARD_DOCX_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"; +const OFFICE_DOCUMENT_RELATIONSHIP = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument"; +const RELATIONSHIPS_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/relationships"; +const CONTENT_TYPES_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/content-types"; + +function relationshipsXml(relationships: string): string { + return `${relationships}`; +} + +function contentTypesXml(declarations: string): string { + return `${declarations}`; +} + +interface DocxOptions { + extra?: ZipEntry[]; + mainContentType?: string; + contentTypes?: string; + packageRelationships?: string; + documentXml?: string; +} + +function docxDocument(text: string, options: DocxOptions = {}): Buffer { + const mainContentType = options.mainContentType ?? STANDARD_DOCX_CONTENT_TYPE; + const contentTypes = options.contentTypes ?? contentTypesXml( + `` + ); + const packageRelationships = options.packageRelationships ?? relationshipsXml( + `` + ); + const documentXml = options.documentXml + ?? `${text}`; + return zip([ + { name: "[Content_Types].xml", data: Buffer.from(contentTypes) }, + { name: "_rels/.rels", data: Buffer.from(packageRelationships) }, + { name: "word/document.xml", data: Buffer.from(documentXml) }, + ...(options.extra ?? []) + ]); +} + +function xlsxAsset(workbookRelationships?: string, extra: ZipEntry[] = []): Buffer { + const relationships = workbookRelationships + ?? ""; + return zip([ + { name: "[Content_Types].xml", data: Buffer.from("") }, + { name: "xl/workbook.xml", data: Buffer.from("") }, + { name: "xl/_rels/workbook.xml.rels", data: Buffer.from(relationships) }, + { name: "xl/worksheets/sheet1.xml", data: Buffer.from("") }, + { name: "xl/worksheets/sheet2.xml", data: Buffer.from("") }, + ...extra + ]); +} + +function png(width: number, height: number): Buffer { + const value = Buffer.alloc(24); + Buffer.from("\x89PNG\r\n\x1a\n", "binary").copy(value, 0); + value.writeUInt32BE(13, 8); + value.write("IHDR", 12, "ascii"); + value.writeUInt32BE(width, 16); + value.writeUInt32BE(height, 20); + return value; +} + +async function expectImportCode(operation: Promise, code: string): Promise { + await expect(operation).rejects.toMatchObject({ code }); +} + +describe("safe package importer", () => { + it("extracts a real PDF and records bounded provenance", async () => { + const root = await tempPackage("pdf"); + const source = pdfDocument(["PDF problem statement"]); + await writeFile(join(root, "problem.pdf"), source); + const imported = await importPackage(root); + expect(imported.problemText).toContain("PDF problem statement"); + expect(imported.problemMetadata).toMatchObject({ format: "pdf", pages: 1, extractedCharacters: expect.any(Number), extractor: expect.stringContaining("pdfjs") }); + expect(imported.problemMetadata?.sha256).toBe(createHash("sha256").update(source).digest("hex")); + expect(imported.problemPath).toBe(join(root, "problem.pdf")); + }); + + it("rejects empty, corrupt, and over-page PDFs with structured failures", async () => { + const empty = await tempPackage("pdf-empty"); + await writeFile(join(empty, "problem.pdf"), pdfDocument([""])); + await expectImportCode(importPackage(empty), "pdf_empty"); + + const corrupt = await tempPackage("pdf-corrupt"); + await writeFile(join(corrupt, "problem.pdf"), Buffer.from("not a pdf")); + await expectImportCode(importPackage(corrupt), "pdf_corrupt"); + + const encrypted = await tempPackage("pdf-encrypted"); + await writeFile(join(encrypted, "problem.pdf"), pdfDocument(["secret"], true)); + await expectImportCode(importPackage(encrypted), "pdf_encrypted"); + + const tooManyPages = await tempPackage("pdf-pages"); + await writeFile(join(tooManyPages, "problem.pdf"), pdfDocument(["one", "two"])); + const limits: Partial = { maxPdfPages: 1 }; + await expectImportCode(importPackage(tooManyPages, { limits }), "pdf_page_limit"); + + const tooManyCharacters = await tempPackage("pdf-characters"); + await writeFile(join(tooManyCharacters, "problem.pdf"), pdfDocument(["0123456789"])); + await expectImportCode(importPackage(tooManyCharacters, { limits: { maxPdfCharacters: 5 } }), "pdf_character_limit"); + + const pageSeparators = await tempPackage("pdf-page-separators"); + await writeFile(join(pageSeparators, "problem.pdf"), pdfDocument(["A", "B"])); + await expectImportCode(importPackage(pageSeparators, { limits: { maxPdfCharacters: 3 } }), "pdf_character_limit"); + const exactPageLimit = await importPackage(pageSeparators, { limits: { maxPdfCharacters: 4 } }); + expect(exactPageLimit.problemText).toBe("A\n\nB"); + expect(exactPageLimit.problemText.length).toBe(4); + + const itemSeparators = await tempPackage("pdf-item-separators"); + await writeFile(join(itemSeparators, "problem.pdf"), pdfDocument([["A", "B"]])); + await expectImportCode(importPackage(itemSeparators, { limits: { maxPdfCharacters: 2 } }), "pdf_character_limit"); + const exactItemLimit = await importPackage(itemSeparators, { limits: { maxPdfCharacters: 3 } }); + expect(exactItemLimit.problemText).toBe("A B"); + expect(exactItemLimit.problemText.length).toBe(3); + + const normalizedItems = await tempPackage("pdf-normalized-items"); + await writeFile(join(normalizedItems, "problem.pdf"), pdfDocument([[" A ", " B "]])); + const normalized = await importPackage(normalizedItems, { limits: { maxPdfCharacters: 3 } }); + expect(normalized.problemText).toBe("A B"); + expect(normalized.problemText.length).toBe(3); + + const tooManyBytes = await tempPackage("pdf-bytes"); + await writeFile(join(tooManyBytes, "problem.pdf"), pdfDocument(["text"])); + await expectImportCode(importPackage(tooManyBytes, { limits: { maxProblemBytes: 10 } }), "problem_file_limit"); + }); + + it("extracts only DOCX body text", async () => { + const root = await tempPackage("docx"); + await writeFile(join(root, "problem.docx"), docxDocument("DOCX body text")); + const imported = await importPackage(root); + expect(imported.problemText).toContain("DOCX body text"); + expect(imported.problemMetadata).toMatchObject({ format: "docx", extractedCharacters: 14, extractor: "ooxml-body-v2" }); + }); + + it("rejects corrupt and macro-enabled DOCX packages", async () => { + const corrupt = await tempPackage("docx-corrupt"); + await writeFile(join(corrupt, "problem.docx"), Buffer.from("not a zip")); + await expectImportCode(importPackage(corrupt), "docx_corrupt"); + + const cfbEncrypted = await tempPackage("docx-cfb-encrypted"); + const privateBody = "PRIVATE-DOCUMENT-BODY-MUST-NOT-LEAK"; + const cfb = Buffer.concat([ + Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]), + Buffer.from(`EncryptedPackage\0${privateBody}`) + ]); + await writeFile(join(cfbEncrypted, "problem.docx"), cfb); + const encryptedError = await importPackage(cfbEncrypted).then( + () => undefined, + (error: unknown) => error + ); + expect(encryptedError).toMatchObject({ code: "docx_encrypted" }); + expect(JSON.stringify(encryptedError)).not.toContain(privateBody); + expect(encryptedError instanceof Error ? encryptedError.message : String(encryptedError)).not.toContain(privateBody); + expect(encryptedError && typeof encryptedError === "object" && "details" in encryptedError + ? (encryptedError as { details?: { cause?: string } }).details?.cause + : undefined).toBeUndefined(); + + const encryptedEntry = await tempPackage("docx-encrypted-entry"); + await writeFile(join(encryptedEntry, "problem.docx"), encryptedZipEntryFixture( + "[Content_Types].xml", + Buffer.from("encrypted") + )); + await expectImportCode(importPackage(encryptedEntry), "docx_encrypted"); + + const crcMismatch = await tempPackage("docx-crc"); + const tampered = docxDocument("body"); + const marker = tampered.indexOf(Buffer.from("body")); + tampered[marker] = "B".charCodeAt(0); + await writeFile(join(crcMismatch, "problem.docx"), tampered); + await expectImportCode(importPackage(crcMismatch), "docx_corrupt"); + + const duplicate = await tempPackage("docx-duplicate"); + await writeFile(join(duplicate, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/document.xml", data: Buffer.from("") }] + })); + await expectImportCode(importPackage(duplicate), "docx_corrupt"); + + const macro = await tempPackage("docx-macro"); + const document = docxDocument("body", { + extra: [{ name: "word/vbaProject.bin", data: Buffer.from("not executed") }], + mainContentType: "application/vnd.ms-word.document.macroEnabled.main+xml" + }); + await writeFile(join(macro, "problem.docx"), document); + await expectImportCode(importPackage(macro), "docx_macro_enabled"); + }); + + it("rejects symlink entries inside DOCX archives", async () => { + const archiveSymlink = await tempPackage("docx-archive-symlink"); + await writeFile(join(archiveSymlink, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/linked.bin", data: Buffer.alloc(0), unixMode: 0o120777 }] + })); + await expectImportCode(importPackage(archiveSymlink), "docx_zip_slip"); + }); + + it("rejects DOCX external relationships and ZIP slip names", async () => { + const external = await tempPackage("docx-external"); + await writeFile(join(external, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) }] + })); + await expectImportCode(importPackage(external), "docx_external_relationship"); + + const paddedExternal = await tempPackage("docx-padded-external"); + const querySecret = "relationship-query-secret"; + await writeFile(join(paddedExternal, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + `` + )) }] + })); + const externalError = await importPackage(paddedExternal).then( + () => undefined, + (error: unknown) => error + ); + expect(externalError).toMatchObject({ code: "docx_external_relationship" }); + expect(JSON.stringify(externalError)).not.toContain(querySecret); + expect(externalError instanceof Error ? externalError.message : String(externalError)).not.toContain(querySecret); + + const unknownMode = await tempPackage("docx-unknown-mode"); + await writeFile(join(unknownMode, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) }] + })); + await expectImportCode(importPackage(unknownMode), "docx_external_relationship"); + + const encodedExternal = await tempPackage("docx-encoded-external"); + await writeFile(join(encodedExternal, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) }] + })); + await expectImportCode(importPackage(encodedExternal), "docx_external_relationship"); + + const encodedEscape = await tempPackage("docx-encoded-escape"); + await writeFile(join(encodedEscape, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) }] + })); + await expectImportCode(importPackage(encodedEscape), "docx_external_relationship"); + + const explicitInternal = await tempPackage("docx-internal-mode"); + await writeFile(join(explicitInternal, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) }] + })); + expect((await importPackage(explicitInternal)).problemText).toBe("body"); + + const entity = await tempPackage("docx-entity"); + await writeFile(join(entity, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from("]>&xxe;") }] + })); + await expectImportCode(importPackage(entity), "docx_corrupt"); + + const traversal = await tempPackage("docx-slip"); + await writeFile(join(traversal, "problem.docx"), docxDocument("body", { + extra: [{ name: "../outside.xml", data: Buffer.from("x") }] + })); + await expectImportCode(importPackage(traversal), "docx_zip_slip"); + }); + + it("rejects ambiguous DOCX relationships and content types", async () => { + const twoMainRelationships = await tempPackage("docx-two-main-relationships"); + await writeFile(join(twoMainRelationships, "problem.docx"), docxDocument("body", { + packageRelationships: relationshipsXml( + `` + + `` + ) + })); + await expectImportCode(importPackage(twoMainRelationships), "docx_corrupt"); + + const normalizedMainRelationship = await tempPackage("docx-normalized-main-relationship"); + await writeFile(join(normalizedMainRelationship, "problem.docx"), docxDocument("body", { + packageRelationships: relationshipsXml( + `` + + `` + ) + })); + await expectImportCode(importPackage(normalizedMainRelationship), "docx_corrupt"); + + const duplicatePackageId = await tempPackage("docx-duplicate-package-id"); + await writeFile(join(duplicatePackageId, "problem.docx"), docxDocument("body", { + packageRelationships: relationshipsXml( + `` + + "" + ) + })); + await expectImportCode(importPackage(duplicatePackageId), "docx_corrupt"); + + const paddedDuplicatePackageId = await tempPackage("docx-padded-duplicate-package-id"); + await writeFile(join(paddedDuplicatePackageId, "problem.docx"), docxDocument("body", { + packageRelationships: relationshipsXml( + `` + + "" + ) + })); + await expectImportCode(importPackage(paddedDuplicatePackageId), "docx_corrupt"); + + const nonCanonicalMainTarget = await tempPackage("docx-noncanonical-main-target"); + await writeFile(join(nonCanonicalMainTarget, "problem.docx"), docxDocument("body", { + packageRelationships: relationshipsXml( + `` + ) + })); + await expectImportCode(importPackage(nonCanonicalMainTarget), "docx_corrupt"); + + const duplicatePartId = await tempPackage("docx-duplicate-part-id"); + await writeFile(join(duplicatePartId, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + + "" + )) }] + })); + await expectImportCode(importPackage(duplicatePartId), "docx_corrupt"); + + const duplicateOverride = await tempPackage("docx-duplicate-override"); + await writeFile(join(duplicateOverride, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + `` + + `` + ) + })); + await expectImportCode(importPackage(duplicateOverride), "docx_corrupt"); + + const conflictingOverride = await tempPackage("docx-conflicting-override"); + await writeFile(join(conflictingOverride, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + `` + + "" + ) + })); + await expectImportCode(importPackage(conflictingOverride), "docx_corrupt"); + + const ambiguousDefault = await tempPackage("docx-ambiguous-default"); + await writeFile(join(ambiguousDefault, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + `` + + `` + ) + })); + await expectImportCode(importPackage(ambiguousDefault), "docx_corrupt"); + + const dotSegmentOverride = await tempPackage("docx-dot-segment-override"); + await writeFile(join(dotSegmentOverride, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + `` + + "" + ) + })); + await expectImportCode(importPackage(dotSegmentOverride), "docx_corrupt"); + + const nonCanonicalOnly = await tempPackage("docx-noncanonical-main-override"); + await writeFile(join(nonCanonicalOnly, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + `` + ) + })); + await expectImportCode(importPackage(nonCanonicalOnly), "docx_corrupt"); + + const caseAmbiguousEntries = await tempPackage("docx-case-ambiguous-entries"); + await writeFile(join(caseAmbiguousEntries, "problem.docx"), docxDocument("body", { + extra: [{ name: "WORD/DOCUMENT.XML", data: Buffer.from("") }] + })); + await expectImportCode(importPackage(caseAmbiguousEntries), "docx_corrupt"); + + const encodedEntry = await tempPackage("docx-encoded-entry"); + await writeFile(join(encodedEntry, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/%2e%2e/hidden.xml", data: Buffer.from("") }] + })); + await expectImportCode(importPackage(encodedEntry), "docx_corrupt"); + }); + + it("rejects macro and VBA indicators anywhere in a DOCX package", async () => { + const macroRelationship = await tempPackage("docx-renamed-macro-relationship"); + await writeFile(join(macroRelationship, "problem.docx"), docxDocument("body", { + extra: [ + { name: "word/hidden-payload.bin", data: Buffer.from("PRIVATE VBA PAYLOAD") }, + { name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) } + ] + })); + await expectImportCode(importPackage(macroRelationship), "docx_macro_enabled"); + + const combinedIndicators = await tempPackage("docx-renamed-macro-combined"); + await writeFile(join(combinedIndicators, "problem.docx"), docxDocument("body", { + extra: [ + { name: "word/hidden-payload.bin", data: Buffer.from("PRIVATE VBA PAYLOAD") }, + { name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) } + ], + contentTypes: contentTypesXml( + `` + + "" + + `` + ) + })); + await expectImportCode(importPackage(combinedIndicators), "docx_macro_enabled"); + + const macroDefault = await tempPackage("docx-macro-default"); + await writeFile(join(macroDefault, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + "" + + `` + ) + })); + await expectImportCode(importPackage(macroDefault), "docx_macro_enabled"); + + const macroOverride = await tempPackage("docx-macro-override"); + await writeFile(join(macroOverride, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + `` + + "" + ) + })); + await expectImportCode(importPackage(macroOverride), "docx_macro_enabled"); + + const renamedEntry = await tempPackage("docx-vba-entry-case"); + await writeFile(join(renamedEntry, "problem.docx"), docxDocument("body", { + extra: [{ name: "custom/VBAPROJECT.BIN", data: Buffer.from("not opened") }] + })); + await expectImportCode(importPackage(renamedEntry), "docx_macro_enabled"); + + const encodedMacroRelationship = await tempPackage("docx-encoded-macro-relationship"); + await writeFile(join(encodedMacroRelationship, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) }] + })); + await expectImportCode(importPackage(encodedMacroRelationship), "docx_macro_enabled"); + + const encodedMacroEntry = await tempPackage("docx-encoded-macro-entry"); + await writeFile(join(encodedMacroEntry, "problem.docx"), docxDocument("body", { + extra: [{ name: "custom/%76baProject.bin", data: Buffer.from("not opened") }] + })); + await expectImportCode(importPackage(encodedMacroEntry), "docx_macro_enabled"); + }); + + it("enforces DOCX entry, decompressed-byte, and character limits", async () => { + const entries = await tempPackage("docx-entries"); + await writeFile(join(entries, "problem.docx"), docxDocument("body", { + extra: [ + { name: "word/extra.xml", data: Buffer.from("x") }, + { name: "word/extra2.xml", data: Buffer.from("x") } + ] + })); + await expectImportCode(importPackage(entries, { limits: { maxDocxZipEntries: 4 } }), "docx_zip_entry_limit"); + + const bytes = await tempPackage("docx-bytes"); + await writeFile(join(bytes, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/large.xml", data: Buffer.alloc(500, 65) }] + })); + await expectImportCode(importPackage(bytes, { limits: { maxDocxUncompressedBytes: 100 } }), "docx_uncompressed_limit"); + + const characters = await tempPackage("docx-characters"); + await writeFile(join(characters, "problem.docx"), docxDocument("0123456789")); + await expectImportCode(importPackage(characters, { limits: { maxDocxCharacters: 5 } }), "docx_character_limit"); + }); + + it("rejects invalid traversal limits before touching the package path", async () => { + await expect(importPackage(join(tmpdir(), "missing-input-package"), { + limits: { maxPackageDepth: 0 } + })).rejects.toThrow(TypeError); + }); + + it("bounds total package traversal entries and depth before filtering", async () => { + const unsupported = await tempPackage("package-entry-limit"); + await writeFile(join(unsupported, "problem.md"), "problem"); + for (let index = 0; index < 20; index += 1) { + await writeFile(join(unsupported, `ignored-${index}.bin`), "ignored"); + } + await expectImportCode(importPackage(unsupported, { limits: { maxPackageEntries: 2 } }), "package_entry_limit"); + + const hidden = await tempPackage("hidden-entry-limit"); + await writeFile(join(hidden, "problem.md"), "problem"); + await mkdir(join(hidden, ".ignored")); + await writeFile(join(hidden, ".ignored", "private.bin"), "private"); + await expectImportCode(importPackage(hidden, { limits: { maxPackageEntries: 1 } }), "package_entry_limit"); + + const hiddenNoRecursion = await tempPackage("hidden-no-recursion"); + await writeFile(join(hiddenNoRecursion, "problem.md"), "problem"); + await mkdir(join(hiddenNoRecursion, ".ignored")); + for (let index = 0; index < 20; index += 1) { + await writeFile(join(hiddenNoRecursion, ".ignored", `private-${index}.bin`), "private"); + } + expect((await importPackage(hiddenNoRecursion, { limits: { maxPackageEntries: 2 } })).problemText).toBe("problem"); + + const deep = await tempPackage("package-depth-limit"); + await mkdir(join(deep, "one", "two", "three"), { recursive: true }); + await writeFile(join(deep, "one", "two", "three", "problem.md"), "problem"); + await expectImportCode(importPackage(deep, { limits: { maxPackageDepth: 2 } }), "package_depth_limit"); + + const exactDepth = await tempPackage("package-depth-exact"); + await mkdir(join(exactDepth, "one", "two"), { recursive: true }); + await writeFile(join(exactDepth, "one", "two", "problem.md"), "problem"); + expect((await importPackage(exactDepth, { limits: { maxPackageDepth: 2 } })).problemText).toBe("problem"); + }); + + it("fails when same-priority problem candidates are ambiguous", async () => { + const root = await tempPackage("ambiguous"); + await writeFile(join(root, "problem.md"), "markdown"); + await writeFile(join(root, "problem.markdown"), "other markdown"); + await expectImportCode(importPackage(root), "ambiguous_problem"); + }); + + it("uses documented precedence and treats lower-priority statements as assets", async () => { + const root = await tempPackage("precedence"); + await writeFile(join(root, "problem.txt"), "text wins"); + await writeFile(join(root, "problem.pdf"), pdfDocument(["pdf loses"])); + await writeFile(join(root, "problem.docx"), docxDocument("docx loses")); + const imported = await importPackage(root); + expect(imported.problemText).toBe("text wins"); + expect(imported.dataAssets.map((asset) => asset.relative_path)).toEqual(["problem.docx", "problem.pdf"]); + }); + + it("rejects root and nested symlinks", async () => { + const target = await tempPackage("symlink-target"); + await writeFile(join(target, "problem.md"), "problem"); + const parent = await tempPackage("symlink-parent"); + const link = join(parent, "linked-package"); + await symlink(target, link, "dir"); + await expectImportCode(importPackage(link), "symlink_input"); + + const root = await tempPackage("symlink-nested"); + await writeFile(join(root, "problem.md"), "problem"); + await symlink(join(root, "problem.md"), join(root, "problem-link.md")); + await expectImportCode(importPackage(root), "symlink_input"); + + const hiddenRoot = await tempPackage("symlink-hidden"); + await writeFile(join(hiddenRoot, "problem.md"), "problem"); + await symlink(join(hiddenRoot, "problem.md"), join(hiddenRoot, ".hidden-link.md")); + await expectImportCode(importPackage(hiddenRoot), "symlink_input"); + }); + + it("keeps raw asset bytes and SHA while exposing bounded XLSX and image metadata", async () => { + const root = await tempPackage("inventory"); + const spreadsheet = xlsxAsset(); + const picture = png(3, 2); + await writeFile(join(root, "problem.md"), "problem"); + await writeFile(join(root, "data.xlsx"), spreadsheet); + await writeFile(join(root, "plot.png"), picture); + const beforeSpreadsheet = await readFile(join(root, "data.xlsx")); + const beforePicture = await readFile(join(root, "plot.png")); + + const imported = await importPackage(root); + const spreadsheetAsset = imported.dataAssets.find((asset) => asset.relative_path === "data.xlsx"); + const pictureAsset = imported.dataAssets.find((asset) => asset.relative_path === "plot.png"); + expect(spreadsheetAsset?.sha256).toBe(createHash("sha256").update(spreadsheet).digest("hex")); + expect(spreadsheetAsset?.size_bytes).toBe(spreadsheet.length); + expect(pictureAsset?.sha256).toBe(createHash("sha256").update(picture).digest("hex")); + expect(pictureAsset?.size_bytes).toBe(picture.length); + expect(imported.assetMetadata?.[spreadsheetAsset?.artifact_id ?? ""]).toMatchObject({ format: "xlsx", sheets: [{ name: "Data", dimension: "A1:C12" }, { name: "Summary", dimension: "B2:D4" }] }); + expect(imported.assetMetadata?.[pictureAsset?.artifact_id ?? ""]).toMatchObject({ format: "png", width: 3, height: 2 }); + expect(await readFile(join(root, "data.xlsx"))).toEqual(beforeSpreadsheet); + expect(await readFile(join(root, "plot.png"))).toEqual(beforePicture); + }); + + it("surfaces unreadable and over-limit metadata as explicit warnings", async () => { + const root = await tempPackage("metadata-warning"); + await writeFile(join(root, "problem.md"), "problem"); + await writeFile(join(root, "bad.png"), Buffer.from("not an image")); + await writeFile(join(root, "bad.xlsx"), Buffer.from("not a workbook")); + await writeFile(join(root, "large.png"), png(10, 10)); + await writeFile(join(root, "external-mode.xlsx"), xlsxAsset( + "" + )); + await writeFile(join(root, "ambiguous-entry.xlsx"), xlsxAsset(undefined, [ + { name: "XL/WORKBOOK.XML", data: Buffer.from("") } + ])); + await writeFile(join(root, "duplicate-id.xlsx"), xlsxAsset( + "" + )); + const imported = await importPackage(root, { limits: { maxImagePixels: 50 } }); + expect(imported.warnings).toEqual(expect.arrayContaining([ + expect.objectContaining({ code: "metadata_unreadable", path: "bad.png" }), + expect.objectContaining({ code: "metadata_unreadable", path: "bad.xlsx" }), + expect.objectContaining({ code: "metadata_limit", path: "large.png" }), + expect.objectContaining({ code: "metadata_unreadable", path: "external-mode.xlsx" }), + expect.objectContaining({ code: "metadata_unreadable", path: "ambiguous-entry.xlsx" }), + expect.objectContaining({ code: "metadata_unreadable", path: "duplicate-id.xlsx" }) + ])); + }); +});