From d99ff68acc08b022b5d8fe85afc02b988a898f49 Mon Sep 17 00:00:00 2001 From: zhanghan <2774877132@qq.com> Date: Thu, 13 Aug 2026 03:29:13 +0800 Subject: [PATCH 1/3] feat: safely import PDF and DOCX packages --- package-lock.json | 359 ++++++++++++++++++++++++++- package.json | 8 +- src/input/README.md | 24 ++ src/input/asset-metadata.ts | 232 +++++++++++++++++ src/input/docx-extractor.ts | 176 +++++++++++++ src/input/package-importer.ts | 312 ++++++++++++++++++++--- src/input/pdf-extractor.ts | 120 +++++++++ src/input/types.ts | 130 ++++++++++ src/input/xml.ts | 32 +++ src/input/zip-reader.ts | 164 ++++++++++++ tests/input-package-importer.test.ts | 331 ++++++++++++++++++++++++ 11 files changed, 1848 insertions(+), 40 deletions(-) create mode 100644 src/input/README.md create mode 100644 src/input/asset-metadata.ts create mode 100644 src/input/docx-extractor.ts create mode 100644 src/input/pdf-extractor.ts create mode 100644 src/input/types.ts create mode 100644 src/input/xml.ts create mode 100644 src/input/zip-reader.ts create mode 100644 tests/input-package-importer.test.ts diff --git a/package-lock.json b/package-lock.json index 72a350e..a9002f2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,11 +12,17 @@ "@earendil-works/pi-coding-agent": "0.84.1", "ajv": "8.20.0", "ajv-formats": "3.0.1", + "buffer-crc32": "0.2.13", "commander": "15.0.0", - "fastify": "5.11.3" + "fastify": "5.11.3", + "pdfjs-dist": "5.4.624", + "saxes": "6.0.0", + "yauzl": "3.1.3" }, "devDependencies": { + "@types/buffer-crc32": "0.2.4", "@types/node": "24.10.1", + "@types/yauzl": "2.10.3", "tsx": "4.23.12", "typescript": "5.9.3", "vitest": "4.1.10", @@ -2465,6 +2471,271 @@ "dev": true, "license": "MIT" }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.100.tgz", + "integrity": "sha512-xglYA6q3XO5P3BNJYxVZ1IV7DLVjp1Py6nwag88YntrS+3vKHyYcMqXVS4ZztJmwz2uGvz1FWhI/4LgbR5uQDA==", + "license": "MIT", + "optional": true, + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.100", + "@napi-rs/canvas-darwin-arm64": "0.1.100", + "@napi-rs/canvas-darwin-x64": "0.1.100", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.100", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.100", + "@napi-rs/canvas-linux-arm64-musl": "0.1.100", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.100", + "@napi-rs/canvas-linux-x64-gnu": "0.1.100", + "@napi-rs/canvas-linux-x64-musl": "0.1.100", + "@napi-rs/canvas-win32-arm64-msvc": "0.1.100", + "@napi-rs/canvas-win32-x64-msvc": "0.1.100" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.100.tgz", + "integrity": "sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.100.tgz", + "integrity": "sha512-2PcswRaC7Ly645DGt88///zuFDhJxJYdKAs1uU3mfk1atYkXufgcgLfBpk6Tm12nCQBaNt1wpybuPZ4qOhTo8A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.100.tgz", + "integrity": "sha512-ePNZtj7pNIva/siZMg+HmbeozkIjqUIYdoymH8HaA3qK7LfzFN4WMBM8G6HQ9ZC+H3+Dnn5pqtiXpgLykaPOhw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.100.tgz", + "integrity": "sha512-d5cDB48oWFGU8/XPhUOFAlySgb/VAu7D+s8fi55K1Pcfg8aPplHWqMgibhVLU8ky7Pyg/fuiVLz4Nf3JrSTuUA==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.100.tgz", + "integrity": "sha512-rDxgxRu69RvDlX/bh9o22DxLsGr8EqsNgotL9+RwQE1S0b0cqeatqsw6aW45mukm0B42DIAaAacKaYQ8cqS1nw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.100.tgz", + "integrity": "sha512-K3mDW66N+xT2/V439u1alFANiBUjdEx2gLiNYnCmUsva5jZMxWTjafBYwTzYK+EMFMHrUoabuU+T1BIP5CgbYQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.100.tgz", + "integrity": "sha512-mooqUBTIsccZpnoQC4NgrC1v6C1vof39etLNMnBwCY+p0gajWJvAHLGQ6g/gGyS5YrpDW+GefSN4+Cvcr08UWw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.100.tgz", + "integrity": "sha512-1eCvkDCazm7FFhsT7DfGOdSaHgZVK3bt/dSBl5EWHOWmnz+I7j8tPseJqqD81NF+MH21jKUK4wQSDjN0mdhnTg==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.100.tgz", + "integrity": "sha512-20arT6lnI19S68qNlii73TSEDbECNgzMz2EpldC1V3mZFuRkeujXkcebRk0LRJe9SEUAooYiLokfMViY8IX7yA==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-arm64-msvc": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.100.tgz", + "integrity": "sha512-DZFFT1wIAg37LJw37yhMRFfjATd3vTQzjZ1Yki8u2vhO6Hi5VE6BVaGQ1aaDu7xb4iMErz+9EOwjpS7xcxFeBw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.100", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.100.tgz", + "integrity": "sha512-MyT1j3mHC2+Lu4pBi9mKyMJhtP6U7k7EldY7sj/uS5gJA65gTXt8MefJQXLJo5d/vZbuWmfxzkEUNc/urV3pHA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, "node_modules/@oxc-project/types": { "version": "0.143.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.143.0.tgz", @@ -2751,6 +3022,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/buffer-crc32": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@types/buffer-crc32/-/buffer-crc32-0.2.4.tgz", + "integrity": "sha512-GSrhSZOK1/wazf2CjDp3CVJQKWzSc5Ugq3NyZ/RQqg1MWtmA9mAT6i6LzGKhzcRxDOl8aLB+AzvObDSlrMpvLw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -2786,6 +3067,16 @@ "undici-types": "~7.16.0" } }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@vitest/expect": { "version": "4.1.10", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.10.tgz", @@ -2977,6 +3268,15 @@ "fastq": "^1.17.1" } }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/chai": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", @@ -3643,6 +3943,13 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/node-readable-to-web-readable-stream": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/node-readable-to-web-readable-stream/-/node-readable-to-web-readable-stream-0.4.2.tgz", + "integrity": "sha512-/cMZNI34v//jUTrI+UIo4ieHAB5EZRY/+7OmXZgBxaWBMcW2tGdceIw06RFxWxrKZ5Jp3sI2i5TsRo+CBhtVLQ==", + "license": "MIT", + "optional": true + }, "node_modules/obug": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.4.tgz", @@ -3673,6 +3980,25 @@ "dev": true, "license": "MIT" }, + "node_modules/pdfjs-dist": { + "version": "5.4.624", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.624.tgz", + "integrity": "sha512-sm6TxKTtWv1Oh6n3C6J6a8odejb5uO4A4zo/2dgkHuC0iu8ZMAXOezEODkVaoVp8nX1Xzr+0WxFJJmUr45hQzg==", + "license": "Apache-2.0", + "engines": { + "node": ">=20.16.0 || >=22.3.0" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.88", + "node-readable-to-web-readable-stream": "^0.4.2" + } + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -3888,6 +4214,18 @@ "node": ">=10" } }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } + }, "node_modules/secure-json-parse": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", @@ -4267,6 +4605,12 @@ "node": ">=8" } }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "license": "MIT" + }, "node_modules/yaml": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", @@ -4282,6 +4626,19 @@ "funding": { "url": "https://github.com/sponsors/eemeli" } + }, + "node_modules/yauzl": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-3.1.3.tgz", + "integrity": "sha512-JCCdmlJJWv7L0q/KylOekyRaUrdEoUxWkWVcgorosTROCFWiS9p2NNPE9Yb91ak7b1N5SxAZEliWpspbZccivw==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "pend": "~1.2.0" + }, + "engines": { + "node": ">=12" + } } } } diff --git a/package.json b/package.json index 299fa93..4a2ce84 100644 --- a/package.json +++ b/package.json @@ -25,11 +25,17 @@ "@earendil-works/pi-coding-agent": "0.84.1", "ajv": "8.20.0", "ajv-formats": "3.0.1", + "buffer-crc32": "0.2.13", "commander": "15.0.0", - "fastify": "5.11.3" + "fastify": "5.11.3", + "pdfjs-dist": "5.4.624", + "saxes": "6.0.0", + "yauzl": "3.1.3" }, "devDependencies": { + "@types/buffer-crc32": "0.2.4", "@types/node": "24.10.1", + "@types/yauzl": "2.10.3", "tsx": "4.23.12", "typescript": "5.9.3", "vitest": "4.1.10", diff --git a/src/input/README.md b/src/input/README.md new file mode 100644 index 0000000..6167f13 --- /dev/null +++ b/src/input/README.md @@ -0,0 +1,24 @@ +# Safe package input + +`importPackage` inventories a local competition package without modifying source files or using the network. Existing callers keep the same required fields (`problemText`, the original `problemPath`, `dataAssets`, and `dataPaths`); extraction and inventory details are optional additions. + +## Problem selection + +Problem statements are selected deterministically by precedence group: + +1. `problem.md` or `problem.markdown` +2. `problem.txt` +3. `problem.pdf` +4. `problem.docx` + +Names are matched case-insensitively at any non-hidden package depth. The first non-empty precedence group wins. If that group contains more than one candidate (including duplicate names in separate directories, or both Markdown spellings), import fails with `PackageImportError.code === "ambiguous_problem"`; lower-priority candidates are data assets only when a higher-priority problem was selected. + +## Safety and limits + +All limits have conservative defaults in `DEFAULT_IMPORT_LIMITS` and may be reduced or raised through `importPackage(path, { limits })`. Invalid limits are rejected. Package roots and nested entries may not be symlinks, paths must remain beneath the package root, hidden directories and files are ignored, and supported file counts and raw sizes are bounded. + +PDF text is extracted locally with the pinned `pdfjs-dist` Node build. Input bytes are passed directly to the parser with fetch, streaming, JavaScript evaluation, system fonts, and WebAssembly disabled. The importer records source SHA-256/size, PDF page count, extracted character count, and extractor version, while explicitly rejecting unavailable tooling, encryption, corruption, empty text, and byte/page/character limits. + +DOCX extraction reads only required OOXML package parts and `word/document.xml` body text. It never executes content, follows external relationships, or opens embedded objects. Macro-enabled content types, encrypted entries, unsafe ZIP names, external relationships, corrupt packages, and ZIP entry/uncompressed/text limits fail with structured codes. + +Data assets retain SHA-256 and size computed from their original bytes. XLSX inspection is limited to workbook, relationship, and bounded worksheet-dimension XML; images are read only far enough to obtain bounded PNG/JPEG format and dimensions. CSV, JSON, legacy XLS, and Parquet are inventoried without parsing or sampling their records. Unreadable or over-limit optional asset metadata is surfaced in `warnings`, never silently treated as successful metadata. diff --git a/src/input/asset-metadata.ts b/src/input/asset-metadata.ts new file mode 100644 index 0000000..e7fa362 --- /dev/null +++ b/src/input/asset-metadata.ts @@ -0,0 +1,232 @@ +import { open as openFile } from "node:fs/promises"; +import { extname } from "node:path"; +import { + PackageImportError, + type AssetMetadata, + type ImageAssetMetadata, + type ImportLimits, + type XlsxSheetMetadata +} from "./types.js"; +import { readBoundedZip } from "./zip-reader.js"; +import { + attributeValue, + PACKAGE_RELATIONSHIPS_NAMESPACE, + parseXml, + SPREADSHEETML_NAMESPACE +} from "./xml.js"; + +const OFFICE_DOCUMENT_RELATIONSHIPS_NAMESPACE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; +const XLSX_ZIP_CODES = { + corrupt: "metadata_unreadable", + encrypted: "metadata_unreadable", + zipSlip: "metadata_unreadable", + entryLimit: "metadata_limit", + uncompressedLimit: "metadata_limit" +} as const; + +function checkedImage(format: ImageAssetMetadata["format"], width: number, height: number, path: string, limits: ImportLimits): ImageAssetMetadata { + const pixels = width * height; + if (!Number.isSafeInteger(width) || !Number.isSafeInteger(height) || width <= 0 || height <= 0) { + throw new PackageImportError("metadata_unreadable", `Invalid ${format.toUpperCase()} dimensions.`, { path }); + } + if (!Number.isSafeInteger(pixels) || pixels > limits.maxImagePixels) { + throw new PackageImportError("metadata_limit", `Image has ${pixels} pixels; limit is ${limits.maxImagePixels}.`, { + path, + actual: pixels, + limit: limits.maxImagePixels + }); + } + return { kind: "image", format, width, height }; +} + +async function readPrefix(path: string, length: number): Promise { + const handle = await openFile(path, "r"); + try { + const buffer = Buffer.alloc(length); + const { bytesRead } = await handle.read(buffer, 0, length, 0); + return buffer.subarray(0, bytesRead); + } finally { + await handle.close(); + } +} + +function pngMetadata(bytes: Buffer, path: string, limits: ImportLimits): ImageAssetMetadata { + const signature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + if (bytes.length < 24 || !bytes.subarray(0, 8).equals(signature) || bytes.toString("ascii", 12, 16) !== "IHDR") { + throw new PackageImportError("metadata_unreadable", "Invalid PNG header.", { path }); + } + return checkedImage("png", bytes.readUInt32BE(16), bytes.readUInt32BE(20), path, limits); +} + +const JPEG_START_OF_FRAME_MARKERS = new Set([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf]); + +function jpegMetadata(bytes: Buffer, path: string, limits: ImportLimits, sourceBytes: number): ImageAssetMetadata { + if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) { + throw new PackageImportError("metadata_unreadable", "Invalid JPEG header.", { path }); + } + let offset = 2; + while (offset + 3 < bytes.length) { + while (bytes[offset] === 0xff) offset += 1; + const marker = bytes[offset]; + offset += 1; + if (marker === undefined || marker === 0xd9 || marker === 0xda) break; + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue; + if (offset + 2 > bytes.length) break; + const segmentLength = bytes.readUInt16BE(offset); + if (segmentLength < 2) throw new PackageImportError("metadata_unreadable", "Invalid JPEG segment length.", { path }); + if (JPEG_START_OF_FRAME_MARKERS.has(marker)) { + if (offset + 7 > bytes.length) break; + return checkedImage("jpeg", bytes.readUInt16BE(offset + 5), bytes.readUInt16BE(offset + 3), path, limits); + } + offset += segmentLength; + } + const code = bytes.length < sourceBytes ? "metadata_limit" : "metadata_unreadable"; + throw new PackageImportError(code, "JPEG dimensions were not found within the bounded header scan.", { + path, + actual: bytes.length, + limit: limits.maxImageHeaderBytes + }); +} + +async function imageMetadata(path: string, sourceBytes: number, extension: string, limits: ImportLimits): Promise { + const bytes = await readPrefix(path, Math.min(sourceBytes, limits.maxImageHeaderBytes)); + if (extension === ".png") return pngMetadata(bytes, path, limits); + return jpegMetadata(bytes, path, limits, sourceBytes); +} + +interface WorkbookSheet { + name: string; + relationshipId: string; +} + +function workbookSheets(xml: Buffer, path: string, sheetLimit: number, byteLimit: number): WorkbookSheet[] { + if (xml.length > byteLimit) { + throw new PackageImportError("metadata_limit", `XLSX workbook metadata exceeds ${byteLimit} bytes.`, { + path, + actual: xml.length, + limit: byteLimit + }); + } + const sheets: WorkbookSheet[] = []; + parseXml(xml, { + openTag(tag) { + if (tag.uri !== SPREADSHEETML_NAMESPACE || tag.local !== "sheet") return; + const name = attributeValue(tag, "name"); + const relationshipId = attributeValue(tag, "id", OFFICE_DOCUMENT_RELATIONSHIPS_NAMESPACE); + if (!name || !relationshipId) throw new PackageImportError("metadata_unreadable", "XLSX sheet is missing a name or relationship id.", { path }); + sheets.push({ name, relationshipId }); + if (sheets.length > sheetLimit) { + throw new PackageImportError("metadata_limit", `XLSX has more than ${sheetLimit} sheets.`, { + path, + actual: sheets.length, + limit: sheetLimit + }); + } + } + }); + return sheets; +} + +function worksheetRelationships(xml: Buffer, path: string, byteLimit: number): Map { + if (xml.length > byteLimit) { + throw new PackageImportError("metadata_limit", `XLSX relationships metadata exceeds ${byteLimit} bytes.`, { + path, + actual: xml.length, + limit: byteLimit + }); + } + const relationships = new Map(); + parseXml(xml, { + openTag(tag) { + if (tag.uri !== PACKAGE_RELATIONSHIPS_NAMESPACE || tag.local !== "Relationship") return; + const id = attributeValue(tag, "Id"); + const target = attributeValue(tag, "Target"); + const type = attributeValue(tag, "Type"); + if (!id || !target || !type?.endsWith("/worksheet")) return; + if (attributeValue(tag, "TargetMode")?.toLowerCase() === "external") { + throw new PackageImportError("metadata_unreadable", "External XLSX worksheet relationship is not followed.", { path }); + } + if (target.startsWith("/") || target.includes("\\") || target.split("/").includes("..")) { + throw new PackageImportError("metadata_unreadable", `Unsafe XLSX worksheet target: ${target}`, { path }); + } + relationships.set(id, `xl/${target}`); + } + }); + return relationships; +} + +function worksheetDimension(xml: Buffer, path: string, limit: number): string | undefined { + if (xml.length > limit) { + throw new PackageImportError("metadata_limit", `XLSX worksheet metadata part exceeds ${limit} bytes.`, { + path, + actual: xml.length, + limit + }); + } + let dimension: string | undefined; + parseXml(xml, { + openTag(tag) { + if (dimension === undefined && tag.uri === SPREADSHEETML_NAMESPACE && tag.local === "dimension") { + dimension = attributeValue(tag, "ref"); + } + } + }); + return dimension; +} + +async function xlsxMetadata(path: string, sourceBytes: number, limits: ImportLimits): Promise { + if (sourceBytes > limits.maxMetadataBytes) { + throw new PackageImportError("metadata_limit", `XLSX is ${sourceBytes} bytes; metadata inspection limit is ${limits.maxMetadataBytes}.`, { + path, + actual: sourceBytes, + limit: limits.maxMetadataBytes + }); + } + try { + const archive = await readBoundedZip( + path, + { maxEntries: limits.maxMetadataZipEntries, maxUncompressedBytes: limits.maxMetadataUncompressedBytes }, + XLSX_ZIP_CODES, + (name) => name === "xl/workbook.xml" || name === "xl/_rels/workbook.xml.rels" || name.startsWith("xl/worksheets/") + ); + const workbook = archive.entries.get("xl/workbook.xml"); + const relationshipXml = archive.entries.get("xl/_rels/workbook.xml.rels"); + if (!workbook || !relationshipXml) throw new PackageImportError("metadata_unreadable", "XLSX is missing workbook metadata parts.", { path }); + const sheetRecords = workbookSheets(workbook, path, limits.maxXlsxSheets, limits.maxXlsxWorkbookBytes); + const relationships = worksheetRelationships(relationshipXml, path, limits.maxXlsxRelationshipsBytes); + const sheets: XlsxSheetMetadata[] = sheetRecords.map((sheet) => { + const target = relationships.get(sheet.relationshipId); + if (!target) throw new PackageImportError("metadata_unreadable", `XLSX relationship ${sheet.relationshipId} is missing.`, { path }); + const worksheet = archive.entries.get(target); + if (!worksheet) throw new PackageImportError("metadata_unreadable", `XLSX worksheet part is missing: ${target}`, { path }); + const dimension = worksheetDimension(worksheet, target, limits.maxXlsxWorksheetBytes); + return dimension === undefined ? { name: sheet.name } : { name: sheet.name, dimension }; + }); + return { + kind: "spreadsheet", + format: "xlsx", + sheets, + zipEntries: archive.entryCount, + uncompressedBytes: archive.uncompressedBytes + }; + } catch (error) { + if (error instanceof PackageImportError) throw error; + const cause = error instanceof Error ? error.message : String(error); + throw new PackageImportError("metadata_unreadable", `Could not read XLSX metadata: ${cause}`, { path, cause }); + } +} + +export async function inspectAssetMetadata( + path: string, + sourceBytes: number, + limits: ImportLimits +): Promise { + const extension = extname(path).toLowerCase(); + if (extension === ".xlsx") return xlsxMetadata(path, sourceBytes, limits); + if (extension === ".png" || extension === ".jpg" || extension === ".jpeg") return imageMetadata(path, sourceBytes, extension, limits); + if (extension === ".csv") return { kind: "bounded-inventory", format: "csv" }; + if (extension === ".json") return { kind: "bounded-inventory", format: "json" }; + if (extension === ".xls") return { kind: "bounded-inventory", format: "xls" }; + if (extension === ".parquet") return { kind: "bounded-inventory", format: "parquet" }; + return undefined; +} diff --git a/src/input/docx-extractor.ts b/src/input/docx-extractor.ts new file mode 100644 index 0000000..6cfdda6 --- /dev/null +++ b/src/input/docx-extractor.ts @@ -0,0 +1,176 @@ +import { PackageImportError, type ImportLimits, type ProblemExtractionMetadata } from "./types.js"; +import { readBoundedZip } from "./zip-reader.js"; +import { attributeValue, PACKAGE_RELATIONSHIPS_NAMESPACE, parseXml, WORDPROCESSINGML_NAMESPACE } from "./xml.js"; + +interface DocxExtraction { + text: string; + metadata: ProblemExtractionMetadata; +} + +const WORD_DOCUMENT_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"; +const OFFICE_DOCUMENT_RELATIONSHIP = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument"; + +const DOCX_ZIP_CODES = { + corrupt: "docx_corrupt", + encrypted: "docx_encrypted", + zipSlip: "docx_zip_slip", + entryLimit: "docx_zip_entry_limit", + uncompressedLimit: "docx_uncompressed_limit" +} as const; + +function causeMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function validateContentTypes(xml: Buffer, path: string): void { + let documentContentType: string | undefined; + parseXml(xml, { + openTag(tag) { + if (tag.local !== "Override" || attributeValue(tag, "PartName") !== "/word/document.xml") return; + documentContentType = attributeValue(tag, "ContentType"); + } + }); + if (documentContentType === undefined) { + throw new PackageImportError("docx_corrupt", "DOCX content types do not declare word/document.xml.", { path }); + } + if (/macroEnabled/i.test(documentContentType)) { + throw new PackageImportError("docx_macro_enabled", "Macro-enabled Word packages are not allowed.", { path }); + } + if (documentContentType !== WORD_DOCUMENT_CONTENT_TYPE) { + throw new PackageImportError("docx_corrupt", `Unexpected DOCX main document content type: ${documentContentType}`, { path }); + } +} + +function validatePackageRelationships(xml: Buffer, path: string): void { + let officeDocumentTarget: string | undefined; + parseXml(xml, { + openTag(tag) { + if (tag.uri !== PACKAGE_RELATIONSHIPS_NAMESPACE || tag.local !== "Relationship") return; + if (attributeValue(tag, "TargetMode")?.toLowerCase() === "external") { + throw new PackageImportError("docx_external_relationship", `External relationship is not allowed in ${path}.`, { path }); + } + if (attributeValue(tag, "Type") === OFFICE_DOCUMENT_RELATIONSHIP) { + officeDocumentTarget = attributeValue(tag, "Target"); + } + } + }); + if (officeDocumentTarget !== "word/document.xml" && officeDocumentTarget !== "/word/document.xml") { + throw new PackageImportError("docx_corrupt", "DOCX package does not point to word/document.xml as its main document.", { path }); + } +} + +function parseRelationships(xml: Buffer, path: string): void { + parseXml(xml, { + openTag(tag) { + if (tag.uri !== PACKAGE_RELATIONSHIPS_NAMESPACE || tag.local !== "Relationship") return; + if (attributeValue(tag, "TargetMode")?.toLowerCase() === "external") { + throw new PackageImportError("docx_external_relationship", `External relationship is not allowed in ${path}.`, { path }); + } + } + }); +} + +function parseBody(xml: Buffer, path: string, characterLimit: number): string { + const pieces: string[] = []; + let collectingText = false; + let characterCount = 0; + const append = (text: string): void => { + characterCount += text.length; + if (characterCount > characterLimit) { + throw new PackageImportError("docx_character_limit", `DOCX body exceeds ${characterLimit} characters.`, { + path, + actual: characterCount, + limit: characterLimit + }); + } + pieces.push(text); + }; + parseXml(xml, { + openTag(tag) { + if (tag.uri !== WORDPROCESSINGML_NAMESPACE) return; + if (tag.local === "t" || tag.local === "instrText") collectingText = true; + else if (tag.local === "tab") append("\t"); + else if (tag.local === "br" || tag.local === "cr") append("\n"); + }, + closeTag(tag) { + if (tag.uri !== WORDPROCESSINGML_NAMESPACE) return; + if (tag.local === "t" || tag.local === "instrText") collectingText = false; + else if (tag.local === "p") append("\n"); + else if (tag.local === "tc") append("\t"); + }, + text(text) { + if (collectingText) append(text); + } + }); + return pieces.join("").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim(); +} + +export async function extractDocx( + path: string, + identity: { sha256: string; sizeBytes: number }, + limits: ImportLimits +): Promise { + if (identity.sizeBytes > limits.maxProblemBytes) { + throw new PackageImportError("problem_file_limit", `DOCX is ${identity.sizeBytes} bytes; limit is ${limits.maxProblemBytes}.`, { + path, + actual: identity.sizeBytes, + limit: limits.maxProblemBytes + }); + } + try { + const archive = await readBoundedZip( + path, + { maxEntries: limits.maxDocxZipEntries, maxUncompressedBytes: limits.maxDocxUncompressedBytes }, + DOCX_ZIP_CODES, + (name) => name === "[Content_Types].xml" || name === "word/document.xml" || name === "word/vbaProject.bin" || name.endsWith(".rels") + ); + const contentTypes = archive.entries.get("[Content_Types].xml"); + const packageRelationships = archive.entries.get("_rels/.rels"); + const document = archive.entries.get("word/document.xml"); + if (!contentTypes || !packageRelationships || !document) { + throw new PackageImportError("docx_corrupt", "DOCX is missing required OOXML parts.", { path }); + } + for (const [name, xml] of archive.entries) { + if (xml.length > limits.maxDocxXmlPartBytes) { + throw new PackageImportError("docx_uncompressed_limit", `DOCX XML part ${name} exceeds ${limits.maxDocxXmlPartBytes} bytes.`, { + path: name, + actual: xml.length, + limit: limits.maxDocxXmlPartBytes + }); + } + } + validateContentTypes(contentTypes, path); + validatePackageRelationships(packageRelationships, path); + if (archive.entries.has("word/vbaProject.bin")) { + throw new PackageImportError("docx_macro_enabled", "Macro-enabled Word packages are not allowed.", { path }); + } + for (const [name, xml] of archive.entries) { + if (name.endsWith(".rels")) parseRelationships(xml, name); + } + const text = parseBody(document, path, limits.maxDocxCharacters); + if (!text) throw new PackageImportError("docx_empty", "DOCX body contains no extractable text.", { path }); + if (text.length > limits.maxDocxCharacters) { + throw new PackageImportError("docx_character_limit", `DOCX body exceeds ${limits.maxDocxCharacters} characters after normalization.`, { + path, + actual: text.length, + limit: limits.maxDocxCharacters + }); + } + return { + text, + metadata: { + format: "docx", + sourceBytes: identity.sizeBytes, + sha256: identity.sha256, + extractedCharacters: text.length, + extractor: "ooxml-body-v1", + zipEntries: archive.entryCount, + uncompressedBytes: archive.uncompressedBytes + } + }; + } catch (error) { + if (error instanceof PackageImportError) throw error; + const message = causeMessage(error); + throw new PackageImportError("docx_corrupt", `Could not parse DOCX: ${message}`, { path, cause: message }); + } +} diff --git a/src/input/package-importer.ts b/src/input/package-importer.ts index 8649835..476ac17 100644 --- a/src/input/package-importer.ts +++ b/src/input/package-importer.ts @@ -1,17 +1,82 @@ -import { lstat, readFile, realpath } from "node:fs/promises"; +import { lstat, readFile, readdir, realpath } from "node:fs/promises"; import { basename, extname, isAbsolute, relative, resolve, sep } from "node:path"; import { fileIdentity, sha256Text } from "../infrastructure/hash.js"; import type { DataAsset } from "../contracts/types.js"; +import { inspectAssetMetadata } from "./asset-metadata.js"; +import { extractDocx } from "./docx-extractor.js"; +import { extractPdf } from "./pdf-extractor.js"; +import { + PackageImportError, + type AssetMetadata, + type ImportedPackage, + type ImportLimits, + type ImportOptions, + type ImportWarning, + type ProblemExtractionMetadata +} from "./types.js"; -const SUPPORTED_EXTENSIONS = new Set([".md", ".markdown", ".txt", ".csv", ".json", ".xlsx", ".xls", ".parquet", ".png", ".jpg", ".jpeg"]); +export { + PackageImportError, + type AssetMetadata, + type ImportedPackage, + type ImportLimits, + type ImportOptions, + type ImportWarning, + type ProblemExtractionMetadata +} from "./types.js"; -export interface ImportedPackage { - rootPath: string; - problemPath: string; - problemText: string; - dataAssets: DataAsset[]; - dataPaths: Map; -} +/** + * Problem candidates are selected by these precedence groups, from highest to + * lowest. Markdown spellings intentionally share one precedence group; if more + * than one candidate exists in the selected group, import fails rather than + * depending on directory enumeration order. + */ +export const PROBLEM_CANDIDATE_PRIORITY = [ + ["problem.md", "problem.markdown"], + ["problem.txt"], + ["problem.pdf"], + ["problem.docx"] +] as const; + +export const DEFAULT_IMPORT_LIMITS: Readonly = Object.freeze({ + maxProblemBytes: 32 * 1024 * 1024, + maxTextCharacters: 2_000_000, + maxPackageEntries: 10_000, + maxAssetBytes: 2 * 1024 * 1024 * 1024, + maxPdfPages: 500, + maxPdfCharacters: 2_000_000, + maxDocxZipEntries: 2_000, + maxDocxUncompressedBytes: 128 * 1024 * 1024, + maxDocxXmlPartBytes: 16 * 1024 * 1024, + maxDocxCharacters: 2_000_000, + maxMetadataBytes: 128 * 1024 * 1024, + maxMetadataZipEntries: 10_000, + maxMetadataUncompressedBytes: 256 * 1024 * 1024, + maxXlsxSheets: 1_000, + maxXlsxWorkbookBytes: 4 * 1024 * 1024, + maxXlsxRelationshipsBytes: 4 * 1024 * 1024, + maxXlsxWorksheetBytes: 1024 * 1024, + maxImageHeaderBytes: 1024 * 1024, + maxImagePixels: 250_000_000 +}); + +const MACRO_ENABLED_EXTENSIONS = new Set([".docm", ".dotm"]); + +const SUPPORTED_EXTENSIONS = new Set([ + ".md", + ".markdown", + ".txt", + ".pdf", + ".docx", + ".csv", + ".json", + ".xlsx", + ".xls", + ".parquet", + ".png", + ".jpg", + ".jpeg" +]); function mediaType(path: string): string { const extension = extname(path).toLowerCase(); @@ -19,6 +84,8 @@ function mediaType(path: string): string { ".md": "text/markdown", ".markdown": "text/markdown", ".txt": "text/plain", + ".pdf": "application/pdf", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", ".csv": "text/csv", ".json": "application/json", ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", @@ -31,67 +98,220 @@ function mediaType(path: string): string { return types[extension] ?? "application/octet-stream"; } +function normalizedLimits(overrides: Partial | undefined): ImportLimits { + const limits = { ...DEFAULT_IMPORT_LIMITS, ...overrides }; + for (const [name, value] of Object.entries(limits)) { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new TypeError(`Import limit ${name} must be a positive safe integer.`); + } + } + return limits; +} + function normalizeRelative(root: string, target: string): string { const value = relative(root, target).split(sep).join("/"); if (!value || value.startsWith("../") || value === ".." || isAbsolute(value)) { - throw new Error(`Unsafe path outside package: ${value || target}`); + throw new PackageImportError("unsafe_path", `Unsafe path outside package: ${value || target}`, { path: value || target }); } return value; } export async function resolveSafePath(rootPath: string, relativePath: string): Promise { - const root = await realpath(rootPath); - if (isAbsolute(relativePath)) throw new Error(`Unsafe absolute path outside package: ${relativePath}`); + const requestedRoot = resolve(rootPath); + const requestedStat = await lstat(requestedRoot); + if (requestedStat.isSymbolicLink()) throw new PackageImportError("symlink_input", `Symlink inputs are not allowed: ${rootPath}`, { path: rootPath }); + const root = await realpath(requestedRoot); + if (isAbsolute(relativePath)) { + throw new PackageImportError("unsafe_path", `Unsafe absolute path outside package: ${relativePath}`, { path: relativePath }); + } const candidate = resolve(root, relativePath); const normalized = normalizeRelative(root, candidate); const parentReal = await realpath(resolve(root, normalized, "..")); if (parentReal !== root && !parentReal.startsWith(`${root}${sep}`)) { - throw new Error(`Unsafe path outside package: ${relativePath}`); + throw new PackageImportError("unsafe_path", `Unsafe path outside package: ${relativePath}`, { path: relativePath }); } - let targetReal: string; try { - targetReal = await realpath(candidate); - } catch { - targetReal = candidate; - } - if (targetReal !== root && !targetReal.startsWith(`${root}${sep}`)) { - throw new Error(`Unsafe path outside package: ${relativePath}`); + const candidateStat = await lstat(candidate); + if (candidateStat.isSymbolicLink()) { + throw new PackageImportError("symlink_input", `Symlink inputs are not allowed: ${relativePath}`, { path: relativePath }); + } + const targetReal = await realpath(candidate); + if (targetReal !== root && !targetReal.startsWith(`${root}${sep}`)) { + throw new PackageImportError("unsafe_path", `Unsafe path outside package: ${relativePath}`, { path: relativePath }); + } + } catch (error) { + if (error instanceof PackageImportError) throw error; + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code !== "ENOENT") throw error; } return candidate; } -async function walk(root: string, directory: string, files: string[]): Promise { - const { readdir } = await import("node:fs/promises"); +async function walk(root: string, directory: string, files: string[], limits: ImportLimits): Promise { const entries = await readdir(directory, { withFileTypes: true }); - for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { - if (entry.name === ".git" || entry.name === "node_modules" || entry.name.startsWith(".")) continue; + for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name, "en"))) { + if (entry.name.startsWith(".")) continue; const absolute = resolve(directory, entry.name); + if (entry.isSymbolicLink()) { + throw new PackageImportError("symlink_input", `Symlink inputs are not allowed: ${normalizeRelative(root, absolute)}`, { + path: normalizeRelative(root, absolute) + }); + } + const extension = extname(entry.name).toLowerCase(); if (entry.isDirectory()) { - await walk(root, absolute, files); - } else if (entry.isFile() && SUPPORTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) { + await walk(root, absolute, files, limits); + } else if (entry.isFile() && MACRO_ENABLED_EXTENSIONS.has(extension)) { + throw new PackageImportError("docx_macro_enabled", `Macro-enabled Word input is not allowed: ${normalizeRelative(root, absolute)}`, { + path: normalizeRelative(root, absolute) + }); + } else if (entry.isFile() && SUPPORTED_EXTENSIONS.has(extension)) { files.push(absolute); - } else if (entry.isSymbolicLink()) { - throw new Error(`Symlink inputs are not allowed: ${entry.name}`); + if (files.length > limits.maxPackageEntries) { + throw new PackageImportError("package_entry_limit", `Package has more than ${limits.maxPackageEntries} supported files.`, { + actual: files.length, + limit: limits.maxPackageEntries + }); + } } } } -export async function importPackage(packagePath: string): Promise { - const root = await realpath(resolve(packagePath)); - const rootStat = await lstat(root); - if (!rootStat.isDirectory()) throw new Error(`Package path is not a directory: ${packagePath}`); +function selectProblem(root: string, files: string[]): string { + const byName = new Map(); + for (const path of files) { + const name = basename(path).toLowerCase(); + const candidates = byName.get(name) ?? []; + candidates.push(path); + byName.set(name, candidates); + } + for (const group of PROBLEM_CANDIDATE_PRIORITY) { + const candidates = group.flatMap((name) => byName.get(name) ?? []).sort((left, right) => left.localeCompare(right, "en")); + if (candidates.length === 0) continue; + if (candidates.length > 1) { + const relativeCandidates = candidates.map((path) => normalizeRelative(root, path)); + throw new PackageImportError( + "ambiguous_problem", + `Multiple problem statements share the selected priority: ${relativeCandidates.join(", ")}`, + { candidates: relativeCandidates } + ); + } + const selected = candidates[0]; + if (selected === undefined) break; + return selected; + } + throw new PackageImportError( + "problem_missing", + "Package must contain problem.md, problem.markdown, problem.txt, problem.pdf, or problem.docx." + ); +} + +function decodeUtf8(bytes: Buffer, path: string): string { + try { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch (error) { + const cause = error instanceof Error ? error.message : String(error); + throw new PackageImportError("problem_encoding", `Problem statement is not valid UTF-8: ${path}`, { path, cause }); + } +} + +async function extractProblem( + problemPath: string, + identity: { sha256: string; sizeBytes: number }, + limits: ImportLimits +): Promise<{ text: string; metadata: ProblemExtractionMetadata }> { + const extension = extname(problemPath).toLowerCase(); + if (extension === ".pdf") return extractPdf(problemPath, identity, limits); + if (extension === ".docx") return extractDocx(problemPath, identity, limits); + if (identity.sizeBytes > limits.maxProblemBytes) { + throw new PackageImportError("problem_file_limit", `Problem statement is ${identity.sizeBytes} bytes; limit is ${limits.maxProblemBytes}.`, { + path: problemPath, + actual: identity.sizeBytes, + limit: limits.maxProblemBytes + }); + } + const text = decodeUtf8(await readFile(problemPath), problemPath); + if (text.length > limits.maxTextCharacters) { + throw new PackageImportError("problem_file_limit", `Problem statement exceeds ${limits.maxTextCharacters} characters.`, { + path: problemPath, + actual: text.length, + limit: limits.maxTextCharacters + }); + } + if (!text.trim()) throw new PackageImportError("problem_empty", "Problem statement is empty.", { path: problemPath }); + return { + text, + metadata: { + format: extension === ".txt" ? "text" : "markdown", + sourceBytes: identity.sizeBytes, + sha256: identity.sha256, + extractedCharacters: text.length, + extractor: "utf8-v1" + } + }; +} + +export async function importPackage(packagePath: string, options: ImportOptions = {}): Promise { + const requestedRoot = resolve(packagePath); + const requestedStat = await lstat(requestedRoot); + if (requestedStat.isSymbolicLink()) { + throw new PackageImportError("symlink_input", `Symlink package roots are not allowed: ${packagePath}`, { path: packagePath }); + } + if (!requestedStat.isDirectory()) throw new Error(`Package path is not a directory: ${packagePath}`); + const root = await realpath(requestedRoot); + const limits = normalizedLimits(options.limits); const files: string[] = []; - await walk(root, root, files); - const problemPath = files.find((path) => ["problem.md", "problem.markdown", "problem.txt"].includes(basename(path).toLowerCase())); - if (!problemPath) throw new Error("Package must contain problem.md, problem.markdown, or problem.txt."); - const problemText = await readFile(problemPath, "utf8"); - if (!problemText.trim()) throw new Error("Problem statement is empty."); + await walk(root, root, files, limits); + const problemPath = selectProblem(root, files); + const problemBeforeStat = await lstat(problemPath); + if (!problemBeforeStat.isFile() || problemBeforeStat.isSymbolicLink()) { + throw new PackageImportError("symlink_input", `Problem input changed or became a symlink during import: ${problemPath}`, { path: problemPath }); + } + if (problemBeforeStat.size > limits.maxProblemBytes) { + throw new PackageImportError("problem_file_limit", `Problem statement is ${problemBeforeStat.size} bytes; limit is ${limits.maxProblemBytes}.`, { + path: problemPath, + actual: problemBeforeStat.size, + limit: limits.maxProblemBytes + }); + } + const problemIdentity = await fileIdentity(problemPath); + const problem = await extractProblem(problemPath, problemIdentity, limits); + const problemAfterStat = await lstat(problemPath); + if (!problemAfterStat.isFile() + || problemAfterStat.isSymbolicLink() + || problemBeforeStat.dev !== problemAfterStat.dev + || problemBeforeStat.ino !== problemAfterStat.ino + || problemBeforeStat.size !== problemAfterStat.size + || problemBeforeStat.mtimeMs !== problemAfterStat.mtimeMs) { + throw new PackageImportError("unsafe_path", `Problem input changed while being extracted: ${problemPath}`, { path: problemPath }); + } const dataAssets: DataAsset[] = []; const dataPaths = new Map(); + const assetMetadata: Record = {}; + const warnings: ImportWarning[] = []; for (const path of files.filter((item) => item !== problemPath)) { const relativePath = normalizeRelative(root, path); + const beforeStat = await lstat(path); + if (!beforeStat.isFile() || beforeStat.isSymbolicLink()) { + throw new PackageImportError("symlink_input", `Input changed or became a symlink during import: ${relativePath}`, { path: relativePath }); + } + if (beforeStat.size > limits.maxAssetBytes) { + throw new PackageImportError("asset_file_limit", `Asset ${relativePath} is ${beforeStat.size} bytes; limit is ${limits.maxAssetBytes}.`, { + path: relativePath, + actual: beforeStat.size, + limit: limits.maxAssetBytes + }); + } const identity = await fileIdentity(path); + const afterStat = await lstat(path); + if (!afterStat.isFile() + || afterStat.isSymbolicLink() + || beforeStat.dev !== afterStat.dev + || beforeStat.ino !== afterStat.ino + || beforeStat.size !== afterStat.size + || beforeStat.mtimeMs !== afterStat.mtimeMs) { + throw new PackageImportError("unsafe_path", `Input changed while being inventoried: ${relativePath}`, { path: relativePath }); + } const asset: DataAsset = { artifact_id: `input-${sha256Text(`${relativePath}:${identity.sha256}`).slice(0, 12)}`, relative_path: relativePath, @@ -101,6 +321,22 @@ export async function importPackage(packagePath: string): Promise { + if (identity.sizeBytes > limits.maxProblemBytes) { + throw new PackageImportError("problem_file_limit", `PDF is ${identity.sizeBytes} bytes; limit is ${limits.maxProblemBytes}.`, { + path, + actual: identity.sizeBytes, + limit: limits.maxProblemBytes + }); + } + const bytes = await readFile(path); + let pdfjs: typeof import("pdfjs-dist/legacy/build/pdf.mjs"); + try { + pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs"); + } catch (error) { + throw new PackageImportError("pdf_tool_unavailable", "The local pdfjs extractor is unavailable.", { + path, + cause: causeMessage(error) + }); + } + + const loadingTask = pdfjs.getDocument({ + data: new Uint8Array(bytes), + disableAutoFetch: true, + disableFontFace: true, + disableRange: true, + disableStream: true, + isEvalSupported: false, + maxImageSize: limits.maxImagePixels, + stopAtErrors: true, + useSystemFonts: false, + useWasm: false, + useWorkerFetch: false, + verbosity: 0 + }); + try { + const document = await loadingTask.promise; + try { + if (document.numPages > limits.maxPdfPages) { + throw new PackageImportError("pdf_page_limit", `PDF has ${document.numPages} pages; limit is ${limits.maxPdfPages}.`, { + path, + actual: document.numPages, + limit: limits.maxPdfPages + }); + } + const pages: string[] = []; + let characters = 0; + for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber += 1) { + const page = await document.getPage(pageNumber); + const content = await page.getTextContent({ disableNormalization: false, includeMarkedContent: false }); + const textItems = content.items.filter(isTextItem); + let pageCharacters = 0; + for (const item of textItems) { + pageCharacters += item.str.length; + if (pageCharacters > limits.maxPdfCharacters || characters + pageCharacters > limits.maxPdfCharacters) { + throw new PackageImportError("pdf_character_limit", `PDF extraction exceeds ${limits.maxPdfCharacters} characters.`, { + path, + actual: characters + pageCharacters, + limit: limits.maxPdfCharacters + }); + } + } + const text = textItems.map((item) => item.str).join(" ").trim(); + characters += text.length; + if (characters > limits.maxPdfCharacters) { + throw new PackageImportError("pdf_character_limit", `PDF extraction exceeds ${limits.maxPdfCharacters} characters.`, { + path, + actual: characters, + limit: limits.maxPdfCharacters + }); + } + pages.push(text); + } + const text = pages.join("\n\n").trim(); + if (!text) throw new PackageImportError("pdf_empty", "PDF contains no extractable text.", { path }); + return { + text, + metadata: { + format: "pdf", + sourceBytes: identity.sizeBytes, + sha256: identity.sha256, + extractedCharacters: text.length, + extractor: `pdfjs-dist/${pdfjs.version}`, + pages: document.numPages + } + }; + } finally { + await document.destroy(); + } + } catch (error) { + if (error instanceof PackageImportError) throw error; + const message = causeMessage(error); + const name = error instanceof Error ? error.name : ""; + if (name === "PasswordException" || /password|encrypted/i.test(message)) { + throw new PackageImportError("pdf_encrypted", "Encrypted PDF files are not allowed.", { path, cause: message }); + } + throw new PackageImportError("pdf_corrupt", `Could not extract PDF text: ${message}`, { path, cause: message }); + } finally { + await loadingTask.destroy(); + } +} diff --git a/src/input/types.ts b/src/input/types.ts new file mode 100644 index 0000000..be8fe5e --- /dev/null +++ b/src/input/types.ts @@ -0,0 +1,130 @@ +import type { DataAsset } from "../contracts/types.js"; + +export type PackageImportErrorCode = + | "ambiguous_problem" + | "problem_missing" + | "problem_empty" + | "problem_file_limit" + | "problem_encoding" + | "package_entry_limit" + | "asset_file_limit" + | "symlink_input" + | "unsafe_path" + | "pdf_tool_unavailable" + | "pdf_corrupt" + | "pdf_encrypted" + | "pdf_empty" + | "pdf_page_limit" + | "pdf_character_limit" + | "docx_corrupt" + | "docx_encrypted" + | "docx_macro_enabled" + | "docx_external_relationship" + | "docx_zip_slip" + | "docx_zip_entry_limit" + | "docx_uncompressed_limit" + | "docx_character_limit" + | "docx_empty" + | "metadata_unreadable" + | "metadata_limit"; + +export interface PackageImportErrorDetails { + path?: string; + candidates?: string[]; + limit?: number; + actual?: number; + cause?: string; +} + +export class PackageImportError extends Error { + readonly code: PackageImportErrorCode; + readonly details: Readonly; + + constructor(code: PackageImportErrorCode, message: string, details: PackageImportErrorDetails = {}) { + super(message, { cause: details.cause }); + this.name = "PackageImportError"; + this.code = code; + this.details = Object.freeze({ ...details }); + } +} + +export interface ImportLimits { + maxProblemBytes: number; + maxTextCharacters: number; + maxPackageEntries: number; + maxAssetBytes: number; + maxPdfPages: number; + maxPdfCharacters: number; + maxDocxZipEntries: number; + maxDocxUncompressedBytes: number; + maxDocxXmlPartBytes: number; + maxDocxCharacters: number; + maxMetadataBytes: number; + maxMetadataZipEntries: number; + maxMetadataUncompressedBytes: number; + maxXlsxSheets: number; + maxXlsxWorkbookBytes: number; + maxXlsxRelationshipsBytes: number; + maxXlsxWorksheetBytes: number; + maxImageHeaderBytes: number; + maxImagePixels: number; +} + +export interface ImportOptions { + limits?: Partial; +} + +export interface ProblemExtractionMetadata { + format: "markdown" | "text" | "pdf" | "docx"; + sourceBytes: number; + sha256: string; + extractedCharacters: number; + extractor: string; + pages?: number; + zipEntries?: number; + uncompressedBytes?: number; +} + +export interface XlsxSheetMetadata { + name: string; + dimension?: string; +} + +export interface XlsxAssetMetadata { + kind: "spreadsheet"; + format: "xlsx"; + sheets: XlsxSheetMetadata[]; + zipEntries: number; + uncompressedBytes: number; +} + +export interface ImageAssetMetadata { + kind: "image"; + format: "png" | "jpeg"; + width: number; + height: number; +} + +export interface BoundedAssetMetadata { + kind: "bounded-inventory"; + format: "csv" | "json" | "xls" | "parquet"; +} + +export type AssetMetadata = XlsxAssetMetadata | ImageAssetMetadata | BoundedAssetMetadata; + +export interface ImportWarning { + code: "metadata_unreadable" | "metadata_limit"; + path: string; + message: string; +} + +export interface ImportedPackage { + rootPath: string; + problemPath: string; + problemText: string; + dataAssets: DataAsset[]; + dataPaths: Map; + problemMetadata?: ProblemExtractionMetadata; + assetMetadata?: Record; + warnings?: ImportWarning[]; +} diff --git a/src/input/xml.ts b/src/input/xml.ts new file mode 100644 index 0000000..d3c3f71 --- /dev/null +++ b/src/input/xml.ts @@ -0,0 +1,32 @@ +import { SaxesParser, type SaxesTagNS } from "saxes"; + +export const WORDPROCESSINGML_NAMESPACE = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"; +export const SPREADSHEETML_NAMESPACE = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; +export const PACKAGE_RELATIONSHIPS_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/relationships"; + +export function parseXml( + xml: Buffer | string, + handlers: { + openTag?: (tag: SaxesTagNS) => void; + closeTag?: (tag: SaxesTagNS) => void; + text?: (text: string) => void; + } +): void { + const parser = new SaxesParser({ xmlns: true, position: true }); + parser.on("opentag", (tag) => handlers.openTag?.(tag)); + parser.on("closetag", (tag) => handlers.closeTag?.(tag)); + parser.on("text", (text) => handlers.text?.(text)); + parser.on("cdata", (text) => handlers.text?.(text)); + parser.on("doctype", () => { + throw new Error("DOCTYPE declarations are not allowed."); + }); + parser.on("error", (error) => { + throw error; + }); + parser.write(xml.toString("utf8")); + parser.close(); +} + +export function attributeValue(tag: SaxesTagNS, local: string, uri?: string): string | undefined { + return Object.values(tag.attributes).find((attribute) => attribute.local === local && (uri === undefined || attribute.uri === uri))?.value; +} diff --git a/src/input/zip-reader.ts b/src/input/zip-reader.ts new file mode 100644 index 0000000..b1ebd31 --- /dev/null +++ b/src/input/zip-reader.ts @@ -0,0 +1,164 @@ +import crc32 from "buffer-crc32"; +import { open, type Entry, type ZipFile } from "yauzl"; +import { PackageImportError, type PackageImportErrorCode } from "./types.js"; + +export interface ZipLimits { + maxEntries: number; + maxUncompressedBytes: number; +} + +export interface ZipContents { + entries: Map; + entryCount: number; + uncompressedBytes: number; +} + +export interface ZipErrorCodes { + corrupt: PackageImportErrorCode; + encrypted: PackageImportErrorCode; + zipSlip: PackageImportErrorCode; + entryLimit: PackageImportErrorCode; + uncompressedLimit: PackageImportErrorCode; +} + +function causeMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function openArchive(path: string): Promise { + return new Promise((resolve, reject) => { + open(path, { + autoClose: false, + lazyEntries: true, + decodeStrings: true, + validateEntrySizes: true, + strictFileNames: true + }, (error, archive) => { + if (error) reject(error); + else resolve(archive); + }); + }); +} + +function readEntry(archive: ZipFile, entry: Entry, maximumBytes: number): Promise { + return new Promise((resolve, reject) => { + archive.openReadStream(entry, (error, stream) => { + if (error) { + reject(error); + return; + } + const chunks: Buffer[] = []; + let size = 0; + stream.on("data", (chunk: Buffer | string) => { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + size += buffer.length; + if (size > maximumBytes) stream.destroy(new Error(`Uncompressed entry exceeds ${maximumBytes} bytes.`)); + else chunks.push(buffer); + }); + stream.once("error", reject); + stream.once("end", () => { + const result = Buffer.concat(chunks); + if (result.length !== entry.uncompressedSize) { + reject(new Error(`ZIP entry size mismatch for ${entry.fileName}: expected ${entry.uncompressedSize}, got ${result.length}.`)); + return; + } + if (crc32.unsigned(result) !== entry.crc32) { + reject(new Error(`ZIP entry CRC mismatch for ${entry.fileName}.`)); + return; + } + resolve(result); + }); + }); + }); +} + +function unsafeEntryName(name: string): boolean { + return name.includes("\\") + || name.startsWith("/") + || /^[A-Za-z]:/.test(name) + || name.split("/").includes(".."); +} + +export async function readBoundedZip( + path: string, + limits: ZipLimits, + codes: ZipErrorCodes, + select: (entryName: string) => boolean +): Promise { + let archive: ZipFile; + try { + archive = await openArchive(path); + } catch (error) { + const message = causeMessage(error); + const code = /invalid relative path|absolute path|invalid characters in fileName/i.test(message) ? codes.zipSlip : codes.corrupt; + throw new PackageImportError(code, `Could not open ZIP package: ${message}`, { path, cause: message }); + } + + if (archive.entryCount > limits.maxEntries) { + archive.close(); + throw new PackageImportError(codes.entryLimit, `ZIP package has ${archive.entryCount} entries; limit is ${limits.maxEntries}.`, { + path, + actual: archive.entryCount, + limit: limits.maxEntries + }); + } + + return new Promise((resolve, reject) => { + const entries = new Map(); + let count = 0; + let total = 0; + let settled = false; + + const fail = (error: unknown): void => { + if (settled) return; + settled = true; + archive.close(); + if (error instanceof PackageImportError) reject(error); + else { + const message = causeMessage(error); + const code = /invalid relative path|absolute path|invalid characters in fileName/i.test(message) ? codes.zipSlip : codes.corrupt; + reject(new PackageImportError(code, `Could not read ZIP package: ${message}`, { path, cause: message })); + } + }; + + archive.on("error", fail); + archive.on("entry", (entry: Entry) => { + void (async () => { + count += 1; + if (count > limits.maxEntries) { + throw new PackageImportError(codes.entryLimit, `ZIP package exceeds ${limits.maxEntries} entries.`, { + path, + actual: count, + limit: limits.maxEntries + }); + } + if (unsafeEntryName(entry.fileName)) { + throw new PackageImportError(codes.zipSlip, `Unsafe ZIP entry path: ${entry.fileName}`, { path: entry.fileName }); + } + if (entry.isEncrypted()) { + throw new PackageImportError(codes.encrypted, `Encrypted ZIP entry is not allowed: ${entry.fileName}`, { path: entry.fileName }); + } + total += entry.uncompressedSize; + if (!Number.isSafeInteger(total) || total > limits.maxUncompressedBytes) { + throw new PackageImportError(codes.uncompressedLimit, `ZIP package expands to more than ${limits.maxUncompressedBytes} bytes.`, { + path, + actual: total, + limit: limits.maxUncompressedBytes + }); + } + if (!entry.fileName.endsWith("/") && select(entry.fileName)) { + const remaining = limits.maxUncompressedBytes - (total - entry.uncompressedSize); + entries.set(entry.fileName, await readEntry(archive, entry, remaining)); + } + archive.readEntry(); + })().catch(fail); + }); + archive.on("end", () => { + if (settled) return; + settled = true; + archive.close(); + resolve({ entries, entryCount: count, uncompressedBytes: total }); + }); + archive.readEntry(); + }); +} diff --git a/tests/input-package-importer.test.ts b/tests/input-package-importer.test.ts new file mode 100644 index 0000000..9261492 --- /dev/null +++ b/tests/input-package-importer.test.ts @@ -0,0 +1,331 @@ +import { describe, expect, it } from "vitest"; +import { deflateRawSync } from "node:zlib"; +import { mkdtemp, mkdir, readFile, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createHash } from "node:crypto"; +import { + importPackage, + type ImportLimits +} from "../src/input/package-importer.js"; + +const tempPackage = async (name: string): Promise => mkdtemp(join(tmpdir(), `modeling-input-${name}-`)); + +function pdfDocument(pages: string[], encrypted = false): Buffer { + const objects: string[] = []; + const pageObjectNumbers: number[] = []; + const fontObject = 3; + const pagesObject = 2; + objects.push("<< /Type /Catalog /Pages 2 0 R >>"); + objects.push(`<< /Type /Pages /Kids [${pages.map((_, index) => `${4 + index * 2} 0 R`).join(" ")}] /Count ${pages.length} >>`); + objects.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"); + for (const [index, text] of pages.entries()) { + const pageObject = 4 + index * 2; + const contentObject = pageObject + 1; + pageObjectNumbers.push(pageObject); + const escaped = text.replaceAll("\\", "\\\\").replaceAll("(", "\\(").replaceAll(")", "\\)"); + const stream = `BT /F1 18 Tf 72 720 Td (${escaped}) Tj ET`; + objects.push(`<< /Type /Page /Parent ${pagesObject} 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 ${fontObject} 0 R >> >> /Contents ${contentObject} 0 R >>`); + objects.push(`<< /Length ${Buffer.byteLength(stream, "ascii")} >>\nstream\n${stream}\nendstream`); + } + const chunks: Buffer[] = [Buffer.from("%PDF-1.4\n", "ascii")]; + const offsets = [0]; + for (const [index, object] of objects.entries()) { + offsets.push(Buffer.concat(chunks).length); + chunks.push(Buffer.from(`${index + 1} 0 obj\n${object}\nendobj\n`, "ascii")); + } + if (encrypted) { + objects.push("<< /Filter /Standard /V 1 /R 2 /Length 40 /O <0000000000000000000000000000000000000000000000000000000000000000> /U <0000000000000000000000000000000000000000000000000000000000000000> /P -4 >>"); + const encryptedObject = objects.length; + offsets.push(Buffer.concat(chunks).length); + chunks.push(Buffer.from(`${encryptedObject} 0 obj\n${objects[encryptedObject - 1]}\nendobj\n`, "ascii")); + } + const xrefOffset = Buffer.concat(chunks).length; + const xref = [`xref`, `0 ${objects.length + 1}`, "0000000000 65535 f "]; + for (const offset of offsets.slice(1)) xref.push(`${String(offset).padStart(10, "0")} 00000 n `); + const encryptionTrailer = encrypted + ? ` /Encrypt ${objects.length} 0 R /ID [<00112233445566778899aabbccddeeff><00112233445566778899aabbccddeeff>]` + : ""; + xref.push(`trailer\n<< /Size ${objects.length + 1} /Root 1 0 R${encryptionTrailer} >>`, `startxref`, String(xrefOffset), "%%EOF\n"); + chunks.push(Buffer.from(`${xref.join("\n")}\n`, "ascii")); + void pageObjectNumbers; + return Buffer.concat(chunks); +} + +type ZipEntry = { name: string; data: Buffer; compression?: "store" | "deflate" }; + +function zip(entries: ZipEntry[]): Buffer { + const local: Buffer[] = []; + const central: Buffer[] = []; + let offset = 0; + for (const entry of entries) { + const name = Buffer.from(entry.name, "utf8"); + const compressed = entry.compression === "deflate" ? deflateRawSync(entry.data) : entry.data; + const method = entry.compression === "deflate" ? 8 : 0; + const crc32 = crc32Of(entry.data); + const header = Buffer.alloc(30 + name.length); + header.writeUInt32LE(0x04034b50, 0); + header.writeUInt16LE(20, 4); + header.writeUInt16LE(0, 6); + header.writeUInt16LE(method, 8); + header.writeUInt16LE(0, 10); + header.writeUInt16LE(0, 12); + header.writeUInt32LE(crc32, 14); + header.writeUInt32LE(compressed.length, 18); + header.writeUInt32LE(entry.data.length, 22); + header.writeUInt16LE(name.length, 26); + header.writeUInt16LE(0, 28); + name.copy(header, 30); + local.push(header, compressed); + const directory = Buffer.alloc(46 + name.length); + directory.writeUInt32LE(0x02014b50, 0); + directory.writeUInt16LE(20, 4); + directory.writeUInt16LE(20, 6); + directory.writeUInt16LE(0, 8); + directory.writeUInt16LE(method, 10); + directory.writeUInt16LE(0, 12); + directory.writeUInt16LE(0, 14); + directory.writeUInt32LE(crc32, 16); + directory.writeUInt32LE(compressed.length, 20); + directory.writeUInt32LE(entry.data.length, 24); + directory.writeUInt16LE(name.length, 28); + directory.writeUInt16LE(0, 30); + directory.writeUInt16LE(0, 32); + directory.writeUInt16LE(0, 34); + directory.writeUInt32LE(0, 36); + directory.writeUInt32LE(offset, 42); + name.copy(directory, 46); + central.push(directory); + offset += header.length + compressed.length; + } + const localBytes = Buffer.concat(local); + const centralBytes = Buffer.concat(central); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(0, 4); + end.writeUInt16LE(0, 6); + end.writeUInt16LE(entries.length, 8); + end.writeUInt16LE(entries.length, 10); + end.writeUInt32LE(centralBytes.length, 12); + end.writeUInt32LE(localBytes.length, 16); + return Buffer.concat([localBytes, centralBytes, end]); +} + +function crc32Of(data: Buffer): number { + let crc = 0xffffffff; + for (const byte of data) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + return (crc ^ 0xffffffff) >>> 0; +} + +function docxDocument( + text: string, + extra: ZipEntry[] = [], + mainContentType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml" +): Buffer { + return zip([ + { name: "[Content_Types].xml", data: Buffer.from(``) }, + { name: "_rels/.rels", data: Buffer.from("") }, + { name: "word/document.xml", data: Buffer.from(`${text}`) }, + ...extra + ]); +} + +function xlsxAsset(): Buffer { + return zip([ + { name: "[Content_Types].xml", data: Buffer.from("") }, + { name: "xl/workbook.xml", data: Buffer.from("") }, + { name: "xl/_rels/workbook.xml.rels", data: Buffer.from("") }, + { name: "xl/worksheets/sheet1.xml", data: Buffer.from("") }, + { name: "xl/worksheets/sheet2.xml", data: Buffer.from("") } + ]); +} + +function png(width: number, height: number): Buffer { + const value = Buffer.alloc(24); + Buffer.from("\x89PNG\r\n\x1a\n", "binary").copy(value, 0); + value.writeUInt32BE(13, 8); + value.write("IHDR", 12, "ascii"); + value.writeUInt32BE(width, 16); + value.writeUInt32BE(height, 20); + return value; +} + +async function expectImportCode(operation: Promise, code: string): Promise { + await expect(operation).rejects.toMatchObject({ code }); +} + +describe("safe package importer", () => { + it("extracts a real PDF and records bounded provenance", async () => { + const root = await tempPackage("pdf"); + const source = pdfDocument(["PDF problem statement"]); + await writeFile(join(root, "problem.pdf"), source); + const imported = await importPackage(root); + expect(imported.problemText).toContain("PDF problem statement"); + expect(imported.problemMetadata).toMatchObject({ format: "pdf", pages: 1, extractedCharacters: expect.any(Number), extractor: expect.stringContaining("pdfjs") }); + expect(imported.problemMetadata?.sha256).toBe(createHash("sha256").update(source).digest("hex")); + expect(imported.problemPath).toBe(join(root, "problem.pdf")); + }); + + it("rejects empty, corrupt, and over-page PDFs with structured failures", async () => { + const empty = await tempPackage("pdf-empty"); + await writeFile(join(empty, "problem.pdf"), pdfDocument([""])); + await expectImportCode(importPackage(empty), "pdf_empty"); + + const corrupt = await tempPackage("pdf-corrupt"); + await writeFile(join(corrupt, "problem.pdf"), Buffer.from("not a pdf")); + await expectImportCode(importPackage(corrupt), "pdf_corrupt"); + + const encrypted = await tempPackage("pdf-encrypted"); + await writeFile(join(encrypted, "problem.pdf"), pdfDocument(["secret"], true)); + await expectImportCode(importPackage(encrypted), "pdf_encrypted"); + + const tooManyPages = await tempPackage("pdf-pages"); + await writeFile(join(tooManyPages, "problem.pdf"), pdfDocument(["one", "two"])); + const limits: Partial = { maxPdfPages: 1 }; + await expectImportCode(importPackage(tooManyPages, { limits }), "pdf_page_limit"); + + const tooManyCharacters = await tempPackage("pdf-characters"); + await writeFile(join(tooManyCharacters, "problem.pdf"), pdfDocument(["0123456789"])); + await expectImportCode(importPackage(tooManyCharacters, { limits: { maxPdfCharacters: 5 } }), "pdf_character_limit"); + + const tooManyBytes = await tempPackage("pdf-bytes"); + await writeFile(join(tooManyBytes, "problem.pdf"), pdfDocument(["text"])); + await expectImportCode(importPackage(tooManyBytes, { limits: { maxProblemBytes: 10 } }), "problem_file_limit"); + }); + + it("extracts only DOCX body text", async () => { + const root = await tempPackage("docx"); + await writeFile(join(root, "problem.docx"), docxDocument("DOCX body text")); + const imported = await importPackage(root); + expect(imported.problemText).toContain("DOCX body text"); + expect(imported.problemMetadata).toMatchObject({ format: "docx", extractedCharacters: 14, extractor: "ooxml-body-v1" }); + }); + + it("rejects corrupt and macro-enabled DOCX packages", async () => { + const corrupt = await tempPackage("docx-corrupt"); + await writeFile(join(corrupt, "problem.docx"), Buffer.from("not a zip")); + await expectImportCode(importPackage(corrupt), "docx_corrupt"); + + const crcMismatch = await tempPackage("docx-crc"); + const tampered = docxDocument("body"); + const marker = tampered.indexOf(Buffer.from("body")); + tampered[marker] = "B".charCodeAt(0); + await writeFile(join(crcMismatch, "problem.docx"), tampered); + await expectImportCode(importPackage(crcMismatch), "docx_corrupt"); + + const macro = await tempPackage("docx-macro"); + const document = docxDocument( + "body", + [{ name: "word/vbaProject.bin", data: Buffer.from("not executed") }], + "application/vnd.ms-word.document.macroEnabled.main+xml" + ); + await writeFile(join(macro, "problem.docx"), document); + await expectImportCode(importPackage(macro), "docx_macro_enabled"); + }); + + it("rejects DOCX external relationships and ZIP slip names", async () => { + const external = await tempPackage("docx-external"); + await writeFile(join(external, "problem.docx"), docxDocument("body", [ + { name: "word/_rels/document.xml.rels", data: Buffer.from("") } + ])); + await expectImportCode(importPackage(external), "docx_external_relationship"); + + const entity = await tempPackage("docx-entity"); + await writeFile(join(entity, "problem.docx"), docxDocument("body", [ + { name: "word/_rels/document.xml.rels", data: Buffer.from("]>&xxe;") } + ])); + await expectImportCode(importPackage(entity), "docx_corrupt"); + + const traversal = await tempPackage("docx-slip"); + await writeFile(join(traversal, "problem.docx"), docxDocument("body", [{ name: "../outside.xml", data: Buffer.from("x") }])); + await expectImportCode(importPackage(traversal), "docx_zip_slip"); + }); + + it("enforces DOCX entry, decompressed-byte, and character limits", async () => { + const entries = await tempPackage("docx-entries"); + await writeFile(join(entries, "problem.docx"), docxDocument("body", [ + { name: "word/extra.xml", data: Buffer.from("x") }, + { name: "word/extra2.xml", data: Buffer.from("x") } + ])); + await expectImportCode(importPackage(entries, { limits: { maxDocxZipEntries: 4 } }), "docx_zip_entry_limit"); + + const bytes = await tempPackage("docx-bytes"); + await writeFile(join(bytes, "problem.docx"), docxDocument("body", [{ name: "word/large.xml", data: Buffer.alloc(500, 65) }])); + await expectImportCode(importPackage(bytes, { limits: { maxDocxUncompressedBytes: 100 } }), "docx_uncompressed_limit"); + + const characters = await tempPackage("docx-characters"); + await writeFile(join(characters, "problem.docx"), docxDocument("0123456789")); + await expectImportCode(importPackage(characters, { limits: { maxDocxCharacters: 5 } }), "docx_character_limit"); + }); + + it("fails when same-priority problem candidates are ambiguous", async () => { + const root = await tempPackage("ambiguous"); + await writeFile(join(root, "problem.md"), "markdown"); + await writeFile(join(root, "problem.markdown"), "other markdown"); + await expectImportCode(importPackage(root), "ambiguous_problem"); + }); + + it("uses documented precedence and treats lower-priority statements as assets", async () => { + const root = await tempPackage("precedence"); + await writeFile(join(root, "problem.txt"), "text wins"); + await writeFile(join(root, "problem.pdf"), pdfDocument(["pdf loses"])); + await writeFile(join(root, "problem.docx"), docxDocument("docx loses")); + const imported = await importPackage(root); + expect(imported.problemText).toBe("text wins"); + expect(imported.dataAssets.map((asset) => asset.relative_path)).toEqual(["problem.docx", "problem.pdf"]); + }); + + it("rejects root and nested symlinks", async () => { + const target = await tempPackage("symlink-target"); + await writeFile(join(target, "problem.md"), "problem"); + const parent = await tempPackage("symlink-parent"); + const link = join(parent, "linked-package"); + await symlink(target, link, "dir"); + await expectImportCode(importPackage(link), "symlink_input"); + + const root = await tempPackage("symlink-nested"); + await writeFile(join(root, "problem.md"), "problem"); + await symlink(join(root, "problem.md"), join(root, "problem-link.md")); + await expectImportCode(importPackage(root), "symlink_input"); + }); + + it("keeps raw asset bytes and SHA while exposing bounded XLSX and image metadata", async () => { + const root = await tempPackage("inventory"); + const spreadsheet = xlsxAsset(); + const picture = png(3, 2); + await writeFile(join(root, "problem.md"), "problem"); + await writeFile(join(root, "data.xlsx"), spreadsheet); + await writeFile(join(root, "plot.png"), picture); + const beforeSpreadsheet = await readFile(join(root, "data.xlsx")); + const beforePicture = await readFile(join(root, "plot.png")); + + const imported = await importPackage(root); + const spreadsheetAsset = imported.dataAssets.find((asset) => asset.relative_path === "data.xlsx"); + const pictureAsset = imported.dataAssets.find((asset) => asset.relative_path === "plot.png"); + expect(spreadsheetAsset?.sha256).toBe(createHash("sha256").update(spreadsheet).digest("hex")); + expect(spreadsheetAsset?.size_bytes).toBe(spreadsheet.length); + expect(pictureAsset?.sha256).toBe(createHash("sha256").update(picture).digest("hex")); + expect(pictureAsset?.size_bytes).toBe(picture.length); + expect(imported.assetMetadata?.[spreadsheetAsset?.artifact_id ?? ""]).toMatchObject({ format: "xlsx", sheets: [{ name: "Data", dimension: "A1:C12" }, { name: "Summary", dimension: "B2:D4" }] }); + expect(imported.assetMetadata?.[pictureAsset?.artifact_id ?? ""]).toMatchObject({ format: "png", width: 3, height: 2 }); + expect(await readFile(join(root, "data.xlsx"))).toEqual(beforeSpreadsheet); + expect(await readFile(join(root, "plot.png"))).toEqual(beforePicture); + }); + + it("surfaces unreadable and over-limit metadata as explicit warnings", async () => { + const root = await tempPackage("metadata-warning"); + await writeFile(join(root, "problem.md"), "problem"); + await writeFile(join(root, "bad.png"), Buffer.from("not an image")); + await writeFile(join(root, "bad.xlsx"), Buffer.from("not a workbook")); + await writeFile(join(root, "large.png"), png(10, 10)); + const imported = await importPackage(root, { limits: { maxImagePixels: 50 } }); + expect(imported.warnings).toEqual(expect.arrayContaining([ + expect.objectContaining({ code: "metadata_unreadable", path: "bad.png" }), + expect.objectContaining({ code: "metadata_unreadable", path: "bad.xlsx" }), + expect.objectContaining({ code: "metadata_limit", path: "large.png" }) + ])); + }); +}); From 93bbb2e6dbbb72d7032909480f1871cb24706b51 Mon Sep 17 00:00:00 2001 From: zhanghan <2774877132@qq.com> Date: Thu, 13 Aug 2026 03:32:59 +0800 Subject: [PATCH 2/3] fix: reject ambiguous OOXML entries --- src/input/docx-extractor.ts | 2 +- src/input/zip-reader.ts | 8 +++++++- tests/input-package-importer.test.ts | 6 ++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/input/docx-extractor.ts b/src/input/docx-extractor.ts index 6cfdda6..57939ea 100644 --- a/src/input/docx-extractor.ts +++ b/src/input/docx-extractor.ts @@ -141,7 +141,7 @@ export async function extractDocx( } validateContentTypes(contentTypes, path); validatePackageRelationships(packageRelationships, path); - if (archive.entries.has("word/vbaProject.bin")) { + if (archive.entryNames.has("word/vbaProject.bin")) { throw new PackageImportError("docx_macro_enabled", "Macro-enabled Word packages are not allowed.", { path }); } for (const [name, xml] of archive.entries) { diff --git a/src/input/zip-reader.ts b/src/input/zip-reader.ts index b1ebd31..0c400e9 100644 --- a/src/input/zip-reader.ts +++ b/src/input/zip-reader.ts @@ -9,6 +9,7 @@ export interface ZipLimits { export interface ZipContents { entries: Map; + entryNames: Set; entryCount: number; uncompressedBytes: number; } @@ -105,6 +106,7 @@ export async function readBoundedZip( return new Promise((resolve, reject) => { const entries = new Map(); + const entryNames = new Set(); let count = 0; let total = 0; let settled = false; @@ -135,6 +137,10 @@ export async function readBoundedZip( if (unsafeEntryName(entry.fileName)) { throw new PackageImportError(codes.zipSlip, `Unsafe ZIP entry path: ${entry.fileName}`, { path: entry.fileName }); } + if (entryNames.has(entry.fileName)) { + throw new PackageImportError(codes.corrupt, `ZIP package contains a duplicate entry: ${entry.fileName}`, { path: entry.fileName }); + } + entryNames.add(entry.fileName); if (entry.isEncrypted()) { throw new PackageImportError(codes.encrypted, `Encrypted ZIP entry is not allowed: ${entry.fileName}`, { path: entry.fileName }); } @@ -157,7 +163,7 @@ export async function readBoundedZip( if (settled) return; settled = true; archive.close(); - resolve({ entries, entryCount: count, uncompressedBytes: total }); + resolve({ entries, entryNames, entryCount: count, uncompressedBytes: total }); }); archive.readEntry(); }); diff --git a/tests/input-package-importer.test.ts b/tests/input-package-importer.test.ts index 9261492..0457627 100644 --- a/tests/input-package-importer.test.ts +++ b/tests/input-package-importer.test.ts @@ -216,6 +216,12 @@ describe("safe package importer", () => { await writeFile(join(crcMismatch, "problem.docx"), tampered); await expectImportCode(importPackage(crcMismatch), "docx_corrupt"); + const duplicate = await tempPackage("docx-duplicate"); + await writeFile(join(duplicate, "problem.docx"), docxDocument("body", [ + { name: "word/document.xml", data: Buffer.from("") } + ])); + await expectImportCode(importPackage(duplicate), "docx_corrupt"); + const macro = await tempPackage("docx-macro"); const document = docxDocument( "body", From 39cf5fc3155a853ac292f0389971572bab3b517f Mon Sep 17 00:00:00 2001 From: zhanghan <2774877132@qq.com> Date: Thu, 13 Aug 2026 18:45:25 +0800 Subject: [PATCH 3/3] fix: harden safe package input boundaries --- src/input/README.md | 6 +- src/input/asset-metadata.ts | 95 +++++- src/input/docx-extractor.ts | 373 +++++++++++++++++++-- src/input/package-importer.ts | 82 +++-- src/input/pdf-extractor.ts | 56 ++-- src/input/types.ts | 77 ++++- src/input/zip-reader.ts | 31 +- tests/input-package-importer.test.ts | 472 ++++++++++++++++++++++++--- 8 files changed, 1055 insertions(+), 137 deletions(-) diff --git a/src/input/README.md b/src/input/README.md index 6167f13..1643e9f 100644 --- a/src/input/README.md +++ b/src/input/README.md @@ -15,10 +15,10 @@ Names are matched case-insensitively at any non-hidden package depth. The first ## Safety and limits -All limits have conservative defaults in `DEFAULT_IMPORT_LIMITS` and may be reduced or raised through `importPackage(path, { limits })`. Invalid limits are rejected. Package roots and nested entries may not be symlinks, paths must remain beneath the package root, hidden directories and files are ignored, and supported file counts and raw sizes are bounded. +All limits have conservative defaults in `DEFAULT_IMPORT_LIMITS` and may be reduced or raised through `importPackage(path, { limits })`. Invalid limits are rejected. Package roots and nested entries may not be symlinks, paths must remain beneath the package root, and total traversal is bounded by `maxPackageEntries` and `maxPackageDepth`. Every encountered child entry (file, directory, unsupported extension, or hidden name) consumes the entry budget; the package root itself does not. Hidden entries are then ignored and hidden directories are never recursed into. -PDF text is extracted locally with the pinned `pdfjs-dist` Node build. Input bytes are passed directly to the parser with fetch, streaming, JavaScript evaluation, system fonts, and WebAssembly disabled. The importer records source SHA-256/size, PDF page count, extracted character count, and extractor version, while explicitly rejecting unavailable tooling, encryption, corruption, empty text, and byte/page/character limits. +PDF text is extracted locally with the pinned `pdfjs-dist` Node build. Input bytes are passed directly to the parser with fetch, streaming, JavaScript evaluation, system fonts, and WebAssembly disabled. The importer records source SHA-256/size, PDF page count, extracted character count, and extractor version, while explicitly rejecting unavailable tooling, encryption, corruption, empty text, and byte/page/character limits. `maxPdfCharacters` applies to the exact returned text, including spaces inserted between text items, blank lines inserted between non-empty pages, and final normalization; construction is checked incrementally and asserted again before return. -DOCX extraction reads only required OOXML package parts and `word/document.xml` body text. It never executes content, follows external relationships, or opens embedded objects. Macro-enabled content types, encrypted entries, unsafe ZIP names, external relationships, corrupt packages, and ZIP entry/uncompressed/text limits fail with structured codes. +DOCX extraction performs a bounded magic-prefix check before opening ZIP data: normal ZIP signatures continue, OLE/CFB Office containers are classified as `docx_encrypted`, and other input is `docx_corrupt`. Extraction reads only required OOXML package parts and `word/document.xml` body text. It never decrypts content, executes payloads, follows non-internal relationships, or opens embedded objects. All relationship files reject duplicate IDs and external, escaping, or unknown normalized `TargetMode` values; package relationships require exactly one normalized main-document relationship; and content-type declarations reject duplicate/case-normalized part ambiguity. Any VBA/macro/ActiveX content type, relationship type, or package-part indicator is rejected before payload expansion; ZIP symlink entries are rejected at the archive boundary. Data assets retain SHA-256 and size computed from their original bytes. XLSX inspection is limited to workbook, relationship, and bounded worksheet-dimension XML; images are read only far enough to obtain bounded PNG/JPEG format and dimensions. CSV, JSON, legacy XLS, and Parquet are inventoried without parsing or sampling their records. Unreadable or over-limit optional asset metadata is surfaced in `warnings`, never silently treated as successful metadata. diff --git a/src/input/asset-metadata.ts b/src/input/asset-metadata.ts index e7fa362..195e5b7 100644 --- a/src/input/asset-metadata.ts +++ b/src/input/asset-metadata.ts @@ -16,6 +16,7 @@ import { } from "./xml.js"; const OFFICE_DOCUMENT_RELATIONSHIPS_NAMESPACE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; +const MAX_CAUSE_LENGTH = 240; const XLSX_ZIP_CODES = { corrupt: "metadata_unreadable", encrypted: "metadata_unreadable", @@ -127,6 +128,74 @@ function workbookSheets(xml: Buffer, path: string, sheetLimit: number, byteLimit return sheets; } +function decodePercentEscapes(value: string): string | undefined { + let decoded = value; + for (let depth = 0; depth < 4 && decoded.includes("%"); depth += 1) { + try { + const next = decodeURIComponent(decoded); + if (next === decoded) return undefined; + decoded = next; + } catch { + return undefined; + } + } + return decoded.includes("%") ? undefined : decoded; +} + +function normalizedOoxmlEntryKey(name: string): string | undefined { + const isDirectory = name.endsWith("/"); + const partName = isDirectory ? name.slice(0, -1) : name; + const decoded = decodePercentEscapes(partName); + if (decoded === undefined + || !decoded + || name.startsWith("/") + || partName.endsWith("/") + || decoded !== partName + || /[\u0000-\u001f\u007f]/.test(decoded) + || decoded.includes("\\") + || decoded.includes("?") + || decoded.includes("#")) return undefined; + const normalized = decoded.normalize("NFC"); + if (normalized !== decoded || normalized.split("/").some((segment) => !segment || segment === "." || segment === "..")) return undefined; + return normalized.toLowerCase(); +} + +function validateOoxmlEntryNames(entryNames: Set, path: string): void { + const keys = new Set(); + for (const name of entryNames) { + const key = normalizedOoxmlEntryKey(name); + if (key === undefined || keys.has(key)) { + throw new PackageImportError("metadata_unreadable", "XLSX contains unsafe or ambiguous package part names.", { path: name }); + } + keys.add(key); + } +} + +function worksheetTargetMode(value: string): string | undefined { + const decoded = decodePercentEscapes(value.trim()); + return decoded?.trim().toLowerCase(); +} + +function worksheetTarget(target: string): string | undefined { + const decoded = decodePercentEscapes(target.trim()); + if (decoded === undefined || !decoded + || decoded.includes("%") + || /[\u0000-\u001f\u007f]/.test(decoded) + || decoded.includes("\\") + || decoded.includes("?") + || decoded.includes("#") + || decoded.startsWith("/") + || decoded.startsWith("//") + || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(decoded)) return undefined; + const segments: string[] = []; + for (const segment of decoded.split("/")) { + if (!segment || segment === ".") return undefined; + if (segment === "..") return undefined; + segments.push(segment); + } + return `xl/${segments.join("/")}`; +} + function worksheetRelationships(xml: Buffer, path: string, byteLimit: number): Map { if (xml.length > byteLimit) { throw new PackageImportError("metadata_limit", `XLSX relationships metadata exceeds ${byteLimit} bytes.`, { @@ -139,17 +208,23 @@ function worksheetRelationships(xml: Buffer, path: string, byteLimit: number): M parseXml(xml, { openTag(tag) { if (tag.uri !== PACKAGE_RELATIONSHIPS_NAMESPACE || tag.local !== "Relationship") return; - const id = attributeValue(tag, "Id"); - const target = attributeValue(tag, "Target"); - const type = attributeValue(tag, "Type"); + const id = attributeValue(tag, "Id")?.trim(); + const target = attributeValue(tag, "Target")?.trim(); + const type = attributeValue(tag, "Type")?.trim(); if (!id || !target || !type?.endsWith("/worksheet")) return; - if (attributeValue(tag, "TargetMode")?.toLowerCase() === "external") { + const rawTargetMode = attributeValue(tag, "TargetMode"); + const targetMode = rawTargetMode === undefined ? undefined : worksheetTargetMode(rawTargetMode); + if (rawTargetMode !== undefined && targetMode !== "internal") { throw new PackageImportError("metadata_unreadable", "External XLSX worksheet relationship is not followed.", { path }); } - if (target.startsWith("/") || target.includes("\\") || target.split("/").includes("..")) { - throw new PackageImportError("metadata_unreadable", `Unsafe XLSX worksheet target: ${target}`, { path }); + const resolvedTarget = worksheetTarget(target); + if (resolvedTarget === undefined) { + throw new PackageImportError("metadata_unreadable", "Unsafe XLSX worksheet target.", { path }); + } + if (relationships.has(id)) { + throw new PackageImportError("metadata_unreadable", "XLSX contains duplicate worksheet relationship IDs.", { path }); } - relationships.set(id, `xl/${target}`); + relationships.set(id, resolvedTarget); } }); return relationships; @@ -189,6 +264,7 @@ async function xlsxMetadata(path: string, sourceBytes: number, limits: ImportLim XLSX_ZIP_CODES, (name) => name === "xl/workbook.xml" || name === "xl/_rels/workbook.xml.rels" || name.startsWith("xl/worksheets/") ); + validateOoxmlEntryNames(archive.entryNames, path); const workbook = archive.entries.get("xl/workbook.xml"); const relationshipXml = archive.entries.get("xl/_rels/workbook.xml.rels"); if (!workbook || !relationshipXml) throw new PackageImportError("metadata_unreadable", "XLSX is missing workbook metadata parts.", { path }); @@ -211,8 +287,9 @@ async function xlsxMetadata(path: string, sourceBytes: number, limits: ImportLim }; } catch (error) { if (error instanceof PackageImportError) throw error; - const cause = error instanceof Error ? error.message : String(error); - throw new PackageImportError("metadata_unreadable", `Could not read XLSX metadata: ${cause}`, { path, cause }); + const value = error instanceof Error ? error.message : String(error); + const cause = value.replace(/[\r\n\t]+/g, " ").slice(0, MAX_CAUSE_LENGTH); + throw new PackageImportError("metadata_unreadable", "Could not read XLSX metadata.", { path, cause }); } } diff --git a/src/input/docx-extractor.ts b/src/input/docx-extractor.ts index 57939ea..bc4de72 100644 --- a/src/input/docx-extractor.ts +++ b/src/input/docx-extractor.ts @@ -1,3 +1,5 @@ +import { open as openFile } from "node:fs/promises"; +import { posix } from "node:path"; import { PackageImportError, type ImportLimits, type ProblemExtractionMetadata } from "./types.js"; import { readBoundedZip } from "./zip-reader.js"; import { attributeValue, PACKAGE_RELATIONSHIPS_NAMESPACE, parseXml, WORDPROCESSINGML_NAMESPACE } from "./xml.js"; @@ -9,6 +11,13 @@ interface DocxExtraction { const WORD_DOCUMENT_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"; const OFFICE_DOCUMENT_RELATIONSHIP = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument"; +const OFFICE_DOCUMENT_RELATIONSHIP_KEY = OFFICE_DOCUMENT_RELATIONSHIP.toLowerCase(); +const CONTENT_TYPES_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/content-types"; +const MAIN_DOCUMENT_PART = "/word/document.xml"; +const PACKAGE_RELATIONSHIPS_PART = "_rels/.rels"; +const CFB_SIGNATURE = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]); +const ZIP_SIGNATURES = [Buffer.from("PK\u0003\u0004", "binary"), Buffer.from("PK\u0005\u0006", "binary"), Buffer.from("PK\u0007\u0008", "binary")]; +const MAX_CAUSE_LENGTH = 240; const DOCX_ZIP_CODES = { corrupt: "docx_corrupt", @@ -19,55 +28,334 @@ const DOCX_ZIP_CODES = { } as const; function causeMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); + const value = error instanceof Error ? error.message : String(error); + return value.replace(/[\r\n\t]+/g, " ").slice(0, MAX_CAUSE_LENGTH); +} + +async function readPrefix(path: string, length: number): Promise { + const handle = await openFile(path, "r"); + try { + const buffer = Buffer.alloc(length); + const { bytesRead } = await handle.read(buffer, 0, length, 0); + return buffer.subarray(0, bytesRead); + } finally { + await handle.close(); + } +} + +function hasSignature(prefix: Buffer, signature: Buffer): boolean { + return prefix.length >= signature.length && prefix.subarray(0, signature.length).equals(signature); +} + +async function classifyContainer(path: string): Promise { + const prefix = await readPrefix(path, CFB_SIGNATURE.length); + if (hasSignature(prefix, CFB_SIGNATURE)) { + throw new PackageImportError("docx_encrypted", "Encrypted Office containers are not allowed.", { path }); + } + if (!ZIP_SIGNATURES.some((signature) => hasSignature(prefix, signature))) { + throw new PackageImportError("docx_corrupt", "DOCX is neither a ZIP package nor an encrypted Office container.", { path }); + } +} + +function macroMarker(value: string | undefined): boolean { + if (value === undefined) return false; + if (/macro|vba|activex/i.test(value)) return true; + const decoded = decodePercentEscapes(value); + return decoded !== undefined && /macro|vba|activex/i.test(decoded.value); +} + +interface DecodedValue { + value: string; + changed: boolean; +} + +function decodePercentEscapes(value: string): DecodedValue | undefined { + if (!value.includes("%")) return { value, changed: false }; + let decoded = value; + let changed = false; + for (let depth = 0; depth < 4 && decoded.includes("%"); depth += 1) { + try { + const next = decodeURIComponent(decoded); + if (next === decoded) return undefined; + decoded = next; + changed = true; + } catch { + return undefined; + } + } + return decoded.includes("%") ? undefined : { value: decoded, changed }; +} + +interface CanonicalPartName { + value: string; + changed: boolean; +} + +function canonicalPartName(value: string): CanonicalPartName | undefined { + const decoded = decodePercentEscapes(value); + if (decoded === undefined + || !decoded.value + || /[\u0000-\u001f\u007f]/.test(decoded.value) + || decoded.value.includes("\\") + || decoded.value.includes("?") + || decoded.value.includes("#")) return undefined; + const normalized = decoded.value.normalize("NFC"); + const withLeadingSlash = normalized.startsWith("/") ? normalized : `/${normalized}`; + const segments = withLeadingSlash.split("/"); + const canonical: string[] = []; + let changed = decoded.changed || normalized !== decoded.value; + for (const [index, segment] of segments.entries()) { + if (segment === "") { + if (index !== 0) changed = true; + continue; + } + if (segment === ".") { + changed = true; + continue; + } + if (segment === "..") return undefined; + canonical.push(segment); + } + const result = `/${canonical.join("/")}`; + return { value: result, changed: changed || result !== withLeadingSlash }; +} + +interface CanonicalArchiveEntry { + key: string; + partName: string; +} + +function canonicalArchiveEntry(name: string): CanonicalArchiveEntry | undefined { + const isDirectory = name.endsWith("/"); + const partName = isDirectory ? name.slice(0, -1) : name; + if (!partName || name.startsWith("/") || partName.endsWith("/")) return undefined; + const canonical = canonicalPartName(partName); + if (canonical === undefined || canonical.changed || canonical.value !== `/${partName}`) return undefined; + const normalized = partName.normalize("NFC"); + return { key: `/${normalized}`.toLowerCase(), partName }; +} + +function entryIndicatesMacro(entry: CanonicalArchiveEntry): boolean { + const partName = entry.partName.normalize("NFC").toLowerCase(); + const baseName = posix.basename(partName); + return macroMarker(partName) || (baseName.endsWith(".bin") && baseName.includes("office")); +} + +function isContentTypesPart(name: string): boolean { + return !name.endsWith("/") && name.toLowerCase() === "[content_types].xml"; +} + +function isRelationshipPart(name: string): boolean { + return !name.endsWith("/") && name.toLowerCase().endsWith(".rels"); } function validateContentTypes(xml: Buffer, path: string): void { + const defaults = new Set(); + const overrides = new Set(); let documentContentType: string | undefined; + let documentOverrideCount = 0; + let depth = 0; + let rootSeen = false; + parseXml(xml, { openTag(tag) { - if (tag.local !== "Override" || attributeValue(tag, "PartName") !== "/word/document.xml") return; - documentContentType = attributeValue(tag, "ContentType"); + const parentDepth = depth; + depth += 1; + if (parentDepth === 0) { + rootSeen = true; + if (tag.uri !== CONTENT_TYPES_NAMESPACE || tag.local !== "Types") { + throw new PackageImportError("docx_corrupt", "DOCX content types XML has an invalid root element.", { path }); + } + return; + } + if (tag.uri !== CONTENT_TYPES_NAMESPACE || (tag.local !== "Default" && tag.local !== "Override")) return; + if (parentDepth !== 1) { + throw new PackageImportError("docx_corrupt", "DOCX content type declarations must be direct children of Types.", { path }); + } + const contentType = attributeValue(tag, "ContentType")?.trim(); + if (!contentType) { + throw new PackageImportError("docx_corrupt", "DOCX content type declaration is incomplete.", { path }); + } + if (macroMarker(contentType)) { + throw new PackageImportError("docx_macro_enabled", "Macro-enabled or VBA content types are not allowed.", { path }); + } + + if (tag.local === "Default") { + const extension = attributeValue(tag, "Extension")?.trim(); + if (!extension || !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(extension)) { + throw new PackageImportError("docx_corrupt", "DOCX default content type has an invalid extension.", { path }); + } + if (macroMarker(extension)) { + throw new PackageImportError("docx_macro_enabled", "Macro-enabled or VBA content types are not allowed.", { path }); + } + const key = extension.toLowerCase(); + if (defaults.has(key)) { + throw new PackageImportError("docx_corrupt", "DOCX contains ambiguous default content type declarations.", { path }); + } + defaults.add(key); + return; + } + + const partName = attributeValue(tag, "PartName")?.trim(); + if (!partName || !partName.startsWith("/")) { + throw new PackageImportError("docx_corrupt", "DOCX override content type has an invalid part name.", { path }); + } + if (macroMarker(partName)) { + throw new PackageImportError("docx_macro_enabled", "VBA package parts are not allowed.", { path }); + } + const canonical = canonicalPartName(partName); + if (canonical === undefined || canonical.changed) { + throw new PackageImportError("docx_corrupt", "DOCX override content type has an unsafe or non-canonical part name.", { path }); + } + const key = canonical.value.toLowerCase(); + if (overrides.has(key)) { + throw new PackageImportError("docx_corrupt", "DOCX contains ambiguous override content type declarations.", { path }); + } + overrides.add(key); + if (key === MAIN_DOCUMENT_PART) { + if (canonical.value !== MAIN_DOCUMENT_PART) { + throw new PackageImportError("docx_corrupt", "DOCX main document override has ambiguous casing.", { path }); + } + documentOverrideCount += 1; + documentContentType = contentType; + } + }, + closeTag() { + depth -= 1; } }); - if (documentContentType === undefined) { - throw new PackageImportError("docx_corrupt", "DOCX content types do not declare word/document.xml.", { path }); - } - if (/macroEnabled/i.test(documentContentType)) { - throw new PackageImportError("docx_macro_enabled", "Macro-enabled Word packages are not allowed.", { path }); + + if (!rootSeen || depth !== 0 || documentOverrideCount !== 1 || documentContentType === undefined) { + throw new PackageImportError("docx_corrupt", "DOCX content types must declare word/document.xml exactly once.", { path }); } if (documentContentType !== WORD_DOCUMENT_CONTENT_TYPE) { - throw new PackageImportError("docx_corrupt", `Unexpected DOCX main document content type: ${documentContentType}`, { path }); + throw new PackageImportError("docx_corrupt", "Unexpected DOCX main document content type.", { path }); } } -function validatePackageRelationships(xml: Buffer, path: string): void { - let officeDocumentTarget: string | undefined; - parseXml(xml, { - openTag(tag) { - if (tag.uri !== PACKAGE_RELATIONSHIPS_NAMESPACE || tag.local !== "Relationship") return; - if (attributeValue(tag, "TargetMode")?.toLowerCase() === "external") { - throw new PackageImportError("docx_external_relationship", `External relationship is not allowed in ${path}.`, { path }); - } - if (attributeValue(tag, "Type") === OFFICE_DOCUMENT_RELATIONSHIP) { - officeDocumentTarget = attributeValue(tag, "Target"); - } +interface ResolvedRelationshipTarget { + partName: string; + changed: boolean; +} + +function relationshipSourceDirectory(path: string): string[] | undefined { + if (path === PACKAGE_RELATIONSHIPS_PART) return []; + const segments = path.split("/"); + const fileName = segments.at(-1); + if (fileName === undefined + || segments.length < 2 + || segments.at(-2) !== "_rels" + || !fileName.toLowerCase().endsWith(".rels") + || fileName.length === ".rels".length) return undefined; + return segments.slice(0, -2); +} + +function resolveInternalTarget(target: string, relationshipPath: string): ResolvedRelationshipTarget | undefined { + const sourceDirectory = relationshipSourceDirectory(relationshipPath); + const trimmed = target.trim(); + const decoded = decodePercentEscapes(trimmed); + if (sourceDirectory === undefined + || decoded === undefined + || !decoded.value + || /[\u0000-\u001f\u007f]/.test(decoded.value) + || decoded.value.includes("\\") + || decoded.value.includes("?") + || decoded.value.includes("#") + || decoded.value.startsWith("//") + || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(decoded.value)) return undefined; + + const segments = decoded.value.startsWith("/") ? [] : [...sourceDirectory]; + let changed = target !== trimmed || decoded.changed; + for (const [index, segment] of decoded.value.split("/").entries()) { + if (segment === "") { + if (index !== 0) changed = true; + continue; } - }); - if (officeDocumentTarget !== "word/document.xml" && officeDocumentTarget !== "/word/document.xml") { - throw new PackageImportError("docx_corrupt", "DOCX package does not point to word/document.xml as its main document.", { path }); + if (segment === ".") { + changed = true; + continue; + } + if (segment === "..") { + changed = true; + if (segments.length === 0) return undefined; + segments.pop(); + continue; + } + segments.push(segment); } + if (segments.length === 0) return undefined; + return { partName: `/${segments.join("/")}`, changed }; +} + +interface RelationshipSummary { + officeDocumentTargets: ResolvedRelationshipTarget[]; } -function parseRelationships(xml: Buffer, path: string): void { +function validateRelationships(xml: Buffer, path: string): RelationshipSummary { + const ids = new Set(); + const officeDocumentTargets: ResolvedRelationshipTarget[] = []; + let depth = 0; + let rootSeen = false; + parseXml(xml, { openTag(tag) { + const parentDepth = depth; + depth += 1; + if (parentDepth === 0) { + rootSeen = true; + if (tag.uri !== PACKAGE_RELATIONSHIPS_NAMESPACE || tag.local !== "Relationships") { + throw new PackageImportError("docx_corrupt", "DOCX relationships XML has an invalid root element.", { path }); + } + return; + } if (tag.uri !== PACKAGE_RELATIONSHIPS_NAMESPACE || tag.local !== "Relationship") return; - if (attributeValue(tag, "TargetMode")?.toLowerCase() === "external") { - throw new PackageImportError("docx_external_relationship", `External relationship is not allowed in ${path}.`, { path }); + if (parentDepth !== 1) { + throw new PackageImportError("docx_corrupt", "DOCX relationships must be direct children of Relationships.", { path }); + } + const rawTargetMode = attributeValue(tag, "TargetMode"); + const targetMode = rawTargetMode === undefined ? undefined : decodePercentEscapes(rawTargetMode.trim())?.value.trim().toLowerCase(); + if (rawTargetMode !== undefined && (targetMode === undefined || targetMode !== "internal")) { + throw new PackageImportError("docx_external_relationship", "Non-internal DOCX relationship mode is not allowed.", { path }); + } + + const id = attributeValue(tag, "Id")?.trim(); + const type = attributeValue(tag, "Type")?.trim(); + const target = attributeValue(tag, "Target"); + if (!id || ids.has(id) || !target?.trim()) { + throw new PackageImportError("docx_corrupt", "DOCX relationships contain a missing or duplicate required value.", { path }); + } + const resolvedTarget = resolveInternalTarget(target, path); + if (resolvedTarget === undefined) { + throw new PackageImportError("docx_external_relationship", "DOCX relationship target is external or escapes the package.", { path }); } + if (!type) { + throw new PackageImportError("docx_corrupt", "DOCX relationships contain a missing or duplicate required value.", { path }); + } + ids.add(id); + if (macroMarker(type) || macroMarker(resolvedTarget.partName)) { + throw new PackageImportError("docx_macro_enabled", "Macro or VBA relationships are not allowed.", { path }); + } + if (decodePercentEscapes(type)?.value.toLowerCase() === OFFICE_DOCUMENT_RELATIONSHIP_KEY) officeDocumentTargets.push(resolvedTarget); + }, + closeTag() { + depth -= 1; } }); + + if (!rootSeen || depth !== 0) { + throw new PackageImportError("docx_corrupt", "DOCX relationships XML is incomplete.", { path }); + } + return { officeDocumentTargets }; +} + +function validatePackageRelationships(summary: RelationshipSummary, path: string): void { + if (summary.officeDocumentTargets.length !== 1) { + throw new PackageImportError("docx_corrupt", "DOCX package must contain exactly one officeDocument relationship.", { path }); + } + const target = summary.officeDocumentTargets[0]; + if (target === undefined || target.changed || target.partName !== MAIN_DOCUMENT_PART) { + throw new PackageImportError("docx_corrupt", "DOCX package does not point to word/document.xml as its main document.", { path }); + } } function parseBody(xml: Buffer, path: string, characterLimit: number): string { @@ -118,11 +406,12 @@ export async function extractDocx( }); } try { + await classifyContainer(path); const archive = await readBoundedZip( path, { maxEntries: limits.maxDocxZipEntries, maxUncompressedBytes: limits.maxDocxUncompressedBytes }, DOCX_ZIP_CODES, - (name) => name === "[Content_Types].xml" || name === "word/document.xml" || name === "word/vbaProject.bin" || name.endsWith(".rels") + (name) => isContentTypesPart(name) || name.toLowerCase() === "word/document.xml" || isRelationshipPart(name) ); const contentTypes = archive.entries.get("[Content_Types].xml"); const packageRelationships = archive.entries.get("_rels/.rels"); @@ -139,13 +428,28 @@ export async function extractDocx( }); } } - validateContentTypes(contentTypes, path); - validatePackageRelationships(packageRelationships, path); - if (archive.entryNames.has("word/vbaProject.bin")) { - throw new PackageImportError("docx_macro_enabled", "Macro-enabled Word packages are not allowed.", { path }); + const canonicalEntries = new Set(); + for (const name of archive.entryNames) { + if (macroMarker(name)) { + throw new PackageImportError("docx_macro_enabled", "Macro or VBA package parts are not allowed.", { path }); + } + const canonical = canonicalArchiveEntry(name); + if (canonical === undefined) { + throw new PackageImportError("docx_corrupt", "DOCX contains an unsafe or non-canonical package part name.", { path: name }); + } + if (canonicalEntries.has(canonical.key)) { + throw new PackageImportError("docx_corrupt", "DOCX contains case-ambiguous package part names.", { path: name }); + } + canonicalEntries.add(canonical.key); + if (entryIndicatesMacro(canonical)) { + throw new PackageImportError("docx_macro_enabled", "Macro or VBA package parts are not allowed.", { path }); + } } + validateContentTypes(contentTypes, path); + const packageSummary = validateRelationships(packageRelationships, "_rels/.rels"); + validatePackageRelationships(packageSummary, "_rels/.rels"); for (const [name, xml] of archive.entries) { - if (name.endsWith(".rels")) parseRelationships(xml, name); + if (name.toLowerCase().endsWith(".rels") && name.toLowerCase() !== "_rels/.rels") validateRelationships(xml, name); } const text = parseBody(document, path, limits.maxDocxCharacters); if (!text) throw new PackageImportError("docx_empty", "DOCX body contains no extractable text.", { path }); @@ -163,14 +467,13 @@ export async function extractDocx( sourceBytes: identity.sizeBytes, sha256: identity.sha256, extractedCharacters: text.length, - extractor: "ooxml-body-v1", + extractor: "ooxml-body-v2", zipEntries: archive.entryCount, uncompressedBytes: archive.uncompressedBytes } }; } catch (error) { if (error instanceof PackageImportError) throw error; - const message = causeMessage(error); - throw new PackageImportError("docx_corrupt", `Could not parse DOCX: ${message}`, { path, cause: message }); + throw new PackageImportError("docx_corrupt", "Could not parse DOCX.", { path, cause: causeMessage(error) }); } } diff --git a/src/input/package-importer.ts b/src/input/package-importer.ts index 476ac17..8c4bd1a 100644 --- a/src/input/package-importer.ts +++ b/src/input/package-importer.ts @@ -42,6 +42,7 @@ export const DEFAULT_IMPORT_LIMITS: Readonly = Object.freeze({ maxProblemBytes: 32 * 1024 * 1024, maxTextCharacters: 2_000_000, maxPackageEntries: 10_000, + maxPackageDepth: 64, maxAssetBytes: 2 * 1024 * 1024 * 1024, maxPdfPages: 500, maxPdfCharacters: 2_000_000, @@ -147,31 +148,64 @@ export async function resolveSafePath(rootPath: string, relativePath: string): P return candidate; } -async function walk(root: string, directory: string, files: string[], limits: ImportLimits): Promise { +interface WalkState { + entries: number; +} + +async function walk( + root: string, + directory: string, + files: string[], + limits: ImportLimits, + state: WalkState, + depth: number +): Promise { const entries = await readdir(directory, { withFileTypes: true }); for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name, "en"))) { - if (entry.name.startsWith(".")) continue; const absolute = resolve(directory, entry.name); - if (entry.isSymbolicLink()) { - throw new PackageImportError("symlink_input", `Symlink inputs are not allowed: ${normalizeRelative(root, absolute)}`, { - path: normalizeRelative(root, absolute) + state.entries += 1; + if (state.entries > limits.maxPackageEntries) { + throw new PackageImportError("package_entry_limit", `Package has more than ${limits.maxPackageEntries} filesystem entries.`, { + actual: state.entries, + limit: limits.maxPackageEntries }); } + const relativePath = normalizeRelative(root, absolute); + const current = await lstat(absolute); + if (current.isSymbolicLink()) { + throw new PackageImportError("symlink_input", `Symlink inputs are not allowed: ${relativePath}`, { path: relativePath }); + } + if (current.isDirectory()) { + const childDepth = depth + 1; + if (childDepth > limits.maxPackageDepth) { + throw new PackageImportError("package_depth_limit", `Package directory depth exceeds ${limits.maxPackageDepth}.`, { + path: relativePath, + actual: childDepth, + limit: limits.maxPackageDepth + }); + } + // Hidden entries consume the traversal budget, but hidden directories are not recursed into. + if (entry.name.startsWith(".")) continue; + const resolvedDirectory = await realpath(absolute); + if (resolvedDirectory !== root && !resolvedDirectory.startsWith(`${root}${sep}`)) { + throw new PackageImportError("unsafe_path", "Directory changed outside package during traversal.", { path: relativePath }); + } + const checked = await lstat(absolute); + if (!checked.isDirectory() || checked.isSymbolicLink() || checked.dev !== current.dev || checked.ino !== current.ino) { + throw new PackageImportError("unsafe_path", "Directory changed during traversal.", { path: relativePath }); + } + await walk(root, resolvedDirectory, files, limits, state, childDepth); + continue; + } + // Hidden entries consume the traversal budget, but hidden files are not inventoried. + if (entry.name.startsWith(".")) continue; const extension = extname(entry.name).toLowerCase(); - if (entry.isDirectory()) { - await walk(root, absolute, files, limits); - } else if (entry.isFile() && MACRO_ENABLED_EXTENSIONS.has(extension)) { - throw new PackageImportError("docx_macro_enabled", `Macro-enabled Word input is not allowed: ${normalizeRelative(root, absolute)}`, { - path: normalizeRelative(root, absolute) + if (current.isFile() && MACRO_ENABLED_EXTENSIONS.has(extension)) { + throw new PackageImportError("docx_macro_enabled", `Macro-enabled Word input is not allowed: ${relativePath}`, { + path: relativePath }); - } else if (entry.isFile() && SUPPORTED_EXTENSIONS.has(extension)) { + } else if (current.isFile() && SUPPORTED_EXTENSIONS.has(extension)) { files.push(absolute); - if (files.length > limits.maxPackageEntries) { - throw new PackageImportError("package_entry_limit", `Package has more than ${limits.maxPackageEntries} supported files.`, { - actual: files.length, - limit: limits.maxPackageEntries - }); - } } } } @@ -209,7 +243,8 @@ function decodeUtf8(bytes: Buffer, path: string): string { try { return new TextDecoder("utf-8", { fatal: true }).decode(bytes); } catch (error) { - const cause = error instanceof Error ? error.message : String(error); + const value = error instanceof Error ? error.message : String(error); + const cause = value.replace(/[\r\n\t]+/g, " ").slice(0, 240); throw new PackageImportError("problem_encoding", `Problem statement is not valid UTF-8: ${path}`, { path, cause }); } } @@ -251,6 +286,7 @@ async function extractProblem( } export async function importPackage(packagePath: string, options: ImportOptions = {}): Promise { + const limits = normalizedLimits(options.limits); const requestedRoot = resolve(packagePath); const requestedStat = await lstat(requestedRoot); if (requestedStat.isSymbolicLink()) { @@ -258,9 +294,15 @@ export async function importPackage(packagePath: string, options: ImportOptions } if (!requestedStat.isDirectory()) throw new Error(`Package path is not a directory: ${packagePath}`); const root = await realpath(requestedRoot); - const limits = normalizedLimits(options.limits); + const checkedRoot = await lstat(requestedRoot); + if (!checkedRoot.isDirectory() + || checkedRoot.isSymbolicLink() + || checkedRoot.dev !== requestedStat.dev + || checkedRoot.ino !== requestedStat.ino) { + throw new PackageImportError("unsafe_path", "Package root changed during import.", { path: packagePath }); + } const files: string[] = []; - await walk(root, root, files, limits); + await walk(root, root, files, limits, { entries: 0 }, 0); const problemPath = selectProblem(root, files); const problemBeforeStat = await lstat(problemPath); if (!problemBeforeStat.isFile() || problemBeforeStat.isSymbolicLink()) { diff --git a/src/input/pdf-extractor.ts b/src/input/pdf-extractor.ts index c7a893f..5f0d963 100644 --- a/src/input/pdf-extractor.ts +++ b/src/input/pdf-extractor.ts @@ -7,8 +7,11 @@ interface PdfExtraction { metadata: ProblemExtractionMetadata; } +const MAX_CAUSE_LENGTH = 240; + function causeMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); + const value = error instanceof Error ? error.message : String(error); + return value.replace(/[\r\n\t]+/g, " ").slice(0, MAX_CAUSE_LENGTH); } function isTextItem(item: unknown): item is TextItem { @@ -64,34 +67,47 @@ export async function extractPdf( } const pages: string[] = []; let characters = 0; + const ensureFits = (base: number, additional: number): void => { + const actual = base + additional; + if (!Number.isSafeInteger(actual) || actual > limits.maxPdfCharacters) { + throw new PackageImportError("pdf_character_limit", `PDF extraction exceeds ${limits.maxPdfCharacters} characters.`, { + path, + actual, + limit: limits.maxPdfCharacters + }); + } + }; for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber += 1) { const page = await document.getPage(pageNumber); const content = await page.getTextContent({ disableNormalization: false, includeMarkedContent: false }); const textItems = content.items.filter(isTextItem); + const leadingPageSeparator = pages.length === 0 ? "" : "\n\n"; + const pagePieces: string[] = []; let pageCharacters = 0; for (const item of textItems) { - pageCharacters += item.str.length; - if (pageCharacters > limits.maxPdfCharacters || characters + pageCharacters > limits.maxPdfCharacters) { - throw new PackageImportError("pdf_character_limit", `PDF extraction exceeds ${limits.maxPdfCharacters} characters.`, { - path, - actual: characters + pageCharacters, - limit: limits.maxPdfCharacters - }); - } + const itemText = item.str.trim(); + if (itemText.length === 0) continue; + const separator = pagePieces.length === 0 ? "" : " "; + const additional = separator.length + itemText.length; + if (!Number.isSafeInteger(pageCharacters + additional)) ensureFits(characters, Number.MAX_SAFE_INTEGER); + ensureFits(characters, leadingPageSeparator.length + pageCharacters + additional); + if (separator) pagePieces.push(separator); + pagePieces.push(itemText); + pageCharacters += additional; } - const text = textItems.map((item) => item.str).join(" ").trim(); - characters += text.length; - if (characters > limits.maxPdfCharacters) { - throw new PackageImportError("pdf_character_limit", `PDF extraction exceeds ${limits.maxPdfCharacters} characters.`, { - path, - actual: characters, - limit: limits.maxPdfCharacters - }); - } - pages.push(text); + if (pageCharacters === 0) continue; + pages.push(pagePieces.join("")); + characters += leadingPageSeparator.length + pageCharacters; } const text = pages.join("\n\n").trim(); if (!text) throw new PackageImportError("pdf_empty", "PDF contains no extractable text.", { path }); + if (text.length > limits.maxPdfCharacters || text.length !== characters) { + throw new PackageImportError("pdf_character_limit", `PDF extraction exceeds ${limits.maxPdfCharacters} characters after normalization.`, { + path, + actual: text.length, + limit: limits.maxPdfCharacters + }); + } return { text, metadata: { @@ -113,7 +129,7 @@ export async function extractPdf( if (name === "PasswordException" || /password|encrypted/i.test(message)) { throw new PackageImportError("pdf_encrypted", "Encrypted PDF files are not allowed.", { path, cause: message }); } - throw new PackageImportError("pdf_corrupt", `Could not extract PDF text: ${message}`, { path, cause: message }); + throw new PackageImportError("pdf_corrupt", "Could not extract PDF text.", { path, cause: message }); } finally { await loadingTask.destroy(); } diff --git a/src/input/types.ts b/src/input/types.ts index be8fe5e..13a5fc7 100644 --- a/src/input/types.ts +++ b/src/input/types.ts @@ -1,5 +1,35 @@ +import { createHash } from "node:crypto"; import type { DataAsset } from "../contracts/types.js"; +const MAX_CAUSE_FINGERPRINT_INPUT = 4096; +const MAX_PUBLIC_PATH_LENGTH = 160; +const MAX_PUBLIC_CANDIDATES = 16; + +function diagnosticFingerprint(value: string): string { + return `sha256:${createHash("sha256").update(value.slice(0, MAX_CAUSE_FINGERPRINT_INPUT)).digest("hex").slice(0, 16)}`; +} + +function isPrivatePath(value: string): boolean { + return value.startsWith("/") + || value.startsWith("\\") + || /^[A-Za-z]:[\\/]/.test(value) + || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(value); +} + +function publicDiagnosticPath(value: string | undefined): string | undefined { + if (value === undefined || value.length === 0) return undefined; + if (/\r|\n|\t/.test(value) || isPrivatePath(value) || value === ".." || value.startsWith("../")) return undefined; + if (value.length > MAX_PUBLIC_PATH_LENGTH + || value.includes("%") + || value.includes("\\") + || value.includes("?") + || value.includes("#") + || value.split("/").some((segment) => segment === ".." || segment === ".")) { + return diagnosticFingerprint(value); + } + return value; +} + export type PackageImportErrorCode = | "ambiguous_problem" | "problem_missing" @@ -7,6 +37,7 @@ export type PackageImportErrorCode = | "problem_file_limit" | "problem_encoding" | "package_entry_limit" + | "package_depth_limit" | "asset_file_limit" | "symlink_input" | "unsafe_path" @@ -28,6 +59,36 @@ export type PackageImportErrorCode = | "metadata_unreadable" | "metadata_limit"; +const PUBLIC_ERROR_MESSAGES: Readonly> = Object.freeze({ + ambiguous_problem: "Problem statement selection is ambiguous.", + problem_missing: "Package does not contain a supported problem statement.", + problem_empty: "Problem statement is empty.", + problem_file_limit: "Problem statement exceeds an import limit.", + problem_encoding: "Problem statement encoding is invalid.", + package_entry_limit: "Package exceeds the filesystem entry limit.", + package_depth_limit: "Package exceeds the filesystem depth limit.", + asset_file_limit: "An input asset exceeds an import limit.", + symlink_input: "Symlink inputs are not allowed.", + unsafe_path: "Unsafe path outside package.", + pdf_tool_unavailable: "The local PDF extractor is unavailable.", + pdf_corrupt: "PDF input could not be parsed.", + pdf_encrypted: "Encrypted PDF files are not allowed.", + pdf_empty: "PDF contains no extractable text.", + pdf_page_limit: "PDF exceeds the page limit.", + pdf_character_limit: "PDF extraction exceeds the character limit.", + docx_corrupt: "DOCX input could not be parsed.", + docx_encrypted: "Encrypted DOCX files are not allowed.", + docx_macro_enabled: "Macro-enabled DOCX files are not allowed.", + docx_external_relationship: "DOCX contains a non-internal relationship.", + docx_zip_slip: "DOCX contains an unsafe archive path.", + docx_zip_entry_limit: "DOCX exceeds the archive entry limit.", + docx_uncompressed_limit: "DOCX exceeds the uncompressed-byte limit.", + docx_character_limit: "DOCX extraction exceeds the character limit.", + docx_empty: "DOCX body contains no extractable text.", + metadata_unreadable: "Input metadata could not be read safely.", + metadata_limit: "Input metadata exceeds an inspection limit." +}); + export interface PackageImportErrorDetails { path?: string; candidates?: string[]; @@ -41,10 +102,21 @@ export class PackageImportError extends Error { readonly details: Readonly; constructor(code: PackageImportErrorCode, message: string, details: PackageImportErrorDetails = {}) { - super(message, { cause: details.cause }); + void message; + const { cause, path, candidates, ...safeDetails } = details; + const normalizedCause = cause?.replace(/[\r\n\t]+/g, " ").slice(0, MAX_CAUSE_FINGERPRINT_INPUT); + const boundedCause = normalizedCause === undefined ? undefined : diagnosticFingerprint(normalizedCause); + const boundedDetails: PackageImportErrorDetails = { ...safeDetails }; + const publicPath = publicDiagnosticPath(path); + if (publicPath !== undefined) boundedDetails.path = publicPath; + if (candidates !== undefined) { + boundedDetails.candidates = candidates.slice(0, MAX_PUBLIC_CANDIDATES).map((candidate) => publicDiagnosticPath(candidate) ?? diagnosticFingerprint(candidate)); + } + if (boundedCause !== undefined) boundedDetails.cause = boundedCause; + super(PUBLIC_ERROR_MESSAGES[code], boundedCause === undefined ? undefined : { cause: boundedCause }); this.name = "PackageImportError"; this.code = code; - this.details = Object.freeze({ ...details }); + this.details = Object.freeze(boundedDetails); } } @@ -52,6 +124,7 @@ export interface ImportLimits { maxProblemBytes: number; maxTextCharacters: number; maxPackageEntries: number; + maxPackageDepth: number; maxAssetBytes: number; maxPdfPages: number; maxPdfCharacters: number; diff --git a/src/input/zip-reader.ts b/src/input/zip-reader.ts index 0c400e9..c6b609e 100644 --- a/src/input/zip-reader.ts +++ b/src/input/zip-reader.ts @@ -22,8 +22,11 @@ export interface ZipErrorCodes { uncompressedLimit: PackageImportErrorCode; } +const MAX_CAUSE_LENGTH = 240; + function causeMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); + const value = error instanceof Error ? error.message : String(error); + return value.replace(/[\r\n\t]+/g, " ").slice(0, MAX_CAUSE_LENGTH); } function openArchive(path: string): Promise { @@ -74,10 +77,23 @@ function readEntry(archive: ZipFile, entry: Entry, maximumBytes: number): Promis } function unsafeEntryName(name: string): boolean { - return name.includes("\\") + return !name + || /[\u0000-\u001f\u007f]/.test(name) + || name.includes("\\") || name.startsWith("/") || /^[A-Za-z]:/.test(name) - || name.split("/").includes(".."); + || name.split("/").some((segment) => segment === ".." || segment === "."); +} + +function isUnixSymlink(entry: Entry): boolean { + const fileType = (entry.externalFileAttributes >>> 16) & 0o170000; + return fileType === 0o120000; +} + +function zipErrorCode(message: string, codes: ZipErrorCodes): PackageImportErrorCode { + if (/encrypt|password/i.test(message)) return codes.encrypted; + if (/invalid relative path|absolute path|invalid characters in fileName/i.test(message)) return codes.zipSlip; + return codes.corrupt; } export async function readBoundedZip( @@ -91,8 +107,7 @@ export async function readBoundedZip( archive = await openArchive(path); } catch (error) { const message = causeMessage(error); - const code = /invalid relative path|absolute path|invalid characters in fileName/i.test(message) ? codes.zipSlip : codes.corrupt; - throw new PackageImportError(code, `Could not open ZIP package: ${message}`, { path, cause: message }); + throw new PackageImportError(zipErrorCode(message, codes), "Could not open ZIP package.", { path, cause: message }); } if (archive.entryCount > limits.maxEntries) { @@ -118,8 +133,7 @@ export async function readBoundedZip( if (error instanceof PackageImportError) reject(error); else { const message = causeMessage(error); - const code = /invalid relative path|absolute path|invalid characters in fileName/i.test(message) ? codes.zipSlip : codes.corrupt; - reject(new PackageImportError(code, `Could not read ZIP package: ${message}`, { path, cause: message })); + reject(new PackageImportError(zipErrorCode(message, codes), "Could not read ZIP package.", { path, cause: message })); } }; @@ -144,6 +158,9 @@ export async function readBoundedZip( if (entry.isEncrypted()) { throw new PackageImportError(codes.encrypted, `Encrypted ZIP entry is not allowed: ${entry.fileName}`, { path: entry.fileName }); } + if (isUnixSymlink(entry)) { + throw new PackageImportError(codes.zipSlip, `ZIP symlink entry is not allowed: ${entry.fileName}`, { path: entry.fileName }); + } total += entry.uncompressedSize; if (!Number.isSafeInteger(total) || total > limits.maxUncompressedBytes) { throw new PackageImportError(codes.uncompressedLimit, `ZIP package expands to more than ${limits.maxUncompressedBytes} bytes.`, { diff --git a/tests/input-package-importer.test.ts b/tests/input-package-importer.test.ts index 0457627..7ac72e3 100644 --- a/tests/input-package-importer.test.ts +++ b/tests/input-package-importer.test.ts @@ -11,7 +11,7 @@ import { const tempPackage = async (name: string): Promise => mkdtemp(join(tmpdir(), `modeling-input-${name}-`)); -function pdfDocument(pages: string[], encrypted = false): Buffer { +function pdfDocument(pages: Array, encrypted = false): Buffer { const objects: string[] = []; const pageObjectNumbers: number[] = []; const fontObject = 3; @@ -19,12 +19,16 @@ function pdfDocument(pages: string[], encrypted = false): Buffer { objects.push("<< /Type /Catalog /Pages 2 0 R >>"); objects.push(`<< /Type /Pages /Kids [${pages.map((_, index) => `${4 + index * 2} 0 R`).join(" ")}] /Count ${pages.length} >>`); objects.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"); - for (const [index, text] of pages.entries()) { + for (const [index, page] of pages.entries()) { const pageObject = 4 + index * 2; const contentObject = pageObject + 1; pageObjectNumbers.push(pageObject); - const escaped = text.replaceAll("\\", "\\\\").replaceAll("(", "\\(").replaceAll(")", "\\)"); - const stream = `BT /F1 18 Tf 72 720 Td (${escaped}) Tj ET`; + const items = typeof page === "string" ? [page] : page; + const operations = items.map((text, itemIndex) => { + const escaped = text.replaceAll("\\", "\\\\").replaceAll("(", "\\(").replaceAll(")", "\\)"); + return `1 0 0 1 ${72 + itemIndex * 72} 720 Tm (${escaped}) Tj`; + }).join(" "); + const stream = `BT /F1 18 Tf ${operations} ET`; objects.push(`<< /Type /Page /Parent ${pagesObject} 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 ${fontObject} 0 R >> >> /Contents ${contentObject} 0 R >>`); objects.push(`<< /Length ${Buffer.byteLength(stream, "ascii")} >>\nstream\n${stream}\nendstream`); } @@ -52,7 +56,7 @@ function pdfDocument(pages: string[], encrypted = false): Buffer { return Buffer.concat(chunks); } -type ZipEntry = { name: string; data: Buffer; compression?: "store" | "deflate" }; +type ZipEntry = { name: string; data: Buffer; compression?: "store" | "deflate"; unixMode?: number }; function zip(entries: ZipEntry[]): Buffer { const local: Buffer[] = []; @@ -79,7 +83,7 @@ function zip(entries: ZipEntry[]): Buffer { local.push(header, compressed); const directory = Buffer.alloc(46 + name.length); directory.writeUInt32LE(0x02014b50, 0); - directory.writeUInt16LE(20, 4); + directory.writeUInt16LE(entry.unixMode === undefined ? 20 : (3 << 8) | 20, 4); directory.writeUInt16LE(20, 6); directory.writeUInt16LE(0, 8); directory.writeUInt16LE(method, 10); @@ -92,7 +96,7 @@ function zip(entries: ZipEntry[]): Buffer { directory.writeUInt16LE(0, 30); directory.writeUInt16LE(0, 32); directory.writeUInt16LE(0, 34); - directory.writeUInt32LE(0, 36); + directory.writeUInt32LE(entry.unixMode === undefined ? 0 : (entry.unixMode << 16) >>> 0, 38); directory.writeUInt32LE(offset, 42); name.copy(directory, 46); central.push(directory); @@ -111,6 +115,15 @@ function zip(entries: ZipEntry[]): Buffer { return Buffer.concat([localBytes, centralBytes, end]); } +function encryptedZipEntryFixture(name: string, data: Buffer): Buffer { + const archive = zip([{ name, data, compression: "deflate" }]); + archive.writeUInt16LE(archive.readUInt16LE(6) | 0x0001, 6); + const centralOffset = archive.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02])); + if (centralOffset < 0) throw new Error("Fixture ZIP has no central directory."); + archive.writeUInt16LE(archive.readUInt16LE(centralOffset + 8) | 0x0001, centralOffset + 8); + return archive; +} + function crc32Of(data: Buffer): number { let crc = 0xffffffff; for (const byte of data) { @@ -120,26 +133,55 @@ function crc32Of(data: Buffer): number { return (crc ^ 0xffffffff) >>> 0; } -function docxDocument( - text: string, - extra: ZipEntry[] = [], - mainContentType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml" -): Buffer { +const STANDARD_DOCX_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"; +const OFFICE_DOCUMENT_RELATIONSHIP = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument"; +const RELATIONSHIPS_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/relationships"; +const CONTENT_TYPES_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/content-types"; + +function relationshipsXml(relationships: string): string { + return `${relationships}`; +} + +function contentTypesXml(declarations: string): string { + return `${declarations}`; +} + +interface DocxOptions { + extra?: ZipEntry[]; + mainContentType?: string; + contentTypes?: string; + packageRelationships?: string; + documentXml?: string; +} + +function docxDocument(text: string, options: DocxOptions = {}): Buffer { + const mainContentType = options.mainContentType ?? STANDARD_DOCX_CONTENT_TYPE; + const contentTypes = options.contentTypes ?? contentTypesXml( + `` + ); + const packageRelationships = options.packageRelationships ?? relationshipsXml( + `` + ); + const documentXml = options.documentXml + ?? `${text}`; return zip([ - { name: "[Content_Types].xml", data: Buffer.from(``) }, - { name: "_rels/.rels", data: Buffer.from("") }, - { name: "word/document.xml", data: Buffer.from(`${text}`) }, - ...extra + { name: "[Content_Types].xml", data: Buffer.from(contentTypes) }, + { name: "_rels/.rels", data: Buffer.from(packageRelationships) }, + { name: "word/document.xml", data: Buffer.from(documentXml) }, + ...(options.extra ?? []) ]); } -function xlsxAsset(): Buffer { +function xlsxAsset(workbookRelationships?: string, extra: ZipEntry[] = []): Buffer { + const relationships = workbookRelationships + ?? ""; return zip([ { name: "[Content_Types].xml", data: Buffer.from("") }, { name: "xl/workbook.xml", data: Buffer.from("") }, - { name: "xl/_rels/workbook.xml.rels", data: Buffer.from("") }, + { name: "xl/_rels/workbook.xml.rels", data: Buffer.from(relationships) }, { name: "xl/worksheets/sheet1.xml", data: Buffer.from("") }, - { name: "xl/worksheets/sheet2.xml", data: Buffer.from("") } + { name: "xl/worksheets/sheet2.xml", data: Buffer.from("") }, + ...extra ]); } @@ -191,6 +233,26 @@ describe("safe package importer", () => { await writeFile(join(tooManyCharacters, "problem.pdf"), pdfDocument(["0123456789"])); await expectImportCode(importPackage(tooManyCharacters, { limits: { maxPdfCharacters: 5 } }), "pdf_character_limit"); + const pageSeparators = await tempPackage("pdf-page-separators"); + await writeFile(join(pageSeparators, "problem.pdf"), pdfDocument(["A", "B"])); + await expectImportCode(importPackage(pageSeparators, { limits: { maxPdfCharacters: 3 } }), "pdf_character_limit"); + const exactPageLimit = await importPackage(pageSeparators, { limits: { maxPdfCharacters: 4 } }); + expect(exactPageLimit.problemText).toBe("A\n\nB"); + expect(exactPageLimit.problemText.length).toBe(4); + + const itemSeparators = await tempPackage("pdf-item-separators"); + await writeFile(join(itemSeparators, "problem.pdf"), pdfDocument([["A", "B"]])); + await expectImportCode(importPackage(itemSeparators, { limits: { maxPdfCharacters: 2 } }), "pdf_character_limit"); + const exactItemLimit = await importPackage(itemSeparators, { limits: { maxPdfCharacters: 3 } }); + expect(exactItemLimit.problemText).toBe("A B"); + expect(exactItemLimit.problemText.length).toBe(3); + + const normalizedItems = await tempPackage("pdf-normalized-items"); + await writeFile(join(normalizedItems, "problem.pdf"), pdfDocument([[" A ", " B "]])); + const normalized = await importPackage(normalizedItems, { limits: { maxPdfCharacters: 3 } }); + expect(normalized.problemText).toBe("A B"); + expect(normalized.problemText.length).toBe(3); + const tooManyBytes = await tempPackage("pdf-bytes"); await writeFile(join(tooManyBytes, "problem.pdf"), pdfDocument(["text"])); await expectImportCode(importPackage(tooManyBytes, { limits: { maxProblemBytes: 10 } }), "problem_file_limit"); @@ -201,7 +263,7 @@ describe("safe package importer", () => { await writeFile(join(root, "problem.docx"), docxDocument("DOCX body text")); const imported = await importPackage(root); expect(imported.problemText).toContain("DOCX body text"); - expect(imported.problemMetadata).toMatchObject({ format: "docx", extractedCharacters: 14, extractor: "ooxml-body-v1" }); + expect(imported.problemMetadata).toMatchObject({ format: "docx", extractedCharacters: 14, extractor: "ooxml-body-v2" }); }); it("rejects corrupt and macro-enabled DOCX packages", async () => { @@ -209,6 +271,31 @@ describe("safe package importer", () => { await writeFile(join(corrupt, "problem.docx"), Buffer.from("not a zip")); await expectImportCode(importPackage(corrupt), "docx_corrupt"); + const cfbEncrypted = await tempPackage("docx-cfb-encrypted"); + const privateBody = "PRIVATE-DOCUMENT-BODY-MUST-NOT-LEAK"; + const cfb = Buffer.concat([ + Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]), + Buffer.from(`EncryptedPackage\0${privateBody}`) + ]); + await writeFile(join(cfbEncrypted, "problem.docx"), cfb); + const encryptedError = await importPackage(cfbEncrypted).then( + () => undefined, + (error: unknown) => error + ); + expect(encryptedError).toMatchObject({ code: "docx_encrypted" }); + expect(JSON.stringify(encryptedError)).not.toContain(privateBody); + expect(encryptedError instanceof Error ? encryptedError.message : String(encryptedError)).not.toContain(privateBody); + expect(encryptedError && typeof encryptedError === "object" && "details" in encryptedError + ? (encryptedError as { details?: { cause?: string } }).details?.cause + : undefined).toBeUndefined(); + + const encryptedEntry = await tempPackage("docx-encrypted-entry"); + await writeFile(join(encryptedEntry, "problem.docx"), encryptedZipEntryFixture( + "[Content_Types].xml", + Buffer.from("encrypted") + )); + await expectImportCode(importPackage(encryptedEntry), "docx_encrypted"); + const crcMismatch = await tempPackage("docx-crc"); const tampered = docxDocument("body"); const marker = tampered.indexOf(Buffer.from("body")); @@ -217,49 +304,296 @@ describe("safe package importer", () => { await expectImportCode(importPackage(crcMismatch), "docx_corrupt"); const duplicate = await tempPackage("docx-duplicate"); - await writeFile(join(duplicate, "problem.docx"), docxDocument("body", [ - { name: "word/document.xml", data: Buffer.from("") } - ])); + await writeFile(join(duplicate, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/document.xml", data: Buffer.from("") }] + })); await expectImportCode(importPackage(duplicate), "docx_corrupt"); const macro = await tempPackage("docx-macro"); - const document = docxDocument( - "body", - [{ name: "word/vbaProject.bin", data: Buffer.from("not executed") }], - "application/vnd.ms-word.document.macroEnabled.main+xml" - ); + const document = docxDocument("body", { + extra: [{ name: "word/vbaProject.bin", data: Buffer.from("not executed") }], + mainContentType: "application/vnd.ms-word.document.macroEnabled.main+xml" + }); await writeFile(join(macro, "problem.docx"), document); await expectImportCode(importPackage(macro), "docx_macro_enabled"); }); + it("rejects symlink entries inside DOCX archives", async () => { + const archiveSymlink = await tempPackage("docx-archive-symlink"); + await writeFile(join(archiveSymlink, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/linked.bin", data: Buffer.alloc(0), unixMode: 0o120777 }] + })); + await expectImportCode(importPackage(archiveSymlink), "docx_zip_slip"); + }); + it("rejects DOCX external relationships and ZIP slip names", async () => { const external = await tempPackage("docx-external"); - await writeFile(join(external, "problem.docx"), docxDocument("body", [ - { name: "word/_rels/document.xml.rels", data: Buffer.from("") } - ])); + await writeFile(join(external, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) }] + })); await expectImportCode(importPackage(external), "docx_external_relationship"); + const paddedExternal = await tempPackage("docx-padded-external"); + const querySecret = "relationship-query-secret"; + await writeFile(join(paddedExternal, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + `` + )) }] + })); + const externalError = await importPackage(paddedExternal).then( + () => undefined, + (error: unknown) => error + ); + expect(externalError).toMatchObject({ code: "docx_external_relationship" }); + expect(JSON.stringify(externalError)).not.toContain(querySecret); + expect(externalError instanceof Error ? externalError.message : String(externalError)).not.toContain(querySecret); + + const unknownMode = await tempPackage("docx-unknown-mode"); + await writeFile(join(unknownMode, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) }] + })); + await expectImportCode(importPackage(unknownMode), "docx_external_relationship"); + + const encodedExternal = await tempPackage("docx-encoded-external"); + await writeFile(join(encodedExternal, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) }] + })); + await expectImportCode(importPackage(encodedExternal), "docx_external_relationship"); + + const encodedEscape = await tempPackage("docx-encoded-escape"); + await writeFile(join(encodedEscape, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) }] + })); + await expectImportCode(importPackage(encodedEscape), "docx_external_relationship"); + + const explicitInternal = await tempPackage("docx-internal-mode"); + await writeFile(join(explicitInternal, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) }] + })); + expect((await importPackage(explicitInternal)).problemText).toBe("body"); + const entity = await tempPackage("docx-entity"); - await writeFile(join(entity, "problem.docx"), docxDocument("body", [ - { name: "word/_rels/document.xml.rels", data: Buffer.from("]>&xxe;") } - ])); + await writeFile(join(entity, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from("]>&xxe;") }] + })); await expectImportCode(importPackage(entity), "docx_corrupt"); const traversal = await tempPackage("docx-slip"); - await writeFile(join(traversal, "problem.docx"), docxDocument("body", [{ name: "../outside.xml", data: Buffer.from("x") }])); + await writeFile(join(traversal, "problem.docx"), docxDocument("body", { + extra: [{ name: "../outside.xml", data: Buffer.from("x") }] + })); await expectImportCode(importPackage(traversal), "docx_zip_slip"); }); + it("rejects ambiguous DOCX relationships and content types", async () => { + const twoMainRelationships = await tempPackage("docx-two-main-relationships"); + await writeFile(join(twoMainRelationships, "problem.docx"), docxDocument("body", { + packageRelationships: relationshipsXml( + `` + + `` + ) + })); + await expectImportCode(importPackage(twoMainRelationships), "docx_corrupt"); + + const normalizedMainRelationship = await tempPackage("docx-normalized-main-relationship"); + await writeFile(join(normalizedMainRelationship, "problem.docx"), docxDocument("body", { + packageRelationships: relationshipsXml( + `` + + `` + ) + })); + await expectImportCode(importPackage(normalizedMainRelationship), "docx_corrupt"); + + const duplicatePackageId = await tempPackage("docx-duplicate-package-id"); + await writeFile(join(duplicatePackageId, "problem.docx"), docxDocument("body", { + packageRelationships: relationshipsXml( + `` + + "" + ) + })); + await expectImportCode(importPackage(duplicatePackageId), "docx_corrupt"); + + const paddedDuplicatePackageId = await tempPackage("docx-padded-duplicate-package-id"); + await writeFile(join(paddedDuplicatePackageId, "problem.docx"), docxDocument("body", { + packageRelationships: relationshipsXml( + `` + + "" + ) + })); + await expectImportCode(importPackage(paddedDuplicatePackageId), "docx_corrupt"); + + const nonCanonicalMainTarget = await tempPackage("docx-noncanonical-main-target"); + await writeFile(join(nonCanonicalMainTarget, "problem.docx"), docxDocument("body", { + packageRelationships: relationshipsXml( + `` + ) + })); + await expectImportCode(importPackage(nonCanonicalMainTarget), "docx_corrupt"); + + const duplicatePartId = await tempPackage("docx-duplicate-part-id"); + await writeFile(join(duplicatePartId, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + + "" + )) }] + })); + await expectImportCode(importPackage(duplicatePartId), "docx_corrupt"); + + const duplicateOverride = await tempPackage("docx-duplicate-override"); + await writeFile(join(duplicateOverride, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + `` + + `` + ) + })); + await expectImportCode(importPackage(duplicateOverride), "docx_corrupt"); + + const conflictingOverride = await tempPackage("docx-conflicting-override"); + await writeFile(join(conflictingOverride, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + `` + + "" + ) + })); + await expectImportCode(importPackage(conflictingOverride), "docx_corrupt"); + + const ambiguousDefault = await tempPackage("docx-ambiguous-default"); + await writeFile(join(ambiguousDefault, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + `` + + `` + ) + })); + await expectImportCode(importPackage(ambiguousDefault), "docx_corrupt"); + + const dotSegmentOverride = await tempPackage("docx-dot-segment-override"); + await writeFile(join(dotSegmentOverride, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + `` + + "" + ) + })); + await expectImportCode(importPackage(dotSegmentOverride), "docx_corrupt"); + + const nonCanonicalOnly = await tempPackage("docx-noncanonical-main-override"); + await writeFile(join(nonCanonicalOnly, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + `` + ) + })); + await expectImportCode(importPackage(nonCanonicalOnly), "docx_corrupt"); + + const caseAmbiguousEntries = await tempPackage("docx-case-ambiguous-entries"); + await writeFile(join(caseAmbiguousEntries, "problem.docx"), docxDocument("body", { + extra: [{ name: "WORD/DOCUMENT.XML", data: Buffer.from("") }] + })); + await expectImportCode(importPackage(caseAmbiguousEntries), "docx_corrupt"); + + const encodedEntry = await tempPackage("docx-encoded-entry"); + await writeFile(join(encodedEntry, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/%2e%2e/hidden.xml", data: Buffer.from("") }] + })); + await expectImportCode(importPackage(encodedEntry), "docx_corrupt"); + }); + + it("rejects macro and VBA indicators anywhere in a DOCX package", async () => { + const macroRelationship = await tempPackage("docx-renamed-macro-relationship"); + await writeFile(join(macroRelationship, "problem.docx"), docxDocument("body", { + extra: [ + { name: "word/hidden-payload.bin", data: Buffer.from("PRIVATE VBA PAYLOAD") }, + { name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) } + ] + })); + await expectImportCode(importPackage(macroRelationship), "docx_macro_enabled"); + + const combinedIndicators = await tempPackage("docx-renamed-macro-combined"); + await writeFile(join(combinedIndicators, "problem.docx"), docxDocument("body", { + extra: [ + { name: "word/hidden-payload.bin", data: Buffer.from("PRIVATE VBA PAYLOAD") }, + { name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) } + ], + contentTypes: contentTypesXml( + `` + + "" + + `` + ) + })); + await expectImportCode(importPackage(combinedIndicators), "docx_macro_enabled"); + + const macroDefault = await tempPackage("docx-macro-default"); + await writeFile(join(macroDefault, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + "" + + `` + ) + })); + await expectImportCode(importPackage(macroDefault), "docx_macro_enabled"); + + const macroOverride = await tempPackage("docx-macro-override"); + await writeFile(join(macroOverride, "problem.docx"), docxDocument("body", { + contentTypes: contentTypesXml( + `` + + `` + + "" + ) + })); + await expectImportCode(importPackage(macroOverride), "docx_macro_enabled"); + + const renamedEntry = await tempPackage("docx-vba-entry-case"); + await writeFile(join(renamedEntry, "problem.docx"), docxDocument("body", { + extra: [{ name: "custom/VBAPROJECT.BIN", data: Buffer.from("not opened") }] + })); + await expectImportCode(importPackage(renamedEntry), "docx_macro_enabled"); + + const encodedMacroRelationship = await tempPackage("docx-encoded-macro-relationship"); + await writeFile(join(encodedMacroRelationship, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/_rels/document.xml.rels", data: Buffer.from(relationshipsXml( + "" + )) }] + })); + await expectImportCode(importPackage(encodedMacroRelationship), "docx_macro_enabled"); + + const encodedMacroEntry = await tempPackage("docx-encoded-macro-entry"); + await writeFile(join(encodedMacroEntry, "problem.docx"), docxDocument("body", { + extra: [{ name: "custom/%76baProject.bin", data: Buffer.from("not opened") }] + })); + await expectImportCode(importPackage(encodedMacroEntry), "docx_macro_enabled"); + }); + it("enforces DOCX entry, decompressed-byte, and character limits", async () => { const entries = await tempPackage("docx-entries"); - await writeFile(join(entries, "problem.docx"), docxDocument("body", [ - { name: "word/extra.xml", data: Buffer.from("x") }, - { name: "word/extra2.xml", data: Buffer.from("x") } - ])); + await writeFile(join(entries, "problem.docx"), docxDocument("body", { + extra: [ + { name: "word/extra.xml", data: Buffer.from("x") }, + { name: "word/extra2.xml", data: Buffer.from("x") } + ] + })); await expectImportCode(importPackage(entries, { limits: { maxDocxZipEntries: 4 } }), "docx_zip_entry_limit"); const bytes = await tempPackage("docx-bytes"); - await writeFile(join(bytes, "problem.docx"), docxDocument("body", [{ name: "word/large.xml", data: Buffer.alloc(500, 65) }])); + await writeFile(join(bytes, "problem.docx"), docxDocument("body", { + extra: [{ name: "word/large.xml", data: Buffer.alloc(500, 65) }] + })); await expectImportCode(importPackage(bytes, { limits: { maxDocxUncompressedBytes: 100 } }), "docx_uncompressed_limit"); const characters = await tempPackage("docx-characters"); @@ -267,6 +601,45 @@ describe("safe package importer", () => { await expectImportCode(importPackage(characters, { limits: { maxDocxCharacters: 5 } }), "docx_character_limit"); }); + it("rejects invalid traversal limits before touching the package path", async () => { + await expect(importPackage(join(tmpdir(), "missing-input-package"), { + limits: { maxPackageDepth: 0 } + })).rejects.toThrow(TypeError); + }); + + it("bounds total package traversal entries and depth before filtering", async () => { + const unsupported = await tempPackage("package-entry-limit"); + await writeFile(join(unsupported, "problem.md"), "problem"); + for (let index = 0; index < 20; index += 1) { + await writeFile(join(unsupported, `ignored-${index}.bin`), "ignored"); + } + await expectImportCode(importPackage(unsupported, { limits: { maxPackageEntries: 2 } }), "package_entry_limit"); + + const hidden = await tempPackage("hidden-entry-limit"); + await writeFile(join(hidden, "problem.md"), "problem"); + await mkdir(join(hidden, ".ignored")); + await writeFile(join(hidden, ".ignored", "private.bin"), "private"); + await expectImportCode(importPackage(hidden, { limits: { maxPackageEntries: 1 } }), "package_entry_limit"); + + const hiddenNoRecursion = await tempPackage("hidden-no-recursion"); + await writeFile(join(hiddenNoRecursion, "problem.md"), "problem"); + await mkdir(join(hiddenNoRecursion, ".ignored")); + for (let index = 0; index < 20; index += 1) { + await writeFile(join(hiddenNoRecursion, ".ignored", `private-${index}.bin`), "private"); + } + expect((await importPackage(hiddenNoRecursion, { limits: { maxPackageEntries: 2 } })).problemText).toBe("problem"); + + const deep = await tempPackage("package-depth-limit"); + await mkdir(join(deep, "one", "two", "three"), { recursive: true }); + await writeFile(join(deep, "one", "two", "three", "problem.md"), "problem"); + await expectImportCode(importPackage(deep, { limits: { maxPackageDepth: 2 } }), "package_depth_limit"); + + const exactDepth = await tempPackage("package-depth-exact"); + await mkdir(join(exactDepth, "one", "two"), { recursive: true }); + await writeFile(join(exactDepth, "one", "two", "problem.md"), "problem"); + expect((await importPackage(exactDepth, { limits: { maxPackageDepth: 2 } })).problemText).toBe("problem"); + }); + it("fails when same-priority problem candidates are ambiguous", async () => { const root = await tempPackage("ambiguous"); await writeFile(join(root, "problem.md"), "markdown"); @@ -296,6 +669,11 @@ describe("safe package importer", () => { await writeFile(join(root, "problem.md"), "problem"); await symlink(join(root, "problem.md"), join(root, "problem-link.md")); await expectImportCode(importPackage(root), "symlink_input"); + + const hiddenRoot = await tempPackage("symlink-hidden"); + await writeFile(join(hiddenRoot, "problem.md"), "problem"); + await symlink(join(hiddenRoot, "problem.md"), join(hiddenRoot, ".hidden-link.md")); + await expectImportCode(importPackage(hiddenRoot), "symlink_input"); }); it("keeps raw asset bytes and SHA while exposing bounded XLSX and image metadata", async () => { @@ -327,11 +705,23 @@ describe("safe package importer", () => { await writeFile(join(root, "bad.png"), Buffer.from("not an image")); await writeFile(join(root, "bad.xlsx"), Buffer.from("not a workbook")); await writeFile(join(root, "large.png"), png(10, 10)); + await writeFile(join(root, "external-mode.xlsx"), xlsxAsset( + "" + )); + await writeFile(join(root, "ambiguous-entry.xlsx"), xlsxAsset(undefined, [ + { name: "XL/WORKBOOK.XML", data: Buffer.from("") } + ])); + await writeFile(join(root, "duplicate-id.xlsx"), xlsxAsset( + "" + )); const imported = await importPackage(root, { limits: { maxImagePixels: 50 } }); expect(imported.warnings).toEqual(expect.arrayContaining([ expect.objectContaining({ code: "metadata_unreadable", path: "bad.png" }), expect.objectContaining({ code: "metadata_unreadable", path: "bad.xlsx" }), - expect.objectContaining({ code: "metadata_limit", path: "large.png" }) + expect.objectContaining({ code: "metadata_limit", path: "large.png" }), + expect.objectContaining({ code: "metadata_unreadable", path: "external-mode.xlsx" }), + expect.objectContaining({ code: "metadata_unreadable", path: "ambiguous-entry.xlsx" }), + expect.objectContaining({ code: "metadata_unreadable", path: "duplicate-id.xlsx" }) ])); }); });