feat: safely import PDF and DOCX packages

package.json

@@ -25,11 +25,17 @@
     "@earendil-works/pi-coding-agent": "0.84.1",
     "ajv": "8.20.0",
     "ajv-formats": "3.0.1",
+    "buffer-crc32": "0.2.13",
     "commander": "15.0.0",
-    "fastify": "5.11.3"
+    "fastify": "5.11.3",
+    "pdfjs-dist": "5.4.624",
+    "saxes": "6.0.0",
+    "yauzl": "3.1.3"
   },
   "devDependencies": {
+    "@types/buffer-crc32": "0.2.4",
     "@types/node": "24.10.1",
+    "@types/yauzl": "2.10.3",
     "tsx": "4.23.12",
     "typescript": "5.9.3",
     "vitest": "4.1.10",

src/input/README.md

@@ -0,0 +1,24 @@
+# Safe package input
+
+`importPackage` inventories a local competition package without modifying source files or using the network. Existing callers keep the same required fields (`problemText`, the original `problemPath`, `dataAssets`, and `dataPaths`); extraction and inventory details are optional additions.
+
+## Problem selection
+
+Problem statements are selected deterministically by precedence group:
+
+1. `problem.md` or `problem.markdown`
+2. `problem.txt`
+3. `problem.pdf`
+4. `problem.docx`
+
+Names are matched case-insensitively at any non-hidden package depth. The first non-empty precedence group wins. If that group contains more than one candidate (including duplicate names in separate directories, or both Markdown spellings), import fails with `PackageImportError.code === "ambiguous_problem"`; lower-priority candidates are data assets only when a higher-priority problem was selected.
+
+## Safety and limits
+
+All limits have conservative defaults in `DEFAULT_IMPORT_LIMITS` and may be reduced or raised through `importPackage(path, { limits })`. Invalid limits are rejected. Package roots and nested entries may not be symlinks, paths must remain beneath the package root, and total traversal is bounded by `maxPackageEntries` and `maxPackageDepth`. Every encountered child entry (file, directory, unsupported extension, or hidden name) consumes the entry budget; the package root itself does not. Hidden entries are then ignored and hidden directories are never recursed into.
+
+PDF text is extracted locally with the pinned `pdfjs-dist` Node build. Input bytes are passed directly to the parser with fetch, streaming, JavaScript evaluation, system fonts, and WebAssembly disabled. The importer records source SHA-256/size, PDF page count, extracted character count, and extractor version, while explicitly rejecting unavailable tooling, encryption, corruption, empty text, and byte/page/character limits. `maxPdfCharacters` applies to the exact returned text, including spaces inserted between text items, blank lines inserted between non-empty pages, and final normalization; construction is checked incrementally and asserted again before return.
+
+DOCX extraction performs a bounded magic-prefix check before opening ZIP data: normal ZIP signatures continue, OLE/CFB Office containers are classified as `docx_encrypted`, and other input is `docx_corrupt`. Extraction reads only required OOXML package parts and `word/document.xml` body text. It never decrypts content, executes payloads, follows non-internal relationships, or opens embedded objects. All relationship files reject duplicate IDs and external, escaping, or unknown normalized `TargetMode` values; package relationships require exactly one normalized main-document relationship; and content-type declarations reject duplicate/case-normalized part ambiguity. Any VBA/macro/ActiveX content type, relationship type, or package-part indicator is rejected before payload expansion; ZIP symlink entries are rejected at the archive boundary.
+
+Data assets retain SHA-256 and size computed from their original bytes. XLSX inspection is limited to workbook, relationship, and bounded worksheet-dimension XML; images are read only far enough to obtain bounded PNG/JPEG format and dimensions. CSV, JSON, legacy XLS, and Parquet are inventoried without parsing or sampling their records. Unreadable or over-limit optional asset metadata is surfaced in `warnings`, never silently treated as successful metadata.

src/input/asset-metadata.ts

@@ -0,0 +1,309 @@
+import { open as openFile } from "node:fs/promises";
+import { extname } from "node:path";
+import {
+  PackageImportError,
+  type AssetMetadata,
+  type ImageAssetMetadata,
+  type ImportLimits,
+  type XlsxSheetMetadata
+} from "./types.js";
+import { readBoundedZip } from "./zip-reader.js";
+import {
+  attributeValue,
+  PACKAGE_RELATIONSHIPS_NAMESPACE,
+  parseXml,
+  SPREADSHEETML_NAMESPACE
+} from "./xml.js";
+
+const OFFICE_DOCUMENT_RELATIONSHIPS_NAMESPACE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships";
+const MAX_CAUSE_LENGTH = 240;
+const XLSX_ZIP_CODES = {
+  corrupt: "metadata_unreadable",
+  encrypted: "metadata_unreadable",
+  zipSlip: "metadata_unreadable",
+  entryLimit: "metadata_limit",
+  uncompressedLimit: "metadata_limit"
+} as const;
+
+function checkedImage(format: ImageAssetMetadata["format"], width: number, height: number, path: string, limits: ImportLimits): ImageAssetMetadata {
+  const pixels = width * height;
+  if (!Number.isSafeInteger(width) || !Number.isSafeInteger(height) || width <= 0 || height <= 0) {
+    throw new PackageImportError("metadata_unreadable", `Invalid ${format.toUpperCase()} dimensions.`, { path });
+  }
+  if (!Number.isSafeInteger(pixels) || pixels > limits.maxImagePixels) {
+    throw new PackageImportError("metadata_limit", `Image has ${pixels} pixels; limit is ${limits.maxImagePixels}.`, {
+      path,
+      actual: pixels,
+      limit: limits.maxImagePixels
+    });
+  }
+  return { kind: "image", format, width, height };
+}
+
+async function readPrefix(path: string, length: number): Promise<Buffer> {
+  const handle = await openFile(path, "r");
+  try {
+    const buffer = Buffer.alloc(length);
+    const { bytesRead } = await handle.read(buffer, 0, length, 0);
+    return buffer.subarray(0, bytesRead);
+  } finally {
+    await handle.close();
+  }
+}
+
+function pngMetadata(bytes: Buffer, path: string, limits: ImportLimits): ImageAssetMetadata {
+  const signature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+  if (bytes.length < 24 || !bytes.subarray(0, 8).equals(signature) || bytes.toString("ascii", 12, 16) !== "IHDR") {
+    throw new PackageImportError("metadata_unreadable", "Invalid PNG header.", { path });
+  }
+  return checkedImage("png", bytes.readUInt32BE(16), bytes.readUInt32BE(20), path, limits);
+}
+
+const JPEG_START_OF_FRAME_MARKERS = new Set([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf]);
+
+function jpegMetadata(bytes: Buffer, path: string, limits: ImportLimits, sourceBytes: number): ImageAssetMetadata {
+  if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) {
+    throw new PackageImportError("metadata_unreadable", "Invalid JPEG header.", { path });
+  }
+  let offset = 2;
+  while (offset + 3 < bytes.length) {
+    while (bytes[offset] === 0xff) offset += 1;
+    const marker = bytes[offset];
+    offset += 1;
+    if (marker === undefined || marker === 0xd9 || marker === 0xda) break;
+    if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue;
+    if (offset + 2 > bytes.length) break;
+    const segmentLength = bytes.readUInt16BE(offset);
+    if (segmentLength < 2) throw new PackageImportError("metadata_unreadable", "Invalid JPEG segment length.", { path });
+    if (JPEG_START_OF_FRAME_MARKERS.has(marker)) {
+      if (offset + 7 > bytes.length) break;
+      return checkedImage("jpeg", bytes.readUInt16BE(offset + 5), bytes.readUInt16BE(offset + 3), path, limits);
+    }
+    offset += segmentLength;
+  }
+  const code = bytes.length < sourceBytes ? "metadata_limit" : "metadata_unreadable";
+  throw new PackageImportError(code, "JPEG dimensions were not found within the bounded header scan.", {
+    path,
+    actual: bytes.length,
+    limit: limits.maxImageHeaderBytes
+  });
+}
+
+async function imageMetadata(path: string, sourceBytes: number, extension: string, limits: ImportLimits): Promise<ImageAssetMetadata> {
+  const bytes = await readPrefix(path, Math.min(sourceBytes, limits.maxImageHeaderBytes));
+  if (extension === ".png") return pngMetadata(bytes, path, limits);
+  return jpegMetadata(bytes, path, limits, sourceBytes);
+}
+
+interface WorkbookSheet {
+  name: string;
+  relationshipId: string;
+}
+
+function workbookSheets(xml: Buffer, path: string, sheetLimit: number, byteLimit: number): WorkbookSheet[] {
+  if (xml.length > byteLimit) {
+    throw new PackageImportError("metadata_limit", `XLSX workbook metadata exceeds ${byteLimit} bytes.`, {
+      path,
+      actual: xml.length,
+      limit: byteLimit
+    });
+  }
+  const sheets: WorkbookSheet[] = [];
+  parseXml(xml, {
+    openTag(tag) {
+      if (tag.uri !== SPREADSHEETML_NAMESPACE || tag.local !== "sheet") return;
+      const name = attributeValue(tag, "name");
+      const relationshipId = attributeValue(tag, "id", OFFICE_DOCUMENT_RELATIONSHIPS_NAMESPACE);
+      if (!name || !relationshipId) throw new PackageImportError("metadata_unreadable", "XLSX sheet is missing a name or relationship id.", { path });
+      sheets.push({ name, relationshipId });
+      if (sheets.length > sheetLimit) {
+        throw new PackageImportError("metadata_limit", `XLSX has more than ${sheetLimit} sheets.`, {
+          path,
+          actual: sheets.length,
+          limit: sheetLimit
+        });
+      }
+    }
+  });
+  return sheets;
+}
+
+function decodePercentEscapes(value: string): string | undefined {
+  let decoded = value;
+  for (let depth = 0; depth < 4 && decoded.includes("%"); depth += 1) {
+    try {
+      const next = decodeURIComponent(decoded);
+      if (next === decoded) return undefined;
+      decoded = next;
+    } catch {
+      return undefined;
+    }
+  }
+  return decoded.includes("%") ? undefined : decoded;
+}
+
+function normalizedOoxmlEntryKey(name: string): string | undefined {
+  const isDirectory = name.endsWith("/");
+  const partName = isDirectory ? name.slice(0, -1) : name;
+  const decoded = decodePercentEscapes(partName);
+  if (decoded === undefined
+    || !decoded
+    || name.startsWith("/")
+    || partName.endsWith("/")
+    || decoded !== partName
+    || /[\u0000-\u001f\u007f]/.test(decoded)
+    || decoded.includes("\\")
+    || decoded.includes("?")
+    || decoded.includes("#")) return undefined;
+  const normalized = decoded.normalize("NFC");
+  if (normalized !== decoded || normalized.split("/").some((segment) => !segment || segment === "." || segment === "..")) return undefined;
+  return normalized.toLowerCase();
+}
+
+function validateOoxmlEntryNames(entryNames: Set<string>, path: string): void {
+  const keys = new Set<string>();
+  for (const name of entryNames) {
+    const key = normalizedOoxmlEntryKey(name);
+    if (key === undefined || keys.has(key)) {
+      throw new PackageImportError("metadata_unreadable", "XLSX contains unsafe or ambiguous package part names.", { path: name });
+    }
+    keys.add(key);
+  }
+}
+
+function worksheetTargetMode(value: string): string | undefined {
+  const decoded = decodePercentEscapes(value.trim());
+  return decoded?.trim().toLowerCase();
+}
+
+function worksheetTarget(target: string): string | undefined {
+  const decoded = decodePercentEscapes(target.trim());
+  if (decoded === undefined || !decoded
+    || decoded.includes("%")
+    || /[\u0000-\u001f\u007f]/.test(decoded)
+    || decoded.includes("\\")
+    || decoded.includes("?")
+    || decoded.includes("#")
+    || decoded.startsWith("/")
+    || decoded.startsWith("//")
+    || /^[A-Za-z][A-Za-z0-9+.-]*:/.test(decoded)) return undefined;
+  const segments: string[] = [];
+  for (const segment of decoded.split("/")) {
+    if (!segment || segment === ".") return undefined;
+    if (segment === "..") return undefined;
+    segments.push(segment);
+  }
+  return `xl/${segments.join("/")}`;
+}
+
+function worksheetRelationships(xml: Buffer, path: string, byteLimit: number): Map<string, string> {
+  if (xml.length > byteLimit) {
+    throw new PackageImportError("metadata_limit", `XLSX relationships metadata exceeds ${byteLimit} bytes.`, {
+      path,
+      actual: xml.length,
+      limit: byteLimit
+    });
+  }
+  const relationships = new Map<string, string>();
+  parseXml(xml, {
+    openTag(tag) {
+      if (tag.uri !== PACKAGE_RELATIONSHIPS_NAMESPACE || tag.local !== "Relationship") return;
+      const id = attributeValue(tag, "Id")?.trim();
+      const target = attributeValue(tag, "Target")?.trim();
+      const type = attributeValue(tag, "Type")?.trim();
+      if (!id || !target || !type?.endsWith("/worksheet")) return;
+      const rawTargetMode = attributeValue(tag, "TargetMode");
+      const targetMode = rawTargetMode === undefined ? undefined : worksheetTargetMode(rawTargetMode);
+      if (rawTargetMode !== undefined && targetMode !== "internal") {
+        throw new PackageImportError("metadata_unreadable", "External XLSX worksheet relationship is not followed.", { path });
+      }
+      const resolvedTarget = worksheetTarget(target);
+      if (resolvedTarget === undefined) {
+        throw new PackageImportError("metadata_unreadable", "Unsafe XLSX worksheet target.", { path });
+      }
+      if (relationships.has(id)) {
+        throw new PackageImportError("metadata_unreadable", "XLSX contains duplicate worksheet relationship IDs.", { path });
+      }
+      relationships.set(id, resolvedTarget);
+    }
+  });
+  return relationships;
+}
+
+function worksheetDimension(xml: Buffer, path: string, limit: number): string | undefined {
+  if (xml.length > limit) {
+    throw new PackageImportError("metadata_limit", `XLSX worksheet metadata part exceeds ${limit} bytes.`, {
+      path,
+      actual: xml.length,
+      limit
+    });
+  }
+  let dimension: string | undefined;
+  parseXml(xml, {
+    openTag(tag) {
+      if (dimension === undefined && tag.uri === SPREADSHEETML_NAMESPACE && tag.local === "dimension") {
+        dimension = attributeValue(tag, "ref");
+      }
+    }
+  });
+  return dimension;
+}
+
+async function xlsxMetadata(path: string, sourceBytes: number, limits: ImportLimits): Promise<AssetMetadata> {
+  if (sourceBytes > limits.maxMetadataBytes) {
+    throw new PackageImportError("metadata_limit", `XLSX is ${sourceBytes} bytes; metadata inspection limit is ${limits.maxMetadataBytes}.`, {
+      path,
+      actual: sourceBytes,
+      limit: limits.maxMetadataBytes
+    });
+  }
+  try {
+    const archive = await readBoundedZip(
+      path,
+      { maxEntries: limits.maxMetadataZipEntries, maxUncompressedBytes: limits.maxMetadataUncompressedBytes },
+      XLSX_ZIP_CODES,
+      (name) => name === "xl/workbook.xml" || name === "xl/_rels/workbook.xml.rels" || name.startsWith("xl/worksheets/")
+    );
+    validateOoxmlEntryNames(archive.entryNames, path);
+    const workbook = archive.entries.get("xl/workbook.xml");
+    const relationshipXml = archive.entries.get("xl/_rels/workbook.xml.rels");
+    if (!workbook || !relationshipXml) throw new PackageImportError("metadata_unreadable", "XLSX is missing workbook metadata parts.", { path });
+    const sheetRecords = workbookSheets(workbook, path, limits.maxXlsxSheets, limits.maxXlsxWorkbookBytes);
+    const relationships = worksheetRelationships(relationshipXml, path, limits.maxXlsxRelationshipsBytes);
+    const sheets: XlsxSheetMetadata[] = sheetRecords.map((sheet) => {
+      const target = relationships.get(sheet.relationshipId);
+      if (!target) throw new PackageImportError("metadata_unreadable", `XLSX relationship ${sheet.relationshipId} is missing.`, { path });
+      const worksheet = archive.entries.get(target);
+      if (!worksheet) throw new PackageImportError("metadata_unreadable", `XLSX worksheet part is missing: ${target}`, { path });
+      const dimension = worksheetDimension(worksheet, target, limits.maxXlsxWorksheetBytes);
+      return dimension === undefined ? { name: sheet.name } : { name: sheet.name, dimension };
+    });
+    return {
+      kind: "spreadsheet",
+      format: "xlsx",
+      sheets,
+      zipEntries: archive.entryCount,
+      uncompressedBytes: archive.uncompressedBytes
+    };
+  } catch (error) {
+    if (error instanceof PackageImportError) throw error;
+    const value = error instanceof Error ? error.message : String(error);
+    const cause = value.replace(/[\r\n\t]+/g, " ").slice(0, MAX_CAUSE_LENGTH);
+    throw new PackageImportError("metadata_unreadable", "Could not read XLSX metadata.", { path, cause });
+  }
+}
+
+export async function inspectAssetMetadata(
+  path: string,
+  sourceBytes: number,
+  limits: ImportLimits
+): Promise<AssetMetadata | undefined> {
+  const extension = extname(path).toLowerCase();
+  if (extension === ".xlsx") return xlsxMetadata(path, sourceBytes, limits);
+  if (extension === ".png" || extension === ".jpg" || extension === ".jpeg") return imageMetadata(path, sourceBytes, extension, limits);
+  if (extension === ".csv") return { kind: "bounded-inventory", format: "csv" };
+  if (extension === ".json") return { kind: "bounded-inventory", format: "json" };
+  if (extension === ".xls") return { kind: "bounded-inventory", format: "xls" };
+  if (extension === ".parquet") return { kind: "bounded-inventory", format: "parquet" };
+  return undefined;
+}