Tenure AI reads every document the viewer can open

apps/web/src/app/(app)/orgs/[slug]/documents/[id]/summary/page.tsx

@@ -11,11 +11,10 @@ import { Card, CardHeader } from "@/components/ui/Card"
 import { AnswerBody } from "@/components/ai/answer-format"
 import { fileTypeLabel } from "@/lib/uploads"
 import { recordAuditEvent } from "@/lib/audit-record"
+import { documentTextFor, MAX_READ_BYTES } from "@/app/api/documents/_lib/text"
 
 export const dynamic = "force-dynamic"
 
-const TEXTUAL = /^(text\/|application\/(json|csv|xml))/
-const MAX_BYTES = 200 * 1024
 
 export default async function DocumentSummaryPage({
   params,
@@ -38,18 +37,32 @@ export default async function DocumentSummaryPage({
 
     let summary: string | null = null
     let note: string | null = null
+    let truncated = false
 
+    /*
+     * THE GATE ASKS THE PARSERS, NOT A REGULAR EXPRESSION.
+     *
+     * This page used to test `^(text\/|application\/(json|csv|xml))` and
+     * refuse everything else, explaining that "PDF and Office support arrives
+     * with the document pipeline". That sentence was not true when it was
+     * written: `_lib/content.ts` has parsed Word, Excel and PowerPoint since
+     * the viewer shipped. Three of the four formats that refusal named could
+     * have been summarised that day.
+     *
+     * The regexp was written before the parsers and never revisited after
+     * them — so the limit was in the gate, not in the product, and a student
+     * was being told a roadmap to explain it.
+     *
+     * `documentTextFor` asks the parsers that already exist. It returns a
+     * REASON rather than a boolean, so the sentence below can say which of the
+     * several different things went wrong.
+     */
     if (!aiConfigured()) {
       note = "Tenure AI is not enabled."
-    } else if (!TEXTUAL.test(doc.mimeType)) {
-      // The TYPE, in the word a person uses for it. The refusal used to name
-      // the Content-Type — "this is application/pdf" — which is an
-      // interchange identifier standing in for an explanation.
-      note = `Tenure AI can only read text documents so far, and this is a ${fileTypeLabel(
-        doc.mimeType
-      )}. Open it to read it in full.`
-    } else if ((doc.sizeBytes ?? 0) > MAX_BYTES) {
-      note = "This document is too large to summarize in the pilot (200 KB limit)."
+    } else if ((doc.sizeBytes ?? 0) > MAX_READ_BYTES) {
+      note =
+        "This document is too large for Tenure AI to read. Open it to read it in full — " +
+        "nothing about it has changed."
     } else if (documentsBucket) {
       // BEFORE the read, not after a successful one.
       //
@@ -89,9 +102,41 @@ export default async function DocumentSummaryPage({
       // arrive, so the stall this page was worried about could still hold the
       // render open. `getDocumentBytes` closes that with an abortSignal, and
       // one reader means one place where that has to be true.
-      const content = (await getDocumentBytes(doc.objectKey)).toString("utf8")
-      summary = await summarizeDocument(doc.title, content)
-      if (!summary) note = "The summary was unavailable. Try again shortly."
+      const read = await documentTextFor(doc.mimeType, await getDocumentBytes(doc.objectKey))
+
+      if (!read.ok) {
+        /*
+         * EACH REFUSAL SAYS WHICH ONE IT IS, and none of them names a
+         * Content-Type. A reader shown "application/pdf" has been handed a
+         * header value and told it is an explanation.
+         *
+         * The PDF sentence promises nothing. The previous one dated the fix to
+         * a "document pipeline" the reader has no way to ask about, and a
+         * promise a product cannot keep is worse than a plain limit.
+         */
+        note =
+          read.reason === "pdf"
+            ? "Tenure AI cannot read a PDF yet. Open it to read it in full — everything else " +
+              "about this document works normally."
+            : read.reason === "image"
+              ? `This is an image, so there is no text for Tenure AI to summarise. Open it to ` +
+                `see it.`
+              : read.reason === "empty"
+                ? "There is no text in this document to summarise. It may be a scan, or empty."
+                : read.reason === "too-large"
+                  ? "This document is too large for Tenure AI to read. Open it to read it in full."
+                  : `Tenure AI cannot read a ${fileTypeLabel(doc.mimeType)}. Open it to read it ` +
+                    `in full.`
+      } else {
+        summary = await summarizeDocument(doc.title, read.text)
+        if (!summary) {
+          note = "The summary was unavailable. Try again shortly."
+        } else if (read.truncated) {
+          // A summary drawn from part of a document, described as one drawn
+          // from the whole, is the failure this whole file is careful about.
+          truncated = true
+        }
+      }
     }
 
     return (
@@ -112,7 +157,13 @@ export default async function DocumentSummaryPage({
         <Card>
           <CardHeader
             title={summary ? "Summary" : "Not available"}
-            subtitle={summary ? "Generated from the document contents" : undefined}
+            subtitle={
+              summary
+                ? truncated
+                  ? "Generated from the beginning of this document"
+                  : "Generated from the document contents"
+                : undefined
+            }
           />
           {/*
             TWO DIFFERENT KINDS OF TEXT, AND ONLY ONE OF THEM IS PROSE A MODEL
@@ -123,9 +174,17 @@ export default async function DocumentSummaryPage({
             would claim a formatting contract that nothing produces.
           */}
           {summary ? (
-            <div className="text-sm text-text-1">
-              <AnswerBody text={summary} />
-            </div>
+            <>
+              <div className="text-sm text-text-1">
+                <AnswerBody text={summary} />
+              </div>
+              {truncated && (
+                <p className="mt-3 text-[12px] text-text-3">
+                  This document was long enough that Tenure AI read only the beginning of it.
+                  The summary covers that much, and the document itself is unchanged.
+                </p>
+              )}
+            </>
           ) : (
             <p className="text-sm text-text-1">{note}</p>
           )}

apps/web/src/app/api/documents/_lib/content.ts

@@ -168,7 +168,110 @@ function linesFromSlideXml(xml: string): string[] {
   return lines
 }
 
-async function extractPptx(bytes: Buffer): Promise<PptxSlide[]> {
+/**
+ * Every sheet as TEXT, for a caller that has to read a workbook rather than
+ * render it — today, Tenure AI's summariser.
+ *
+ * ── Why this lives HERE and not beside its caller ──────────────────────────
+ *
+ * `xlsx@0.18.5` carries two HIGH advisories that are reachable and that npm
+ * cannot fix: GHSA-4r6h-8v6p-xvw6 (prototype pollution) and GHSA-5pgg-2g8v-p4x9
+ * (ReDoS). The registry's newest xlsx IS 0.18.5; the patched builds exist only
+ * on cdn.sheetjs.com. So the repo ACCEPTS them, and the acceptance is bounded
+ * by an enumerated list of parse sites in `scripts/advisory-rechecks.mjs` —
+ * "the blast radius has not grown since the exposure was written down".
+ *
+ * A second `XLSX.read` beside the summariser would have grown it. The recheck
+ * failed the first version of this work and named the file, and it was right:
+ * same bytes and same parser, but a new entry on that list is a new thing to
+ * re-argue every time somebody audits this.
+ *
+ * `sheet_to_csv` is not a parse entry point — the advisories exempt workflows
+ * that do not read arbitrary files — so this file keeps the READ and callers
+ * keep their own shaping. One `XLSX.read` in the server path, in the file whose
+ * own header already says heavy parsers live here and nowhere a client bundle
+ * can reach.
+ *
+ * NOT capped at MAX_SHEETS / MAX_ROWS like the viewer's grid: those bounds
+ * exist so a browser does not render 40,000 rows, and a reader wants the whole
+ * workbook. `MAX_PARSE_BYTES` still bounds the input, and the caller caps the
+ * text it keeps.
+ */
+/**
+ * Rows and columns converted from any ONE worksheet, whatever its `!ref` claims.
+ *
+ * Sized against what a READER gets, not against what a spreadsheet may hold.
+ * The summariser is given 24,000 characters; a thousand rows of sixty-four
+ * columns is already several times that, so the cap is generous for every real
+ * workbook and still bounds a hostile one.
+ *
+ * The first attempt used 5,000 x 256. That is 1.28 million cells, and building
+ * that string to then slice it to 24,000 took SEVENTEEN SECONDS in the test
+ * that proved the clamp worked — a bound that is technically finite and
+ * practically a stall. The test time was the measurement that said so.
+ */
+const MAX_SHEET_ROWS = 1_000
+const MAX_SHEET_COLS = 64
+
+export function sheetsToText(bytes: Buffer, budgetChars: number): string {
+  const wb = XLSX.read(bytes, { type: "buffer" })
+
+  /*
+   * BOUNDED PER SHEET *AND* WITHIN EACH SHEET.
+   *
+   * The first version checked the budget between sheets, which bounds the
+   * NUMBER of worksheets converted and nothing about the size of any one of
+   * them. `sheet_to_csv` walks the whole `!ref` range in a single call before
+   * it returns, so one sparse worksheet declaring A1:ZZ1000000 — tiny on disk,
+   * because almost every cell is absent — generates and allocates all of it
+   * synchronously on the request thread, and the loop only notices afterwards.
+   *
+   * So the RANGE is clamped before conversion.
+   *
+   * ── The option that does not work, measured rather than assumed ───────────
+   *
+   * `sheet_to_csv(ws, { range })` is documented and is silently ignored by this
+   * build — with a Range object, an encoded string, and a numeric start row,
+   * all three returned every row. Mutating `!ref` on the worksheet before the
+   * call is what actually clamps it, so that is what this does. The worksheet
+   * object is ours, parsed a few lines above and discarded after.
+   */
+  const out: string[] = []
+  let used = 0
+  for (const name of wb.SheetNames) {
+    if (used > budgetChars) break
+
+    const ws = wb.Sheets[name]
+    const ref = ws?.["!ref"]
+    // A worksheet with no `!ref` has no cells at all.
+    if (!ref) continue
+
+    const range = XLSX.utils.decode_range(ref)
+    range.e.r = Math.min(range.e.r, range.s.r + MAX_SHEET_ROWS - 1)
+    range.e.c = Math.min(range.e.c, range.s.c + MAX_SHEET_COLS - 1)
+    ws["!ref"] = XLSX.utils.encode_range(range)
+
+    const csv = XLSX.utils.sheet_to_csv(ws)
+    /*
+     * A HEADING IS NOT CONTENT.
+     *
+     * `--- Sheet1 ---` is non-empty, so a workbook of nothing but blank sheets
+     * used to come back as readable text and get sent to a model to summarise.
+     * The caller distinguishes "nothing to summarise" from "cannot open this"
+     * and says different things for each; a blank workbook must reach the first.
+     */
+    if (csv.trim() === "") continue
+
+    // The sheet's NAME matters to a reader — "Q3 budget" versus "Sheet1" is
+    // most of what a spreadsheet's structure says.
+    const block = `--- ${name} ---\n${csv}`
+    out.push(block)
+    used += block.length + 2
+  }
+  return out.join("\n\n")
+}
+
+export async function extractPptx(bytes: Buffer): Promise<PptxSlide[]> {
   const zip = await JSZip.loadAsync(bytes)
 
   const slideFiles = Object.keys(zip.files)