From ba4878b78032fdc0f99c6e09cf2c9c1cb9e7172e Mon Sep 17 00:00:00 2001 From: Wooseong Kim Date: Thu, 6 Aug 2026 20:52:13 +0900 Subject: [PATCH] feat(read): extract PDF text via pdftotext instead of base64 dump (#236) When pdftotext is available, the read tool returns extracted text (capped at 30K chars) so the model can actually read PDF contents. Falls back to the binary warning when unavailable. (cherry picked from commit e94ededbd465931747e2b4db40d0eab43c270782) --- packages/core/src/tests/read-handler.test.ts | 96 ++++++++++++++++++++ packages/core/src/tools/read-handler.ts | 38 ++++++++ 2 files changed, 134 insertions(+) create mode 100644 packages/core/src/tests/read-handler.test.ts diff --git a/packages/core/src/tests/read-handler.test.ts b/packages/core/src/tests/read-handler.test.ts new file mode 100644 index 00000000..68e5a6a7 --- /dev/null +++ b/packages/core/src/tests/read-handler.test.ts @@ -0,0 +1,96 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import type { ToolExecutionContext } from "../tools/executor"; +import { handleReadTool } from "../tools/read-handler"; + +const tempDirs: string[] = []; + +test.afterEach(() => { + while (tempDirs.length > 0) { + const dir = tempDirs.pop(); + if (dir) { + fs.rmSync(dir, { recursive: true, force: true }); + } + } +}); + +function createContext(projectRoot: string, model: string): ToolExecutionContext { + return { + sessionId: "read-test", + projectRoot, + toolCall: { + id: "tool-call-id", + type: "function", + function: { + name: "read", + arguments: "{}", + }, + }, + createOpenAIClient: () => ({ + client: null, + model, + thinkingEnabled: false, + }), + }; +} + +test( + "read tool extracts PDF text when pdftotext is available (#236)", + { skip: process.platform === "win32" }, + async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "deepcode-read-")); + tempDirs.push(workspace); + + const binDir = fs.mkdtempSync(path.join(os.tmpdir(), "deepcode-bin-")); + tempDirs.push(binDir); + const fakeBin = path.join(binDir, "pdftotext"); + fs.writeFileSync(fakeBin, '#!/bin/sh\nprintf "extracted pdf text from %s\\n" "$2"\n', { mode: 0o755 }); + + const filePath = path.join(workspace, "sample.pdf"); + fs.writeFileSync(filePath, Buffer.from("%PDF-1.4 fake pdf bytes", "latin1")); + + const originalPath = process.env.PATH; + process.env.PATH = `${binDir}${path.delimiter}${originalPath ?? ""}`; + try { + const result = await handleReadTool({ file_path: filePath }, createContext(workspace, "deepseek-v4-flash")); + assert.equal(result.ok, true); + assert.match(result.output ?? "", /extracted pdf text from/); + assert.equal((result.metadata as { encoding?: string }).encoding, "text"); + } finally { + if (originalPath === undefined) { + delete process.env.PATH; + } else { + process.env.PATH = originalPath; + } + } + } +); + +test("read tool falls back to binary warning when pdftotext is unavailable (#236)", async () => { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "deepcode-read-")); + tempDirs.push(workspace); + + const emptyBin = fs.mkdtempSync(path.join(os.tmpdir(), "deepcode-emptybin-")); + tempDirs.push(emptyBin); + + const filePath = path.join(workspace, "sample.pdf"); + fs.writeFileSync(filePath, Buffer.from("%PDF-1.4 fake pdf bytes", "latin1")); + + const originalPath = process.env.PATH; + process.env.PATH = emptyBin; + try { + const result = await handleReadTool({ file_path: filePath }, createContext(workspace, "deepseek-v4-flash")); + assert.equal(result.ok, true); + assert.equal(result.output, "WARNING: File is binary."); + assert.equal((result.metadata as { encoding?: string }).encoding, "base64"); + } finally { + if (originalPath === undefined) { + delete process.env.PATH; + } else { + process.env.PATH = originalPath; + } + } +}); diff --git a/packages/core/src/tools/read-handler.ts b/packages/core/src/tools/read-handler.ts index 94be54b4..48aa28f8 100644 --- a/packages/core/src/tools/read-handler.ts +++ b/packages/core/src/tools/read-handler.ts @@ -1,5 +1,6 @@ import * as fs from "fs"; import * as path from "path"; +import { spawnSync } from "child_process"; import ignore from "ignore"; import type { ToolExecutionContext, ToolExecutionFollowUpMessage, ToolExecutionResult } from "./executor"; import { readTextFileWithMetadata } from "../common/file-utils"; @@ -14,6 +15,8 @@ import { const DEFAULT_LINE_LIMIT = 2000; const MAX_LINE_LENGTH = 2000; const LINE_NUMBER_WIDTH = 6; +// Cap for text extracted from PDFs via pdftotext (#236). +const MAX_PDF_TEXT_CHARS = 30000; const DEFAULT_GITIGNORE = [ "node_modules/", ".git/", @@ -158,6 +161,21 @@ export async function handleReadTool( timestamp: Math.floor(stat.mtimeMs), isPartialView: true, }); + const extracted = extractPdfText(filePath); + if (extracted) { + return { + ok: true, + name: "read", + output: extracted, + metadata: { + mime: "application/pdf", + encoding: "text", + bytes: buffer.length, + pageCount, + truncated: extracted.length >= MAX_PDF_TEXT_CHARS, + }, + }; + } return { ok: true, name: "read", @@ -483,6 +501,26 @@ function countPdfPages(buffer: Buffer): number | null { } } +/** Extract PDF text via the external `pdftotext` binary; returns null when unavailable or empty. (#236) */ +function extractPdfText(filePath: string): string | null { + try { + const result = spawnSync("pdftotext", ["-layout", filePath, "-"], { + encoding: "utf8", + maxBuffer: 10 * 1024 * 1024, + timeout: 10000, + }); + const text = result.stdout?.trim(); + if (result.status !== 0 || !text) { + return null; + } + return text.length > MAX_PDF_TEXT_CHARS + ? `${text.slice(0, MAX_PDF_TEXT_CHARS)}\n\n[truncated: PDF text exceeds ${MAX_PDF_TEXT_CHARS} characters]` + : text; + } catch { + return null; + } +} + function readNotebook(filePath: string): string { const raw = fs.readFileSync(filePath, "utf8"); if (!raw) {