diff --git a/.github/workflows/windows-distribution.yml b/.github/workflows/windows-distribution.yml index d69d868..0386e91 100644 --- a/.github/workflows/windows-distribution.yml +++ b/.github/workflows/windows-distribution.yml @@ -49,6 +49,10 @@ jobs: if ($LASTEXITCODE -ne 0 -or $version -ne "Interly 0.6.0") { throw "Standalone executable smoke test failed: $version" } + $documentSupport = & .\dist\interly.exe --check-document-support + if ($LASTEXITCODE -ne 0 -or $documentSupport -ne "Document support: PDF DOCX XLSX XLSM PPTX") { + throw "Standalone document backend smoke test failed: $documentSupport" + } - name: Install Inno Setup run: choco install innosetup --no-progress --yes diff --git a/README.md b/README.md index cb43d10..24ac061 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,9 @@ Development installations upgrade through pipx. Restart Interly after a successf - Automatic isolated-browser cleanup after every browser-assisted request - Guarded file search, bounded text reads, exact text edits, comparison, creation, copying, moving, renaming, and folder creation +- Read-only structured understanding of PDF, Word (`.docx`), Excel (`.xlsx`/`.xlsm`), and + PowerPoint (`.pptx`) documents, including pages or slides, headings/text blocks, tables, + spreadsheet sheets/ranges/formulas, speaker notes, and metadata where the format exposes it - Approved direct-file downloads with public-URL validation, a 1 GB limit, overwrite protection, temporary-file cleanup, final content type, byte count, and SHA-256 reporting - Visible top-level Windows window enumeration with exact handles, PIDs, titles, and rectangles @@ -140,16 +143,30 @@ Sensitive local-read prompts use a different meaning: Sensitive `A` approval applies to one command only; it is never remembered for the session. Raw process lists, application matches, IP and Wi-Fi configuration, users, routes, performance -metrics, installed-application reports, desktop window listings, OCR results, control inspection, -and clipboard reads default to local-only output. With `Y`, Interly sends Groq only a short -completion status. With `A`, the user explicitly authorizes that one output to be included in the -Groq conversation. +metrics, installed-application reports, structured document contents, desktop window listings, OCR +results, control inspection, and clipboard reads default to local-only output. With `Y`, Interly +sends Groq only a short completion status. With `A`, the user explicitly authorizes that one output +to be included in the Groq conversation. `set-free 1` through `set-free 30` temporarily removes repeated prompts for ordinary actions. `set-free 0` disables the window immediately. Emergency stop remains active, sensitive local output still stays local unless explicitly shared with `A`, and destructive Windows actions still require individual confirmation. +## Structured document model + +Structured document reads reuse the same permission path as ordinary approved file reads. Interly +opens supported documents read-only and converts their useful structure into bounded local output. +PDF reads expose pages, extracted text, heuristic headings, tables, and metadata. Word reads preserve +ordered headings, paragraphs, and tables. Excel reads expose sheets, used ranges, headers, rows, and +formula cells. PowerPoint reads expose slides, titles, text blocks, tables, speaker notes, and core +metadata. + +The reader limits input size, Office package entry count and uncompressed size, page/slide/sheet and +table dimensions, and total extracted text. Corrupt or invalid Office Open XML packages fail without +being parsed. Document contents remain terminal-only unless the user explicitly chooses `A` at the +sensitive read approval prompt. + ## Desktop interaction model Interly is instructed to inspect before acting. For window operations it first resolves exact native @@ -173,16 +190,17 @@ other sensitive machine inspection tools. ## Important limitations -Interly is alpha software. Model responses can be wrong, and read-only system or desktop output may -contain private information. Review proposed actions. Forced process termination can lose unsaved -work. Generic mouse and keyboard actions affect the currently visible/focused desktop state. Web -searches and selected page text are sent to external services and Groq. +Interly is alpha software. Model responses can be wrong, and read-only system, document, or desktop +output may contain private information. Review proposed actions. Forced process termination can lose +unsaved work. Generic mouse and keyboard actions affect the currently visible/focused desktop state. +Web searches and selected page text are sent to external services and Groq. Personal-browser access, file deletion, webpage video extraction, streaming-platform downloads, -structured PDF/Word/Excel/PowerPoint parsing, uploads, logins, purchases, messaging, volume control, -brightness control, and speech input/output are not implemented. Direct downloads currently require -a public URL that returns the file itself. The emergency stop prevents additional actions, but an -operating-system call that has already completed cannot be reversed. +uploads, logins, purchases, messaging, volume control, brightness control, and speech input/output +are not implemented. Structured document understanding is read-only and currently supports modern +Open XML Office formats rather than legacy `.doc`, `.xls`, or `.ppt` files. Direct downloads currently +require a public URL that returns the file itself. The emergency stop prevents additional actions, +but an operating-system call that has already completed cannot be reversed. ## Development diff --git a/ROADMAP.md b/ROADMAP.md index 3b50eaf..a3b7e35 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -6,7 +6,7 @@ Status rule: - ✅ = the complete item is implemented and verified. - ❌ = the item is not fully complete. Partial implementation still counts as ❌. -Current verified status: **88/101 complete**. +Current verified status: **89/101 complete**. ## 1. Core agent foundation @@ -94,7 +94,7 @@ Current verified status: **88/101 complete**. 64. ✅ Preview and apply exact text edits to existing files. 65. ✅ Create, copy, move, and rename files or folders. 66. ❌ Add file deletion with Windows Recycle Bin support. -67. ❌ Add structured PDF, Word, Excel, and PowerPoint understanding. +67. ✅ Add structured PDF, Word, Excel, and PowerPoint understanding. 68. ✅ Compare two approved text files and return their differences. 69. ✅ Download direct public file URLs with type, size, hash, and destination approval. 70. ❌ Add malware scanning and quarantine checks for downloads. diff --git a/interly-priority-ownership-plan.md b/interly-priority-ownership-plan.md index d9b0498..07ab31a 100644 --- a/interly-priority-ownership-plan.md +++ b/interly-priority-ownership-plan.md @@ -75,27 +75,33 @@ Interly can inspect the visible Windows desktop, capture it, extract text throug # Priority 3 — Documents & Safe Filesystem Completion **Owner: Joint / Convergence Point** -**Status: Active — both streams meet here** +**Status: Active — structured documents complete; filesystem safety remains** This is the current shared workstream after the parallel implementation passes. Goal: complete Interly's existing filesystem layer and expand it from text-file access into structured document work. -1. [ ] **#67 — Structured PDF, Word, Excel and PowerPoint understanding** +1. [x] **#67 — Structured PDF, Word, Excel and PowerPoint understanding** + - PDF pages, text, heuristic headings, tables and metadata + - Word ordered headings, paragraphs, tables and metadata + - Excel sheets, used ranges, headers, rows and formulas + - PowerPoint slides, titles, text blocks, tables, speaker notes and metadata + - read-only and local-only by default through the existing sensitive file-read approval path + - verified in the packaged Windows executable 2. [ ] **#66 — File deletion through the Windows Recycle Bin** 3. [ ] **#70 — Malware scanning and quarantine checks for downloads** -### Intended file flow +### Current file flow -`find → read → understand → compare → create/edit → move/copy → delete safely` +`find → read → understand ✅ → compare → create/edit → move/copy → delete safely` -### Intended download flow +### Remaining download flow `download → inspect → scan → accept / quarantine` -### Completion outcome +### Current outcome -Interly can work with common structured documents, complete ordinary filesystem management safely, and treat downloaded files as a security-sensitive pipeline. +Interly can now understand common structured documents instead of treating them as opaque files. Priority 3 remains active until ordinary file deletion uses the Windows Recycle Bin and downloaded files have a malware scanning/quarantine path. --- @@ -165,7 +171,7 @@ Goal: turn Interly's existing machine and file capabilities into a coherent soft **Status: After Priority 3 convergence** 1. [ ] **#78 — Volume and mute control** -2. [ ] **#79 — Brightness control** +2. [ ] **#79 — Brightness control where supported** 3. [ ] **#80 — Speech input** 4. [ ] **#81 — Speech output and optional wake phrase** @@ -194,7 +200,7 @@ This comes after the major capability and security architecture stabilises so th |---|---|---|---| | 1 | Execution Governance | **ChatGPT** | Complete ✅ | | 2 | Desktop Perception & Interaction | **ChatGPT** | Complete ✅ | -| 3 | Documents & Safe Filesystem Completion | **Joint** | **Active convergence point** | +| 3 | Documents & Safe Filesystem Completion | **Joint** | **Active — #67 complete; #66/#70 open** | | 4 | Persistent Memory & Reusable Work | **User** | Checkpoint complete; #88 open | | 5 | Developer Agent | **User** | Checkpoint complete; #91 and #98 open | | 6 | Release & Supply-Chain Security | Deferred | After Priority 3 | @@ -210,7 +216,7 @@ Completed parallel passes: Current shared position: -**Both:** `→ Priority 3` +**Both:** `→ Priority 3 (#67 ✅; #66/#70 next)` --- diff --git a/packaging/interlink.spec b/packaging/interlink.spec index d8dc1ce..81168bd 100644 --- a/packaging/interlink.spec +++ b/packaging/interlink.spec @@ -7,6 +7,10 @@ rapidocr_data, rapidocr_binaries, rapidocr_hidden = collect_all("rapidocr") onnx_data, onnx_binaries, onnx_hidden = collect_all("onnxruntime") pillow_data, pillow_binaries, pillow_hidden = collect_all("PIL") uia_data, uia_binaries, uia_hidden = collect_all("uiautomation") +pdf_data, pdf_binaries, pdf_hidden = collect_all("pdfplumber") +docx_data, docx_binaries, docx_hidden = collect_all("docx") +excel_data, excel_binaries, excel_hidden = collect_all("openpyxl") +pptx_data, pptx_binaries, pptx_hidden = collect_all("pptx") a = Analysis( ["entrypoint.py"], @@ -18,6 +22,10 @@ a = Analysis( + onnx_binaries + pillow_binaries + uia_binaries + + pdf_binaries + + docx_binaries + + excel_binaries + + pptx_binaries ), datas=( playwright_data @@ -26,7 +34,15 @@ a = Analysis( + onnx_data + pillow_data + uia_data + + pdf_data + + docx_data + + excel_data + + pptx_data + copy_metadata("interly") + + copy_metadata("pdfplumber") + + copy_metadata("python-docx") + + copy_metadata("openpyxl") + + copy_metadata("python-pptx") ), hiddenimports=( playwright_hidden @@ -35,6 +51,10 @@ a = Analysis( + onnx_hidden + pillow_hidden + uia_hidden + + pdf_hidden + + docx_hidden + + excel_hidden + + pptx_hidden ), noarchive=False, ) diff --git a/pyproject.toml b/pyproject.toml index c4d4058..910f5c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,12 +20,16 @@ dependencies = [ "beautifulsoup4==4.15.0", "groq==1.6.0", "onnxruntime==1.27.0", + "openpyxl==3.1.5", + "pdfplumber==0.11.7", "pillow==12.3.0", "playwright==1.62.0", "psutil==7.2.2", "pynput==1.8.2", "pydantic-settings==2.14.2", + "python-docx==1.2.0", "python-dotenv==1.2.2", + "python-pptx==1.0.2", "rapidocr==3.9.2", "uiautomation==2.0.29", ] diff --git a/requirements.txt b/requirements.txt index 97e556e..d77cb17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,16 @@ beautifulsoup4==4.15.0 groq==1.6.0 onnxruntime==1.27.0 +openpyxl==3.1.5 +pdfplumber==0.11.7 pillow==12.3.0 playwright==1.62.0 psutil==7.2.2 pynput==1.8.2 pydantic-settings==2.14.2 +python-docx==1.2.0 python-dotenv==1.2.2 +python-pptx==1.0.2 rapidocr==3.9.2 uiautomation==2.0.29 # Development and verification diff --git a/src/computer_agent/__main__.py b/src/computer_agent/__main__.py index 91918fe..105f3f9 100644 --- a/src/computer_agent/__main__.py +++ b/src/computer_agent/__main__.py @@ -10,6 +10,7 @@ from computer_agent.browser import BROWSER from computer_agent.chat import run_chat from computer_agent.config import load_settings, save_api_key +from computer_agent.documents import document_support_status from computer_agent.emergency import EmergencyStop from computer_agent.models import GroqModel from computer_agent.updater import update_interly @@ -49,6 +50,9 @@ def main() -> None: if "--version" in sys.argv[1:]: print(f"Interly {__version__}") return + if "--check-document-support" in sys.argv[1:]: + print(document_support_status()) + return settings = load_settings() if not settings.groq_api_key: @@ -69,6 +73,7 @@ def reconfigure_groq() -> bool: return False model.update_api_key(api_key) return True + emergency_stop = EmergencyStop() if emergency_stop.start(): print("Emergency stop: press Esc to cancel the current request.") diff --git a/src/computer_agent/documents.py b/src/computer_agent/documents.py new file mode 100644 index 0000000..49f6743 --- /dev/null +++ b/src/computer_agent/documents.py @@ -0,0 +1,502 @@ +"""Read PDF and Office documents into one bounded, read-only structure.""" + +from __future__ import annotations + +import json +import re +import zipfile +from dataclasses import dataclass +from datetime import date, datetime, time +from pathlib import Path +from statistics import median +from typing import Any + +import pdfplumber +from docx import Document as WordDocument +from docx.oxml.table import CT_Tbl +from docx.oxml.text.paragraph import CT_P +from docx.table import Table +from docx.text.paragraph import Paragraph +from openpyxl import load_workbook +from openpyxl.utils import get_column_letter +from pptx import Presentation + +SUPPORTED_EXTENSIONS = { + ".pdf": "pdf", + ".docx": "word", + ".xlsx": "excel", + ".xlsm": "excel", + ".pptx": "powerpoint", +} +MAX_DOCUMENT_BYTES = 50 * 1024 * 1024 +MAX_OFFICE_UNCOMPRESSED_BYTES = 250 * 1024 * 1024 +MAX_OFFICE_ZIP_ENTRIES = 10_000 +MAX_TOTAL_TEXT_CHARS = 90_000 +MAX_TEXT_ITEM_CHARS = 8_000 +MAX_PAGES = 200 +MAX_SLIDES = 300 +MAX_SHEETS = 50 +MAX_TABLE_ROWS = 200 +MAX_TABLE_COLUMNS = 50 +MAX_BLOCKS = 2_000 + + +@dataclass +class OutputBudget: + """Bound document output before it is exposed to the terminal or model.""" + + remaining_chars: int = MAX_TOTAL_TEXT_CHARS + truncated: bool = False + + def take(self, value: Any, limit: int = MAX_TEXT_ITEM_CHARS) -> str: + text = "" if value is None else str(value) + if not text: + return "" + allowed = min(limit, self.remaining_chars) + if allowed <= 0: + self.truncated = True + return "" + if len(text) > allowed: + self.truncated = True + suffix = "\n[truncated]" + if allowed > len(suffix): + text = text[: allowed - len(suffix)] + suffix + else: + text = text[:allowed] + self.remaining_chars -= len(text) + return text + + +def document_support_status() -> str: + """Return a frozen-build smoke-test string after all document backends import.""" + backends = [ + pdfplumber.__name__, + WordDocument.__module__, + load_workbook.__module__, + Presentation.__module__, + ] + if not all(backends): + raise RuntimeError("One or more document backends failed to import.") + return "Document support: PDF DOCX XLSX XLSM PPTX" + + +def read_structured_document(path: str) -> str: + """Read one supported document into a common bounded JSON representation.""" + target = Path(path).expanduser() + if not target.is_file(): + return f"Document not found: {target}" + suffix = target.suffix.casefold() + kind = SUPPORTED_EXTENSIONS.get(suffix) + if kind is None: + supported = ", ".join(sorted(SUPPORTED_EXTENSIONS)) + return f"Unsupported document type {suffix or '(no extension)'}. Supported: {supported}" + try: + size = target.stat().st_size + except OSError as error: + return f"Could not inspect document: {error}" + if size > MAX_DOCUMENT_BYTES: + return ( + f"Document is too large ({size} bytes). " + f"Interly limits structured document reads to {MAX_DOCUMENT_BYTES} bytes." + ) + if suffix in {".docx", ".xlsx", ".xlsm", ".pptx"}: + validation_error = _validate_office_container(target) + if validation_error: + return validation_error + + budget = OutputBudget() + try: + if kind == "pdf": + result = _read_pdf(target, budget) + elif kind == "word": + result = _read_word(target, budget) + elif kind == "excel": + result = _read_excel(target, budget) + else: + result = _read_powerpoint(target, budget) + except (OSError, ValueError, KeyError, TypeError, zipfile.BadZipFile) as error: + return f"Structured document read failed safely: {error}" + + result["path"] = str(target.resolve()) + result["size_bytes"] = size + result["read_only"] = True + result["truncated"] = bool(result.get("truncated") or budget.truncated) + result["limits"] = { + "max_document_bytes": MAX_DOCUMENT_BYTES, + "max_total_text_chars": MAX_TOTAL_TEXT_CHARS, + "max_table_rows": MAX_TABLE_ROWS, + "max_table_columns": MAX_TABLE_COLUMNS, + } + return json.dumps(result, indent=2, ensure_ascii=False, default=_json_default) + + +def _validate_office_container(path: Path) -> str | None: + try: + with zipfile.ZipFile(path) as archive: + members = archive.infolist() + except zipfile.BadZipFile: + return "Office document is encrypted, corrupt, or not a valid Open XML package." + if len(members) > MAX_OFFICE_ZIP_ENTRIES: + return "Office document contains too many package entries; nothing was read." + uncompressed = sum(member.file_size for member in members) + if uncompressed > MAX_OFFICE_UNCOMPRESSED_BYTES: + return ( + "Office document expands beyond Interly's safe read limit " + f"({uncompressed} bytes uncompressed); nothing was read." + ) + return None + + +def _read_pdf(path: Path, budget: OutputBudget) -> dict[str, Any]: + pages: list[dict[str, Any]] = [] + with pdfplumber.open(path) as document: + metadata = { + str(key): budget.take(value, 1_000) + for key, value in (document.metadata or {}).items() + if value not in {None, ""} + } + page_count = len(document.pages) + for index, page in enumerate(document.pages[:MAX_PAGES], start=1): + page_text = page.extract_text() or "" + paragraphs = [ + budget.take(block.strip()) + for block in re.split(r"\n\s*\n", page_text) + if block.strip() + ] + if len(paragraphs) <= 1 and page_text: + paragraphs = [ + budget.take(line.strip()) + for line in page_text.splitlines() + if line.strip() + ] + headings = _pdf_headings(page, budget) + tables = _bounded_tables(page.extract_tables() or [], budget) + pages.append( + { + "kind": "page", + "index": index, + "width": round(float(page.width), 2), + "height": round(float(page.height), 2), + "headings": headings, + "paragraphs": paragraphs[:MAX_BLOCKS], + "tables": tables, + } + ) + if budget.remaining_chars <= 0: + break + return { + "document_type": "pdf", + "metadata": metadata, + "page_count": page_count, + "pages": pages, + "truncated": page_count > len(pages), + } + + +def _pdf_headings(page: Any, budget: OutputBudget) -> list[dict[str, Any]]: + words = page.extract_words(extra_attrs=["size"], use_text_flow=True) or [] + sizes = [float(word.get("size", 0) or 0) for word in words if word.get("size")] + if not sizes: + return [] + body_size = median(sizes) + lines: list[list[dict[str, Any]]] = [] + sorted_words = sorted( + words, + key=lambda item: (float(item.get("top", 0)), float(item.get("x0", 0))), + ) + for word in sorted_words: + top = float(word.get("top", 0)) + if not lines or abs(top - float(lines[-1][0].get("top", 0))) > 3: + lines.append([word]) + else: + lines[-1].append(word) + headings: list[dict[str, Any]] = [] + for line in lines: + text = " ".join(str(word.get("text", "")) for word in line).strip() + line_size = max(float(word.get("size", 0) or 0) for word in line) + if text and len(text) <= 180 and line_size >= max(body_size + 1.0, body_size * 1.12): + headings.append( + { + "text": budget.take(text, 500), + "font_size": round(line_size, 2), + "top": round(float(line[0].get("top", 0)), 2), + } + ) + if len(headings) >= 100 or budget.remaining_chars <= 0: + break + return headings + + +def _read_word(path: Path, budget: OutputBudget) -> dict[str, Any]: + document = WordDocument(path) + blocks: list[dict[str, Any]] = [] + for child in document.element.body.iterchildren(): + if len(blocks) >= MAX_BLOCKS or budget.remaining_chars <= 0: + budget.truncated = True + break + if isinstance(child, CT_P): + paragraph = Paragraph(child, document) + text = paragraph.text.strip() + if not text: + continue + style = paragraph.style.name if paragraph.style is not None else "" + heading_match = re.match(r"Heading\s+(\d+)", style or "", re.IGNORECASE) + if heading_match: + blocks.append( + { + "kind": "heading", + "level": int(heading_match.group(1)), + "text": budget.take(text), + } + ) + else: + blocks.append( + {"kind": "paragraph", "style": style or None, "text": budget.take(text)} + ) + elif isinstance(child, CT_Tbl): + table = Table(child, document) + table_rows = list(table.rows) + rows = [] + for row in table_rows[:MAX_TABLE_ROWS]: + cells = list(row.cells) + rows.append( + [ + budget.take(cell.text.strip(), 2_000) + for cell in cells[:MAX_TABLE_COLUMNS] + ] + ) + blocks.append( + { + "kind": "table", + "rows": rows, + "row_count": len(table_rows), + "column_count": max( + (len(list(row.cells)) for row in table_rows), + default=0, + ), + "truncated": len(table_rows) > MAX_TABLE_ROWS + or any(len(list(row.cells)) > MAX_TABLE_COLUMNS for row in table_rows), + } + ) + core = document.core_properties + metadata = { + "title": budget.take(core.title, 1_000), + "subject": budget.take(core.subject, 1_000), + "author": budget.take(core.author, 1_000), + "keywords": budget.take(core.keywords, 1_000), + "comments": budget.take(core.comments, 2_000), + "created": _json_default(core.created) if core.created else None, + "modified": _json_default(core.modified) if core.modified else None, + } + return { + "document_type": "word", + "metadata": { + key: value for key, value in metadata.items() if value not in {None, ""} + }, + "blocks": blocks, + "paragraph_count": len(document.paragraphs), + "table_count": len(document.tables), + "truncated": budget.truncated, + } + + +def _read_excel(path: Path, budget: OutputBudget) -> dict[str, Any]: + workbook = load_workbook(path, read_only=True, data_only=False, keep_links=False) + sheets: list[dict[str, Any]] = [] + try: + sheet_names = workbook.sheetnames + for sheet_name in sheet_names[:MAX_SHEETS]: + if budget.remaining_chars <= 0: + budget.truncated = True + break + sheet = workbook[sheet_name] + captured_rows: list[list[Any]] = [] + formula_cells: list[dict[str, Any]] = [] + header_row: list[Any] | None = None + row_count = 0 + max_columns_seen = 0 + for row in sheet.iter_rows(): + row_count += 1 + values: list[Any] = [] + for cell in row[:MAX_TABLE_COLUMNS]: + value = _excel_value(cell.value, budget) + values.append(value) + if ( + isinstance(cell.value, str) + and cell.value.startswith("=") + and len(formula_cells) < 500 + ): + formula_cells.append( + { + "cell": cell.coordinate, + "formula": budget.take(cell.value, 2_000), + } + ) + max_columns_seen = max(max_columns_seen, len(row)) + if header_row is None and any(value not in {None, ""} for value in values): + header_row = values + if len(captured_rows) < MAX_TABLE_ROWS: + captured_rows.append(values) + else: + budget.truncated = True + if row_count >= 5_000: + budget.truncated = True + break + if budget.remaining_chars <= 0: + break + max_row = int(getattr(sheet, "max_row", row_count) or row_count) + max_column = int(getattr(sheet, "max_column", max_columns_seen) or max_columns_seen) + dimension = ( + f"A1:{get_column_letter(max(1, max_column))}{max(1, max_row)}" + if max_row or max_column + else "A1" + ) + sheets.append( + { + "kind": "sheet", + "name": budget.take(sheet_name, 500), + "state": str(getattr(sheet, "sheet_state", "visible")), + "range": dimension, + "row_count": max_row, + "column_count": max_column, + "headers": header_row or [], + "rows": captured_rows, + "formula_cells": formula_cells, + "truncated": max_row > len(captured_rows) + or max_column > MAX_TABLE_COLUMNS, + } + ) + finally: + workbook.close() + return { + "document_type": "excel", + "sheet_count": len(sheet_names), + "sheets": sheets, + "truncated": len(sheet_names) > len(sheets) or budget.truncated, + } + + +def _read_powerpoint(path: Path, budget: OutputBudget) -> dict[str, Any]: + presentation = Presentation(path) + slides: list[dict[str, Any]] = [] + slide_list = list(presentation.slides) + for index, slide in enumerate(slide_list[:MAX_SLIDES], start=1): + if budget.remaining_chars <= 0: + budget.truncated = True + break + title_shape = slide.shapes.title + title = ( + budget.take(title_shape.text.strip(), 1_000) + if title_shape is not None and title_shape.text + else "" + ) + blocks: list[dict[str, Any]] = [] + tables: list[dict[str, Any]] = [] + for shape in slide.shapes: + if getattr(shape, "has_table", False): + table_rows = list(shape.table.rows) + rows = [] + for row in table_rows[:MAX_TABLE_ROWS]: + cells = list(row.cells) + rows.append( + [ + budget.take(cell.text.strip(), 2_000) + for cell in cells[:MAX_TABLE_COLUMNS] + ] + ) + tables.append( + { + "rows": rows, + "row_count": len(table_rows), + "column_count": len(shape.table.columns), + "truncated": len(table_rows) > MAX_TABLE_ROWS + or len(shape.table.columns) > MAX_TABLE_COLUMNS, + } + ) + continue + if not getattr(shape, "has_text_frame", False): + continue + if title_shape is not None and shape == title_shape: + continue + paragraphs = [] + for paragraph in shape.text_frame.paragraphs: + text = paragraph.text.strip() + if text: + paragraphs.append( + {"level": int(paragraph.level), "text": budget.take(text, 4_000)} + ) + if paragraphs: + blocks.append({"kind": "text", "paragraphs": paragraphs}) + notes = "" + if slide.has_notes_slide: + notes_frame = getattr(slide.notes_slide, "notes_text_frame", None) + if notes_frame is not None: + notes = budget.take((notes_frame.text or "").strip(), 8_000) + slides.append( + { + "kind": "slide", + "index": index, + "title": title or None, + "blocks": blocks, + "tables": tables, + "speaker_notes": notes or None, + } + ) + core = presentation.core_properties + metadata = { + "title": budget.take(core.title, 1_000), + "subject": budget.take(core.subject, 1_000), + "author": budget.take(core.author, 1_000), + "keywords": budget.take(core.keywords, 1_000), + "comments": budget.take(core.comments, 2_000), + "created": _json_default(core.created) if core.created else None, + "modified": _json_default(core.modified) if core.modified else None, + } + return { + "document_type": "powerpoint", + "metadata": { + key: value for key, value in metadata.items() if value not in {None, ""} + }, + "slide_count": len(slide_list), + "slides": slides, + "truncated": len(slide_list) > len(slides) or budget.truncated, + } + + +def _bounded_tables(tables: list[Any], budget: OutputBudget) -> list[dict[str, Any]]: + output = [] + for table in tables[:50]: + rows = [] + for row in table[:MAX_TABLE_ROWS]: + rows.append( + [ + budget.take(cell, 2_000) if cell is not None else None + for cell in row[:MAX_TABLE_COLUMNS] + ] + ) + output.append( + { + "rows": rows, + "row_count": len(table), + "column_count": max((len(row) for row in table), default=0), + "truncated": len(table) > MAX_TABLE_ROWS + or any(len(row) > MAX_TABLE_COLUMNS for row in table), + } + ) + if budget.remaining_chars <= 0: + break + return output + + +def _excel_value(value: Any, budget: OutputBudget) -> Any: + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, (datetime, date, time)): + return value.isoformat() + return budget.take(value, 4_000) + + +def _json_default(value: Any) -> str: + if isinstance(value, (datetime, date, time)): + return value.isoformat() + return str(value) diff --git a/src/computer_agent/runtime_tools.py b/src/computer_agent/runtime_tools.py index 0507c15..8ad4172 100644 --- a/src/computer_agent/runtime_tools.py +++ b/src/computer_agent/runtime_tools.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from pathlib import Path from typing import Any from computer_agent.desktop import ( @@ -16,6 +17,7 @@ window_action, write_clipboard, ) +from computer_agent.documents import SUPPORTED_EXTENSIONS, read_structured_document from computer_agent.tools import TOOL_SCHEMAS as BASE_TOOL_SCHEMAS from computer_agent.tools import LocalOnlyResult from computer_agent.tools import describe_tool as describe_base_tool @@ -33,7 +35,28 @@ def _function(name: str, description: str, parameters: dict[str, Any]) -> dict[s } +BASE_RUNTIME_TOOL_SCHEMAS = [ + schema + for schema in BASE_TOOL_SCHEMAS + if schema.get("function", {}).get("name") != "read_text_file" +] + ADDITIONAL_TOOL_SCHEMAS: list[dict[str, Any]] = [ + _function( + "read_text_file", + ( + "Read one approved local file. Plain text and source files use the bounded text reader. " + "PDF, Word (.docx), Excel (.xlsx/.xlsm), and PowerPoint (.pptx) files are parsed " + "read-only into structured pages/blocks/sheets/slides, headings, tables, formulas, " + "and metadata where available." + ), + { + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + "additionalProperties": False, + }, + ), _function( "propose_plan", ( @@ -200,7 +223,7 @@ def _function(name: str, description: str, parameters: dict[str, Any]) -> dict[s ), ] -TOOL_SCHEMAS = [*BASE_TOOL_SCHEMAS, *ADDITIONAL_TOOL_SCHEMAS] +TOOL_SCHEMAS = [*BASE_RUNTIME_TOOL_SCHEMAS, *ADDITIONAL_TOOL_SCHEMAS] TOOL_NAMES = {schema["function"]["name"] for schema in TOOL_SCHEMAS} DESKTOP_TOOL_NAMES = { "desktop_list_windows", @@ -215,15 +238,29 @@ def _function(name: str, description: str, parameters: dict[str, Any]) -> dict[s } +def _is_structured_document(path: str) -> bool: + return Path(path).suffix.casefold() in SUPPORTED_EXTENSIONS + + def describe_tool(name: str, arguments: str) -> tuple[str, str, str | None]: """Describe a runtime tool for the host approval UI.""" - if name not in DESKTOP_TOOL_NAMES and name != "propose_plan": - return describe_base_tool(name, arguments) try: parsed = json.loads(arguments or "{}") except json.JSONDecodeError: parsed = {} + if name == "read_text_file" and _is_structured_document(str(parsed.get("path", ""))): + return ( + f"Read structured document: {parsed.get('path', '')}", + "Extract bounded document structure and content without editing the source file", + ( + "Choose Y to keep document content terminal-only, or A to explicitly allow " + "Groq access. Office package expansion and output are bounded before parsing." + ), + ) + if name not in DESKTOP_TOOL_NAMES and name != "propose_plan": + return describe_base_tool(name, arguments) + if name == "propose_plan": return ("Present a multi-step execution plan", "Request one-request scoped approval", None) if name == "desktop_list_windows": @@ -310,6 +347,23 @@ def describe_tool(name: str, arguments: str) -> tuple[str, str, str | None]: def execute_tool(name: str, arguments: str = "{}") -> str | LocalOnlyResult: """Execute a runtime tool after host-side governance has approved it.""" + if name == "read_text_file": + try: + parsed = json.loads(arguments or "{}") + except json.JSONDecodeError: + return "Tool arguments were invalid JSON; nothing was executed." + path = str(parsed.get("path", "")) + if _is_structured_document(path): + return LocalOnlyResult( + ( + "The approved structured document was read and its extracted content was " + "displayed only in the user's terminal. Do not infer its contents unless " + "the user explicitly allows Groq access." + ), + read_structured_document(path), + ) + return execute_base_tool(name, arguments) + if name not in DESKTOP_TOOL_NAMES: return execute_base_tool(name, arguments) try: diff --git a/tests/test_documents.py b/tests/test_documents.py new file mode 100644 index 0000000..c6f8fbb --- /dev/null +++ b/tests/test_documents.py @@ -0,0 +1,175 @@ +import json +from pathlib import Path + +from docx import Document +from openpyxl import Workbook +from pptx import Presentation +from pptx.util import Inches + +from computer_agent.documents import document_support_status, read_structured_document +from computer_agent.runtime_tools import TOOL_SCHEMAS, LocalOnlyResult, execute_tool + + +def _write_test_pdf(path: Path) -> None: + stream = b"\n".join( + [ + b"BT /F1 18 Tf 72 720 Td (Quarterly Report) Tj ET", + b"BT /F1 11 Tf 72 690 Td (Revenue increased this quarter.) Tj ET", + b"0.5 w", + b"72 640 m 300 640 l S", + b"72 620 m 300 620 l S", + b"72 600 m 300 600 l S", + b"72 600 m 72 640 l S", + b"180 600 m 180 640 l S", + b"300 600 m 300 640 l S", + b"BT /F1 10 Tf 80 627 Td (Item) Tj ET", + b"BT /F1 10 Tf 188 627 Td (Value) Tj ET", + b"BT /F1 10 Tf 80 607 Td (Revenue) Tj ET", + b"BT /F1 10 Tf 188 607 Td (120) Tj ET", + ] + ) + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>" + ), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" + stream + b"\nendstream", + ] + + pdf = bytearray(b"%PDF-1.4\n") + offsets = [0] + for index, obj in enumerate(objects, start=1): + offsets.append(len(pdf)) + pdf.extend(f"{index} 0 obj\n".encode()) + pdf.extend(obj) + pdf.extend(b"\nendobj\n") + xref = len(pdf) + pdf.extend(f"xref\n0 {len(objects) + 1}\n".encode()) + pdf.extend(b"0000000000 65535 f \n") + for offset in offsets[1:]: + pdf.extend(f"{offset:010d} 00000 n \n".encode()) + pdf.extend( + ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" + f"startxref\n{xref}\n%%EOF\n" + ).encode() + ) + path.write_bytes(pdf) + + +def test_document_support_status_lists_all_backends() -> None: + assert document_support_status() == "Document support: PDF DOCX XLSX XLSM PPTX" + + +def test_pdf_reader_extracts_pages_headings_and_tables(tmp_path: Path) -> None: + path = tmp_path / "report.pdf" + _write_test_pdf(path) + + result = json.loads(read_structured_document(str(path))) + + assert result["document_type"] == "pdf" + assert result["page_count"] == 1 + page = result["pages"][0] + assert any("Quarterly Report" in text for text in page["paragraphs"]) + assert any(item["text"] == "Quarterly Report" for item in page["headings"]) + assert "tables" in page + assert result["read_only"] is True + + +def test_word_reader_preserves_heading_paragraph_and_table_order(tmp_path: Path) -> None: + path = tmp_path / "brief.docx" + document = Document() + document.core_properties.title = "Client Brief" + document.add_heading("Scope", level=1) + document.add_paragraph("Build the document layer.") + table = document.add_table(rows=2, cols=2) + table.cell(0, 0).text = "Owner" + table.cell(0, 1).text = "Status" + table.cell(1, 0).text = "Interly" + table.cell(1, 1).text = "Active" + document.save(path) + + result = json.loads(read_structured_document(str(path))) + + assert result["document_type"] == "word" + assert result["metadata"]["title"] == "Client Brief" + assert result["blocks"][0] == {"kind": "heading", "level": 1, "text": "Scope"} + assert result["blocks"][1]["text"] == "Build the document layer." + table_block = next(block for block in result["blocks"] if block["kind"] == "table") + assert table_block["rows"][1] == ["Interly", "Active"] + + +def test_excel_reader_exposes_sheet_headers_range_and_formulas(tmp_path: Path) -> None: + path = tmp_path / "sales.xlsx" + workbook = Workbook() + sheet = workbook.active + sheet.title = "Sales" + sheet.append(["Item", "Qty", "Total"]) + sheet.append(["Widget", 2, "=B2*10"]) + workbook.save(path) + + result = json.loads(read_structured_document(str(path))) + + assert result["document_type"] == "excel" + assert result["sheet_count"] == 1 + parsed_sheet = result["sheets"][0] + assert parsed_sheet["name"] == "Sales" + assert parsed_sheet["range"] == "A1:C2" + assert parsed_sheet["headers"] == ["Item", "Qty", "Total"] + assert parsed_sheet["formula_cells"] == [{"cell": "C2", "formula": "=B2*10"}] + + +def test_powerpoint_reader_exposes_slide_text_and_tables(tmp_path: Path) -> None: + path = tmp_path / "roadmap.pptx" + presentation = Presentation() + slide = presentation.slides.add_slide(presentation.slide_layouts[5]) + slide.shapes.title.text = "Priority 3" + textbox = slide.shapes.add_textbox(Inches(1), Inches(2), Inches(6), Inches(1)) + textbox.text_frame.text = "Documents become understandable." + table_shape = slide.shapes.add_table(2, 2, Inches(1), Inches(3), Inches(6), Inches(1.5)) + table_shape.table.cell(0, 0).text = "Format" + table_shape.table.cell(0, 1).text = "State" + table_shape.table.cell(1, 0).text = "PDF" + table_shape.table.cell(1, 1).text = "Ready" + presentation.save(path) + + result = json.loads(read_structured_document(str(path))) + + assert result["document_type"] == "powerpoint" + assert result["slide_count"] == 1 + parsed_slide = result["slides"][0] + assert parsed_slide["title"] == "Priority 3" + assert parsed_slide["blocks"][0]["paragraphs"][0]["text"] == "Documents become understandable." + assert parsed_slide["tables"][0]["rows"][1] == ["PDF", "Ready"] + + +def test_structured_read_reuses_sensitive_read_text_file_tool(tmp_path: Path) -> None: + path = tmp_path / "memory.xlsx" + workbook = Workbook() + workbook.active["A1"] = "private value" + workbook.save(path) + + result = execute_tool("read_text_file", json.dumps({"path": str(path)})) + + assert isinstance(result, LocalOnlyResult) + assert "private value" in result.terminal_output + assert "private value" not in result.model_status + schema = next( + item["function"] + for item in TOOL_SCHEMAS + if item["function"]["name"] == "read_text_file" + ) + assert "PDF" in schema["description"] + assert "PowerPoint" in schema["description"] + + +def test_corrupt_office_package_fails_without_parsing(tmp_path: Path) -> None: + path = tmp_path / "bad.docx" + path.write_text("not an Office package", encoding="utf-8") + + result = read_structured_document(str(path)) + + assert "encrypted, corrupt, or not a valid Open XML package" in result