diff --git a/src/scrapers/codex.ts b/src/scrapers/codex.ts index fda0c34..6e1131f 100644 --- a/src/scrapers/codex.ts +++ b/src/scrapers/codex.ts @@ -606,9 +606,22 @@ export function isKnownBulkyRecord(line: string): boolean { * still consumes an index so chunk identity stays stable between a full and * an incremental scrape; anything else is the message. * - * `response_item` events with role "user" are deliberately not handled here: - * they are system-injected context (AGENTS.md, permissions, environment), - * not something the person typed. + * Two shapes, because Codex changed one and kept writing the other for a + * while. The original is `payload.type === "user_message"` with the text on + * `payload.message`. The current one wraps it: `payload.type === + * "item_completed"` with `payload.item.type === "UserMessage"`. + * + * Reading only the original meant every human turn in a current transcript + * was dropped, silently — `event_msg` is a known type and `payload.type` was + * present, so the drift guard below had nothing to complain about. Measured + * across a real store of 825 session files: 41 turns in the old shape, 15,169 + * in the new one. A scrape of that history returned the assistant side of + * every Codex conversation and almost none of what the person asked for. + * + * `response_item` events with role "user" are still deliberately not handled + * here: they are system-injected context (AGENTS.md, permissions, + * environment), not something the person typed, and they outnumber the real + * turns in a file that carries both. */ function userMessageContent(payload: unknown, filePath: string): string | undefined { if (!isRecord(payload)) { @@ -616,16 +629,37 @@ function userMessageContent(payload: unknown, filePath: string): string | undefi return undefined; } - if (payload.type !== "user_message") { - // A payload with no `type` at all is drift worth seeing; a payload with a - // different one is just an event this scraper does not read. - if (!("type" in payload)) { - warnDrift(filePath, "event_msg payload missing 'type' key — likely renamed"); + if (payload.type === "user_message") { + return toStringValue(payload.message) ?? ""; + } + + if (payload.type === "item_completed") { + const item = payload.item; + if (!isRecord(item) || item.type !== "UserMessage") { + // Every other item type is an event this scraper reads elsewhere or + // does not read at all; neither is drift. + return undefined; } - return undefined; + // The text sits in a parts array, the way the assistant side does: + // `content: [{ type: "text", text: "..." }]`. A flat `text` is accepted + // too, so an older or simpler record still reads. + if (Array.isArray(item.content)) { + const text = item.content + .filter((part): part is Record => isRecord(part)) + .map((part) => toStringValue(part.text) ?? "") + .filter((part) => part.length > 0) + .join("\n"); + return text; + } + return toStringValue(item.text) ?? ""; } - return toStringValue(payload.message) ?? ""; + // A payload with no `type` at all is drift worth seeing; a payload with a + // different one is just an event this scraper does not read. + if (!("type" in payload)) { + warnDrift(filePath, "event_msg payload missing 'type' key — likely renamed"); + } + return undefined; } /** diff --git a/tests/scrapers/codex-user-turns.test.ts b/tests/scrapers/codex-user-turns.test.ts new file mode 100644 index 0000000..e3ed547 --- /dev/null +++ b/tests/scrapers/codex-user-turns.test.ts @@ -0,0 +1,160 @@ +/** + * Codex renamed the shape a human turn arrives in, and the reader kept looking + * for the old one. + * + * Real turns used to be `event_msg` with `payload.type === "user_message"` and + * the text on `payload.message`. Current Codex wraps them: + * `payload.type === "item_completed"`, `payload.item.type === "UserMessage"`, + * and the text inside a `content` parts array. + * + * Nothing caught the change. `event_msg` is a known event type and + * `payload.type` was present, so the drift guard had nothing to report — the + * turns simply stopped arriving. Measured across a real store of 825 session + * files: 41 turns still in the old shape, 15,169 in the new one. A scrape of + * that history returned the assistant side of every conversation and almost + * none of what the person actually asked for, which is the half a handoff is + * for. + * + * Verified against the newest real transcript on the machine that exposed it: + * 32 human turns in the file, 0 emitted before, 32 after. + */ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { CodexCliScraper } from "@xtctx/scrapers/codex"; +import type { CodexChunk } from "@xtctx/types/scraper"; + +describe("a human turn in a codex transcript", () => { + let storeDir = ""; + let stateDir = ""; + const projectRoot = join("H:", "projects", "app"); + + beforeEach(async () => { + storeDir = await mkdtemp(join(tmpdir(), "xtctx-codex-user-")); + stateDir = await mkdtemp(join(tmpdir(), "xtctx-codex-user-state-")); + }); + + afterEach(async () => { + for (const dir of [storeDir, stateDir]) { + await rm(dir, { recursive: true, force: true }); + } + }); + + async function writeSession(records: unknown[]): Promise { + const dir = join(storeDir, "2026", "09", "07"); + await mkdir(dir, { recursive: true }); + await writeFile( + join(dir, "rollout-user-turns.jsonl"), + records.map((record) => JSON.stringify(record)).join("\n") + "\n", + "utf-8", + ); + } + + async function scrape(): Promise { + const chunks: CodexChunk[] = []; + for await (const chunk of new CodexCliScraper(storeDir, stateDir, projectRoot).fullSync()) { + chunks.push(chunk); + } + return chunks; + } + + const meta = { + timestamp: "2026-09-07T10:00:00.000Z", + type: "session_meta", + payload: { id: "user-turns", cwd: projectRoot, originator: "codex_cli_rs" }, + }; + + it("is read from the shape Codex writes now", async () => { + // Copied from a real record, down to the parts array. + await writeSession([ + meta, + { + timestamp: "2026-09-07T10:00:01.000Z", + type: "event_msg", + payload: { + type: "item_completed", + item: { + type: "UserMessage", + id: "01a07cd1", + content: [{ type: "text", text: "Can you audit this project?\n", text_elements: [] }], + }, + }, + }, + ]); + + const users = (await scrape()).filter((chunk) => chunk.role === "user"); + + expect(users).toHaveLength(1); + expect(users[0].content).toContain("Can you audit this project?"); + }); + + it("is still read from the shape Codex used to write", async () => { + // The old shape has not vanished — a real store carried both. + await writeSession([ + meta, + { + timestamp: "2026-09-07T10:00:01.000Z", + type: "event_msg", + payload: { type: "user_message", message: "the older shape" }, + }, + ]); + + const users = (await scrape()).filter((chunk) => chunk.role === "user"); + + expect(users).toHaveLength(1); + expect(users[0].content).toBe("the older shape"); + }); + + it("joins a turn split across several text parts", async () => { + await writeSession([ + meta, + { + timestamp: "2026-09-07T10:00:01.000Z", + type: "event_msg", + payload: { + type: "item_completed", + item: { + type: "UserMessage", + content: [ + { type: "text", text: "first part" }, + { type: "text", text: "second part" }, + ], + }, + }, + }, + ]); + + const [user] = (await scrape()).filter((chunk) => chunk.role === "user"); + + expect(user.content).toContain("first part"); + expect(user.content).toContain("second part"); + }); + + it("leaves the other item types to the readers that handle them", async () => { + // `item_completed` also wraps the assistant side and tool activity. Only + // UserMessage is a human turn; treating the rest as one would invent + // turns nobody typed. + await writeSession([ + meta, + { + timestamp: "2026-09-07T10:00:01.000Z", + type: "event_msg", + payload: { + type: "item_completed", + item: { type: "AgentMessage", content: [{ type: "text", text: "assistant text" }] }, + }, + }, + { + timestamp: "2026-09-07T10:00:02.000Z", + type: "event_msg", + payload: { + type: "item_completed", + item: { type: "CommandExecution", content: [{ type: "text", text: "ls -la" }] }, + }, + }, + ]); + + expect((await scrape()).filter((chunk) => chunk.role === "user")).toHaveLength(0); + }); +});