fix(codex): read the human turns Codex writes now, not only the ones it used to

src/scrapers/codex.ts

@@ -606,26 +606,60 @@ export function isKnownBulkyRecord(line: string): boolean {
  * still consumes an index so chunk identity stays stable between a full and
  * an incremental scrape; anything else is the message.
  *
- * `response_item` events with role "user" are deliberately not handled here:
- * they are system-injected context (AGENTS.md, permissions, environment),
- * not something the person typed.
+ * Two shapes, because Codex changed one and kept writing the other for a
+ * while. The original is `payload.type === "user_message"` with the text on
+ * `payload.message`. The current one wraps it: `payload.type ===
+ * "item_completed"` with `payload.item.type === "UserMessage"`.
+ *
+ * Reading only the original meant every human turn in a current transcript
+ * was dropped, silently — `event_msg` is a known type and `payload.type` was
+ * present, so the drift guard below had nothing to complain about. Measured
+ * across a real store of 825 session files: 41 turns in the old shape, 15,169
+ * in the new one. A scrape of that history returned the assistant side of
+ * every Codex conversation and almost none of what the person asked for.
+ *
+ * `response_item` events with role "user" are still deliberately not handled
+ * here: they are system-injected context (AGENTS.md, permissions,
+ * environment), not something the person typed, and they outnumber the real
+ * turns in a file that carries both.
  */
 function userMessageContent(payload: unknown, filePath: string): string | undefined {
   if (!isRecord(payload)) {
     warnDrift(filePath, `event_msg payload is not an object (got ${describeType(payload)})`);
     return undefined;
   }
 
-  if (payload.type !== "user_message") {
-    // A payload with no `type` at all is drift worth seeing; a payload with a
-    // different one is just an event this scraper does not read.
-    if (!("type" in payload)) {
-      warnDrift(filePath, "event_msg payload missing 'type' key — likely renamed");
+  if (payload.type === "user_message") {
+    return toStringValue(payload.message) ?? "";
+  }
+
+  if (payload.type === "item_completed") {
+    const item = payload.item;
+    if (!isRecord(item) || item.type !== "UserMessage") {
+      // Every other item type is an event this scraper reads elsewhere or
+      // does not read at all; neither is drift.
+      return undefined;
     }
-    return undefined;
+    // The text sits in a parts array, the way the assistant side does:
+    // `content: [{ type: "text", text: "..." }]`. A flat `text` is accepted
+    // too, so an older or simpler record still reads.
+    if (Array.isArray(item.content)) {
+      const text = item.content
+        .filter((part): part is Record<string, unknown> => isRecord(part))
+        .map((part) => toStringValue(part.text) ?? "")
+        .filter((part) => part.length > 0)
+        .join("\n");
+      return text;
+    }
+    return toStringValue(item.text) ?? "";
   }
 
-  return toStringValue(payload.message) ?? "";
+  // A payload with no `type` at all is drift worth seeing; a payload with a
+  // different one is just an event this scraper does not read.
+  if (!("type" in payload)) {
+    warnDrift(filePath, "event_msg payload missing 'type' key — likely renamed");
+  }
+  return undefined;
 }
 
 /**

tests/scrapers/codex-user-turns.test.ts

@@ -0,0 +1,160 @@
+/**
+ * Codex renamed the shape a human turn arrives in, and the reader kept looking
+ * for the old one.
+ *
+ * Real turns used to be `event_msg` with `payload.type === "user_message"` and
+ * the text on `payload.message`. Current Codex wraps them:
+ * `payload.type === "item_completed"`, `payload.item.type === "UserMessage"`,
+ * and the text inside a `content` parts array.
+ *
+ * Nothing caught the change. `event_msg` is a known event type and
+ * `payload.type` was present, so the drift guard had nothing to report — the
+ * turns simply stopped arriving. Measured across a real store of 825 session
+ * files: 41 turns still in the old shape, 15,169 in the new one. A scrape of
+ * that history returned the assistant side of every conversation and almost
+ * none of what the person actually asked for, which is the half a handoff is
+ * for.
+ *
+ * Verified against the newest real transcript on the machine that exposed it:
+ * 32 human turns in the file, 0 emitted before, 32 after.
+ */
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { CodexCliScraper } from "@xtctx/scrapers/codex";
+import type { CodexChunk } from "@xtctx/types/scraper";
+
+describe("a human turn in a codex transcript", () => {
+  let storeDir = "";
+  let stateDir = "";
+  const projectRoot = join("H:", "projects", "app");
+
+  beforeEach(async () => {
+    storeDir = await mkdtemp(join(tmpdir(), "xtctx-codex-user-"));
+    stateDir = await mkdtemp(join(tmpdir(), "xtctx-codex-user-state-"));
+  });
+
+  afterEach(async () => {
+    for (const dir of [storeDir, stateDir]) {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+
+  async function writeSession(records: unknown[]): Promise<void> {
+    const dir = join(storeDir, "2026", "09", "07");
+    await mkdir(dir, { recursive: true });
+    await writeFile(
+      join(dir, "rollout-user-turns.jsonl"),
+      records.map((record) => JSON.stringify(record)).join("\n") + "\n",
+      "utf-8",
+    );
+  }
+
+  async function scrape(): Promise<CodexChunk[]> {
+    const chunks: CodexChunk[] = [];
+    for await (const chunk of new CodexCliScraper(storeDir, stateDir, projectRoot).fullSync()) {
+      chunks.push(chunk);
+    }
+    return chunks;
+  }
+
+  const meta = {
+    timestamp: "2026-09-07T10:00:00.000Z",
+    type: "session_meta",
+    payload: { id: "user-turns", cwd: projectRoot, originator: "codex_cli_rs" },
+  };
+
+  it("is read from the shape Codex writes now", async () => {
+    // Copied from a real record, down to the parts array.
+    await writeSession([
+      meta,
+      {
+        timestamp: "2026-09-07T10:00:01.000Z",
+        type: "event_msg",
+        payload: {
+          type: "item_completed",
+          item: {
+            type: "UserMessage",
+            id: "01a07cd1",
+            content: [{ type: "text", text: "Can you audit this project?\n", text_elements: [] }],
+          },
+        },
+      },
+    ]);
+
+    const users = (await scrape()).filter((chunk) => chunk.role === "user");
+
+    expect(users).toHaveLength(1);
+    expect(users[0].content).toContain("Can you audit this project?");
+  });
+
+  it("is still read from the shape Codex used to write", async () => {
+    // The old shape has not vanished — a real store carried both.
+    await writeSession([
+      meta,
+      {
+        timestamp: "2026-09-07T10:00:01.000Z",
+        type: "event_msg",
+        payload: { type: "user_message", message: "the older shape" },
+      },
+    ]);
+
+    const users = (await scrape()).filter((chunk) => chunk.role === "user");
+
+    expect(users).toHaveLength(1);
+    expect(users[0].content).toBe("the older shape");
+  });
+
+  it("joins a turn split across several text parts", async () => {
+    await writeSession([
+      meta,
+      {
+        timestamp: "2026-09-07T10:00:01.000Z",
+        type: "event_msg",
+        payload: {
+          type: "item_completed",
+          item: {
+            type: "UserMessage",
+            content: [
+              { type: "text", text: "first part" },
+              { type: "text", text: "second part" },
+            ],
+          },
+        },
+      },
+    ]);
+
+    const [user] = (await scrape()).filter((chunk) => chunk.role === "user");
+
+    expect(user.content).toContain("first part");
+    expect(user.content).toContain("second part");
+  });
+
+  it("leaves the other item types to the readers that handle them", async () => {
+    // `item_completed` also wraps the assistant side and tool activity. Only
+    // UserMessage is a human turn; treating the rest as one would invent
+    // turns nobody typed.
+    await writeSession([
+      meta,
+      {
+        timestamp: "2026-09-07T10:00:01.000Z",
+        type: "event_msg",
+        payload: {
+          type: "item_completed",
+          item: { type: "AgentMessage", content: [{ type: "text", text: "assistant text" }] },
+        },
+      },
+      {
+        timestamp: "2026-09-07T10:00:02.000Z",
+        type: "event_msg",
+        payload: {
+          type: "item_completed",
+          item: { type: "CommandExecution", content: [{ type: "text", text: "ls -la" }] },
+        },
+      },
+    ]);
+
+    expect((await scrape()).filter((chunk) => chunk.role === "user")).toHaveLength(0);
+  });
+});