diff --git a/apps/desktop/src/main/e2e-fixture.ts b/apps/desktop/src/main/e2e-fixture.ts index 9e971d2033..08dc5b226d 100644 --- a/apps/desktop/src/main/e2e-fixture.ts +++ b/apps/desktop/src/main/e2e-fixture.ts @@ -27,6 +27,7 @@ import { AGENT_GRAPH_CLIENT_PROJECTION_SCHEMA_VERSION } from '@maka/core/agent-g import { MODEL_CALL_ATTEMPT_EVENT_TYPE } from '@maka/core/model-call-attempt'; import type { UiLocale } from '@maka/core/ui-locale'; import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; +import { createWorkspaceRuntimeStore } from '@maka/storage/runtime-event-persistence'; import { createProjectCatalog } from '@maka/storage/project-catalog'; import { resolveStorageRoot, @@ -305,13 +306,14 @@ export async function seedE2eFixture(input: { // below. It MUST be the lease's canonicalPath, not the raw workspaceRoot — // a /var vs /private/var realpath difference would open a different DB. const runStore = createSqliteAgentRunStore(owner.lease.canonicalPath); + const runtimeEventStore = createWorkspaceRuntimeStore(owner.lease.canonicalPath); try { const records = usageStatsRecords(now); // Model calls seed the CANONICAL ledger through the AgentRun event stream; // tools stay on the legacy telemetry table (there is no canonical tool // ledger). This is what actually exercises the canonical merge branch. - for (const { header: runHeader, attempt } of records.modelCalls) { - await runStore.createRun(runHeader); + for (const { opening, attempt } of records.modelCalls) { + await runtimeEventStore.appendRuntimeEvent(attempt.sessionId, attempt.runId, opening); await runStore.appendEvent(attempt.sessionId, attempt.runId, { id: attempt.attemptId, type: MODEL_CALL_ATTEMPT_EVENT_TYPE, @@ -323,6 +325,7 @@ export async function seedE2eFixture(input: { }); } for (const record of records.tools) await usage.telemetry.recordToolInvocation(record); + runtimeEventStore.close(); await runStore.close?.(); // Fold the appended attempts into the read model so the page's first read // sees canonical usage (production's readCanonicalUsage also repairs). diff --git a/apps/desktop/src/main/e2e-fixture/scenarios-usage.ts b/apps/desktop/src/main/e2e-fixture/scenarios-usage.ts index 14af6b6249..c51430eb6e 100644 --- a/apps/desktop/src/main/e2e-fixture/scenarios-usage.ts +++ b/apps/desktop/src/main/e2e-fixture/scenarios-usage.ts @@ -17,7 +17,8 @@ * under the License. */ -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeEvent } from '@maka/core/runtime-event'; +import { buildInvocationOpenedEvent } from '@maka/core/runtime-invocation'; import { MODEL_CALL_ATTEMPT_SCHEMA_VERSION, type ModelCallAttempt, @@ -229,11 +230,11 @@ export function usageStatsSessions( } export function usageStatsRecords(now: number): { - modelCalls: Array<{ header: AgentRunHeader; attempt: ModelCallAttempt }>; + modelCalls: Array<{ opening: RuntimeEvent; attempt: ModelCallAttempt }>; tools: PersistedToolInvocationRecord[]; } { const sessions = usageStatsSessions(now); - const modelCalls: Array<{ header: AgentRunHeader; attempt: ModelCallAttempt }> = []; + const modelCalls: Array<{ opening: RuntimeEvent; attempt: ModelCallAttempt }> = []; const tools: PersistedToolInvocationRecord[] = []; for (const { header: session, messages } of sessions) { const modelByTurn = new Map( @@ -257,19 +258,37 @@ export function usageStatsRecords(now: number): { // Run/attempt ids must match SAFE_ID_PATTERN ([A-Za-z0-9_-]); no colons. const runId = `run-${message.id}`; modelCalls.push({ - header: { - runId, - sessionId: session.id, - turnId: message.turnId, - status: 'created', - backendKind: 'fake', - llmConnectionSlug: session.llmConnectionSlug, - modelId, - cwd: '/tmp/e2e-usage', - permissionMode: 'ask', - createdAt: message.ts - 2_000, - updatedAt: message.ts, - }, + opening: buildInvocationOpenedEvent({ + id: `${runId}-open`, + run: { + sessionId: session.id, + invocationId: runId, + runId, + turnId: message.turnId, + }, + openedAt: message.ts - 2_000, + opening: { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: session.llmConnectionSlug, + llmConnectionSlug: session.llmConnectionSlug, + modelId, + }, + configuration: { + cwd: '/tmp/e2e-usage', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + root: { kind: 'user' }, + source: { kind: 'fresh' }, + }, + }), attempt: { schemaVersion: MODEL_CALL_ATTEMPT_SCHEMA_VERSION, logicalCallId: message.id, diff --git a/docs/architecture/runtime-core-architecture-draft.md b/docs/architecture/runtime-core-architecture-draft.md index b7408c1c3b..7d10db4e31 100644 --- a/docs/architecture/runtime-core-architecture-draft.md +++ b/docs/architecture/runtime-core-architecture-draft.md @@ -156,7 +156,7 @@ Maka is not implementing Kafka inside one process, nor does it claim that Runtim > **Log is the source of truth; state is a materialized view.** -That principle directly explains the most important terminal invariant later in this chapter: a Run header cannot declare completion on its own; a terminal RuntimeEvent must support it. +That principle directly explains the most important terminal invariant later in this chapter: nothing declares that a Run ended except the Run's own terminal RuntimeEvent. ## Three lifecycle identities, plus one correlation field @@ -221,13 +221,12 @@ It is an orchestration boundary, not the model loop. A Backend should not own th `AgentRun` gives one execution a durable identity and lifecycle. At startup it: -1. creates an `AgentRunHeader` in `created` state; +1. commits the invocation's opening fact as a RuntimeEvent; 2. writes the user message and a `running` Turn projection for a top-level Run; 3. writes the initial user `RuntimeEvent`; 4. locks the Session's connection configuration; 5. ensures a Backend exists and registers the active Run; -6. marks the Run as `running`; -7. builds model history from earlier RuntimeEvent ledgers. +6. builds model history from earlier RuntimeEvent ledgers. While execution is active, `AgentRun` receives both legacy `SessionEvent`s and canonical `RuntimeEvent`s and writes each to the projection or ledger it belongs to. At the end, it unregisters the active Run, converges Session and Turn state, and commits the final Run state. @@ -323,12 +322,12 @@ The important point is that permission is not a UI-only pause. Requests and deci ## One semantic truth, two supporting forms of state -Maka currently maintains three forms of durable data. They are not three equal sources of truth, nor do they store the same chat three times. `RuntimeEventStore` is the canonical semantic log of AI interaction; the other stores carry product projections and operational Run state. +Maka currently maintains three forms of durable data. They are not three equal sources of truth, nor do they store the same chat three times. `RuntimeEventStore` is the canonical semantic log of AI interaction; the other stores carry product projections and the operational record of what the runtime did. | Store | Main contents | Question it answers best | |---|---|---| | `SessionStore` | `StoredMessage`s for users, assistants, tools, and Turn state | What should the UI and compatibility APIs display? What is the current in-flight projection? | -| `AgentRunStore` | Run header and operational Run events | When did this Run start, what is its state, and at which model or tool stage did it fail? | +| `AgentRunStore` | operational Run events | At which model or tool stage did this Run do what, and where did it fail? | | `RuntimeEventStore` | canonical RuntimeEvents plus bounded partial snapshots | Which semantic facts occurred, and how should other state be rebuilt from them? | The current implementation is backed by SQLite rather than a directory per Run: `AgentRunStore` and `RuntimeEventStore` both sit on the same operational state database, and RuntimeEvents land in the `runtime_events` table. Order is carried by that table's `event_seq` under a `(invocation_id, event_seq)` uniqueness constraint, so sequence numbers never repeat within one correlated execution stream — that constraint is what "ordered log" means at the storage layer. Session, Turn, Run, and the compatibility correlation field each occupy their own column, so "what happened in this Turn of this Run of this Session" is an indexed lookup. @@ -343,18 +342,15 @@ Streaming text and thinking deltas are not appended forever to immutable JSONL. One of the hardest runtime failure classes is disagreement about whether an execution ended. For example: -- The Run header says completed, but the RuntimeEvent ledger has no terminal event; - the user stopped the Run, but a late complete event rewrites the Session to active; - the Backend stream exhausts without saying whether it succeeded or failed; -- the terminal event is durable, but the process crashes before updating the Run header. +- a second writer tries to end a Run that has already ended. Maka protects this core invariant: -> A terminal Run must have exactly one valid terminal RuntimeEvent, and a terminal Run header must be supported by that terminal fact. +> A Run ends exactly once, and its terminal RuntimeEvent is the only statement that it ended. -`AgentRun` therefore requires the terminal RuntimeEvent to be durable before committing a terminal Run header. A Backend stream without a terminal event becomes a `missing_terminal_event` failure. Duplicate terminal events are coalesced. Terminal events with a mismatched status, a different Run identity, or `partial: true` are rejected. - -If the terminal RuntimeEvent exists but an interrupted header remains `running`, the read model can treat the event as the stronger fact and recovery can repair the header. In the opposite direction, if a header claims termination without a trustworthy terminal fact, the system does not blindly trust the header; it conservatively repairs the Run as a `missing_terminal_event` failure. +There is no separate record of the outcome to keep in step, so a crash cannot leave one saying the Run finished while the other says it is still running. A Backend stream without a terminal event becomes a `missing_terminal_event` failure. Duplicate terminal events are coalesced. Terminal events with a mismatched status, a different Run identity, or `partial: true` are rejected. This invariant means recovery does not need to guess what the model intended to do next. It only needs to determine which facts are durable and converge all projections on one explainable outcome. @@ -362,7 +358,7 @@ This invariant means recovery does not need to guess what the model intended to ### User stop -`RuntimeKernel.stopSession()` first marks active `AgentRun`s as stopped, then calls Backend `stop()`. `AiSdkBackend` aborts the provider stream, ends any pending sandbox boundary or user question, and emits abort/complete events. Even if a provider later produces a complete or error event, `RuntimeKernel` and `AgentRun` do not allow it to overwrite the established aborted semantics. The stop source, such as the renderer stop button, is retained in the terminal fact and Run header for diagnostics. +`RuntimeKernel.stopSession()` first marks active `AgentRun`s as stopped, then calls Backend `stop()`. `AiSdkBackend` aborts the provider stream, ends any pending sandbox boundary or user question, and emits abort/complete events. Even if a provider later produces a complete or error event, `RuntimeKernel` and `AgentRun` do not allow it to overwrite the established aborted semantics. The stop source, such as the renderer stop button, is retained in the terminal fact for diagnostics. ### Provider or runtime error @@ -393,7 +389,7 @@ Continuing execution is a separate path. `safe_boundary_continuation` resumes fr - `AiSdkBackend` remains large and coordinates history, context budgets, tool availability, the step loop, usage, and telemetry. - The mapper is still a legacy-to-canonical bridge rather than consuming native RuntimeEvents from the Backend. - `SessionStore` and RuntimeEvent projection must cooperate for active and in-flight reads. -- Startup recovery performs deterministic termination and repair, not arbitrary warm resume. Continuation is a separate path: `safe_boundary_continuation` resumes from a verified safe boundary, is marked by `continuationSource` on the Run header, and is admitted and dispatched by `RuntimeKernel`; see [Chapter 8](./runtime-resume-architecture.md) for the difference. +- Startup recovery performs deterministic termination and repair, not arbitrary warm resume. Continuation is a separate path: `safe_boundary_continuation` resumes from a verified safe boundary, is marked by the continuation source on the invocation's opening fact, and is admitted and dispatched by `RuntimeKernel`; see [Chapter 8](./runtime-resume-architecture.md) for the difference. These are real architecture boundaries, not details to hide. Future Backend decomposition or checkpoint work must preserve request shape, tool visibility, event order, and the terminal invariant before optimizing for smaller files. diff --git a/docs/architecture/runtime-core-architecture-draft.zh-CN.md b/docs/architecture/runtime-core-architecture-draft.zh-CN.md index 8ac2ea9eeb..65d5c8698f 100644 --- a/docs/architecture/runtime-core-architecture-draft.zh-CN.md +++ b/docs/architecture/runtime-core-architecture-draft.zh-CN.md @@ -156,7 +156,7 @@ Maka 并不是在进程内实现了 Kafka，也没有声称 RuntimeEventStore > **Log is the source of truth; state is a materialized view.** -这一原则直接解释了后文最重要的 terminal invariant：Run header 不能凭自己宣布完成，它必须得到 terminal RuntimeEvent 的支持。 +这一原则直接解释了后文最重要的 terminal invariant：除了这次 Run 自己的 terminal RuntimeEvent，没有别的东西能宣布它结束。 ## 三种生命周期身份，加一个关联字段 @@ -221,13 +221,12 @@ flowchart LR `AgentRun` 让一次执行在持久世界里有身份和生命周期。开始运行时，它会： -1. 创建 `AgentRunHeader`，初始状态为 `created`； +1. 把这次 invocation 的开场事实作为 RuntimeEvent 提交； 2. 对顶层 Run 写入用户消息和 `running` Turn 投影； 3. 写入本轮初始用户 `RuntimeEvent`； 4. 锁定本 Session 的连接配置； 5. 确保 Backend 已创建并注册为活跃 Run； -6. 将 Run 标记为 `running`； -7. 从此前的 RuntimeEvent ledger 构造模型历史。 +6. 从此前的 RuntimeEvent ledger 构造模型历史。 运行过程中，`AgentRun` 同时接收旧的 `SessionEvent` 与新的 `RuntimeEvent`，并把它们写入各自所属的投影或账本。结束时，它注销活跃 Run、收敛 Session/Turn 状态，并提交最终 Run 状态。 @@ -321,12 +320,12 @@ AI SDK 的 step 是这个循环的自然节拍。Maka 会按 step 持久化 assi ## 一份语义事实，两类辅助状态 -Maka 当前同时维护三类持久数据。它们不是三个地位相同的“真相”，也不是重复保存同一份聊天。`RuntimeEventStore` 是 AI 交互的 canonical semantic log；另外两类存储承担产品投影与运行运维状态。 +Maka 当前同时维护三类持久数据。它们不是三个地位相同的“真相”，也不是重复保存同一份聊天。`RuntimeEventStore` 是 AI 交互的 canonical semantic log；另外两类存储承担产品投影，以及 Runtime 做过什么的运维记录。 | 存储 | 主要内容 | 它最适合回答的问题 | |---|---|---| | `SessionStore` | 用户、assistant、工具和 turn-state 等 `StoredMessage` | UI 与兼容接口要展示什么？活跃流有哪些即时投影？ | -| `AgentRunStore` | Run header 与 operational Run events | 这次 Run 何时开始、当前状态、在哪个模型或工具阶段失败？ | +| `AgentRunStore` | operational Run events | 这次 Run 在哪个模型或工具阶段做了什么、又在哪里失败？ | | `RuntimeEventStore` | canonical RuntimeEvent 与有界 partial snapshots | Agent 交互发生过哪些语义事实，其他状态应如何重建？ | 当前实现由 SQLite 承载，而不是每个 Run 一个目录：`AgentRunStore` 与 `RuntimeEventStore` 都建立在同一份 operational state 数据库之上，RuntimeEvent 落在 `runtime_events` 表。顺序由该表的 `event_seq` 承担，并以 `(invocation_id, event_seq)` 唯一约束保证一次 Invocation 内序号不重复——“有序日志”在存储层就是这条约束。四个身份各占一列，因此“这个 Session 的这次 Run 的这个 Turn 发生了什么”是一次索引查询。 @@ -341,18 +340,15 @@ Maka 当前同时维护三类持久数据。它们不是三个地位相同的“ Runtime 最容易出现的一类故障，是不同存储对“是否结束”给出不同答案。例如： -- Run header 写成 completed，但 RuntimeEvent ledger 没有 terminal event； - 用户已经 stop，但迟到的 complete 又把 Session 写回 active； - Backend 流耗尽，却从未说明它是成功还是失败； -- terminal event 已写入，但进程在更新 Run header 前崩溃。 +- 已经结束的 Run，又有第二个写入方想再结束它一次。 Maka 当前保护的核心不变量是： -> 一个终止的 Run 必须有且只有一个有效 terminal RuntimeEvent；终止的 Run header 必须能够由这个 terminal fact 支撑。 +> 一个 Run 只结束一次，而它的 terminal RuntimeEvent 是唯一说它结束了的事实。 -因此，`AgentRun` 在提交 terminal Run header 前，先要求 terminal RuntimeEvent 成功落盘。没有终态的 Backend stream 会被合成为 `missing_terminal_event` 失败；重复终态由 Kernel 合并；状态不匹配、来自其他 Run 或标记为 partial 的 terminal event 都会被拒绝。 - -如果 terminal RuntimeEvent 已经存在，但 Run header 因中断仍是 `running`，read model 可以把 terminal event 作为更强事实来解释运行结果，并在恢复时修复 header。反过来，如果 header 声称已经结束却没有可信 terminal fact，系统不会盲目信任 header，而会保守地修复为 `missing_terminal_event` 失败。 +因为结果没有第二份记录要同步，崩溃也就不可能留下一份说“已完成”、另一份说“还在跑”的状态。没有终态的 Backend stream 会被合成为 `missing_terminal_event` 失败；重复终态由 Kernel 合并；状态不匹配、来自其他 Run 或标记为 partial 的 terminal event 都会被拒绝。 这条不变量让恢复不必“猜模型当时准备做什么”。系统只需要判断哪些事实已经 durable，然后把各个投影收敛到同一个可解释终态。 @@ -360,7 +356,7 @@ Maka 当前保护的核心不变量是： ### 用户停止 -`RuntimeKernel.stopSession()` 会先把所有活跃 `AgentRun` 标记为 stopped，再调用 Backend 的 `stop()`。`AiSdkBackend` 会中止 provider stream、结束正在等待的 sandbox boundary 或用户提问，并产生 abort/complete 事件。即使 provider 随后发送迟到的 complete 或 error，RuntimeKernel 与 AgentRun 也不会允许它覆盖已经确定的 aborted 语义。停止来源，例如 renderer stop button，会进入 terminal fact 与 Run header，供诊断使用。 +`RuntimeKernel.stopSession()` 会先把所有活跃 `AgentRun` 标记为 stopped，再调用 Backend 的 `stop()`。`AiSdkBackend` 会中止 provider stream、结束正在等待的 sandbox boundary 或用户提问，并产生 abort/complete 事件。即使 provider 随后发送迟到的 complete 或 error，RuntimeKernel 与 AgentRun 也不会允许它覆盖已经确定的 aborted 语义。停止来源，例如 renderer stop button，会进入 terminal fact，供诊断使用。 ### Provider 或 Runtime 错误 @@ -391,7 +387,7 @@ Maka 当前保护的核心不变量是： - `AiSdkBackend` 仍然很重，同时组织 history、context budget、tool availability、step loop、usage 与 telemetry； - `SessionEvent Runtime mapper` 仍承担 legacy-to-canonical adapter 角色，而不是 Backend 原生产 canonical events； - `SessionStore` 与 RuntimeEvent projection 需要在 active/in-flight 场景中协同； -- 启动恢复是确定性终结与修复，不是从任意位置热续跑。续跑走另一条路：`safe_boundary_continuation` 从一个经过校验的安全边界接着跑，Run header 上是 `continuationSource`，由 `RuntimeKernel` 完成准入和 dispatch；两者的区别见[第八章](./runtime-resume-architecture.zh-CN.md)。 +- 启动恢复是确定性终结与修复，不是从任意位置热续跑。续跑走另一条路：`safe_boundary_continuation` 从一个经过校验的安全边界接着跑，它在 invocation 开场事实里记着自己的续跑来源，由 `RuntimeKernel` 完成准入和 dispatch；两者的区别见[第八章](./runtime-resume-architecture.zh-CN.md)。 这些不是应该隐藏的实现细节，而是当前架构的真实边界。未来拆分 Backend 或加入 checkpoint 时，首要目标不是减少文件行数，而是保持 request shape、工具可见性、事件顺序和 terminal invariant 不变。 diff --git a/docs/architecture/runtime-resume-architecture.md b/docs/architecture/runtime-resume-architecture.md index 89650fba4b..3b3cec5fb3 100644 --- a/docs/architecture/runtime-resume-architecture.md +++ b/docs/architecture/runtime-resume-architecture.md @@ -133,7 +133,7 @@ These three words are easy to mix up: | Term | Subject | Result | |---|---|---| -| Repair | Durable state of an old Run | Align terminal RuntimeEvent, Run header, and Turn state | +| Repair | Durable state of an old Run | Give an interrupted Run its terminal RuntimeEvent and align Turn state | | Resume / Continuation | A history boundary already proved safe | Create fresh identities and continue the provider loop | | Reconcile | A tool operation with T1 but no T2 outcome | Observe the external world and commit either completed or parked | @@ -204,7 +204,8 @@ Safety does not come merely from putting everything in SQLite. It comes from ass | Data | Nature | Purpose | |---|---|---| | Immutable `RuntimeEvent` | Canonical semantic fact | Model history, tool call/dispatch/outcome, recovery observation/decision, terminal fact | -| `AgentRunHeader` and AgentRun events | Durable operational envelope | Attempt identity, status, lineage, and diagnostics | +| Invocation opening fact | Immutable statement of one attempt | Identity, route, configuration, root authority, lineage | +| AgentRun events | Durable operational record | What the runtime did, stage by stage, and its diagnostics | | `tool_operations` | SQLite projection | Fast current-state lookup for an operation | | `tool_journal_events` | SQLite projection | Fast prepared/outcome/recovery transition lookup | | Session messages / Turn state | Product and UI projection | Conversation and Turn display, not recovery judgment | @@ -401,14 +402,11 @@ sequenceDiagram participant UI as Renderer App->>SM: recoverInterruptedSessions() - SM->>RS: list non-terminal / suspicious AgentRuns + SM->>ES: list invocations with no terminal event SM->>ES: read immutable RuntimeEvents - SM->>SM: compare terminal ledger and Run header - alt terminal RuntimeEvent exists, header lags - SM->>RS: repair the matching Run header - else no terminal RuntimeEvent - SM->>ES: commit recovered terminal RuntimeEvent first - SM->>RS: then commit matching failed/cancelled header + SM->>RS: read the operational events for the Run + alt no terminal RuntimeEvent + SM->>ES: commit a recovered terminal RuntimeEvent else ledger is ambiguous / unreadable SM-->>UI: preserve inspectable state and fail closed end @@ -418,9 +416,9 @@ sequenceDiagram The invariant is: -> The terminal RuntimeEvent commits before the terminal Run header. A header cannot declare completion without its semantic fact. +> A Run has ended exactly when its terminal RuntimeEvent is durable, and nothing else records that it ended. -A second crash between those commits remains repairable from the terminal event. Desktop also recovers Graph coordination. Automatic continuation is considered only after those repairs and only when the feature flag is enabled. +There is no second commit for a crash to land between. Desktop also recovers Graph coordination. Automatic continuation is considered only after those repairs and only when the feature flag is enabled. ## Phase 1: create a new execution at a safe boundary @@ -429,7 +427,7 @@ Phase 1 does not resolve unknown side effects. It continues only when every acce Planner gates include: - readable source Run and RuntimeEvent ledger; -- exactly one terminal event matching the Run header; +- exactly one terminal event for the source invocation; - one source execution identity across events; - Phase 0 `safe_replay`; - no pending permission; @@ -791,7 +789,7 @@ Eval does not resume or reconstruct Runtime execution. It asks Runtime Host to e 4. Atomically commit call, dispatch, and projection at T1. 5. Execute the external effect without a long database transaction. 6. Atomically commit T2 before publishing the result. -7. Commit terminal RuntimeEvent before terminal Run header. +7. End a Run by committing exactly one terminal RuntimeEvent. 8. On restart, repair the old Run first. 9. Resolve immutable facts into completed / not-dispatched / indeterminate / parked / corruption. 10. If a production reconciler exists, commit one atomic recovery bundle; otherwise park. diff --git a/docs/architecture/runtime-resume-architecture.zh-CN.md b/docs/architecture/runtime-resume-architecture.zh-CN.md index e180a54dd5..e4348a0061 100644 --- a/docs/architecture/runtime-resume-architecture.zh-CN.md +++ b/docs/architecture/runtime-resume-architecture.zh-CN.md @@ -133,7 +133,7 @@ flowchart TD | 词 | 处理对象 | 结果 | |---|---|---| -| Repair | 旧 Run 的持久化状态 | 补齐或对齐 terminal RuntimeEvent、Run header 和 Turn 状态 | +| Repair | 旧 Run 的持久化状态 | 给被中断的 Run 补上 terminal RuntimeEvent，并对齐 Turn 状态 | | Resume / Continuation | 一段已经证明安全的历史边界 | 创建新身份，继续 provider loop | | Reconcile | T1 已派发但没有 T2 outcome 的工具操作 | 观察外部世界，提交 completed 或 parked recovery decision | @@ -206,7 +206,8 @@ Resume 安全性的核心不是“数据都写进 SQLite”，而是每类数据 | 数据 | 性质 | 用途 | |---|---|---| | Immutable `RuntimeEvent` | canonical semantic fact | 模型历史、工具 call/dispatch/outcome、recovery observation/decision、terminal fact | -| `AgentRunHeader` 与 AgentRun events | durable operational envelope | 一次执行尝试的身份、状态、lineage、诊断 | +| invocation 开场事实 | 一次执行尝试的不可变声明 | 身份、route、配置、root authority、lineage | +| AgentRun events | durable operational record | Runtime 逐阶段做了什么，以及诊断 | | `tool_operations` | SQLite projection | 快速读取某个 operation 当前状态 | | `tool_journal_events` | SQLite projection | 快速查看 prepared/outcome/recovery 状态变化 | | Session messages / Turn state | 产品与 UI 投影 | 展示对话和 Turn 状态，不参与工具恢复裁决 | @@ -407,14 +408,11 @@ sequenceDiagram participant UI as Renderer App->>SM: recoverInterruptedSessions() - SM->>RS: 列出非终态 / 可疑 AgentRun + SM->>ES: 列出没有 terminal event 的 invocation SM->>ES: 读取 immutable RuntimeEvents - SM->>SM: 检查 terminal ledger 与 run header - alt 已有 terminal RuntimeEvent，header 落后 - SM->>RS: 修复 matching run header - else 没有 terminal RuntimeEvent - SM->>ES: 先提交 recovered terminal RuntimeEvent - SM->>RS: 再提交 matching failed/cancelled header + SM->>RS: 读取这次 Run 的 operational events + alt 没有 terminal RuntimeEvent + SM->>ES: 提交 recovered terminal RuntimeEvent else ledger ambiguous / unreadable SM-->>UI: 保留可检查状态，fail closed end @@ -424,9 +422,9 @@ sequenceDiagram 这里保护一个贯穿 Runtime 的不变量： -> terminal RuntimeEvent 必须先于 terminal Run header 提交；header 不能凭自己宣布一次执行已经结束。 +> 一次执行结束，当且仅当它的 terminal RuntimeEvent 已经落盘；没有别的东西记录它结束了。 -如果在两次提交之间再次崩溃，下次启动仍能从 terminal RuntimeEvent 修好 header。反过来先写 header，就会出现一个没有语义事实支持的“完成”状态。 +因为不存在第二次提交，崩溃也就没有可以落进去的缝隙。 Desktop 还会恢复 Graph coordinator 和 supervisor wake。只有这些 startup repair 完成，并且 safe-boundary flag 开启后，才会尝试自动 continuation。 @@ -437,7 +435,7 @@ Phase 1 不处理未知副作用。它只允许“所有工具都已经有 commi Planner 需要同时通过这些 gate： - source Run 与 RuntimeEvent ledger 可读； -- Run header 与唯一 terminal RuntimeEvent 一致； +- source invocation 有且只有一个 terminal event； - 所有事件属于同一个 source execution identity； - Phase 0 得到 `safe_replay`； - 没有 pending permission； @@ -525,7 +523,7 @@ Host 投影和 CLI 展示，不改变 planner、durable continuation claim 或 f 1. 不创建第二条相同的 user event； 2. 先提交一个 system-owned、model-invisible 的 continuation-start RuntimeEvent； -3. 在新 Run header 中记录 source identity 和 high-water； +3. 在新 invocation 的开场事实里记录 source identity 和 high-water； 4. 直接把验证过的 history 交给 provider。 这样既避免模型看到重复请求，也避免 completed tool call 因为“新建了一轮”而再次执行。 @@ -819,7 +817,7 @@ Eval 不恢复或重建 Runtime execution，只请求 Runtime Host 执行 Maka s 4. T1 原子提交 call、dispatch 和 projection。 5. 执行外部副作用，不持有数据库长事务。 6. T2 原子提交 outcome，再把结果交给模型。 -7. terminal RuntimeEvent 先提交，Run header 后提交。 +7. 一次执行只以提交唯一一个 terminal RuntimeEvent 来结束。 8. 崩溃重启后先 repair 旧 Run。 9. Resolver 只读 immutable facts，判定 completed / not-dispatched / indeterminate / parked / corruption。 10. 有 production reconciler 时，对 indeterminate 提交一个原子 recovery bundle；没有时 park。 diff --git a/docs/architecture/runtime-resume-extraction-ledger.zh-CN.md b/docs/architecture/runtime-resume-extraction-ledger.zh-CN.md index 340230e21e..e0a8d99af6 100644 --- a/docs/architecture/runtime-resume-extraction-ledger.zh-CN.md +++ b/docs/architecture/runtime-resume-extraction-ledger.zh-CN.md @@ -78,7 +78,7 @@ Phase 3B/4A 的 workspace checkpoint 是后续独立切片，不进入 PR A。 - SQLite 与 JSONL 共享唯一 lossless canonical RuntimeEvent codec；validator 消费 codec 返回的 event，store 持久化同一次编码返回的稳定 JSON bytes； - SQLite 对每个 invocation 强制唯一 `(sessionId, runId, turnId)` execution spine； -- JSONL immutable append 对 exact retry 物理去重，并在落盘前验证目标 Run header； +- JSONL immutable append 对 exact retry 物理去重，并在落盘前验证目标 invocation 身份； - projection-local journal ID 由 operation/event 派生，调用者不能选择； - schema 4 的 nullable-dispatch legacy projection 可读但隔离，不进入 recovery 或 canonical rebuild。 @@ -171,7 +171,7 @@ future newer schema -> fail closed | decoder canonical persistence 与有损 JSON 拒绝 | storage authority test | 已覆盖 | | nested undefined、provider `toJSON`、recovery evidence 改写 | storage authority test | 已覆盖 | | JSONL ordinary/tool exact retry 与 conflicting retry | JSONL storage test | 已覆盖 | -| JSONL event 与目标 Run header identity | JSONL storage test | 已覆盖 | +| JSONL event 与目标 invocation identity | JSONL storage test | 已覆盖 | | invocation 跨 session/run/turn 漂移 | core scanner + SQLite authority test | 已覆盖 | | unrelated session corruption 阻断新 session tool boundary | storage authority test | 已覆盖 | | corrupt ledger 上的 T1/T2/recovery exact retry | storage authority test | 已覆盖 | @@ -268,7 +268,7 @@ claim race 与 provider-call T1 测试，再补满足不变量的最小生产路 - **B1 — immutable boundary 与 replay**：物理 `event_seq`、canonical RuntimeEvent bytes、 segment digest、ordered manifest、provider replay digest； - **B2 — durable authority 与 provider T1**：SQLite unique claim、执行前完整重验证、 - exact target Run header、store-owned live start、一次性 admission proof/receipt，然后才允许 + exact target invocation、store-owned live start、一次性 admission proof/receipt，然后才允许 backend/provider 启动； - **B2.1 — pre-provider crash convergence**：claim-only/created-without-start 通过 deterministic repair start + terminal 收敛；normal start/no-terminal 无 owner proof 时只 park。 diff --git a/docs/architecture/runtime-resume-phase1-safe-boundary-contract.md b/docs/architecture/runtime-resume-phase1-safe-boundary-contract.md index 27f905e131..6472391cb2 100644 --- a/docs/architecture/runtime-resume-phase1-safe-boundary-contract.md +++ b/docs/architecture/runtime-resume-phase1-safe-boundary-contract.md @@ -55,11 +55,11 @@ The continuation-start event must be durable before the provider is called. ## Planner gates -`RuntimeContinuationPlanner` reads the source AgentRun and RuntimeEvent ledger. +`RuntimeContinuationPlanner` reads the source invocation and its RuntimeEvent ledger. The plan is `continue` only when all of the following are true: - the source run and RuntimeEvent ledger are readable; -- the run header has exactly one matching, non-partial terminal RuntimeEvent; +- the source invocation has exactly one matching, non-partial terminal RuntimeEvent; - every RuntimeEvent belongs to one source Session, Invocation, Run, and Turn; - the Phase 0 projection is `safe_replay`; - every accepted tool call has a committed matching response; @@ -111,10 +111,9 @@ from being executed merely because a new model turn was created. If continuation-start persistence fails: 1. the provider is not called; -2. no terminal AgentRun header is committed without a terminal RuntimeEvent; -3. the incomplete target Run remains recoverable; -4. existing startup recovery later writes a recovered terminal RuntimeEvent - and then commits the matching failed run header. +2. the incomplete target Run remains recoverable; +3. existing startup recovery later writes a recovered terminal RuntimeEvent, + which is the whole of ending that Run. The source ledger is never mutated by continuation execution. diff --git a/docs/architecture/runtime-resume-phase3-phase4-workspace-checkpoint-design.zh-CN.md b/docs/architecture/runtime-resume-phase3-phase4-workspace-checkpoint-design.zh-CN.md index a62763cc51..d62d783bbd 100644 --- a/docs/architecture/runtime-resume-phase3-phase4-workspace-checkpoint-design.zh-CN.md +++ b/docs/architecture/runtime-resume-phase3-phase4-workspace-checkpoint-design.zh-CN.md @@ -78,7 +78,7 @@ RuntimeEvent 是语义事实的唯一权威，但不能替代执行所有权的 11. strict args identity 明确处理 `__proto__` 并拒绝 sparse/accessor/custom array； 12. 唯一 canonical RuntimeEvent codec 负责 decode、normalization、strict JSON、稳定 bytes 与 lossless round-trip；SQLite/JSONL、未来 prefix digest 均复用它； -13. JSONL immutable exact retry 物理去重，写前验证 Run header identity； +13. JSONL immutable exact retry 物理去重，写前验证 invocation identity； 14. SQLite 强制一个 invocation 只对应一个 `(sessionId, runId, turnId)`； 15. journal ID 只由 store 派生；正式 schema 4 的无 dispatch legacy rows保守隔离。 @@ -184,7 +184,7 @@ schema 6 增加 `runtime_continuation_claims` 与 capability - immediate source execution identity、physical high-water、prefix digest； - provider projection version 与 provider replay digest； - fresh target session/invocation/run/turn； -- target Run 的完整、严格解码 `AgentRunHeader`（含 V2 continuation source）； +- target invocation 的完整、严格解码开场事实（含 continuation source）； - claim id、claimed-at、protocol version； - 可空、唯一的 continuation-start event id； - 与 start 同生存期的 store-owned `start_kind`：`runtime_admission | claim_repair`。 @@ -288,8 +288,8 @@ retry 入口。历史 `linked_child_resume` / `linked_child_provider_retry` desc `retriedFromRunId` 只保留重启关闭、查询和展示兼容，不会重新触发 provider。 live continuation-start 同时绑定 claim id、boundary digest、immediate source identity/high-water/prefix -digest、replay manifest、provider projection version 和 provider replay digest。V2 AgentRun header 的 -`continuationSource` 必须与首条 continuation-start 完全一致。若当前执行使用 +digest、replay manifest、provider projection version 和 provider replay digest。target invocation 开场事实里的 +continuation source 必须与首条 continuation-start 完全一致。若当前执行使用 `t1_after_preflight_v1`，该 marker 也写在同一 event-seq 1；repair start 不得携带它。 只有 `RuntimeKernel` 能 dispatch durable continuation。AgentRun 仅在 live start 返回 diff --git a/packages/core/package.json b/packages/core/package.json index 194034d62b..a862fe2ef6 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -30,6 +30,8 @@ "./collaboration": "./dist/collaboration.js", "./orchestration": "./dist/orchestration.js", "./tool-mode": "./dist/tool-mode.js", + "./record-schema": "./dist/record-schema.js", + "./runtime-invocation": "./dist/runtime-invocation.js", "./plan": "./dist/plan.js", "./agent-run": "./dist/agent-run.js", "./subagent-workspace": "./dist/subagent-workspace.js", diff --git a/packages/core/src/__tests__/agent-run-authority.test.ts b/packages/core/src/__tests__/agent-run-authority.test.ts deleted file mode 100644 index 7a5f7de488..0000000000 --- a/packages/core/src/__tests__/agent-run-authority.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import assert from 'node:assert/strict'; -import { test } from 'node:test'; -import { - decodeAgentRunHeader, - decodePersistedAgentRunHeader, - type AgentRunHeader, -} from '../agent-run.js'; -import { markPersisted } from '../persisted-value.js'; - -test('rejects a Run header with multiple hosted root authorities', () => { - assert.throws( - () => - decodeAgentRunHeader({ - ...runHeader(), - scheduledTaskId: 'scheduled-task-1', - goalId: 'goal-1', - }), - /Invalid AgentRun header schema/, - ); -}); - -test('decodes a released Automation Run as read-only legacy provenance', () => { - const decoded = decodePersistedAgentRunHeader( - markPersisted({ - ...runHeader(), - automationId: 'automation-1', - }), - ); - assert.equal(decoded.legacyAutomationId, 'automation-1'); - assert.equal(Object.hasOwn(decoded, 'automationId'), false); -}); - -test('folds all retired AgentRun values only at the persistence boundary', () => { - const persisted = { - ...runHeader(), - status: 'waiting_permission', - permissionMode: 'execute', - }; - - const decoded = decodePersistedAgentRunHeader(markPersisted(persisted)); - assert.equal(decoded.status, 'waiting_for_user'); - assert.equal(decoded.permissionMode, 'ask'); - - assert.throws(() => decodeAgentRunHeader(persisted), /Invalid AgentRun header schema/); - assert.throws( - () => decodeAgentRunHeader({ ...runHeader(), automationId: 'automation-1' }), - /Invalid AgentRun header schema/, - ); - assert.throws( - () => decodeAgentRunHeader({ ...runHeader(), permissionMode: 'execute' }), - /Invalid AgentRun header schema/, - ); -}); - -test('accepts both bound and legacy AgentRun connection identity', () => { - const legacy = decodePersistedAgentRunHeader(markPersisted(runHeader())); - assert.equal(legacy.llmConnectionId, undefined); - - const bound = decodeAgentRunHeader({ - ...runHeader(), - llmConnectionId: '11111111-1111-4111-8111-111111111111', - }); - assert.equal(bound.llmConnectionId, '11111111-1111-4111-8111-111111111111'); - assert.throws( - () => decodeAgentRunHeader({ ...runHeader(), llmConnectionId: '' }), - /Invalid AgentRun header schema/, - ); -}); - -function runHeader(): AgentRunHeader { - return { - runId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - status: 'created', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/workspace', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 1, - }; -} diff --git a/packages/core/src/__tests__/agent-run-continuation-source.test.ts b/packages/core/src/__tests__/agent-run-continuation-source.test.ts deleted file mode 100644 index b8c1eba5c2..0000000000 --- a/packages/core/src/__tests__/agent-run-continuation-source.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; -import { decodeAgentRunHeader, type AgentRunHeader } from '../agent-run.js'; - -describe('AgentRun continuation source decoding', () => { - it('rejects an empty V2 continuation claim identity', () => { - assert.throws( - () => - decodeAgentRunHeader( - headerWithContinuation({ - ...validV2ContinuationSource(), - claimId: '', - }), - ), - /Invalid AgentRun header schema/, - ); - }); - - it('rejects a zero V2 source high-water', () => { - assert.throws( - () => - decodeAgentRunHeader( - headerWithContinuation({ - ...validV2ContinuationSource(), - sourceRuntimeEventHighWater: 0, - }), - ), - /Invalid AgentRun header schema/, - ); - }); - - for (const field of ['sourceInvocationId', 'sourceRunId', 'sourceTurnId'] as const) { - it(`rejects an empty V2 ${field}`, () => { - assert.throws( - () => - decodeAgentRunHeader( - headerWithContinuation({ - ...validV2ContinuationSource(), - [field]: '', - }), - ), - /Invalid AgentRun header schema/, - ); - }); - } - - it('rejects a V2 replay manifest that does not identify its boundary', () => { - assert.throws( - () => - decodeAgentRunHeader( - headerWithContinuation({ - ...validV2ContinuationSource(), - replayManifestDigest: `sha256:${'c'.repeat(64)}`, - }), - ), - /Invalid AgentRun header schema/, - ); - }); -}); - -function headerWithContinuation( - continuationSource: AgentRunHeader['continuationSource'], -): AgentRunHeader { - return { - runId: 'target-run', - invocationId: 'target-invocation', - sessionId: 'session-1', - turnId: 'target-turn', - status: 'created', - backendKind: 'fake', - llmConnectionSlug: 'test', - modelId: 'test-model', - cwd: '/workspace', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 1, - continuationSource, - }; -} - -function validV2ContinuationSource(): Extract< - NonNullable, - { protocol: 'continuation_source_v2' } -> { - return { - protocol: 'continuation_source_v2', - claimId: 'claim-1', - boundaryDigest: `sha256:${'a'.repeat(64)}`, - sourceInvocationId: 'source-invocation', - sourceRunId: 'source-run', - sourceTurnId: 'source-turn', - sourceRuntimeEventHighWater: 1, - sourcePrefixDigest: `sha256:${'b'.repeat(64)}`, - replayManifestDigest: `sha256:${'a'.repeat(64)}`, - }; -} diff --git a/packages/core/src/__tests__/agent-run-event-contract.test.ts b/packages/core/src/__tests__/agent-run-event-contract.test.ts index 406ea7a5cf..0be3f94129 100644 --- a/packages/core/src/__tests__/agent-run-event-contract.test.ts +++ b/packages/core/src/__tests__/agent-run-event-contract.test.ts @@ -73,7 +73,7 @@ test('AgentRun closes its write contract against a type this build does not emit store.appendEvent('session-1', 'run-1', retired); assert.equal(typeof appendRetired, 'function'); - const emitted: EmittedAgentRunEvent = { ...retired, type: 'run_started' }; + const emitted: EmittedAgentRunEvent = { ...retired, type: 'turn_started' }; const appendEmitted = (store: AgentRunStore) => store.appendEvent('session-1', 'run-1', emitted); assert.equal(typeof appendEmitted, 'function'); }); diff --git a/packages/core/src/__tests__/agent-run-hosted-root.test.ts b/packages/core/src/__tests__/agent-run-hosted-root.test.ts deleted file mode 100644 index 17a876a9cf..0000000000 --- a/packages/core/src/__tests__/agent-run-hosted-root.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import assert from 'node:assert/strict'; -import { test } from 'node:test'; -import { agentRunMatchesHostedRootExecution, type AgentRunHeader } from '../agent-run.js'; - -test('regenerate hosted root identity requires both source lineage fields', () => { - const run: AgentRunHeader = { - runId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-2', - status: 'completed', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/workspace', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 2, - completedAt: 2, - parentTurnId: 'turn-1', - regeneratedFromTurnId: 'turn-1', - }; - - assert.equal( - agentRunMatchesHostedRootExecution(run, { - kind: 'regenerate', - sourceTurnId: 'turn-1', - }), - true, - ); - assert.equal( - agentRunMatchesHostedRootExecution( - { ...run, regeneratedFromTurnId: 'turn-other' }, - { kind: 'regenerate', sourceTurnId: 'turn-1' }, - ), - false, - ); - assert.equal( - agentRunMatchesHostedRootExecution( - { ...run, scheduledTaskId: 'scheduled-task-1' }, - { kind: 'regenerate', sourceTurnId: 'turn-1' }, - ), - false, - ); -}); - -test('context compact hosted root identity rejects message lineage', () => { - const run: AgentRunHeader = { - runId: 'run-compact', - sessionId: 'session-1', - turnId: 'turn-compact', - status: 'completed', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/workspace', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 2, - completedAt: 2, - rootExecutionKind: 'context_compact', - }; - - assert.equal(agentRunMatchesHostedRootExecution(run, { kind: 'context_compact' }), true); - const { rootExecutionKind: _, ...ordinaryRun } = run; - assert.equal(agentRunMatchesHostedRootExecution(ordinaryRun, { kind: 'context_compact' }), false); - assert.equal( - agentRunMatchesHostedRootExecution( - { ...run, parentTurnId: 'turn-1' }, - { kind: 'context_compact' }, - ), - false, - ); -}); diff --git a/packages/core/src/__tests__/runtime-boundary.test.ts b/packages/core/src/__tests__/runtime-boundary.test.ts index 5ec71103a3..4705864e6e 100644 --- a/packages/core/src/__tests__/runtime-boundary.test.ts +++ b/packages/core/src/__tests__/runtime-boundary.test.ts @@ -24,6 +24,7 @@ import { buildImmutableRuntimePrefix, createRuntimeBoundaryCursor, decodeContinuationClaim, + invocationMatchesClaimTarget, runtimePrefixSegment, type RuntimeBoundaryCursorV1, type RuntimePrefixIdentityV1, @@ -298,6 +299,55 @@ describe('immutable RuntimeEvent boundary', () => { /target turnId reuses source identity/, ); }); + + it('rejects a target opening that does not name the boundary the claim holds', () => { + const boundary = boundaryForRuns('run-source'); + const claim = claimForBoundary(boundary); + + assert.throws( + () => + decodeContinuationClaim({ + ...claim, + targetOpening: { + ...claim.targetOpening, + source: { ...claim.targetOpening.source, sourceRunId: 'run-elsewhere' }, + }, + }), + /target opening mismatch/, + ); + assert.throws( + () => + decodeContinuationClaim({ + ...claim, + targetOpening: { ...claim.targetOpening, source: { kind: 'fresh' } }, + }), + /target opening mismatch/, + ); + }); + + it('recognises the invocation the claim authorises', () => { + const boundary = boundaryForRuns('run-source'); + const claim = decodeContinuationClaim(claimForBoundary(boundary)); + const invocation = { ...claim.target, opening: claim.targetOpening }; + + assert.ok(invocationMatchesClaimTarget(invocation, claim)); + assert.ok( + !invocationMatchesClaimTarget( + { + ...invocation, + opening: { + ...invocation.opening, + configuration: { ...invocation.opening.configuration, cwd: '/elsewhere' }, + }, + }, + claim, + ), + ); + assert.ok( + !invocationMatchesClaimTarget({ ...invocation, runId: 'another-run' }, claim), + 'the claim fixes the target identity as well as its opening', + ); + }); }); function runtimeIdentity(runId: string): RuntimePrefixIdentityV1 { @@ -353,34 +403,36 @@ function claimForBoundary(boundary: RuntimeBoundaryCursorV1) { providerProjectionVersion: 1, providerReplayDigest: `sha256:${'b'.repeat(64)}`, target, - targetRunHeader: { - runId: target.runId, - invocationId: target.invocationId, - sessionId: target.sessionId, - turnId: target.turnId, - status: 'created', - backendKind: 'fake', - llmConnectionSlug: 'connection-1', - modelId: 'model-1', - cwd: '/workspace', - permissionMode: 'ask', - collaborationMode: 'agent', - orchestrationMode: 'default', - orchestrationSource: 'session', - createdAt: 1, - updatedAt: 1, - parentRunId: source.identity.runId, - parentTurnId: source.identity.turnId, - continuationSource: { - protocol: 'continuation_source_v2', - claimId: 'claim-1', - boundaryDigest: boundary.manifestDigest, + targetOpening: { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: { + provenance: 'unknown', + backendKind: 'fake', + llmConnectionSlug: 'connection-1', + modelId: 'model-1', + }, + configuration: { + cwd: '/workspace', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + root: { kind: 'user' }, + source: { + kind: 'continuation', sourceInvocationId: source.identity.invocationId, sourceRunId: source.identity.runId, sourceTurnId: source.identity.turnId, sourceRuntimeEventHighWater: source.position.lastEventSeq, - sourcePrefixDigest: source.prefixDigest, - replayManifestDigest: boundary.manifestDigest, + claimId: 'claim-1', + boundaryDigest: boundary.manifestDigest, + }, + lineage: { + parentRunId: source.identity.runId, + parentTurnId: source.identity.turnId, }, }, claimedAt: 1, diff --git a/packages/core/src/__tests__/runtime-invocation-hosted-root.test.ts b/packages/core/src/__tests__/runtime-invocation-hosted-root.test.ts new file mode 100644 index 0000000000..d9a5073b31 --- /dev/null +++ b/packages/core/src/__tests__/runtime-invocation-hosted-root.test.ts @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { invocationMatchesHostedRootExecution } from '../runtime-invocation.js'; +import type { + RuntimeEventInvocationOpenedContent, + RuntimeInvocationLineage, + RuntimeInvocationOpenSource, + RuntimeInvocationRootAuthority, +} from '../runtime-event.js'; + +const BOUNDARY_DIGEST = `sha256:${'a'.repeat(64)}` as const; + +function invocation( + root: RuntimeInvocationRootAuthority, + overrides: { + invocationId?: string; + source?: RuntimeInvocationOpenSource; + lineage?: RuntimeInvocationLineage; + orchestrationMode?: 'default' | 'graph'; + orchestrationSource?: 'session' | 'turn_override'; + } = {}, +): { invocationId: string; opening: RuntimeEventInvocationOpenedContent } { + const lineage = overrides.lineage; + return { + invocationId: overrides.invocationId ?? 'invocation-1', + opening: { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: { + provenance: 'unknown', + backendKind: 'fake', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: '/workspace', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: overrides.orchestrationMode ?? 'default', + orchestrationSource: overrides.orchestrationSource ?? 'session', + toolMode: 'direct', + agentSwarmAuthorization: 'none', + }, + root, + source: overrides.source ?? { kind: 'fresh' }, + ...(lineage ? { lineage } : {}), + }, + }; +} + +test('regenerate root identity requires exactly its own turn lineage', () => { + const lineage = { parentTurnId: 'turn-1', regeneratedFromTurnId: 'turn-1' }; + const execution = { kind: 'regenerate', sourceTurnId: 'turn-1' } as const; + + assert.equal( + invocationMatchesHostedRootExecution(invocation({ kind: 'user' }, { lineage }), execution), + true, + ); + assert.equal( + invocationMatchesHostedRootExecution( + invocation({ kind: 'user' }, { lineage: { ...lineage, regeneratedFromTurnId: 'turn-x' } }), + execution, + ), + false, + ); + // One extra lineage edge is one edge too many: a regenerate root has no parent + // run, no agent and no branch. + assert.equal( + invocationMatchesHostedRootExecution( + invocation({ kind: 'user' }, { lineage: { ...lineage, parentRunId: 'run-0' } }), + execution, + ), + false, + ); + assert.equal( + invocationMatchesHostedRootExecution( + invocation({ kind: 'scheduled_task', scheduledTaskId: 'task-1' }, { lineage }), + execution, + ), + false, + ); +}); + +test('context compact root identity rejects any lineage and any other root', () => { + const execution = { kind: 'context_compact' } as const; + + assert.equal( + invocationMatchesHostedRootExecution(invocation({ kind: 'context_compact' }), execution), + true, + ); + assert.equal( + invocationMatchesHostedRootExecution(invocation({ kind: 'user' }), execution), + false, + ); + assert.equal( + invocationMatchesHostedRootExecution( + invocation({ kind: 'context_compact' }, { lineage: { parentTurnId: 'turn-1' } }), + execution, + ), + false, + ); +}); + +test('each host authority root matches only its own kind and id', () => { + assert.equal( + invocationMatchesHostedRootExecution( + invocation({ kind: 'scheduled_task', scheduledTaskId: 'task-1' }), + { kind: 'scheduled_task', scheduledTaskId: 'task-1' }, + ), + true, + ); + assert.equal( + invocationMatchesHostedRootExecution( + invocation({ kind: 'scheduled_task', scheduledTaskId: 'task-2' }), + { kind: 'scheduled_task', scheduledTaskId: 'task-1' }, + ), + false, + ); + assert.equal( + invocationMatchesHostedRootExecution( + invocation({ kind: 'legacy_automation', legacyAutomationId: 'automation-1' }), + { kind: 'legacy_automation', automationId: 'automation-1' }, + ), + true, + ); + assert.equal( + invocationMatchesHostedRootExecution(invocation({ kind: 'goal', goalId: 'goal-1' }), { + kind: 'goal', + goalId: 'goal-1', + }), + true, + ); + // A Goal root is not a ScheduledTask root, and the union says so directly. + assert.equal( + invocationMatchesHostedRootExecution(invocation({ kind: 'goal', goalId: 'goal-1' }), { + kind: 'scheduled_task', + scheduledTaskId: 'goal-1', + }), + false, + ); +}); + +test('a supervisor wake root carries its graph prefix and graph orchestration', () => { + const root = { + kind: 'agent_graph_supervisor_wake', + wakeId: 'graph-1:wake-1', + attemptId: 'attempt-1', + } as const; + const execution = { + kind: 'agent_graph_supervisor_wake', + graphId: 'graph-1', + wakeId: 'graph-1:wake-1', + attemptId: 'attempt-1', + } as const; + const graphConfiguration = { + orchestrationMode: 'graph', + orchestrationSource: 'turn_override', + } as const; + + assert.equal( + invocationMatchesHostedRootExecution(invocation(root, graphConfiguration), execution), + true, + ); + assert.equal(invocationMatchesHostedRootExecution(invocation(root), execution), false); + assert.equal( + invocationMatchesHostedRootExecution(invocation(root, graphConfiguration), { + ...execution, + graphId: 'graph-2', + }), + false, + ); +}); + +test('a safe boundary continuation root matches its claim and its own invocation', () => { + const source = { + kind: 'continuation', + sourceInvocationId: 'invocation-0', + sourceRunId: 'run-0', + sourceTurnId: 'turn-0', + sourceRuntimeEventHighWater: 4, + claimId: 'claim-1', + boundaryDigest: BOUNDARY_DIGEST, + } as const; + const lineage = { parentRunId: 'run-0', parentTurnId: 'turn-0' }; + const execution = { + kind: 'safe_boundary_continuation', + sourceInvocationId: 'invocation-0', + sourceRunId: 'run-0', + sourceTurnId: 'turn-0', + sourceRuntimeEventHighWater: 4, + claimId: 'claim-1', + boundaryDigest: BOUNDARY_DIGEST, + providerReplayDigest: BOUNDARY_DIGEST, + safetyDigest: BOUNDARY_DIGEST, + targetInvocationId: 'invocation-1', + } as const; + + assert.equal( + invocationMatchesHostedRootExecution( + invocation({ kind: 'user' }, { source, lineage }), + execution, + ), + true, + ); + assert.equal( + invocationMatchesHostedRootExecution( + invocation({ kind: 'user' }, { source, lineage, invocationId: 'invocation-other' }), + execution, + ), + false, + ); + assert.equal( + invocationMatchesHostedRootExecution(invocation({ kind: 'user' }, { lineage }), execution), + false, + ); +}); diff --git a/packages/core/src/__tests__/runtime-invocation-opened.test.ts b/packages/core/src/__tests__/runtime-invocation-opened.test.ts new file mode 100644 index 0000000000..205b345a3d --- /dev/null +++ b/packages/core/src/__tests__/runtime-invocation-opened.test.ts @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { describe, test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + decodeRuntimeEvent, + decodeRuntimeInvocationOpened, + runtimeEventHasModelVisibleContent, + runtimeEventInvocationOpening, + RUNTIME_EVENT_CONTENT_KINDS, + type RuntimeEvent, + type RuntimeEventInvocationOpenedContent, +} from '../runtime-event.js'; + +const DIGEST = `sha256:${'a'.repeat(64)}` as const; + +function opening( + overrides: Partial = {}, +): RuntimeEventInvocationOpenedContent { + return { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: { + provenance: 'runtime', + backendKind: 'ai-sdk', + llmConnectionId: 'conn-1', + llmConnectionSlug: 'anthropic', + modelId: 'claude-x', + providerStateIdentity: DIGEST, + }, + configuration: { + cwd: '/repo', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + root: { kind: 'user' }, + source: { kind: 'fresh' }, + ...overrides, + }; +} + +function openingEvent(content: unknown): unknown { + return { + id: 'evt-open', + invocationId: 'inv-1', + runId: 'inv-1', + sessionId: 'sess-1', + turnId: 'turn-1', + ts: 10, + partial: false, + role: 'system', + author: 'system', + modelVisibility: 'hidden', + content, + }; +} + +describe('invocation_opened content contract', () => { + test('is a runtime event content kind', () => { + assert.ok(RUNTIME_EVENT_CONTENT_KINDS.includes('invocation_opened')); + }); + + test('decodes as RuntimeEvent content and narrows back out', () => { + const event = decodeRuntimeEvent(openingEvent(opening())); + const fact = runtimeEventInvocationOpening(event); + assert.ok(fact); + assert.equal(fact.protocol, 'invocation_opened_v1'); + assert.equal(fact.route.provenance, 'runtime'); + assert.equal(fact.route.modelId, 'claude-x'); + }); + + test('is never model visible', () => { + const event = decodeRuntimeEvent(openingEvent(opening())) as RuntimeEvent; + assert.equal(runtimeEventHasModelVisibleContent(event), false); + }); + + test('binds connection identity to where the route came from, both ways', () => { + const unknownRoute = { + provenance: 'unknown', + backendKind: 'ai-sdk', + llmConnectionSlug: 'legacy', + modelId: 'legacy-model', + } as const; + assert.equal( + decodeRuntimeInvocationOpened(opening({ route: unknownRoute })).route.provenance, + 'unknown', + ); + assert.throws(() => + decodeRuntimeInvocationOpened( + opening({ route: { ...unknownRoute, llmConnectionId: 'conn-1' } as never }), + ), + ); + assert.throws(() => + decodeRuntimeInvocationOpened( + opening({ route: { ...unknownRoute, provenance: 'runtime' } as never }), + ), + ); + }); + + test('accepts every root authority the runtime can open, and no mixture of them', () => { + for (const root of [ + { kind: 'user' }, + { kind: 'context_compact' }, + { kind: 'scheduled_task', scheduledTaskId: 'task-1' }, + { kind: 'goal', goalId: 'goal-1' }, + { kind: 'agent_graph_supervisor_wake', wakeId: 'w1', attemptId: 'a1' }, + { kind: 'legacy_automation', legacyAutomationId: 'auto-1' }, + ] as const) { + assert.equal(decodeRuntimeInvocationOpened(opening({ root })).root.kind, root.kind); + } + assert.throws(() => + decodeRuntimeInvocationOpened( + opening({ root: { kind: 'goal', goalId: 'g1', scheduledTaskId: 's1' } as never }), + ), + ); + }); + + test('carries a continuation source only with the boundary position it resumes from', () => { + const source = { + kind: 'continuation', + sourceInvocationId: 'inv-0', + sourceRunId: 'inv-0', + sourceTurnId: 'turn-0', + } as const; + const fact = decodeRuntimeInvocationOpened( + opening({ + source: { + ...source, + sourceRuntimeEventHighWater: 7, + claimId: 'claim-1', + boundaryDigest: DIGEST, + }, + }), + ); + assert.equal(fact.source.kind, 'continuation'); + assert.throws(() => decodeRuntimeInvocationOpened(opening({ source: source as never }))); + }); + + test('rejects anything the closed shape does not name', () => { + assert.throws(() => + decodeRuntimeInvocationOpened({ ...opening(), runComposition: {} } as never), + ); + assert.throws(() => + decodeRuntimeInvocationOpened( + opening({ + configuration: { ...opening().configuration, sessionMode: 'agent' } as never, + }), + ), + ); + }); + + test('a malformed opening fact fails the whole RuntimeEvent decode', () => { + assert.throws(() => + decodeRuntimeEvent( + openingEvent({ kind: 'invocation_opened', protocol: 'invocation_opened_v1' }), + ), + ); + }); +}); diff --git a/packages/core/src/agent-run.ts b/packages/core/src/agent-run.ts index ffc7e44d4c..cf13e687ec 100644 --- a/packages/core/src/agent-run.ts +++ b/packages/core/src/agent-run.ts @@ -17,22 +17,15 @@ * under the License. */ -import { - decodePersistedPermissionMode, - isPermissionMode, - type PermissionMode, -} from './permission.js'; -import type { PersistedValue } from './persisted-value.js'; -import { isCollaborationMode, type CollaborationMode } from './collaboration.js'; -import { - isAgentSwarmAuthorizationSource, - isEffectiveOrchestrationSource, - isOrchestrationMode, - type AgentSwarmAuthorizationSource, - type EffectiveOrchestrationSource, - type OrchestrationMode, -} from './orchestration.js'; -import type { PersistedBackendKind } from './session.js'; +/** + * The operational ledger one invocation writes beside its canonical events. + * + * These records are metering, request attempts, permission decisions and + * diagnostics: facts with an operational demand of their own. What an + * invocation *is* — its route, configuration, lineage and outcome — belongs to + * the event spine in `runtime-invocation.ts`, not here. + */ + import { defineObjectShape, hasExactShape, @@ -40,354 +33,14 @@ import { isOptionalString, isRecord, } from './record-schema.js'; -import type { AgentGraphIntentClaim } from './agent-graph-control.js'; -import { isToolMode, type ToolMode } from './tool-mode.js'; import { decodeRunCompositionSnapshot, type RunCompositionSnapshot } from './run-composition.js'; -export const AGENT_RUN_STATUSES = [ - 'created', - 'running', - 'waiting_for_user', - 'completed', - 'failed', - 'cancelled', -] as const; - -export type AgentRunStatus = (typeof AGENT_RUN_STATUSES)[number]; - -export interface AgentRunContinuationSourceV1 { - sourceInvocationId: string; - sourceRunId: string; - sourceTurnId: string; - sourceRuntimeEventHighWater: number; -} - -export interface AgentRunContinuationSourceV2 extends AgentRunContinuationSourceV1 { - protocol: 'continuation_source_v2'; - claimId: string; - boundaryDigest: `sha256:${string}`; - sourcePrefixDigest: `sha256:${string}`; - replayManifestDigest: `sha256:${string}`; -} - -export type AgentRunContinuationSource = - | AgentRunContinuationSourceV1 - | AgentRunContinuationSourceV2; - -export type RootExecutionDescriptor = - | { - kind: 'external_message'; - inputDigest?: `sha256:${string}`; - maxSteps?: number; - } - | { - /** Tool-free conversational execution admitted only by WorkHub authority. */ - kind: 'workhub_coordination'; - inputDigest: `sha256:${string}`; - } - | { kind: 'regenerate'; sourceTurnId: string } - | { kind: 'context_compact' } - | { - kind: 'scheduled_task'; - scheduledTaskId: string; - /** Includes the immutable Connection target for Agent ScheduledTasks. */ - executionFingerprint?: `sha256:${string}`; - } - | { kind: 'legacy_automation'; automationId: string } - | { kind: 'goal'; goalId: string } - | { - kind: 'agent_graph_supervisor_wake'; - graphId: string; - wakeId: string; - attemptId: string; - } - | { - kind: 'safe_boundary_continuation'; - sourceInvocationId: string; - sourceRunId: string; - sourceTurnId: string; - sourceRuntimeEventHighWater: number; - claimId: string; - boundaryDigest: `sha256:${string}`; - providerReplayDigest: `sha256:${string}`; - safetyDigest: `sha256:${string}`; - targetInvocationId: string; - } - | { - kind: 'linked_child_initial'; - agentId: string; - agentName: string; - } - | { - kind: 'linked_child_resume'; - agentId: string; - agentName: string; - sourceRunId: string; - } - | { - kind: 'linked_child_provider_retry'; - agentId: string; - agentName: string; - sourceRunId: string; - } - | { - kind: 'claimed_agent_graph_intent'; - claim: AgentGraphIntentClaim; - agentId: string; - agentName: string; - }; - -const AGENT_RUN_CONTINUATION_SOURCE_V1_SHAPE = defineObjectShape()( - ['sourceInvocationId', 'sourceRunId', 'sourceTurnId', 'sourceRuntimeEventHighWater'], - [], -); -const AGENT_RUN_CONTINUATION_SOURCE_V2_SHAPE = defineObjectShape()( - [ - 'protocol', - 'claimId', - 'boundaryDigest', - 'sourceInvocationId', - 'sourceRunId', - 'sourceTurnId', - 'sourceRuntimeEventHighWater', - 'sourcePrefixDigest', - 'replayManifestDigest', - ], - [], -); - -export interface AgentRunHeader { - runId: string; - /** Durable Runtime invocation spine. Optional only for legacy run headers. */ - invocationId?: string; - sessionId: string; - turnId: string; - status: AgentRunStatus; - backendKind: PersistedBackendKind; - /** Immutable Connection entity identity. Optional only on legacy run headers. */ - llmConnectionId?: string; - /** - * Opaque identity of the provider endpoint and credential ownership frozen - * before this run's first provider dispatch. Optional only on legacy or - * non-provider run headers. - */ - providerStateIdentity?: `sha256:${string}`; - llmConnectionSlug: string; - modelId: string; - cwd: string; - /** Authoritative host identity for the workspace observed when the run was created. */ - workspaceIdentity?: string; - permissionMode: PermissionMode; - /** Snapshot of the session collaboration mode. Optional on legacy runs. */ - collaborationMode?: CollaborationMode; - /** Effective orchestration mode for this run. Optional on legacy runs. */ - orchestrationMode?: OrchestrationMode; - /** Whether the effective mode came from the session or this turn. */ - orchestrationSource?: EffectiveOrchestrationSource; - /** Narrow authority for the parent agent_swarm envelope. */ - agentSwarmAuthorization?: AgentSwarmAuthorizationSource; - /** Effective tool protocol for this run. Optional on legacy runs. */ - toolMode?: ToolMode; - /** Immutable composer-owned prompt and tool-surface snapshot committed before provider dispatch. */ - runComposition?: RunCompositionSnapshot; - createdAt: number; - updatedAt: number; - completedAt?: number; - parentRunId?: string; - /** Immediate child AgentRun continued by this run. */ - resumedFromRunId?: string; - /** Immediate child AgentRun whose provider step is retried by this run. */ - retriedFromRunId?: string; - agentId?: string; - agentName?: string; - parentTurnId?: string; - retriedFromTurnId?: string; - regeneratedFromTurnId?: string; - branchOfTurnId?: string; - parentSessionId?: string; - /** Durable claim that this run is the continuation child for one source boundary. */ - continuationSource?: AgentRunContinuationSource; - /** ScheduledTask that triggered this host-authored Run. */ - scheduledTaskId?: string; - /** Removed Automation authority that triggered this historical Run. */ - legacyAutomationId?: string; - /** Host-owned Goal generation that triggered this continuation Run. */ - goalId?: string; - /** Durable graph milestone that caused this host-authored supervisor turn. */ - agentGraphWakeId?: string; - /** Durable delivery attempt for this host-authored supervisor turn. */ - agentGraphWakeAttemptId?: string; - /** Positive identity for a host-authored root that has no message lineage. */ - rootExecutionKind?: 'context_compact'; - failureClass?: string; - failureMessage?: string; - abortSource?: string; - traceWriteError?: string; -} - -type HostedRootExecutionDescriptor = Extract< - RootExecutionDescriptor, - { - kind: - | 'regenerate' - | 'context_compact' - | 'scheduled_task' - | 'legacy_automation' - | 'goal' - | 'agent_graph_supervisor_wake' - | 'safe_boundary_continuation'; - } ->; - -export function agentRunMatchesHostedRootExecution( - run: AgentRunHeader, - execution: HostedRootExecutionDescriptor, -): boolean { - if (execution.kind !== 'context_compact' && run.rootExecutionKind !== undefined) return false; - if (execution.kind === 'regenerate') { - return ( - run.parentTurnId === execution.sourceTurnId && - run.regeneratedFromTurnId === execution.sourceTurnId && - run.parentRunId === undefined && - run.resumedFromRunId === undefined && - run.retriedFromRunId === undefined && - run.agentId === undefined && - run.agentName === undefined && - run.retriedFromTurnId === undefined && - run.branchOfTurnId === undefined && - run.parentSessionId === undefined && - run.continuationSource === undefined && - run.scheduledTaskId === undefined && - run.legacyAutomationId === undefined && - run.goalId === undefined && - run.agentGraphWakeId === undefined && - run.agentGraphWakeAttemptId === undefined - ); - } - if (execution.kind === 'context_compact') { - return ( - run.rootExecutionKind === 'context_compact' && - run.parentTurnId === undefined && - run.regeneratedFromTurnId === undefined && - run.parentRunId === undefined && - run.resumedFromRunId === undefined && - run.retriedFromRunId === undefined && - run.agentId === undefined && - run.agentName === undefined && - run.retriedFromTurnId === undefined && - run.branchOfTurnId === undefined && - run.parentSessionId === undefined && - run.continuationSource === undefined && - run.scheduledTaskId === undefined && - run.legacyAutomationId === undefined && - run.goalId === undefined && - run.agentGraphWakeId === undefined && - run.agentGraphWakeAttemptId === undefined - ); - } - if (execution.kind === 'safe_boundary_continuation') { - const source = run.continuationSource; - return ( - run.invocationId === execution.targetInvocationId && - run.parentRunId === execution.sourceRunId && - run.parentTurnId === execution.sourceTurnId && - source !== undefined && - 'protocol' in source && - source.protocol === 'continuation_source_v2' && - source.sourceInvocationId === execution.sourceInvocationId && - source.sourceRunId === execution.sourceRunId && - source.sourceTurnId === execution.sourceTurnId && - source.sourceRuntimeEventHighWater === execution.sourceRuntimeEventHighWater && - source.claimId === execution.claimId && - source.boundaryDigest === execution.boundaryDigest && - source.replayManifestDigest === execution.boundaryDigest && - run.resumedFromRunId === undefined && - run.retriedFromRunId === undefined && - run.agentId === undefined && - run.agentName === undefined && - run.retriedFromTurnId === undefined && - run.regeneratedFromTurnId === undefined && - run.branchOfTurnId === undefined && - run.parentSessionId === undefined && - run.scheduledTaskId === undefined && - run.legacyAutomationId === undefined && - run.goalId === undefined && - run.agentGraphWakeId === undefined && - run.agentGraphWakeAttemptId === undefined - ); - } - const authorityMatches = hostedRootAuthorityMatches(run, execution); - return ( - authorityMatches && - run.parentRunId === undefined && - run.resumedFromRunId === undefined && - run.retriedFromRunId === undefined && - run.agentId === undefined && - run.agentName === undefined && - run.parentTurnId === undefined && - run.retriedFromTurnId === undefined && - run.regeneratedFromTurnId === undefined && - run.branchOfTurnId === undefined && - run.parentSessionId === undefined && - run.continuationSource === undefined - ); -} - -function hostedRootAuthorityMatches( - run: AgentRunHeader, - execution: Exclude< - HostedRootExecutionDescriptor, - { kind: 'regenerate' | 'context_compact' | 'safe_boundary_continuation' } - >, -): boolean { - switch (execution.kind) { - case 'scheduled_task': - return ( - run.scheduledTaskId === execution.scheduledTaskId && - run.legacyAutomationId === undefined && - run.goalId === undefined && - run.agentGraphWakeId === undefined && - run.agentGraphWakeAttemptId === undefined - ); - case 'legacy_automation': - return ( - run.legacyAutomationId === execution.automationId && - run.scheduledTaskId === undefined && - run.goalId === undefined && - run.agentGraphWakeId === undefined && - run.agentGraphWakeAttemptId === undefined - ); - case 'goal': - return ( - run.goalId === execution.goalId && - run.scheduledTaskId === undefined && - run.legacyAutomationId === undefined && - run.agentGraphWakeId === undefined && - run.agentGraphWakeAttemptId === undefined - ); - case 'agent_graph_supervisor_wake': - return ( - execution.wakeId.startsWith(`${execution.graphId}:`) && - run.agentGraphWakeId === execution.wakeId && - run.agentGraphWakeAttemptId === execution.attemptId && - run.orchestrationMode === 'graph' && - run.orchestrationSource === 'turn_override' && - run.agentSwarmAuthorization === 'none' && - run.scheduledTaskId === undefined && - run.legacyAutomationId === undefined && - run.goalId === undefined - ); - } -} - export interface AgentRunInputSummary { textLength: number; attachmentCount: number; } export const AGENT_RUN_EVENT_TYPES = [ - 'run_created', - 'run_started', 'turn_started', 'plan_context_resolved', 'plan_submitted', @@ -399,7 +52,6 @@ export const AGENT_RUN_EVENT_TYPES = [ 'plan_execution_resumed', 'plan_transition_failed', 'graph_supervisor_yielded', - 'run_status_changed', 'model_resolved', 'model_resolve_failed', 'model_stream_started', @@ -427,16 +79,11 @@ export const AGENT_RUN_EVENT_TYPES = [ 'sandbox_escalation_applied', 'sandbox_escalation_failed', 'sandbox_denial_detected', - 'provider_request_captured', - 'provider_request_attempt_recorded', 'model_call_attempt_recorded', 'history_compact_checkpoint_recorded', 'model_projection_transition_recorded', - 'task_gate_decided', + 'run_composition_recorded', 'abort_requested', - 'run_completed', - 'run_failed', - 'run_cancelled', 'trace_write_failed', 'event_corrupt', ] as const; @@ -563,197 +210,28 @@ export function isEmittedAgentRunEventType(type: string): type is AgentRunEventT return EMITTED_AGENT_RUN_EVENT_TYPES.has(type); } -const AGENT_RUN_HEADER_SHAPE = defineObjectShape()( - [ - 'runId', - 'sessionId', - 'turnId', - 'status', - 'backendKind', - 'llmConnectionSlug', - 'modelId', - 'cwd', - 'permissionMode', - 'createdAt', - 'updatedAt', - ], - [ - 'invocationId', - 'llmConnectionId', - 'providerStateIdentity', - 'completedAt', - 'parentRunId', - 'resumedFromRunId', - 'retriedFromRunId', - 'agentId', - 'agentName', - 'parentTurnId', - 'retriedFromTurnId', - 'regeneratedFromTurnId', - 'branchOfTurnId', - 'parentSessionId', - 'workspaceIdentity', - 'continuationSource', - 'scheduledTaskId', - 'legacyAutomationId', - 'goalId', - 'agentGraphWakeId', - 'agentGraphWakeAttemptId', - 'rootExecutionKind', - 'failureClass', - 'failureMessage', - 'abortSource', - 'traceWriteError', - 'collaborationMode', - 'orchestrationMode', - 'orchestrationSource', - 'agentSwarmAuthorization', - 'toolMode', - 'runComposition', - ], -); - const AGENT_RUN_EVENT_SHAPE = defineObjectShape()( ['type', 'id', 'runId', 'sessionId', 'turnId', 'ts'], ['message', 'data'], ); -const RETIRED_AGENT_RUN_STATUSES: Readonly> = { - waiting_permission: 'waiting_for_user', -}; - -export function decodePersistedAgentRunHeader( - persisted: PersistedValue, -): AgentRunHeader { - let value = persisted as unknown; - if ( - isRecord(value) && - value.automationId !== undefined && - value.legacyAutomationId === undefined - ) { - const { automationId, ...current } = value; - value = { ...current, legacyAutomationId: automationId }; - } - if (isRecord(value)) { - const status = - typeof value.status === 'string' - ? (RETIRED_AGENT_RUN_STATUSES[value.status] ?? value.status) - : value.status; - const permissionMode = decodePersistedPermissionMode(value.permissionMode); - if (status !== value.status || permissionMode !== value.permissionMode) { - value = { ...value, status, permissionMode }; - } - } - return decodeAgentRunHeader(value); -} - -export function decodeAgentRunHeader(value: unknown): AgentRunHeader { - if (!isRecord(value) || !hasExactShape(value, AGENT_RUN_HEADER_SHAPE)) { - throw new Error('Invalid AgentRun header schema'); - } - const valid = - typeof value.runId === 'string' && - typeof value.sessionId === 'string' && - typeof value.turnId === 'string' && - (AGENT_RUN_STATUSES as readonly unknown[]).includes(value.status) && - isPersistedBackendKind(value.backendKind) && - (value.llmConnectionId === undefined || - (typeof value.llmConnectionId === 'string' && value.llmConnectionId.length > 0)) && - (value.providerStateIdentity === undefined || - (typeof value.providerStateIdentity === 'string' && - /^sha256:[0-9a-f]{64}$/.test(value.providerStateIdentity))) && - typeof value.llmConnectionSlug === 'string' && - typeof value.modelId === 'string' && - typeof value.cwd === 'string' && - isPermissionMode(value.permissionMode) && - (value.collaborationMode === undefined || isCollaborationMode(value.collaborationMode)) && - (value.orchestrationMode === undefined || isOrchestrationMode(value.orchestrationMode)) && - (value.orchestrationSource === undefined || - isEffectiveOrchestrationSource(value.orchestrationSource)) && - (value.agentSwarmAuthorization === undefined || - isAgentSwarmAuthorizationSource(value.agentSwarmAuthorization)) && - (value.rootExecutionKind === undefined || value.rootExecutionKind === 'context_compact') && - Number(value.scheduledTaskId !== undefined) + - Number(value.legacyAutomationId !== undefined) + - Number(value.goalId !== undefined) + - Number(value.agentGraphWakeId !== undefined) <= - 1 && - (value.toolMode === undefined || isToolMode(value.toolMode)) && - (value.runComposition === undefined || isRunCompositionSnapshot(value.runComposition)) && - isFiniteNumber(value.createdAt) && - isFiniteNumber(value.updatedAt) && - isOptionalString(value.invocationId) && - (value.completedAt === undefined || isFiniteNumber(value.completedAt)) && - [ - value.parentRunId, - value.resumedFromRunId, - value.retriedFromRunId, - value.agentId, - value.agentName, - value.parentTurnId, - value.retriedFromTurnId, - value.regeneratedFromTurnId, - value.branchOfTurnId, - value.parentSessionId, - value.workspaceIdentity, - value.scheduledTaskId, - value.legacyAutomationId, - value.goalId, - value.agentGraphWakeId, - value.agentGraphWakeAttemptId, - value.failureClass, - value.failureMessage, - value.abortSource, - value.traceWriteError, - ].every(isOptionalString) && - (value.continuationSource === undefined || - isAgentRunContinuationSource(value.continuationSource)); - if (!valid) throw new Error('Invalid AgentRun header schema'); - return value as unknown as AgentRunHeader; -} +export const RUN_COMPOSITION_RECORDED_EVENT_TYPE = 'run_composition_recorded' as const; -function isRunCompositionSnapshot(value: unknown): value is RunCompositionSnapshot { - try { - decodeRunCompositionSnapshot(value); - return true; - } catch { - return false; +/** + * Read a run's composer snapshot back out of its ledger. + * + * The composition is written once, before provider dispatch, and the store + * refuses a second append that disagrees with the first. So the earliest + * matching row is the whole answer, and a reader never has to reduce a stream. + */ +export function agentRunCompositionFromEvents( + events: readonly AgentRunEvent[], +): RunCompositionSnapshot | undefined { + for (const event of events) { + if (event.type !== RUN_COMPOSITION_RECORDED_EVENT_TYPE) continue; + return decodeRunCompositionSnapshot(event.data?.runComposition); } -} - -function isAgentRunContinuationSource(value: unknown): value is AgentRunContinuationSource { - if (!isRecord(value)) return false; - const common = - typeof value.sourceInvocationId === 'string' && - typeof value.sourceRunId === 'string' && - typeof value.sourceTurnId === 'string' && - typeof value.sourceRuntimeEventHighWater === 'number' && - Number.isSafeInteger(value.sourceRuntimeEventHighWater) && - value.sourceRuntimeEventHighWater >= 0; - if (!common) return false; - if (hasExactShape(value, AGENT_RUN_CONTINUATION_SOURCE_V1_SHAPE)) return true; - return ( - hasExactShape(value, AGENT_RUN_CONTINUATION_SOURCE_V2_SHAPE) && - value.protocol === 'continuation_source_v2' && - typeof value.claimId === 'string' && - value.claimId.length > 0 && - typeof value.sourceInvocationId === 'string' && - value.sourceInvocationId.length > 0 && - typeof value.sourceRunId === 'string' && - value.sourceRunId.length > 0 && - typeof value.sourceTurnId === 'string' && - value.sourceTurnId.length > 0 && - typeof value.sourceRuntimeEventHighWater === 'number' && - value.sourceRuntimeEventHighWater > 0 && - isSha256Digest(value.boundaryDigest) && - isSha256Digest(value.sourcePrefixDigest) && - isSha256Digest(value.replayManifestDigest) && - value.replayManifestDigest === value.boundaryDigest - ); -} - -function isSha256Digest(value: unknown): value is `sha256:${string}` { - return typeof value === 'string' && /^sha256:[0-9a-f]{64}$/.test(value); + return undefined; } export function decodeAgentRunEvent(value: unknown): AgentRunEvent { @@ -775,24 +253,7 @@ export function decodeAgentRunEvent(value: unknown): AgentRunEvent { return value as unknown as AgentRunEvent; } -/** - * Decode guard for a durable run header. `'fake'` stays accepted: runs written - * by builds that shipped FakeBackend must keep decoding (#3211). - */ -function isPersistedBackendKind(value: unknown): value is PersistedBackendKind { - return value === 'ai-sdk' || value === 'fake'; -} - export interface AgentRunStore { - createRun(header: AgentRunHeader, options?: { durable?: boolean }): Promise; - updateRun( - sessionId: string, - runId: string, - patch: Partial, - options?: { durable?: boolean }, - ): Promise; - readRun(sessionId: string, runId: string): Promise; - listSessionRuns(sessionId: string): Promise; appendEvent( sessionId: string, runId: string, @@ -820,21 +281,3 @@ export interface AgentRunStore { options: { ifLedgerRevision: string; replaceEventId?: string }, ): Promise; } - -/** - * Whether a run contributes directly to the owning session's transcript. - * Top-level continuations carry parent lineage for recovery, but unlike - * child-agent runs their output remains part of the parent session - * conversation. A legacy child retry may also carry continuation authority; - * its agent identity keeps it outside the owning session transcript. - */ -export function isSessionInlineRun(run: { - readonly parentRunId?: string; - readonly continuationSource?: unknown; - readonly agentId?: string; -}): boolean { - return ( - run.parentRunId === undefined || - (run.continuationSource !== undefined && run.agentId === undefined) - ); -} diff --git a/packages/core/src/backend-types.ts b/packages/core/src/backend-types.ts index 7b9d45854f..ef98240d7e 100644 --- a/packages/core/src/backend-types.ts +++ b/packages/core/src/backend-types.ts @@ -42,7 +42,7 @@ import type { InteractionClosureReason, InteractionFormResult } from './interact import type { RuntimeEvent } from './runtime-event.js'; import type { SandboxBoundaryResponse, SandboxBoundarySettlement } from './sandbox-boundary.js'; import type { StoredMessage, PersistedBackendKind } from './session.js'; -import type { AgentRunHeader } from './agent-run.js'; +import type { RuntimeInvocationRecord } from './runtime-invocation.js'; import type { UserQuestionResponse } from './user-question.js'; import type { ContextBudgetDiagnostic } from './usage-stats/types.js'; import type { EffectiveOrchestration } from './orchestration.js'; @@ -94,11 +94,11 @@ export interface BackendSendInput { */ runtimeContext?: RuntimeEvent[]; /** - * Existing durable run headers for `runtimeContext`, used only to verify + * The invocations `runtimeContext` came from, used only to verify * provider-owned replay against the current model route. RuntimeEvents stay - * the transcript authority; route provenance remains owned by AgentRun. + * the transcript authority; route provenance is read off each opening fact. */ - runtimeContextRunHeaders?: readonly AgentRunHeader[]; + runtimeContextInvocations?: readonly RuntimeInvocationRecord[]; /** Continue from an already committed RuntimeEvent boundary without adding another user turn. */ continuation?: RuntimeContinuationMetadata; /** @@ -189,8 +189,8 @@ export interface BackendCompactHistoryInput { */ runId: string; runtimeContext: readonly RuntimeEvent[]; - /** Source-run route authority for provider-owned history projected into the compaction call. */ - runtimeContextRunHeaders?: readonly AgentRunHeader[]; + /** Source-invocation route authority for provider-owned history projected into the compaction call. */ + runtimeContextInvocations?: readonly RuntimeInvocationRecord[]; } export interface BackendCompactHistoryResult { diff --git a/packages/core/src/execution-inspect.ts b/packages/core/src/execution-inspect.ts index 5022e1b23e..e2fff784d9 100644 --- a/packages/core/src/execution-inspect.ts +++ b/packages/core/src/execution-inspect.ts @@ -17,7 +17,6 @@ * under the License. */ -import { AGENT_RUN_STATUSES, type AgentRunHeader } from './agent-run.js'; import { EXECUTION_LOG_LEDGERS, type ExecutionLogCoverage } from './execution-log-coverage.js'; import { SESSION_STATUSES, type SessionHeader } from './session.js'; @@ -36,20 +35,22 @@ export interface ExecutionInspectDiagnostic { eventId?: string; } +const AGENT_RUN_INSPECT_STATUSES = ['running', 'completed', 'failed', 'cancelled'] as const; + export interface AgentRunInspectIdentity { sessionId: string; agentRunId: string; - invocationId?: string; + invocationId: string; turnId: string; parentRunId?: string; resumedFromRunId?: string; retriedFromRunId?: string; parentTurnId?: string; agentId?: string; - status: AgentRunHeader['status']; - createdAt: number; - updatedAt: number; - completedAt?: number; + /** Derived from the terminal RuntimeEvent; `running` means there is none yet. */ + status: (typeof AGENT_RUN_INSPECT_STATUSES)[number]; + openedAt: number; + endedAt?: number; failureClass?: string; abortSource?: string; } @@ -80,8 +81,6 @@ export interface AgentRunInspectCompactionCheckpoint { export interface AgentRunInspectSourceHealth { runtimeLedger: 'present' | 'missing' | 'read_failed'; runtimeTerminalPresent: boolean; - operationalTerminalPresent: boolean; - statusConsistency: 'consistent' | 'inconsistent' | 'incomplete'; } export interface AgentRunInspectDocument { @@ -194,28 +193,28 @@ function isAgentRunIdentity(value: unknown): value is AgentRunInspectIdentity { return ( hasShape( value, - ['sessionId', 'agentRunId', 'turnId', 'status', 'createdAt', 'updatedAt'], + ['sessionId', 'agentRunId', 'invocationId', 'turnId', 'status', 'openedAt'], [ - 'invocationId', 'parentRunId', 'resumedFromRunId', 'retriedFromRunId', 'parentTurnId', 'agentId', - 'completedAt', + 'endedAt', 'failureClass', 'abortSource', ], ) && isString(value.sessionId) && isString(value.agentRunId) && + isString(value.invocationId) && isString(value.turnId) && - AGENT_RUN_STATUSES.includes(value.status as (typeof AGENT_RUN_STATUSES)[number]) && - isCount(value.createdAt) && - isCount(value.updatedAt) && - isOptionalCount(value.completedAt) && + AGENT_RUN_INSPECT_STATUSES.includes( + value.status as (typeof AGENT_RUN_INSPECT_STATUSES)[number], + ) && + isCount(value.openedAt) && + isOptionalCount(value.endedAt) && [ - value.invocationId, value.parentRunId, value.resumedFromRunId, value.retriedFromRunId, @@ -237,24 +236,11 @@ function isAgentRunSources(value: unknown): boolean { isCount(value.operationalEventCount) && isCount(value.runtimeEventCount) && (value.runtimeCoverage === undefined || isCoverage(value.runtimeCoverage)) && - hasShape( - value.health, - [ - 'runtimeLedger', - 'runtimeTerminalPresent', - 'operationalTerminalPresent', - 'statusConsistency', - ], - [], - ) && + hasShape(value.health, ['runtimeLedger', 'runtimeTerminalPresent'], []) && (value.health.runtimeLedger === 'present' || value.health.runtimeLedger === 'missing' || value.health.runtimeLedger === 'read_failed') && - typeof value.health.runtimeTerminalPresent === 'boolean' && - typeof value.health.operationalTerminalPresent === 'boolean' && - (value.health.statusConsistency === 'consistent' || - value.health.statusConsistency === 'inconsistent' || - value.health.statusConsistency === 'incomplete') + typeof value.health.runtimeTerminalPresent === 'boolean' ); } diff --git a/packages/core/src/runtime-boundary.ts b/packages/core/src/runtime-boundary.ts index 6408b67b3b..8d9ff1ae68 100644 --- a/packages/core/src/runtime-boundary.ts +++ b/packages/core/src/runtime-boundary.ts @@ -19,10 +19,10 @@ import * as nodeCrypto from 'node:crypto'; import type { Hash } from 'node:crypto'; -import { decodeAgentRunHeader, type AgentRunHeader } from './agent-run.js'; import { encodeCanonicalRuntimeEvent } from './canonical-runtime-event.js'; import { isRecord } from './record-schema.js'; -import type { RuntimeEvent } from './runtime-event.js'; +import { decodeRuntimeInvocationOpened, TOOL_BOUNDARY_PROTOCOL_V1 } from './runtime-event.js'; +import type { RuntimeEvent, RuntimeEventInvocationOpenedContent } from './runtime-event.js'; import { stableJsonStringify } from './tool-args-identity.js'; export type RuntimeBoundaryDigest = `sha256:${string}`; @@ -79,8 +79,16 @@ export interface ContinuationClaimV1 { runId: string; turnId: string; }; - /** Exact pre-provider target Run header used by both normal admission and crash repair. */ - targetRunHeader: AgentRunHeader; + /** + * The opening fact the target invocation's first event must carry. + * + * This is what the claim is actually for: a continuation's start event is + * event 1 of its target, so it is also that invocation's opening fact, and the + * claim has to say in advance exactly what that fact will be. Everything else + * about the target is fixed by the claim's own fields, so the pre-provider Run + * header is a projection of this rather than a second record of it. + */ + targetOpening: RuntimeEventInvocationOpenedContent; claimedAt: number; } @@ -231,7 +239,7 @@ export function decodeContinuationClaim(value: unknown): ContinuationClaimV1 { 'providerProjectionVersion', 'providerReplayDigest', 'target', - 'targetRunHeader', + 'targetOpening', 'claimedAt', ]) || value.protocol !== 'continuation_claim_v1' || @@ -270,32 +278,18 @@ export function decodeContinuationClaim(value: unknown): ContinuationClaimV1 { if (boundary.segments.some((segment) => segment.identity.turnId === targetTurnId)) { throw new Error('Continuation claim target turnId reuses source identity'); } - const targetRunHeader = decodeAgentRunHeader(value.targetRunHeader); - const continuationSource = targetRunHeader.continuationSource; + const targetOpening = decodeRuntimeInvocationOpened(value.targetOpening); + const openSource = targetOpening.source; if ( - targetRunHeader.runId !== targetRunId || - targetRunHeader.invocationId !== targetInvocationId || - targetRunHeader.sessionId !== value.target.sessionId || - targetRunHeader.turnId !== targetTurnId || - targetRunHeader.status !== 'created' || - targetRunHeader.createdAt !== value.claimedAt || - targetRunHeader.updatedAt !== value.claimedAt || - targetRunHeader.completedAt !== undefined || - targetRunHeader.failureClass !== undefined || - targetRunHeader.failureMessage !== undefined || - !continuationSource || - !('protocol' in continuationSource) || - continuationSource.protocol !== 'continuation_source_v2' || - continuationSource.claimId !== value.claimId || - continuationSource.boundaryDigest !== boundaryDigest || - continuationSource.sourceInvocationId !== source.identity.invocationId || - continuationSource.sourceRunId !== source.identity.runId || - continuationSource.sourceTurnId !== source.identity.turnId || - continuationSource.sourceRuntimeEventHighWater !== source.position.lastEventSeq || - continuationSource.sourcePrefixDigest !== source.prefixDigest || - continuationSource.replayManifestDigest !== boundary.manifestDigest + openSource.kind !== 'continuation' || + openSource.claimId !== value.claimId || + openSource.boundaryDigest !== boundaryDigest || + openSource.sourceInvocationId !== source.identity.invocationId || + openSource.sourceRunId !== source.identity.runId || + openSource.sourceTurnId !== source.identity.turnId || + openSource.sourceRuntimeEventHighWater !== source.position.lastEventSeq ) { - throw new Error('Continuation claim target Run header mismatch'); + throw new Error('Continuation claim target opening mismatch'); } return { protocol: 'continuation_claim_v1', @@ -310,11 +304,94 @@ export function decodeContinuationClaim(value: unknown): ContinuationClaimV1 { runId: value.target.runId, turnId: value.target.turnId, }, - targetRunHeader, + targetOpening, claimedAt: value.claimedAt as number, }; } +/** + * Is this the invocation the claim opened? + * + * The claim froze the target's opening, so the check is that the invocation + * still carries it, plus the identity the claim fixed. There is nothing else to + * compare: an invocation's lifecycle lives in its events, not in a record that + * a frozen copy could go stale against. + */ +export function invocationMatchesClaimTarget( + invocation: { + sessionId: string; + invocationId: string; + runId: string; + turnId: string; + opening: RuntimeEventInvocationOpenedContent; + }, + claim: ContinuationClaimV1, +): boolean { + return ( + invocation.sessionId === claim.target.sessionId && + invocation.invocationId === claim.target.invocationId && + invocation.runId === claim.target.runId && + invocation.turnId === claim.target.turnId && + stableJsonStringify(invocation.opening) === stableJsonStringify(claim.targetOpening) + ); +} + +/** + * Does this event discharge the claim as its target's first event? + * + * One rule, one implementation. The store refuses a start that fails it and the + * runtime refuses to resume across one; when those were two copies of the same + * predicate, a fix to either left the other admitting what the other rejected. + */ +export function continuationStartEventMatchesClaim( + event: RuntimeEvent | undefined, + claim: ContinuationClaimV1, + /** Undefined means the claim has not recorded a start yet, so nothing matches. */ + startKind: 'runtime_admission' | 'claim_repair' | undefined, +): boolean { + if (!event?.actions) return false; + const start = event.actions.continuationStart; + const runtimeProtocol = event.actions.runtimeProtocol; + const actionKeys = Object.keys(event.actions); + const source = claim.boundary.segments.at(-1)!; + return Boolean( + event.sessionId === claim.target.sessionId && + event.invocationId === claim.target.invocationId && + event.runId === claim.target.runId && + event.turnId === claim.target.turnId && + event.ts >= claim.claimedAt && + event.partial !== true && + event.role === 'system' && + event.author === 'system' && + event.status === undefined && + // Event 1 of a continuation target is also that invocation's opening fact, + // which is why the claim names it in advance. + stableJsonStringify(event.content) === stableJsonStringify(claim.targetOpening) && + actionKeys.includes('continuationStart') && + actionKeys.every((key) => key === 'continuationStart' || key === 'runtimeProtocol') && + actionKeys.length === (runtimeProtocol === undefined ? 1 : 2) && + (runtimeProtocol === undefined || + (startKind === 'runtime_admission' && + runtimeProtocol.toolBoundary === TOOL_BOUNDARY_PROTOCOL_V1)) && + start?.protocol === 'continuation_start_v2' && + start.provenance === startKind && + start.claimId === claim.claimId && + start.boundaryDigest === claim.boundaryDigest && + start.replayManifestDigest === claim.boundary.manifestDigest && + start.providerProjectionVersion === claim.providerProjectionVersion && + start.providerReplayDigest === claim.providerReplayDigest && + stableJsonStringify(start.immediateSource) === + stableJsonStringify({ + sessionId: source.identity.sessionId, + invocationId: source.identity.invocationId, + runId: source.identity.runId, + turnId: source.identity.turnId, + highWater: source.position.lastEventSeq, + prefixDigest: source.prefixDigest, + }), + ); +} + function canonicalizePrefixRows( identity: RuntimePrefixIdentityV1, rows: readonly RuntimePrefixRowV1[], diff --git a/packages/core/src/runtime-event-store.ts b/packages/core/src/runtime-event-store.ts index 1593689f0e..9fa17f304e 100644 --- a/packages/core/src/runtime-event-store.ts +++ b/packages/core/src/runtime-event-store.ts @@ -18,6 +18,7 @@ */ import type { RuntimeEvent } from './runtime-event.js'; +import type { RuntimeInvocationRecord } from './runtime-invocation.js'; import type { ContinuationClaimV1, ImmutableRuntimePrefixV1, @@ -72,6 +73,46 @@ export class DurableStoreWriteError extends Error { export interface RuntimeEventStore { /** Canonical stores fail the active run closed on every durable write error. */ readonly durability?: 'best_effort' | 'canonical'; + /** + * Enumerate a Session's invocations from the canonical events. + * + * This is a query, not a table. Nothing writes it and nothing repairs it, so + * clearing any physical index and rebuilding from the events produces the + * same inventory. Reserved control-plane invocation streams have no opening + * fact and therefore never appear here. + * + * One exception, and it is a durable one: an invocation that predates the + * opening fact could not be given one without rewriting an immutable + * sequence, so a store that migrated such a Session keeps that opening + * outside the events and merges it in here. Those invocations cannot be + * rebuilt from events alone, and never will be. + * + * An invocation's `terminalEvent` is its first terminal event. Sealing makes + * that the only one for anything written through this interface; a ledger + * from before the seal can carry a straggler after it, and the ending is + * still the terminal event. + */ + listSessionInvocations(sessionId: string): Promise; + /** + * One invocation by run id, absent when no opening fact names it. A store + * that indexes openings answers this in one read; stores without the fast + * path are answered from the inventory by `readRunInvocation`. + */ + readRunInvocation?( + sessionId: string, + runId: string, + ): Promise; + /** + * Append one event to a run. + * + * Every implementation seals: once a run holds a terminal event, appending + * any event the store does not already have must throw `RunSealedError`. An + * exact-id replay of an event already stored stays idempotent. This is what + * makes a run's ending single and final, so it is an obligation of this + * interface rather than a detail of one store — a test double that skips it + * is manufacturing a ledger no supported store can produce. Tests that need a + * corrupt ledger should build it beneath this interface, not through it. + */ appendRuntimeEvent( sessionId: string, runId: string, @@ -82,14 +123,21 @@ export interface RuntimeEventStore { * Coalesce one already-admitted mutable presentation stream into one store * transaction. Callers must preserve provider order and flush before every * immutable execution boundary. Stores that do not implement this optional - * fast path continue to receive one append per partial event. + * fast path continue to receive one append per partial event. The seal on + * `appendRuntimeEvent` applies here too. */ appendRuntimePartialBatch?( sessionId: string, runId: string, events: readonly RuntimeEvent[], ): Promise; - /** Append the terminal event if absent, or re-establish its stable-storage barrier if present. */ + /** + * Append the terminal event if absent, or re-establish its stable-storage + * barrier if present. This is the one writer the seal admits: it must commit + * the terminal event and the seal check in the same transaction, so two + * callers racing to end one run produce one terminal event and a + * `RunSealedError` for the loser. + */ ensureTerminalRuntimeEventDurable( sessionId: string, runId: string, @@ -111,6 +159,18 @@ export interface RuntimeEventStore { readSessionRuntimeEvents(sessionId: string): Promise; } +/** One invocation by run id, through the store's fast path when it has one. */ +export async function readRunInvocation( + store: Pick, + sessionId: string, + runId: string, +): Promise { + if (store.readRunInvocation) return store.readRunInvocation(sessionId, runId); + return (await store.listSessionInvocations(sessionId)).find( + (invocation) => invocation.runId === runId, + ); +} + export interface RuntimeRecoveryBundleStore extends RuntimeEventStore { readonly recoveryBundleCapability: typeof TOOL_RECOVERY_BUNDLE_CAPABILITY_V1; commitToolRecoveryBundle(input: RuntimeRecoveryBundleCommit): Promise; diff --git a/packages/core/src/runtime-event.ts b/packages/core/src/runtime-event.ts index 4365735f99..8323d289b4 100644 --- a/packages/core/src/runtime-event.ts +++ b/packages/core/src/runtime-event.ts @@ -45,7 +45,23 @@ import { decodeInteractionRequest, type InteractionFormInput, } from './interaction.js'; -import type { PermissionRequestPayload, PermissionResponse } from './permission.js'; +import { + isPermissionMode, + type PermissionMode, + type PermissionRequestPayload, + type PermissionResponse, +} from './permission.js'; +import { isCollaborationMode, type CollaborationMode } from './collaboration.js'; +import { + isAgentSwarmAuthorizationSource, + isEffectiveOrchestrationSource, + isOrchestrationMode, + type AgentSwarmAuthorizationSource, + type EffectiveOrchestrationSource, + type OrchestrationMode, +} from './orchestration.js'; +import { isToolMode, type ToolMode } from './tool-mode.js'; +import type { PersistedBackendKind } from './session.js'; import { decodeTurnOrigin, type TurnOrigin } from './turn-origin.js'; import type { UserQuestionRequest } from './user-question.js'; import { @@ -211,17 +227,117 @@ export interface RuntimeEventErrorContent { details?: string[] | Record; } +/** + * Where an invocation's provider route came from. `unknown` is the fail-closed + * marker for data that predates the opening fact: the transcript and tool + * evidence stay readable, but nothing may treat the route as authenticated. + */ +export type RuntimeInvocationRoute = + | { + provenance: 'runtime'; + backendKind: PersistedBackendKind; + llmConnectionId: string; + llmConnectionSlug: string; + modelId: string; + /** Frozen provider endpoint and credential ownership; absent on non-provider runs. */ + providerStateIdentity?: `sha256:${string}`; + } + | { + provenance: 'unknown'; + backendKind: PersistedBackendKind; + llmConnectionSlug: string; + modelId: string; + }; + +/** Execution configuration frozen before an invocation's first dispatch. */ +export interface RuntimeInvocationConfiguration { + cwd: string; + permissionMode: PermissionMode; + collaborationMode: CollaborationMode; + orchestrationMode: OrchestrationMode; + orchestrationSource: EffectiveOrchestrationSource; + toolMode: ToolMode; + agentSwarmAuthorization?: AgentSwarmAuthorizationSource; + /** Authoritative host identity for the workspace observed at open. */ + workspaceIdentity?: string; +} + +/** + * The authority that caused this invocation to exist. Closed and discriminated, + * so a reader names the root it wants instead of asserting that every other + * optional root field is absent. + */ +export type RuntimeInvocationRootAuthority = + | { kind: 'user' } + | { kind: 'context_compact' } + | { kind: 'scheduled_task'; scheduledTaskId: string } + | { kind: 'goal'; goalId: string } + | { kind: 'agent_graph_supervisor_wake'; wakeId: string; attemptId: string } + | { kind: 'legacy_automation'; legacyAutomationId: string }; + +/** Turn/session lineage that is immutable once the invocation opens. */ +export interface RuntimeInvocationLineage { + parentRunId?: string; + /** The run this one continues, and the run it re-attempts. Never both. */ + resumedFromRunId?: string; + retriedFromRunId?: string; + parentTurnId?: string; + parentSessionId?: string; + retriedFromTurnId?: string; + regeneratedFromTurnId?: string; + branchOfTurnId?: string; + agentId?: string; + agentName?: string; +} + +/** + * How this invocation was opened. `continuation` carries the same source + * identity the continuation-start action authenticates, so a migrated opening + * fact keeps the lineage edge even where no start event exists. + */ +export type RuntimeInvocationOpenSource = + | { kind: 'fresh' } + | { + kind: 'continuation'; + sourceInvocationId: string; + sourceRunId: string; + sourceTurnId: string; + sourceRuntimeEventHighWater: number; + claimId?: string; + boundaryDigest?: `sha256:${string}`; + }; + +/** + * The one immutable opening fact of a run-kind invocation, committed before any + * provider or tool dispatch. Route provenance lives here once per invocation + * and is joined by `invocationId`; it is never copied onto other events. + * + * Reserved control-plane streams (history compaction checkpoints, workspace + * version authority) have no run and therefore no opening fact. + */ +export interface RuntimeEventInvocationOpenedContent { + kind: 'invocation_opened'; + protocol: 'invocation_opened_v1'; + route: RuntimeInvocationRoute; + configuration: RuntimeInvocationConfiguration; + root: RuntimeInvocationRootAuthority; + source: RuntimeInvocationOpenSource; + /** Omitted entirely when the invocation has no lineage edges. */ + lineage?: RuntimeInvocationLineage; +} + /** * Content union for user/model text, model thinking, function call, - * function response, and error payloads. Discriminated by `kind` to - * match the existing ToolResultContent convention. + * function response, error payloads, and the invocation opening fact. + * Discriminated by `kind` to match the existing ToolResultContent convention. */ export type RuntimeEventContent = | RuntimeEventTextContent | RuntimeEventThinkingContent | RuntimeEventFunctionCallContent | RuntimeEventFunctionResponseContent - | RuntimeEventErrorContent; + | RuntimeEventErrorContent + | RuntimeEventInvocationOpenedContent; export const RUNTIME_EVENT_CONTENT_KINDS = [ 'text', @@ -229,6 +345,7 @@ export const RUNTIME_EVENT_CONTENT_KINDS = [ 'function_call', 'function_response', 'error', + 'invocation_opened', ] as const; export type RuntimeEventContentKind = (typeof RUNTIME_EVENT_CONTENT_KINDS)[number]; @@ -485,7 +602,7 @@ export interface RuntimeEvent { id: string; /** Durable invocation spine id; groups every run/turn of one request. */ invocationId: string; - /** Durable operational run identity (maps to AgentRunHeader.runId). */ + /** Durable operational run identity; names one execution of the invocation. */ runId: string; sessionId: string; /** Groups all events from one agent turn (maps to StoredMessage.turnId). */ @@ -579,6 +696,76 @@ const ERROR_CONTENT_SHAPE = defineObjectShape()( ['kind', 'message'], ['code', 'reason', 'details'], ); +const INVOCATION_OPENED_CONTENT_SHAPE = defineObjectShape()( + ['kind', 'protocol', 'route', 'configuration', 'root', 'source'], + ['lineage'], +); +const INVOCATION_ROUTE_RUNTIME_SHAPE = defineObjectShape< + Extract +>()( + ['provenance', 'backendKind', 'llmConnectionId', 'llmConnectionSlug', 'modelId'], + ['providerStateIdentity'], +); +const INVOCATION_ROUTE_UNKNOWN_SHAPE = defineObjectShape< + Extract +>()(['provenance', 'backendKind', 'llmConnectionSlug', 'modelId'], []); +const INVOCATION_CONFIGURATION_SHAPE = defineObjectShape()( + [ + 'cwd', + 'permissionMode', + 'collaborationMode', + 'orchestrationMode', + 'orchestrationSource', + 'toolMode', + ], + ['agentSwarmAuthorization', 'workspaceIdentity'], +); +const INVOCATION_LINEAGE_SHAPE = defineObjectShape()( + [], + [ + 'parentRunId', + 'resumedFromRunId', + 'retriedFromRunId', + 'parentTurnId', + 'parentSessionId', + 'retriedFromTurnId', + 'regeneratedFromTurnId', + 'branchOfTurnId', + 'agentId', + 'agentName', + ], +); +const INVOCATION_CONTINUATION_SOURCE_SHAPE = defineObjectShape< + Extract +>()( + ['kind', 'sourceInvocationId', 'sourceRunId', 'sourceTurnId', 'sourceRuntimeEventHighWater'], + ['claimId', 'boundaryDigest'], +); +const INVOCATION_FRESH_SOURCE_SHAPE = defineObjectShape< + Extract +>()(['kind'], []); +const INVOCATION_ROOT_SHAPES = { + user: defineObjectShape>()( + ['kind'], + [], + ), + context_compact: defineObjectShape< + Extract + >()(['kind'], []), + scheduled_task: defineObjectShape< + Extract + >()(['kind', 'scheduledTaskId'], []), + goal: defineObjectShape>()( + ['kind', 'goalId'], + [], + ), + agent_graph_supervisor_wake: defineObjectShape< + Extract + >()(['kind', 'wakeId', 'attemptId'], []), + legacy_automation: defineObjectShape< + Extract + >()(['kind', 'legacyAutomationId'], []), +} as const; const RUNTIME_ACTIONS_SHAPE = defineObjectShape()( [], [ @@ -835,11 +1022,150 @@ function isRuntimeEventContent(value: unknown): value is RuntimeEventContent { typeof value.message === 'string' && (value.details === undefined || isStringArray(value.details) || isRecord(value.details)) ); + case 'invocation_opened': + return isRuntimeInvocationOpened(value); default: return false; } } +/** + * True when the event is the immutable opening fact of its invocation. + * Narrowing here keeps every reader off a hand-rolled `content.kind` test. + */ +export function runtimeEventInvocationOpening( + event: RuntimeEvent, +): RuntimeEventInvocationOpenedContent | undefined { + return event.content?.kind === 'invocation_opened' ? event.content : undefined; +} + +/** Strict decode for one persisted opening fact; throws on any drift. */ +export function decodeRuntimeInvocationOpened(value: unknown): RuntimeEventInvocationOpenedContent { + if (!isRuntimeInvocationOpened(value)) { + throw new Error('Invalid RuntimeEvent invocation_opened schema'); + } + return value; +} + +function isRuntimeInvocationOpened(value: unknown): value is RuntimeEventInvocationOpenedContent { + return ( + isRecord(value) && + value.kind === 'invocation_opened' && + hasExactShape(value, INVOCATION_OPENED_CONTENT_SHAPE) && + value.protocol === 'invocation_opened_v1' && + isRuntimeInvocationRoute(value.route) && + isRuntimeInvocationConfiguration(value.configuration) && + isRuntimeInvocationRootAuthority(value.root) && + isRuntimeInvocationOpenSource(value.source) && + (value.lineage === undefined || isRuntimeInvocationLineage(value.lineage)) + ); +} + +function isRuntimeInvocationRoute(value: unknown): value is RuntimeInvocationRoute { + if (!isRecord(value)) return false; + if ( + !isPersistedBackendKind(value.backendKind) || + !isNonEmptyString(value.llmConnectionSlug) || + !isNonEmptyString(value.modelId) + ) { + return false; + } + if (value.provenance === 'runtime') { + return ( + hasExactShape(value, INVOCATION_ROUTE_RUNTIME_SHAPE) && + isNonEmptyString(value.llmConnectionId) && + (value.providerStateIdentity === undefined || isSha256Digest(value.providerStateIdentity)) + ); + } + return value.provenance === 'unknown' && hasExactShape(value, INVOCATION_ROUTE_UNKNOWN_SHAPE); +} + +function isPersistedBackendKind(value: unknown): value is PersistedBackendKind { + return value === 'ai-sdk' || value === 'fake'; +} + +function isRuntimeInvocationConfiguration(value: unknown): value is RuntimeInvocationConfiguration { + return ( + isRecord(value) && + hasExactShape(value, INVOCATION_CONFIGURATION_SHAPE) && + typeof value.cwd === 'string' && + isPermissionMode(value.permissionMode) && + isCollaborationMode(value.collaborationMode) && + isOrchestrationMode(value.orchestrationMode) && + isEffectiveOrchestrationSource(value.orchestrationSource) && + isToolMode(value.toolMode) && + (value.agentSwarmAuthorization === undefined || + isAgentSwarmAuthorizationSource(value.agentSwarmAuthorization)) && + (value.workspaceIdentity === undefined || isNonEmptyString(value.workspaceIdentity)) + ); +} + +function isRuntimeInvocationRootAuthority(value: unknown): value is RuntimeInvocationRootAuthority { + if (!isRecord(value)) return false; + switch (value.kind) { + case 'user': + return hasExactShape(value, INVOCATION_ROOT_SHAPES.user); + case 'context_compact': + return hasExactShape(value, INVOCATION_ROOT_SHAPES.context_compact); + case 'scheduled_task': + return ( + hasExactShape(value, INVOCATION_ROOT_SHAPES.scheduled_task) && + isNonEmptyString(value.scheduledTaskId) + ); + case 'goal': + return hasExactShape(value, INVOCATION_ROOT_SHAPES.goal) && isNonEmptyString(value.goalId); + case 'agent_graph_supervisor_wake': + return ( + hasExactShape(value, INVOCATION_ROOT_SHAPES.agent_graph_supervisor_wake) && + isNonEmptyString(value.wakeId) && + isNonEmptyString(value.attemptId) + ); + case 'legacy_automation': + return ( + hasExactShape(value, INVOCATION_ROOT_SHAPES.legacy_automation) && + isNonEmptyString(value.legacyAutomationId) + ); + default: + return false; + } +} + +function isRuntimeInvocationOpenSource(value: unknown): value is RuntimeInvocationOpenSource { + if (!isRecord(value)) return false; + if (value.kind === 'fresh') return hasExactShape(value, INVOCATION_FRESH_SOURCE_SHAPE); + return ( + value.kind === 'continuation' && + hasExactShape(value, INVOCATION_CONTINUATION_SOURCE_SHAPE) && + isNonEmptyString(value.sourceInvocationId) && + isNonEmptyString(value.sourceRunId) && + isNonEmptyString(value.sourceTurnId) && + Number.isSafeInteger(value.sourceRuntimeEventHighWater) && + (value.sourceRuntimeEventHighWater as number) >= 0 && + (value.claimId === undefined || isNonEmptyString(value.claimId)) && + (value.boundaryDigest === undefined || isSha256Digest(value.boundaryDigest)) + ); +} + +function isRuntimeInvocationLineage(value: unknown): value is RuntimeInvocationLineage { + return ( + isRecord(value) && + hasExactShape(value, INVOCATION_LINEAGE_SHAPE) && + Object.keys(value).length > 0 && + [ + value.parentRunId, + value.resumedFromRunId, + value.retriedFromRunId, + value.parentTurnId, + value.parentSessionId, + value.retriedFromTurnId, + value.regeneratedFromTurnId, + value.branchOfTurnId, + value.agentId, + value.agentName, + ].every(isOptionalString) + ); +} + function decodesDurableToolResultProjection(value: unknown): boolean { try { decodeDurableToolResultProjection(value); @@ -1159,6 +1485,7 @@ export function runtimeEventHasModelVisibleContent(event: RuntimeEvent): boolean case 'function_response': return true; case 'error': + case 'invocation_opened': return false; } } diff --git a/packages/core/src/runtime-invocation.ts b/packages/core/src/runtime-invocation.ts new file mode 100644 index 0000000000..e020d7f14f --- /dev/null +++ b/packages/core/src/runtime-invocation.ts @@ -0,0 +1,404 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * One physical execution attempt, as the event spine describes it. + * + * An invocation is an opening fact, the events that follow it, and — once it + * has ended — a terminal event. Everything a reader used to take from a mutable + * Run header is a projection of those three, so there is nothing here that a + * writer could set independently of the events. + */ + +import type { AgentGraphIntentClaim } from './agent-graph-control.js'; +import type { + RuntimeEvent, + RuntimeEventInvocationOpenedContent, + RuntimeInvocationLineage, +} from './runtime-event.js'; +import { isTerminalRuntimeEvent } from './runtime-event.js'; + +export interface RuntimeInvocationRecord { + sessionId: string; + invocationId: string; + runId: string; + turnId: string; + /** Timestamp of the opening fact's own event. */ + openedAt: number; + opening: RuntimeEventInvocationOpenedContent; + terminalEvent?: RuntimeEvent; +} + +/** + * Rebuild a Session's invocation inventory from its events alone. + * + * This is the definition of the inventory, not a cache of it: a store that + * holds the Session's events can answer `listSessionInvocations` with this and + * get exactly what an indexed store returns. Events whose invocation never + * opened are control-plane streams and are absent by construction. + */ +export function runtimeInvocationsFromSessionEvents( + sessionId: string, + events: readonly RuntimeEvent[], +): RuntimeInvocationRecord[] { + const byInvocation = new Map(); + for (const event of events) { + if (event.sessionId !== sessionId || event.partial === true) continue; + if (event.content?.kind === 'invocation_opened') { + byInvocation.set(event.invocationId, { + sessionId, + invocationId: event.invocationId, + runId: event.runId, + turnId: event.turnId, + openedAt: event.ts, + opening: event.content, + }); + } + } + // An invocation ends at its first terminal event. Sealing makes that the only + // one for any ledger this codebase wrote; one written before the seal existed + // can carry a straggler after it, and the ending is still the terminal event. + // A Session-ordered read may place it anywhere relative to other invocations' + // events, so this scans rather than looking at the tail. + for (const event of events) { + if (event.sessionId !== sessionId || event.partial === true) continue; + if (!isTerminalRuntimeEvent(event)) continue; + const record = byInvocation.get(event.invocationId); + if (record && !record.terminalEvent) record.terminalEvent = event; + } + return [...byInvocation.values()].sort( + (a, b) => a.openedAt - b.openedAt || a.invocationId.localeCompare(b.invocationId), + ); +} + +/** + * Wrap an opening fact in the event that carries it. + * + * Every writer that opens an invocation goes through here, so the envelope the + * inventory reads back is decided once. It is hidden from the model: the + * opening is a fact about the run, not something the run said. + */ +export function buildInvocationOpenedEvent(input: { + id: string; + run: { sessionId: string; invocationId: string; runId: string; turnId: string }; + openedAt: number; + opening: RuntimeEventInvocationOpenedContent; +}): RuntimeEvent { + return { + id: input.id, + sessionId: input.run.sessionId, + invocationId: input.run.invocationId, + runId: input.run.runId, + turnId: input.run.turnId, + ts: input.openedAt, + partial: false, + role: 'system', + author: 'system', + modelVisibility: 'hidden', + content: input.opening, + }; +} + +export interface BuildSyntheticTerminalRuntimeEventInput { + id: string; + invocationId: string; + run: { sessionId: string; runId: string; turnId: string }; + status: RuntimeInvocationOutcome; + ts: number; + failureClass?: string; + abortSource?: string; + recoveryReason?: string; + diagnostic?: Record; + message?: string; +} + +/** + * The terminal event a writer states on the run's behalf, when the run did not + * state its own: recovery after a crash, a copy, or the migration of a header + * whose run never wrote an event. One envelope, decided here. + */ +export function buildSyntheticTerminalRuntimeEvent( + input: BuildSyntheticTerminalRuntimeEventInput, +): RuntimeEvent { + const failureClass = input.status === 'failed' ? (input.failureClass ?? 'unknown') : undefined; + const abortSource = input.status === 'cancelled' ? input.abortSource : undefined; + return { + id: input.id, + invocationId: input.invocationId, + runId: input.run.runId, + sessionId: input.run.sessionId, + turnId: input.run.turnId, + ts: input.ts, + partial: false, + role: 'system', + author: 'system', + status: input.status === 'cancelled' ? 'aborted' : input.status, + ...(failureClass + ? { + content: { + kind: 'error', + code: failureClass, + reason: failureClass, + message: input.message ?? failureClass, + }, + } + : {}), + actions: { + endInvocation: true, + stateDelta: { + ...(input.recoveryReason ? { recovered: true, recoveryReason: input.recoveryReason } : {}), + ...(input.diagnostic ?? {}), + ...(failureClass ? { failureClass } : {}), + ...(abortSource ? { abortSource } : {}), + }, + }, + }; +} + +/** One invocation's position in a Session's opening order. */ +export interface RuntimeInvocationPageCursor { + readonly openedAt: number; + readonly invocationId: string; +} + +export interface RuntimeInvocationPageInput { + readonly before?: RuntimeInvocationPageCursor; + readonly limit: number; +} + +export interface RuntimeInvocationPageResult { + readonly invocations: readonly RuntimeInvocationRecord[]; + readonly nextCursor: RuntimeInvocationPageCursor | null; +} + +export interface RuntimeInvocationSearchResult { + readonly invocations: readonly RuntimeInvocationRecord[]; + readonly truncated: boolean; +} + +export type RuntimeInvocationOutcome = 'completed' | 'failed' | 'cancelled'; + +/** + * How the invocation ended, according to the only fact that decides it. + * + * `undefined` covers both an invocation still running and one whose terminal + * event ends the stream without stating an outcome; a caller that needs to tell + * those apart looks at `terminalEvent` itself. + */ +export function runtimeInvocationOutcome(record: { + terminalEvent?: RuntimeEvent; +}): RuntimeInvocationOutcome | undefined { + switch (record.terminalEvent?.status) { + case 'completed': + return 'completed'; + case 'failed': + return 'failed'; + case 'aborted': + case 'cancelled': + return 'cancelled'; + default: + return undefined; + } +} + +/** + * Whether this invocation contributes directly to the owning session's + * transcript. Top-level continuations carry parent lineage for recovery, but + * unlike child-agent invocations their output remains part of the parent + * session conversation. A legacy child retry may also carry continuation + * authority; its agent identity keeps it outside the owning session transcript. + */ +export function isSessionInlineInvocation(opening: RuntimeEventInvocationOpenedContent): boolean { + const lineage = opening.lineage; + return ( + lineage?.parentRunId === undefined || + (opening.source.kind === 'continuation' && lineage.agentId === undefined) + ); +} + +export type RootExecutionDescriptor = + | { + kind: 'external_message'; + inputDigest?: `sha256:${string}`; + maxSteps?: number; + } + | { + /** Tool-free conversational execution admitted only by WorkHub authority. */ + kind: 'workhub_coordination'; + inputDigest: `sha256:${string}`; + } + | { kind: 'regenerate'; sourceTurnId: string } + | { kind: 'context_compact' } + | { + kind: 'scheduled_task'; + scheduledTaskId: string; + /** Includes the immutable Connection target for Agent ScheduledTasks. */ + executionFingerprint?: `sha256:${string}`; + } + | { kind: 'legacy_automation'; automationId: string } + | { kind: 'goal'; goalId: string } + | { + kind: 'agent_graph_supervisor_wake'; + graphId: string; + wakeId: string; + attemptId: string; + } + | { + kind: 'safe_boundary_continuation'; + sourceInvocationId: string; + sourceRunId: string; + sourceTurnId: string; + sourceRuntimeEventHighWater: number; + claimId: string; + boundaryDigest: `sha256:${string}`; + providerReplayDigest: `sha256:${string}`; + safetyDigest: `sha256:${string}`; + targetInvocationId: string; + } + | { + kind: 'linked_child_initial'; + agentId: string; + agentName: string; + } + | { + kind: 'linked_child_resume'; + agentId: string; + agentName: string; + sourceRunId: string; + } + | { + kind: 'linked_child_provider_retry'; + agentId: string; + agentName: string; + sourceRunId: string; + } + | { + kind: 'claimed_agent_graph_intent'; + claim: AgentGraphIntentClaim; + agentId: string; + agentName: string; + }; + +type HostedRootExecutionDescriptor = Extract< + RootExecutionDescriptor, + { + kind: + | 'regenerate' + | 'context_compact' + | 'scheduled_task' + | 'legacy_automation' + | 'goal' + | 'agent_graph_supervisor_wake' + | 'safe_boundary_continuation'; + } +>; + +/** + * Is this invocation the one the Host admitted for that root execution? + * + * The opening fact names its root as a closed union, so each arm names the root + * it wants instead of asserting that every other root marker is absent. What + * remains is lineage, and the rule there is exactness: an admitted root has the + * lineage its kind implies and no other, so one comparison replaces a list of + * per-field negatives that had to be extended every time a lineage field was + * added. + */ +export function invocationMatchesHostedRootExecution( + invocation: { invocationId: string; opening: RuntimeEventInvocationOpenedContent }, + execution: HostedRootExecutionDescriptor, +): boolean { + const { root, source, configuration, lineage } = invocation.opening; + switch (execution.kind) { + case 'regenerate': + return ( + root.kind === 'user' && + source.kind === 'fresh' && + lineageIsExactly(lineage, { + parentTurnId: execution.sourceTurnId, + regeneratedFromTurnId: execution.sourceTurnId, + }) + ); + case 'context_compact': + return ( + root.kind === 'context_compact' && source.kind === 'fresh' && lineageIsExactly(lineage, {}) + ); + case 'safe_boundary_continuation': + return ( + root.kind === 'user' && + source.kind === 'continuation' && + invocation.invocationId === execution.targetInvocationId && + source.sourceInvocationId === execution.sourceInvocationId && + source.sourceRunId === execution.sourceRunId && + source.sourceTurnId === execution.sourceTurnId && + source.sourceRuntimeEventHighWater === execution.sourceRuntimeEventHighWater && + source.claimId === execution.claimId && + source.boundaryDigest === execution.boundaryDigest && + lineageIsExactly(lineage, { + parentRunId: execution.sourceRunId, + parentTurnId: execution.sourceTurnId, + }) + ); + case 'scheduled_task': + return ( + root.kind === 'scheduled_task' && + root.scheduledTaskId === execution.scheduledTaskId && + source.kind === 'fresh' && + lineageIsExactly(lineage, {}) + ); + case 'legacy_automation': + return ( + root.kind === 'legacy_automation' && + root.legacyAutomationId === execution.automationId && + source.kind === 'fresh' && + lineageIsExactly(lineage, {}) + ); + case 'goal': + return ( + root.kind === 'goal' && + root.goalId === execution.goalId && + source.kind === 'fresh' && + lineageIsExactly(lineage, {}) + ); + case 'agent_graph_supervisor_wake': + return ( + root.kind === 'agent_graph_supervisor_wake' && + execution.wakeId.startsWith(`${execution.graphId}:`) && + root.wakeId === execution.wakeId && + root.attemptId === execution.attemptId && + configuration.orchestrationMode === 'graph' && + configuration.orchestrationSource === 'turn_override' && + configuration.agentSwarmAuthorization === 'none' && + source.kind === 'fresh' && + lineageIsExactly(lineage, {}) + ); + } +} + +/** An admitted root has the lineage its kind implies, and no other edge. */ +function lineageIsExactly( + lineage: RuntimeInvocationLineage | undefined, + expected: RuntimeInvocationLineage, +): boolean { + const actual = (lineage ?? {}) as Record; + const wanted = expected as Record; + const keys = Object.keys(wanted); + return ( + Object.keys(actual).length === keys.length && keys.every((key) => actual[key] === wanted[key]) + ); +} diff --git a/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts b/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts index 06eff1cae3..dd8c16dc9b 100644 --- a/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts +++ b/packages/runtime-host/src/__tests__/canonical-session-projection.test.ts @@ -22,7 +22,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import { seedInvocation } from '@maka/runtime/test-only/invocation-fixture'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { openInteractiveExecutionStoresForWrite, @@ -90,18 +90,28 @@ test('projects the canonical root lifecycle and the attachment queue from real S assert.ok(admittedProjection); assert.equal(admittedProjection.rootTurn?.status, 'admitted'); - await stores.agentRunStore.createRun(runHeader(session.id)); - await stores.agentRunStore.appendEvent(session.id, 'run-1', { - type: 'run_started', - id: 'run-started-1', + await seedInvocation(stores.runtimeEventStore, { sessionId: session.id, - turnId: 'turn-1', runId: 'run-1', - ts: 11, - }); - await stores.agentRunStore.updateRun(session.id, 'run-1', { - status: 'running', - updatedAt: 11, + turnId: 'turn-1', + openedAt: 10, + opening: { + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: '/private/runtime-cwd', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + }, }); messages.reserveRootTurn({ sessionId: session.id, turnId: 'turn-1', runId: 'run-1' }); @@ -130,11 +140,6 @@ test('projects the canonical root lifecycle and the attachment queue from real S const terminal = terminalEvent(session.id); await stores.runtimeEventStore.appendRuntimeEvent(session.id, 'run-1', terminal); - await stores.agentRunStore.updateRun(session.id, 'run-1', { - status: 'completed', - updatedAt: 12, - completedAt: 12, - }); const completed = await reader.read(session.id); assert.ok(completed); assert.deepEqual(completed.rootTurn, { @@ -397,13 +402,6 @@ test('projects a failed Turn message from the canonical terminal event', async ( ); await stores.runtimeEventStore.appendRuntimeEvent(sessionId, 'run-1', errorEvent); await stores.runtimeEventStore.appendRuntimeEvent(sessionId, 'run-1', terminalEvent); - await stores.agentRunStore.updateRun(sessionId, 'run-1', { - status: 'failed', - updatedAt: 13, - completedAt: 13, - failureClass: 'provider_error', - failureMessage: 'stale Run header failure', - }); const reader = new CanonicalSessionProjectionReader({ stores, @@ -469,12 +467,6 @@ test('a legacy context_budget_exhausted terminal event still projects, as a cont }, }; await stores.runtimeEventStore.appendRuntimeEvent(sessionId, 'run-1', terminalEvent); - await stores.agentRunStore.updateRun(sessionId, 'run-1', { - status: 'failed', - updatedAt: 13, - completedAt: 13, - failureClass: 'context_budget_exhausted', - }); const reader = new CanonicalSessionProjectionReader({ stores, @@ -637,24 +629,6 @@ function sessionInput(root: string) { }; } -function runHeader(sessionId: string): AgentRunHeader { - return { - runId: 'run-1', - invocationId: 'run-1', - sessionId, - turnId: 'turn-1', - status: 'created', - backendKind: 'fake', - llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/private/runtime-cwd', - permissionMode: 'ask', - createdAt: 10, - updatedAt: 10, - }; -} - async function createRunningRoot( root: string, stores: ExecutionStoresWriter<'interactive'>, @@ -672,18 +646,28 @@ async function createRunningRoot( sourceMessages: [], admittedAt: 10, }); - await stores.agentRunStore.createRun(runHeader(session.id)); - await stores.agentRunStore.appendEvent(session.id, 'run-1', { - type: 'run_started', - id: 'run-started-1', + await seedInvocation(stores.runtimeEventStore, { sessionId: session.id, - turnId: 'turn-1', runId: 'run-1', - ts: 11, - }); - await stores.agentRunStore.updateRun(session.id, 'run-1', { - status: 'running', - updatedAt: 11, + turnId: 'turn-1', + openedAt: 10, + opening: { + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: '/private/runtime-cwd', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + }, }); return { sessionId: session.id, rootAdmissions }; } diff --git a/packages/runtime-host/src/__tests__/daily-review-coordinator.test.ts b/packages/runtime-host/src/__tests__/daily-review-coordinator.test.ts index 9df8437004..6eb94115c2 100644 --- a/packages/runtime-host/src/__tests__/daily-review-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/daily-review-coordinator.test.ts @@ -447,8 +447,8 @@ function appendCorruptAuthorityEvent(root: string, sessionId: string, runId: str lease.transaction('write', () => { lease.database .prepare(` - INSERT INTO core_agent_runs(session_id, run_id, created_at, record_json) - VALUES (?, ?, 0, '{}') + INSERT INTO core_agent_runs(session_id, run_id, created_at) + VALUES (?, ?, 0) `) .run(sessionId, runId); lease.database diff --git a/packages/runtime-host/src/__tests__/execution-composition.test.ts b/packages/runtime-host/src/__tests__/execution-composition.test.ts index a21b24c4bf..f9a68677ec 100644 --- a/packages/runtime-host/src/__tests__/execution-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-composition.test.ts @@ -18,6 +18,8 @@ */ import assert from 'node:assert/strict'; +import { runtimeInvocationOutcome } from '@maka/core/runtime-invocation'; +import { runtimeInvocationFailureClass } from '@maka/runtime/runtime-event-read-model'; import { parseNoRealConnectionError } from '@maka/core/connection-error-copy'; import { createRequire } from 'node:module'; import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'; @@ -465,10 +467,16 @@ test('production recovery preserves legacy Automation history and closes an orph kind: 'legacy_automation', automationId: 'historical-automation', }); - const recoveredRun = await stores.agentRunStore.readRun(pending.id, 'legacy-automation-run'); - assert.equal(recoveredRun.status, 'failed'); - assert.equal(recoveredRun.legacyAutomationId, 'legacy-automation'); - assert.equal(recoveredRun.failureClass, 'app_restarted'); + const recoveredRun = (await stores.runtimeEventStore.listSessionInvocations(pending.id)).find( + (candidate) => candidate.runId === 'legacy-automation-run', + ); + assert.ok(recoveredRun); + assert.equal(recoveredRun && runtimeInvocationOutcome(recoveredRun), 'failed'); + assert.deepEqual(recoveredRun?.opening.root, { + kind: 'legacy_automation', + legacyAutomationId: 'legacy-automation', + }); + assert.equal(recoveredRun && runtimeInvocationFailureClass(recoveredRun), 'app_restarted'); } finally { await composition.close(); } @@ -1398,26 +1406,22 @@ test('production composition validates graph stop before aborting a claimed chil ); assert.ok(abortedAdmission?.userMessageId); assert.deepEqual(abortedAdmission?.execution, graphExecutionDescriptor(abortedClaim)); - const abortedRun = await stores.agentRunStore.readRun( - abortedClaim.targetSessionId, - abortedClaim.targetRunId, - ); - assert.equal(abortedRun.status, 'cancelled'); + const abortedRun = ( + await stores.runtimeEventStore.listSessionInvocations(abortedClaim.targetSessionId) + ).find((candidate) => candidate.runId === abortedClaim.targetRunId); + assert.ok(abortedRun); + assert.equal(abortedRun && runtimeInvocationOutcome(abortedRun), 'cancelled'); await assertUniqueGraphExecutionFacts( stores, abortedClaim, abortedAdmission.userMessageId, - 'run_cancelled', - ); - assert.equal( - ( - await stores.agentRunStore.readRun( - completedClaim.targetSessionId, - completedClaim.targetRunId, - ) - ).status, - 'completed', + 'cancelled', ); + const completedRun = ( + await stores.runtimeEventStore.listSessionInvocations(completedClaim.targetSessionId) + ).find((candidate) => candidate.runId === completedClaim.targetRunId); + assert.ok(completedRun); + assert.equal(completedRun && runtimeInvocationOutcome(completedRun), 'completed'); } catch (error) { journeyError = error; throw error; @@ -1690,12 +1694,11 @@ async function assertUniqueGraphExecutionFacts( stores: Awaited>, claim: AgentGraphIntentClaim, userMessageId: string, - expectedTerminal: 'run_completed' | 'run_cancelled' = 'run_completed', + expectedOutcome: 'completed' | 'cancelled' = 'completed', ): Promise { - const [runs, messages, runEvents, runtimeEvents] = await Promise.all([ - stores.agentRunStore.listSessionRuns(claim.targetSessionId), + const [runs, messages, runtimeEvents] = await Promise.all([ + stores.runtimeEventStore.listSessionInvocations(claim.targetSessionId), stores.sessionStore.readMessages(claim.targetSessionId), - stores.agentRunStore.readEvents(claim.targetSessionId, claim.targetRunId), stores.runtimeEventStore.readImmutableRuntimeEvents(claim.targetSessionId, claim.targetRunId), ]); assert.deepEqual( @@ -1708,11 +1711,13 @@ async function assertUniqueGraphExecutionFacts( .map((message) => message.id), [userMessageId], ); - assert.equal(runEvents.filter((event) => event.type === 'run_started').length, 1); - assert.equal(runEvents.filter((event) => event.type === expectedTerminal).length, 1); + assert.equal( + runtimeEvents.filter((event) => event.content?.kind === 'invocation_opened').length, + 1, + ); assert.equal( runtimeEvents.filter( - (event) => event.status === (expectedTerminal === 'run_cancelled' ? 'aborted' : 'completed'), + (event) => event.status === (expectedOutcome === 'cancelled' ? 'aborted' : 'completed'), ).length, 1, ); diff --git a/packages/runtime-host/src/__tests__/execution-host-continuation.test.ts b/packages/runtime-host/src/__tests__/execution-host-continuation.test.ts index 6b4a1039b9..b93591d3ec 100644 --- a/packages/runtime-host/src/__tests__/execution-host-continuation.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-continuation.test.ts @@ -96,21 +96,19 @@ test('two Clients idempotently start one Host-owned safe-boundary continuation', if (admission?.execution.kind !== 'safe_boundary_continuation') return; assert.equal(admission.execution.sourceRunId, source.sourceRunId); assert.equal(admission.execution.sourceInvocationId, source.sourceInvocationId); - assert.equal(admission.execution.sourceRuntimeEventHighWater, 2); + assert.equal(admission.execution.sourceRuntimeEventHighWater, 3); const ledger = await fixture.readTurn(turnId); assert.equal(ledger.runs.length, 1); const run = ledger.runs[0]; - assert.equal(run?.parentRunId, source.sourceRunId); - assert.equal(run?.parentTurnId, source.sourceTurnId); + assert.equal(run?.opening.lineage?.parentRunId, source.sourceRunId); + assert.equal(run?.opening.lineage?.parentTurnId, source.sourceTurnId); assert.equal(run?.invocationId, admission.execution.targetInvocationId); - assert.equal(run?.continuationSource?.sourceRunId, source.sourceRunId); - assert.equal( - run?.continuationSource && 'protocol' in run.continuationSource - ? run.continuationSource.claimId - : undefined, - admission.execution.claimId, - ); + const openSource = run?.opening.source; + assert.equal(openSource?.kind, 'continuation'); + if (openSource?.kind !== 'continuation') return; + assert.equal(openSource.sourceRunId, source.sourceRunId); + assert.equal(openSource.claimId, admission.execution.claimId); } finally { if (!clientsClosed) { await first.close(); @@ -121,38 +119,6 @@ test('two Clients idempotently start one Host-owned safe-boundary continuation', }); }); -test('startup repairs a continuation Run created before its durable start', async () => { - await withExecutionRoot(async (fixture) => { - const crash = await fixture.seedSafeBoundaryContinuationCrash('after_run_created'); - const host = await fixture.startHost(); - const client = await connectClient(fixture.root); - try { - const repaired = await client.request('turn.query', { - sessionId: fixture.sessionId, - turnId: crash.targetTurnId, - }); - assert.equal(repaired.runId, crash.targetRunId); - assert.equal(repaired.status, 'failed'); - assert.equal(repaired.failureClass, 'continuation_abandoned_before_provider_dispatch'); - assert.deepEqual( - await client.request('turn.resume.query', { - sessionId: fixture.sessionId, - sourceRunId: crash.sourceRunId, - expectedRuntimeEventHighWater: crash.sourceRuntimeEventHighWater, - }), - { - sessionId: fixture.sessionId, - disposition: 'parked', - reason: 'continuation_already_exists', - }, - ); - } finally { - await client.close(); - await fixture.stopHost(host); - } - }); -}); - test('startup repairs a continuation claim committed before its target Run', async () => { await withExecutionRoot(async (fixture) => { const crash = await fixture.seedSafeBoundaryContinuationCrash( diff --git a/packages/runtime-host/src/__tests__/execution-host-message.test.ts b/packages/runtime-host/src/__tests__/execution-host-message.test.ts index 76f171a765..e3d544a23d 100644 --- a/packages/runtime-host/src/__tests__/execution-host-message.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-message.test.ts @@ -38,7 +38,6 @@ import { dirname, join } from 'node:path'; import { test } from 'node:test'; import { TOOL_BOUNDARY_PROTOCOL_V1 } from '@maka/core/runtime-event'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; -import type { AgentRunHeader } from '@maka/core/agent-run'; import type { MessageContent } from '@maka/core/events'; import type { ConnectionCatalogEntry } from '@maka/core/runtime-policy'; import type { StoredMessage } from '@maka/core/session'; diff --git a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts index b67513f8dd..25052409c6 100644 --- a/packages/runtime-host/src/__tests__/execution-host-queue.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-queue.test.ts @@ -38,12 +38,16 @@ import { dirname, join } from 'node:path'; import { test } from 'node:test'; import { TOOL_BOUNDARY_PROTOCOL_V1 } from '@maka/core/runtime-event'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import { + runtimeInvocationOutcome, + type RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; +import { runtimeInvocationFailureClass } from '@maka/runtime/runtime-event-read-model'; import type { MessageContent } from '@maka/core/events'; import type { ConnectionCatalogEntry } from '@maka/core/runtime-policy'; import type { StoredMessage } from '@maka/core/session'; import { isTerminalRuntimeEvent } from '@maka/core/runtime-event'; -import type { RuntimeEvent } from '@maka/core/runtime-event'; +import type { RuntimeEvent, RuntimeInvocationLineage } from '@maka/core/runtime-event'; import { buildRecoveredTerminalRuntimeEvent, classifyTerminalRuntimeLedger, @@ -727,21 +731,25 @@ test('startup recovery canonically closes pending linked child admissions withou try { stores = await openInteractiveExecutionStoresForRead(reader.lease); for (const recovered of [initial, resume, retry, graph]) { - const run = await stores.agentRunStore.readRun(recovered.sessionId, recovered.runId); - assert.equal(run.status, 'failed'); - assert.equal(run.failureClass, 'app_restarted'); - assert.equal(run.agentId, recovered.agentId); - assert.equal(run.agentName, recovered.agentName); - assert.equal(run.workspaceIdentity, undefined); + const run: RuntimeInvocationRecord | undefined = ( + await stores.runtimeEventStore.listSessionInvocations(recovered.sessionId) + ).find((candidate) => candidate.runId === recovered.runId); + assert.ok(run); + assert.equal(runtimeInvocationOutcome(run), 'failed'); + assert.equal(runtimeInvocationFailureClass(run), 'app_restarted'); + const lineage: RuntimeInvocationLineage | undefined = run.opening.lineage; + assert.equal(lineage?.agentId, recovered.agentId); + assert.equal(lineage?.agentName, recovered.agentName); + assert.equal(run.opening.configuration.workspaceIdentity, undefined); if (recovered.kind === 'linked_child_resume') { - assert.equal(run.resumedFromRunId, recovered.sourceRunId); - assert.equal(run.retriedFromRunId, undefined); + assert.equal(lineage?.resumedFromRunId, recovered.sourceRunId); + assert.equal(lineage?.retriedFromRunId, undefined); } else if (recovered.kind === 'linked_child_provider_retry') { - assert.equal(run.retriedFromRunId, recovered.sourceRunId); - assert.equal(run.resumedFromRunId, undefined); + assert.equal(lineage?.retriedFromRunId, recovered.sourceRunId); + assert.equal(lineage?.resumedFromRunId, undefined); } else { - assert.equal(run.resumedFromRunId, undefined); - assert.equal(run.retriedFromRunId, undefined); + assert.equal(lineage?.resumedFromRunId, undefined); + assert.equal(lineage?.retriedFromRunId, undefined); } const runtimeEvents = await stores.runtimeEventStore.readImmutableRuntimeEvents( recovered.sessionId, diff --git a/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts b/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts index c06127a15b..7288617e0d 100644 --- a/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts @@ -29,7 +29,6 @@ import { join } from 'node:path'; import { test } from 'node:test'; import { TOOL_BOUNDARY_PROTOCOL_V1 } from '@maka/core/runtime-event'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; -import type { AgentRunHeader } from '@maka/core/agent-run'; import type { MessageContent } from '@maka/core/events'; import type { ConnectionCatalogEntry } from '@maka/core/runtime-policy'; import type { StoredMessage } from '@maka/core/session'; @@ -205,8 +204,8 @@ test('startup recovery replays an admitted regenerate with its source lineage', const ledger = await fixture.readTurn(regeneratedTurnId); assert.equal(ledger.runs.length, 1); assert.equal(ledger.userMessages.length, 1); - assert.equal(ledger.runs[0]?.parentTurnId, sourceTurnId); - assert.equal(ledger.runs[0]?.regeneratedFromTurnId, sourceTurnId); + assert.equal(ledger.runs[0]?.opening.lineage?.parentTurnId, sourceTurnId); + assert.equal(ledger.runs[0]?.opening.lineage?.regeneratedFromTurnId, sourceTurnId); }); }); @@ -328,7 +327,7 @@ test('startup recovery rejects an unproven legacy non-terminal Run before closin await fixture.assertOwnerAvailable(); const ledger = await fixture.readTurn(legacy.turnId); assert.equal(ledger.runs.length, 1); - assert.equal(ledger.runs[0]?.status, 'created'); + assert.equal(ledger.runs[0]?.terminalEvent, undefined); assert.equal(ledger.terminalEvents.length, 0); assert.deepEqual( (await fixture.readSessionUserMessages()).filter((message) => diff --git a/packages/runtime-host/src/__tests__/execution-host.test.ts b/packages/runtime-host/src/__tests__/execution-host.test.ts index cf523aaa67..096479011b 100644 --- a/packages/runtime-host/src/__tests__/execution-host.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host.test.ts @@ -39,7 +39,6 @@ import { DatabaseSync } from 'node:sqlite'; import { test } from 'node:test'; import { TOOL_BOUNDARY_PROTOCOL_V1 } from '@maka/core/runtime-event'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; -import type { AgentRunHeader } from '@maka/core/agent-run'; import type { MessageContent } from '@maka/core/events'; import type { ConnectionCatalogEntry } from '@maka/core/runtime-policy'; import { @@ -920,8 +919,8 @@ test('regenerate replays the durable source content with one recoverable root id const ledger = await fixture.readTurn(regeneratedTurnId); assert.equal(ledger.runs.length, 1); assert.equal(ledger.userMessages.length, 1); - assert.equal(ledger.runs[0]?.parentTurnId, sourceTurnId); - assert.equal(ledger.runs[0]?.regeneratedFromTurnId, sourceTurnId); + assert.equal(ledger.runs[0]?.opening.lineage?.parentTurnId, sourceTurnId); + assert.equal(ledger.runs[0]?.opening.lineage?.regeneratedFromTurnId, sourceTurnId); assert.deepEqual( { text: ledger.userMessages[0]?.text, diff --git a/packages/runtime-host/src/__tests__/execution-inspect-coordinator.test.ts b/packages/runtime-host/src/__tests__/execution-inspect-coordinator.test.ts index 753368305b..093fe57a1a 100644 --- a/packages/runtime-host/src/__tests__/execution-inspect-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/execution-inspect-coordinator.test.ts @@ -23,7 +23,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; import { describe, test } from 'node:test'; -import type { AgentRunHeader, EmittedAgentRunEvent } from '@maka/core/agent-run'; +import type { EmittedAgentRunEvent } from '@maka/core/agent-run'; +import { seedInvocation } from '@maka/runtime/test-only/invocation-fixture'; import { MODEL_CALL_ATTEMPT_SCHEMA_VERSION } from '@maka/core/model-call-attempt'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { openInteractiveExecutionStoresForWrite } from '@maka/storage/execution-stores'; @@ -41,7 +42,7 @@ describe('HostExecutionInspectCoordinator', () => { await withCoordinator(async ({ root, stores, coordinator }) => { const session = await stores.sessionStore.create(sessionInput('Corrupt model call')); const runId = 'corrupt-model-call-run'; - await stores.agentRunStore.createRun(runHeader(session.id, runId, 1)); + await seedInvocation(stores.runtimeEventStore, runHeader(session.id, runId, 1)); await stores.agentRunStore.appendEvent(session.id, runId, { type: 'model_call_attempt_recorded', id: 'corrupt-model-call-event', @@ -77,7 +78,7 @@ describe('HostExecutionInspectCoordinator', () => { const session = await stores.sessionStore.create(sessionInput('Compaction diagnostics')); const runId = 'compact-run'; const turnId = `turn-${runId}`; - await stores.agentRunStore.createRun(runHeader(session.id, runId, 1)); + await seedInvocation(stores.runtimeEventStore, runHeader(session.id, runId, 1)); await stores.agentRunStore.appendEvent(session.id, runId, { type: 'model_call_attempt_recorded', id: 'attempt-compact-1', @@ -147,8 +148,13 @@ describe('HostExecutionInspectCoordinator', () => { await withCoordinator(async ({ stores, coordinator }) => { const first = await stores.sessionStore.create(sessionInput('First')); const second = await stores.sessionStore.create(sessionInput('Second')); - await stores.agentRunStore.createRun(runHeader(first.id, 'shared-run', 1)); - await stores.agentRunStore.createRun(runHeader(second.id, 'shared-run', 2)); + await seedInvocation(stores.runtimeEventStore, runHeader(first.id, 'shared-run', 1)); + // One invocation id names one execution everywhere, so two Sessions that + // reuse a run id still open separate invocations. + await seedInvocation( + stores.runtimeEventStore, + runHeader(second.id, 'shared-run', 2, 'shared-run-second'), + ); const run = await coordinator.handlers['execution.inspect.query']( { kind: 'agent_run', sessionId: second.id, agentRunId: 'shared-run' }, @@ -176,7 +182,7 @@ describe('HostExecutionInspectCoordinator', () => { 'shared-run', runtimeEvent(first.id, 'shared-run', 4), ); - await stores.agentRunStore.createRun(runHeader(first.id, 'older-run', 0)); + await seedInvocation(stores.runtimeEventStore, runHeader(first.id, 'older-run', 0)); await stores.runtimeEventStore.appendRuntimeEvent( first.id, 'older-run', @@ -201,7 +207,10 @@ describe('HostExecutionInspectCoordinator', () => { await withCoordinator(async ({ stores, coordinator }) => { const session = await stores.sessionStore.create(sessionInput('Large')); for (let index = 0; index <= EXECUTION_INSPECT_SESSION_MAX_RUNS; index += 1) { - await stores.agentRunStore.createRun(runHeader(session.id, `run-${index}`, index)); + await seedInvocation( + stores.runtimeEventStore, + runHeader(session.id, `run-${index}`, index), + ); } const oversized = await coordinator.handlers['execution.inspect.query']( @@ -232,7 +241,7 @@ describe('HostExecutionInspectCoordinator', () => { const runCount = EXECUTION_INSPECT_TRACE_PAGE_MAX_TURNS + 8; for (let index = 0; index < runCount; index += 1) { const runId = `paged-run-${index}`; - await stores.agentRunStore.createRun(runHeader(session.id, runId, index)); + await seedInvocation(stores.runtimeEventStore, runHeader(session.id, runId, index)); await stores.runtimeEventStore.appendRuntimeEvent( session.id, runId, @@ -272,7 +281,8 @@ describe('HostExecutionInspectCoordinator', () => { await withCoordinator(async ({ stores, coordinator }) => { const session = await stores.sessionStore.create(sessionInput('Legacy timestamps')); for (let index = 0; index <= EXECUTION_INSPECT_TRACE_PAGE_MAX_TURNS; index += 1) { - await stores.agentRunStore.createRun( + await seedInvocation( + stores.runtimeEventStore, runHeader(session.id, `legacy-run-${index}`, index + 0.5), ); } @@ -301,7 +311,10 @@ describe('HostExecutionInspectCoordinator', () => { await withCoordinator(async ({ stores, coordinator }) => { const session = await stores.sessionStore.create(sessionInput('Target Turn')); for (let index = 0; index <= EXECUTION_INSPECT_SESSION_MAX_RUNS; index += 1) { - await stores.agentRunStore.createRun(runHeader(session.id, `unrelated-${index}`, index)); + await seedInvocation( + stores.runtimeEventStore, + runHeader(session.id, `unrelated-${index}`, index), + ); } const runId = 'target-run'; const turnId = `turn-${runId}`; @@ -316,7 +329,7 @@ describe('HostExecutionInspectCoordinator', () => { sourceMessages: [], admittedAt: 100, }); - await stores.agentRunStore.createRun(runHeader(session.id, runId, 100)); + await seedInvocation(stores.runtimeEventStore, runHeader(session.id, runId, 100)); await stores.runtimeEventStore.appendRuntimeEvent( session.id, runId, @@ -344,9 +357,9 @@ describe('HostExecutionInspectCoordinator', () => { test('rejects oversized evidence at the bounded Store read boundary', async () => { await withCoordinator(async ({ stores, coordinator }) => { const session = await stores.sessionStore.create(sessionInput('Large evidence')); - await stores.agentRunStore.createRun(runHeader(session.id, 'large-run', 1)); + await seedInvocation(stores.runtimeEventStore, runHeader(session.id, 'large-run', 1)); await stores.agentRunStore.appendEvent(session.id, 'large-run', { - type: 'run_started', + type: 'turn_started', id: 'large-event', sessionId: session.id, runId: 'large-run', @@ -374,9 +387,9 @@ describe('HostExecutionInspectCoordinator', () => { test('does not charge unrelated AgentRun diagnostics to the Session trace budget', async () => { await withCoordinator(async ({ stores, coordinator }) => { const session = await stores.sessionStore.create(sessionInput('Trace evidence')); - await stores.agentRunStore.createRun(runHeader(session.id, 'trace-run', 1)); + await seedInvocation(stores.runtimeEventStore, runHeader(session.id, 'trace-run', 1)); await stores.agentRunStore.appendEvent(session.id, 'trace-run', { - type: 'run_started', + type: 'turn_started', id: 'large-unrelated-event', sessionId: session.id, runId: 'trace-run', @@ -404,12 +417,12 @@ describe('HostExecutionInspectCoordinator', () => { test('keeps a Session trace pageable when one run exceeds the evidence budget', async () => { await withCoordinator(async ({ stores, coordinator }) => { const session = await stores.sessionStore.create(sessionInput('Oversized trace page')); - await stores.agentRunStore.createRun(runHeader(session.id, 'oversized-run', 2)); + await seedInvocation(stores.runtimeEventStore, runHeader(session.id, 'oversized-run', 2)); await stores.runtimeEventStore.appendRuntimeEvent(session.id, 'oversized-run', { ...runtimeEvent(session.id, 'oversized-run', 2), content: { kind: 'text', text: 'x'.repeat(EXECUTION_INSPECT_EVIDENCE_MAX_BYTES) }, }); - await stores.agentRunStore.createRun(runHeader(session.id, 'older-run', 1)); + await seedInvocation(stores.runtimeEventStore, runHeader(session.id, 'older-run', 1)); await stores.runtimeEventStore.appendRuntimeEvent( session.id, 'older-run', @@ -445,7 +458,10 @@ describe('HostExecutionInspectCoordinator', () => { test('keeps earlier Session history reachable when one projected page exceeds the result limit', async () => { await withCoordinator(async ({ stores, coordinator }) => { const session = await stores.sessionStore.create(sessionInput('Oversized trace result')); - await stores.agentRunStore.createRun(runHeader(session.id, 'oversized-result-run', 2)); + await seedInvocation( + stores.runtimeEventStore, + runHeader(session.id, 'oversized-result-run', 2), + ); for (let index = 0; index < 128; index += 1) { await stores.runtimeEventStore.appendRuntimeEvent(session.id, 'oversized-result-run', { ...runtimeEvent(session.id, 'oversized-result-run', index + 2), @@ -453,7 +469,7 @@ describe('HostExecutionInspectCoordinator', () => { content: { kind: 'error', message: 'x'.repeat(EXECUTION_INSPECT_RESULT_MAX_BYTES / 64) }, }); } - await stores.agentRunStore.createRun(runHeader(session.id, 'older-run', 1)); + await seedInvocation(stores.runtimeEventStore, runHeader(session.id, 'older-run', 1)); await stores.runtimeEventStore.appendRuntimeEvent( session.id, 'older-run', @@ -509,9 +525,9 @@ describe('HostExecutionInspectCoordinator', () => { test('accepts evidence that exactly consumes the shared byte budget before an empty ledger', async () => { await withCoordinator(async ({ stores, coordinator }) => { const session = await stores.sessionStore.create(sessionInput('Exact evidence budget')); - await stores.agentRunStore.createRun(runHeader(session.id, 'exact-run', 1)); + await seedInvocation(stores.runtimeEventStore, runHeader(session.id, 'exact-run', 1)); const baseEvent: EmittedAgentRunEvent = { - type: 'run_started', + type: 'turn_started', id: 'exact-event', sessionId: session.id, runId: 'exact-run', @@ -520,23 +536,31 @@ describe('HostExecutionInspectCoordinator', () => { data: { payload: '' }, }; const baseBytes = Buffer.byteLength(JSON.stringify(baseEvent), 'utf8'); - assert.ok(baseBytes < EXECUTION_INSPECT_EVIDENCE_MAX_BYTES); + // One query charges both ledgers to the same budget, and the invocation's + // opening fact is already on the RuntimeEvent ledger. The operational + // event is sized to exactly the rest. + const opening = await stores.runtimeEventStore.readRuntimeEventsBounded( + session.id, + 'exact-run', + { maxRecords: 8, maxBytes: EXECUTION_INSPECT_EVIDENCE_MAX_BYTES }, + ); + assert.equal(opening.status, 'complete'); + const operationalBytes = EXECUTION_INSPECT_EVIDENCE_MAX_BYTES - opening.storedBytes; + assert.ok(baseBytes < operationalBytes); const event = { ...baseEvent, - data: { - payload: 'x'.repeat(EXECUTION_INSPECT_EVIDENCE_MAX_BYTES - baseBytes), - }, + data: { payload: 'x'.repeat(operationalBytes - baseBytes) }, }; await stores.agentRunStore.appendEvent(session.id, 'exact-run', event); const exact = await stores.agentRunStore.readEventsBounded(session.id, 'exact-run', { maxRecords: 1, - maxBytes: EXECUTION_INSPECT_EVIDENCE_MAX_BYTES, + maxBytes: operationalBytes, }); assert.equal(exact.status, 'complete'); const oneByteShort = await stores.agentRunStore.readEventsBounded(session.id, 'exact-run', { maxRecords: 1, - maxBytes: EXECUTION_INSPECT_EVIDENCE_MAX_BYTES - 1, + maxBytes: operationalBytes - 1, }); assert.equal(oneByteShort.status, 'limit_exceeded'); @@ -552,7 +576,7 @@ describe('HostExecutionInspectCoordinator', () => { await withCoordinator(async ({ stores, coordinator }) => { const session = await stores.sessionStore.create(sessionInput('Aggregate evidence')); for (const [index, runId] of ['aggregate-run-1', 'aggregate-run-2'].entries()) { - await stores.agentRunStore.createRun(runHeader(session.id, runId, index + 1)); + await seedInvocation(stores.runtimeEventStore, runHeader(session.id, runId, index + 1)); await stores.runtimeEventStore.appendRuntimeEvent(session.id, runId, { id: `aggregate-event-${index + 1}`, invocationId: runId, @@ -580,7 +604,12 @@ describe('HostExecutionInspectCoordinator', () => { ); assert.equal(first.ok, true); if (!first.ok || first.result.kind !== 'session_trace_page') return; - assert.equal(first.result.turns.length, 0); + // Only the newer run's evidence fits one budget. Its page carries the turn + // its opening fact projects, and the older run waits behind the cursor. + assert.deepEqual( + first.result.turns.map((turn) => turn.runId), + ['aggregate-run-2'], + ); assert.ok(first.result.nextCursor !== null); }); }); @@ -598,21 +627,30 @@ function sessionInput(name: string) { } as const; } -function runHeader(sessionId: string, runId: string, createdAt: number): AgentRunHeader { +function runHeader(sessionId: string, runId: string, createdAt: number, invocationId?: string) { return { sessionId, runId, + ...(invocationId ? { invocationId } : {}), turnId: `turn-${runId}`, - status: 'completed', - backendKind: 'fake', - llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/workspace', - permissionMode: 'ask', - createdAt, - updatedAt: createdAt, - completedAt: createdAt, + openedAt: createdAt, + opening: { + route: { + provenance: 'runtime' as const, + backendKind: 'fake' as const, + llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: '/tmp/workspace', + permissionMode: 'ask' as const, + collaborationMode: 'agent' as const, + orchestrationMode: 'default' as const, + orchestrationSource: 'session' as const, + toolMode: 'direct' as const, + }, + }, }; } diff --git a/packages/runtime-host/src/__tests__/execution-inspect-protocol.test.ts b/packages/runtime-host/src/__tests__/execution-inspect-protocol.test.ts index 7febc0924d..944f2deecf 100644 --- a/packages/runtime-host/src/__tests__/execution-inspect-protocol.test.ts +++ b/packages/runtime-host/src/__tests__/execution-inspect-protocol.test.ts @@ -263,11 +263,11 @@ function agentRunDocument(sessionId = 'session-1', agentRunId = 'run-1'): AgentR agentRun: { sessionId, agentRunId, + invocationId: agentRunId, turnId: 'turn-1', status: 'completed', - createdAt: 1, - updatedAt: 2, - completedAt: 2, + openedAt: 1, + endedAt: 2, }, sources: { operationalEventCount: 0, @@ -275,8 +275,6 @@ function agentRunDocument(sessionId = 'session-1', agentRunId = 'run-1'): AgentR health: { runtimeLedger: 'missing', runtimeTerminalPresent: false, - operationalTerminalPresent: false, - statusConsistency: 'incomplete', }, }, tools: { diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index 8408d2f940..67abcacd27 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -40,7 +40,9 @@ import { import { PROVIDER_REGISTRY } from '@maka/core/llm-connections'; import { createWorkspaceWritePermissionProfile } from '@maka/core/permission-profile'; import { decodeRunCompositionSnapshot } from '@maka/core/run-composition'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import { readInvocation, testInvocationRecord } from '@maka/runtime/test-only/invocation-fixture'; +import { runtimeInvocationOutcome } from '@maka/core/runtime-invocation'; +import { agentRunCompositionFromEvents } from '@maka/core/agent-run'; import type { BackendCompactHistoryInput } from '@maka/core/backend-types'; import { decodeCanonicalToolResultContent } from '@maka/core/tool-result-record-schema'; import { type ModelCallAttempt, type ModelCallKind } from '@maka/core/model-call-attempt'; @@ -328,7 +330,7 @@ test('production Host executes Bash against the current live sandbox boundary', ), context, ); - const firstRun = await execution.agentRunStore.readRun(session.id, firstTerminal.runId); + const firstRun = await readInvocation(execution, session.id, firstTerminal.runId); const firstRunEvents = await execution.agentRunStore.readEvents( session.id, firstTerminal.runId, @@ -1041,38 +1043,58 @@ test('Codex OAuth history compaction falls back to a text checkpoint after nativ turnId: 'turn-compact', runId: 'run-compact', runtimeContext, - runtimeContextRunHeaders: [ - { - runId: 'compact-source-run', + runtimeContextInvocations: [ + testInvocationRecord({ sessionId: 'backend-creation-session', + runId: 'compact-source-run', turnId: 'turn-old-model', - status: 'completed', - backendKind: 'ai-sdk', - llmConnectionId: '11111111-1111-4111-8111-111111111111', - llmConnectionSlug: 'backend-creation-connection', - modelId: 'gpt-5.2', - cwd: '/workspace', - permissionMode: 'bypass', - createdAt: 1, - updatedAt: 2, - completedAt: 2, - } satisfies AgentRunHeader, - { - runId: 'compact-same-route-run', + openedAt: 1, + closedAt: 2, + outcome: 'completed', + opening: { + route: { + provenance: 'runtime', + backendKind: 'ai-sdk', + llmConnectionId: '11111111-1111-4111-8111-111111111111', + llmConnectionSlug: 'backend-creation-connection', + modelId: 'gpt-5.2', + }, + configuration: { + cwd: '/workspace', + permissionMode: 'bypass', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + }, + }), + testInvocationRecord({ sessionId: 'backend-creation-session', + runId: 'compact-same-route-run', turnId: 'turn-current-route-model', - status: 'completed', - backendKind: 'ai-sdk', - llmConnectionId: '11111111-1111-4111-8111-111111111111', - llmConnectionSlug: 'backend-creation-connection', - modelId, - providerStateIdentity, - cwd: '/workspace', - permissionMode: 'bypass', - createdAt: 2, - updatedAt: 3, - completedAt: 3, - } satisfies AgentRunHeader, + openedAt: 2, + closedAt: 3, + outcome: 'completed', + opening: { + route: { + provenance: 'runtime', + backendKind: 'ai-sdk', + llmConnectionId: '11111111-1111-4111-8111-111111111111', + llmConnectionSlug: 'backend-creation-connection', + modelId, + providerStateIdentity, + }, + configuration: { + cwd: '/workspace', + permissionMode: 'bypass', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + }, + }), ], } satisfies BackendCompactHistoryInput; const result = await backend.compactHistory(compactInput); @@ -1898,6 +1920,9 @@ test('production Host executes a canonical ai-sdk Session against a real provide const hostedCheckpoints = await loadHistoryCompactCheckpointsFromRunLedger( execution.agentRunStore, session.id, + (await execution.runtimeEventStore.listSessionInvocations(session.id)).map( + (invocation) => invocation.runId, + ), ); const hostedMemoryBoundary = hostedCheckpoints.find( (checkpoint) => checkpoint.memoryExtractionBoundary, @@ -2201,20 +2226,22 @@ test('production Host executes and durably supervises an Agent Graph over a real graphStore = createAgentGraphControlStore(root); const graphId = agentGraphIdForRootSession(session.id); let updates = await graphStore.listAgentGraphScheduleUpdates(graphId); - let runs = await execution.agentRunStore.listSessionRuns(session.id); + let runs = await execution.runtimeEventStore.listSessionInvocations(session.id); for (let attempt = 0; attempt < 400; attempt += 1) { - const wakeRuns = runs.filter((run) => run.agentGraphWakeAttemptId !== undefined); + const wakeRuns = runs.filter( + (run) => run.opening.root.kind === 'agent_graph_supervisor_wake', + ); if ( updates.at(-1)?.finish && wakeRuns.length > 0 && - wakeRuns.every((run) => ['completed', 'failed', 'cancelled'].includes(run.status)) && + wakeRuns.every((run) => runtimeInvocationOutcome(run) !== undefined) && liveResidencies === 0 ) { break; } await new Promise((resolve) => setTimeout(resolve, 10)); updates = await graphStore.listAgentGraphScheduleUpdates(graphId); - runs = await execution.agentRunStore.listSessionRuns(session.id); + runs = await execution.runtimeEventStore.listSessionInvocations(session.id); } const finish = updates.at(-1)?.finish; @@ -2225,22 +2252,26 @@ test('production Host executes and durably supervises an Agent Graph over a real lastUpdate: updates.at(-1), runs: runs.map((run) => ({ runId: run.runId, - status: run.status, - wakeAttemptId: run.agentGraphWakeAttemptId, + status: runtimeInvocationOutcome(run) ?? 'running', + root: run.opening.root, })), requests: providerRequestTrace(provider.requests), }), ); assert.equal(finish?.resultIds.length, 1); const rootRun = runs.find((run) => run.runId === initialTerminal.runId); - assert.equal(rootRun?.runComposition?.composerId, 'maka.interactive'); - assert.equal(rootRun?.runComposition?.contextWindow, 32_768); - assert.match(rootRun?.runComposition?.baseSystemPromptHash ?? '', /^sha256:[a-f0-9]{64}$/u); - assert.ok(rootRun?.runComposition?.toolNames.includes('view_agent_graph')); - const wakeRuns = runs.filter((run) => run.agentGraphWakeAttemptId !== undefined); + assert.ok(rootRun); + const rootComposition = agentRunCompositionFromEvents( + await execution.agentRunStore.readEvents(session.id, rootRun.runId), + ); + assert.equal(rootComposition?.composerId, 'maka.interactive'); + assert.equal(rootComposition?.contextWindow, 32_768); + assert.match(rootComposition?.baseSystemPromptHash ?? '', /^sha256:[a-f0-9]{64}$/u); + assert.ok(rootComposition?.toolNames.includes('view_agent_graph')); + const wakeRuns = runs.filter((run) => run.opening.root.kind === 'agent_graph_supervisor_wake'); assert.ok(wakeRuns.length > 0); - assert.ok(wakeRuns.every((run) => run.status === 'completed')); - assert.ok(wakeRuns.every((run) => run.orchestrationMode === 'graph')); + assert.ok(wakeRuns.every((run) => runtimeInvocationOutcome(run) === 'completed')); + assert.ok(wakeRuns.every((run) => run.opening.configuration.orchestrationMode === 'graph')); assert.equal(liveResidencies, 0); const sessions = await execution.sessionStore.listForRecovery(); @@ -2250,9 +2281,11 @@ test('production Host executes and durably supervises an Agent Graph over a real assert.ok(child); assert.equal(child?.subagentRuntime?.profile, 'local_read'); assert.equal(child?.subagentParent?.parentSessionId, session.id); - const childRuns = child ? await execution.agentRunStore.listSessionRuns(child.id) : []; + const childRuns = child + ? await execution.runtimeEventStore.listSessionInvocations(child.id) + : []; assert.equal(childRuns.length, 1); - assert.equal(childRuns[0]?.status, 'completed'); + assert.equal(childRuns[0] && runtimeInvocationOutcome(childRuns[0]), 'completed'); const graphRequests = provider.requests.filter( (request) => @@ -2379,7 +2412,7 @@ test('production Host executes a durable runnable child with an exact tool ceili ), context, ); - const parentRun = await execution.agentRunStore.readRun(parent.id, terminal.runId); + const parentRun = await readInvocation(execution, parent.id, terminal.runId); const parentRunEvents = await execution.agentRunStore.readEvents(parent.id, terminal.runId); assert.equal( terminal.status, @@ -2425,10 +2458,10 @@ test('production Host executes a durable runnable child with an exact tool ceili if (!child) return; assert.equal(child.subagentWorkspace, undefined); assert.equal(child.cwd, project); - const childRuns = await execution.agentRunStore.listSessionRuns(child.id); + const childRuns = await execution.runtimeEventStore.listSessionInvocations(child.id); assert.equal(childRuns.length, 1); - assert.equal(childRuns[0]?.status, 'completed'); - assert.equal(childRuns[0]?.parentRunId, undefined); + assert.equal(childRuns[0] && runtimeInvocationOutcome(childRuns[0]), 'completed'); + assert.equal(childRuns[0]?.opening.lineage?.parentRunId, undefined); const childMessages = await execution.sessionStore.readMessagesSnapshot(child.id); assert.equal( childMessages.find((message) => message.type === 'assistant')?.text, @@ -2577,7 +2610,7 @@ test('production Host publishes and retires an implementation child patch', asyn ), context, ); - const parentRun = await execution.agentRunStore.readRun(parent.id, terminal.runId); + const parentRun = await readInvocation(execution, parent.id, terminal.runId); const parentRunEvents = await execution.agentRunStore.readEvents(parent.id, terminal.runId); assert.equal( terminal.status, @@ -2642,10 +2675,10 @@ test('production Host publishes and retires an implementation child patch', asyn assert.equal(child.cwd, child.subagentWorkspace?.worktreePath); assert.equal(await fileExists(join(project, 'implementation.txt')), false); assert.equal(await fileExists(join(child.cwd, 'implementation.txt')), true); - const childRuns = await execution.agentRunStore.listSessionRuns(child.id); + const childRuns = await execution.runtimeEventStore.listSessionInvocations(child.id); assert.equal(childRuns.length, 1); - assert.equal(childRuns[0]?.status, 'completed'); - assert.equal(childRuns[0]?.parentRunId, undefined); + assert.equal(childRuns[0] && runtimeInvocationOutcome(childRuns[0]), 'completed'); + assert.equal(childRuns[0]?.opening.lineage?.parentRunId, undefined); const childMessages = await execution.sessionStore.readMessagesSnapshot(child.id); assert.equal( childMessages.find((message) => message.type === 'assistant')?.text, diff --git a/packages/runtime-host/src/__tests__/fixtures/agent-graph-provider-scenario.ts b/packages/runtime-host/src/__tests__/fixtures/agent-graph-provider-scenario.ts index 3c64dea057..20edbf923f 100644 --- a/packages/runtime-host/src/__tests__/fixtures/agent-graph-provider-scenario.ts +++ b/packages/runtime-host/src/__tests__/fixtures/agent-graph-provider-scenario.ts @@ -140,7 +140,11 @@ export class AgentGraphProviderScenario { requireRecord(output.execution, 'agent output execution').kind, 'child_session', ); - assert.equal(requireRecord(output.header, 'agent output header').status, 'completed'); + const invocation = requireRecord(output.invocation, 'agent output invocation'); + assert.equal( + requireRecord(invocation.terminalEvent, 'agent output terminal event').status, + 'completed', + ); const result = requireRecord(output.result, 'agent output payload'); assert.equal(result.status, 'completed'); assert.equal(result.text, this.childResultText); diff --git a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts index 1cdf00afd2..520c689a22 100644 --- a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts +++ b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts @@ -39,7 +39,11 @@ import { DatabaseSync } from 'node:sqlite'; import { test } from 'node:test'; import { TOOL_BOUNDARY_PROTOCOL_V1 } from '@maka/core/runtime-event'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import { + runtimeInvocationOutcome, + type RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; +import { seedInvocation } from '@maka/runtime/test-only/invocation-fixture'; import { aggregateMessageContents, messageContentDigest, @@ -121,7 +125,7 @@ export interface ExecutionHostHandle { } export interface TurnLedger { - runs: AgentRunHeader[]; + runs: RuntimeInvocationRecord[]; userMessages: Array>; runtimeEvents: RuntimeEvent[]; terminalEvents: RuntimeEvent[]; @@ -176,24 +180,31 @@ export class ExecutionFixture { const sourceTurnId = randomUUID(); const createdAt = Date.now(); const workspace = await resolveWorkspaceIdentity({ path: this.root }); - const sourceRun: AgentRunHeader = { - runId: sourceRunId, - invocationId: sourceInvocationId, + const sourceRun = await seedInvocation(stores.runtimeEventStore, { sessionId: this.sessionId, + invocationId: sourceInvocationId, + runId: sourceRunId, turnId: sourceTurnId, - status: 'created', - backendKind: 'fake', - llmConnectionId: FAKE_CONNECTION_ID, - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: this.root, - workspaceIdentity: workspace.workspaceIdentity, - permissionMode: 'ask', - collaborationMode: 'agent', - createdAt, - updatedAt: createdAt, - }; - await stores.agentRunStore.createRun(sourceRun, { durable: true }); + openedAt: createdAt, + opening: { + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: FAKE_CONNECTION_ID, + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: this.root, + workspaceIdentity: workspace.workspaceIdentity, + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + }, + }); await stores.runtimeEventStore.appendRuntimeEvent(this.sessionId, sourceRunId, { id: randomUUID(), sessionId: this.sessionId, @@ -251,7 +262,6 @@ export class ExecutionFixture { recoveryReason: 'test_safe_boundary_source', }); await commitTerminalRunWithRuntimeFact({ - runStore: stores.agentRunStore, runtimeEventStore: stores.runtimeEventStore, newId: randomUUID, sessionId: this.sessionId, @@ -266,7 +276,9 @@ export class ExecutionFixture { sourceInvocationId, sourceRunId, sourceTurnId, - sourceRuntimeEventHighWater: requiredToolName ? 4 : 2, + // The opening fact is event 1 of the invocation, ahead of the user event, + // any tool pair, and the terminal event. + sourceRuntimeEventHighWater: requiredToolName ? 5 : 3, }; } finally { await stores?.sessionStore.close?.(); @@ -275,10 +287,7 @@ export class ExecutionFixture { } async seedSafeBoundaryContinuationCrash( - failpoint: - | 'after_continuation_claim_committed' - | 'after_run_created' - | 'after_continuation_start_committed', + failpoint: 'after_continuation_claim_committed' | 'after_continuation_start_committed', ): Promise<{ sourceRunId: string; sourceRuntimeEventHighWater: number; @@ -549,25 +558,31 @@ export class ExecutionFixture { assert.equal(child.created, true); if (sourceRunId) { const sourceTs = Date.now(); - const sourceRun: AgentRunHeader = { - runId: sourceRunId, - invocationId: sourceRunId, + const sourceRun = await seedInvocation(stores.runtimeEventStore, { sessionId: child.header.id, + invocationId: sourceRunId, + runId: sourceRunId, turnId: `source-turn-${kind}`, - status: 'created', - backendKind: 'fake', - llmConnectionId: FAKE_CONNECTION_ID, - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: this.root, - permissionMode: 'explore', - collaborationMode: 'agent', - createdAt: sourceTs, - updatedAt: sourceTs, - agentId, - agentName, - }; - await stores.agentRunStore.createRun(sourceRun, { durable: true }); + openedAt: sourceTs, + opening: { + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: FAKE_CONNECTION_ID, + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: this.root, + permissionMode: 'explore', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + lineage: { agentId, agentName }, + }, + }); const sourceTerminal = buildRecoveredTerminalRuntimeEvent({ id: randomUUID(), run: sourceRun, @@ -577,7 +592,6 @@ export class ExecutionFixture { recoveryReason: 'test_source_terminal', }); await commitTerminalRunWithRuntimeFact({ - runStore: stores.agentRunStore, runtimeEventStore: stores.runtimeEventStore, newId: randomUUID, sessionId: child.header.id, @@ -649,28 +663,35 @@ export class ExecutionFixture { try { stores = await openInteractiveExecutionStoresForWrite(owner.lease); const ts = Date.now(); - await stores.agentRunStore.createRun( - { - runId: graph.runId, - invocationId: graph.runId, - sessionId: graph.sessionId, - turnId: graph.turnId, - status: 'created', - backendKind: 'fake', - llmConnectionId: FAKE_CONNECTION_ID, - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: this.root, - permissionMode: 'explore', - collaborationMode: 'agent', - createdAt: ts, - updatedAt: ts, - resumedFromRunId: randomUUID(), - agentId: graph.agentId, - agentName: graph.agentName, + await seedInvocation(stores.runtimeEventStore, { + sessionId: graph.sessionId, + invocationId: graph.runId, + runId: graph.runId, + turnId: graph.turnId, + openedAt: ts, + opening: { + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: FAKE_CONNECTION_ID, + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: this.root, + permissionMode: 'explore', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + lineage: { + resumedFromRunId: randomUUID(), + agentId: graph.agentId, + agentName: graph.agentName, + }, }, - { durable: true }, - ); + }); } finally { await stores?.sessionStore.close?.(); await owner.close(); @@ -798,22 +819,31 @@ export class ExecutionFixture { admittedAt, }); assert.equal(admission.kind, 'admitted'); - const run: AgentRunHeader = { - runId, - invocationId: runId, - sessionId: this.sessionId, - turnId, - status: 'created', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: this.root, - permissionMode: 'ask', - createdAt: admittedAt, - updatedAt: admittedAt, - }; + const run = { runId, invocationId: runId, sessionId: this.sessionId, turnId }; if (runState !== 'missing') { - await stores.agentRunStore.createRun(run, { durable: true }); + await seedInvocation(stores.runtimeEventStore, { + sessionId: this.sessionId, + invocationId: runId, + runId, + turnId, + openedAt: admittedAt, + opening: { + route: { + provenance: 'unknown', + backendKind: 'fake', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: this.root, + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + }, + }); } if (runState === 'terminal') { const terminalAt = admittedAt + 1; @@ -826,7 +856,6 @@ export class ExecutionFixture { recoveryReason: 'test_legacy_terminal_root', }); await commitTerminalRunWithRuntimeFact({ - runStore: stores.agentRunStore, runtimeEventStore: stores.runtimeEventStore, newId: randomUUID, sessionId: this.sessionId, @@ -1002,20 +1031,29 @@ export class ExecutionFixture { }); assert.equal(result.kind, 'admitted'); if (createRun) { - await stores.agentRunStore.createRun({ - runId: result.admission.runId, - invocationId: result.admission.runId, + await seedInvocation(stores.runtimeEventStore, { sessionId: this.sessionId, + invocationId: result.admission.runId, + runId: result.admission.runId, turnId, - status: 'created', - backendKind: 'fake', - llmConnectionId: FAKE_CONNECTION_ID, - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: this.root, - permissionMode: 'ask', - createdAt: admittedAt, - updatedAt: admittedAt, + openedAt: admittedAt, + opening: { + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: FAKE_CONNECTION_ID, + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: this.root, + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + }, }); } assert.ok(result.admission.userMessageId); @@ -1107,10 +1145,10 @@ export class ExecutionFixture { stores = await openInteractiveExecutionStoresForRead(reader.lease); const admission = await stores.agentRunStore.readRootTurnAdmission(this.sessionId, turnId); assert.ok(admission); - const runs = (await stores.agentRunStore.listSessionRuns(this.sessionId)).filter( - (candidate) => candidate.turnId === turnId, - ); - const run = await stores.agentRunStore.readRun(this.sessionId, admission.runId); + const invocations = await stores.runtimeEventStore.listSessionInvocations(this.sessionId); + const runs = invocations.filter((candidate) => candidate.turnId === turnId); + const run = invocations.find((candidate) => candidate.runId === admission.runId); + assert.ok(run); const messages = await stores.sessionStore.readMessages(this.sessionId); const runtimeEvents = await stores.runtimeEventStore.readImmutableRuntimeEvents( this.sessionId, @@ -1149,7 +1187,7 @@ export class ExecutionFixture { let stores: Awaited> | undefined; try { stores = await openInteractiveExecutionStoresForRead(reader.lease); - return (await stores.agentRunStore.listSessionRuns(this.sessionId)).filter( + return (await stores.runtimeEventStore.listSessionInvocations(this.sessionId)).filter( (candidate) => candidate.turnId === turnId, ); } finally { @@ -1197,7 +1235,7 @@ export class ExecutionFixture { stores = await openInteractiveExecutionStoresForRead(reader.lease); const [admission, runs, messages] = await Promise.all([ stores.agentRunStore.readRootTurnAdmission(this.sessionId, turnId), - stores.agentRunStore.listSessionRuns(this.sessionId), + stores.runtimeEventStore.listSessionInvocations(this.sessionId), stores.sessionStore.readMessages(this.sessionId), ]); return { diff --git a/packages/runtime-host/src/__tests__/goal-coordinator.test.ts b/packages/runtime-host/src/__tests__/goal-coordinator.test.ts index 09b7ea9953..b69e4e0538 100644 --- a/packages/runtime-host/src/__tests__/goal-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/goal-coordinator.test.ts @@ -23,6 +23,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; import type { GoalAuthorityRecord } from '@maka/core/goal'; +import { seedInvocation } from '@maka/runtime/test-only/invocation-fixture'; import type { GoalTurnOutcome } from '@maka/runtime/goal-continuation'; import { openInteractiveExecutionStoresForWrite } from '@maka/storage/execution-stores'; import { openInteractiveGoalAuthorityForWrite } from '@maka/storage/goal-authority'; @@ -354,21 +355,30 @@ test('restart settles the durable current Goal execution through Hosted Executio admittedAt: 1, }); assert.equal(admission.kind, 'admitted'); - await stores.agentRunStore.createRun({ - runId: execution.runId, - invocationId: execution.runId, + await seedInvocation(stores.runtimeEventStore, { sessionId: session.id, + invocationId: execution.runId, + runId: execution.runId, turnId: execution.turnId, - status: 'created', - backendKind: 'fake', - llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: capability.canonicalPath, - permissionMode: 'ask', - goalId: record.goal.id, - createdAt: 2, - updatedAt: 2, + openedAt: 2, + opening: { + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: capability.canonicalPath, + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + root: { kind: 'goal', goalId: record.goal.id }, + }, }); await stores.runtimeEventStore.appendRuntimeEvent(session.id, execution.runId, { id: 'goal_recovery_terminal', @@ -383,11 +393,6 @@ test('restart settles the durable current Goal execution through Hosted Executio author: 'agent', content: { kind: 'text', text: 'done' }, }); - await stores.agentRunStore.updateRun(session.id, execution.runId, { - status: 'completed', - updatedAt: 3, - completedAt: 3, - }); let drainRequested = false; const executionProjection = new HostedExecutionProjectionReader(stores); diff --git a/packages/runtime-host/src/__tests__/goal-root-authority.test.ts b/packages/runtime-host/src/__tests__/goal-root-authority.test.ts index fcbc66a1a3..c6e89e8376 100644 --- a/packages/runtime-host/src/__tests__/goal-root-authority.test.ts +++ b/packages/runtime-host/src/__tests__/goal-root-authority.test.ts @@ -24,7 +24,12 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import { + runtimeInvocationOutcome, + type RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; +import { runtimeInvocationFailureClass } from '@maka/runtime/runtime-event-read-model'; +import { seedInvocation } from '@maka/runtime/test-only/invocation-fixture'; import { BackendRegistry, SessionManager } from '@maka/runtime/session-manager'; import { FakeBackend } from '@maka/runtime/test-only/fake-backend'; import { GOAL_SET_TOOL_NAME } from '@maka/runtime/goal-tools'; @@ -98,11 +103,8 @@ test('Goal continuation uses the canonical root admission and durable origin', { assert.deepEqual(durableAdmission?.execution, { kind: 'goal', goalId: created.id }); assert.ok(durableAdmission); if (!durableAdmission) return; - const run = await fixture.stores.agentRunStore.readRun( - fixture.sessionId, - durableAdmission.runId, - ); - assert.equal(run.goalId, created.id); + const run = await readInvocation(fixture, durableAdmission.runId); + assert.deepEqual(run?.opening.root, { kind: 'goal', goalId: created.id }); const user = (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).find( (message) => message.type === 'user' && message.turnId === admission.turnId, ); @@ -164,8 +166,8 @@ test('queued Goal control revokes a prepared root before durable admission', asy undefined, ); assert.equal( - (await fixture.stores.agentRunStore.listSessionRuns(fixture.sessionId)).some( - (run) => run.goalId === created.id, + (await fixture.stores.runtimeEventStore.listSessionInvocations(fixture.sessionId)).some( + (run) => run.opening.root.kind === 'goal' && run.opening.root.goalId === created.id, ), false, ); @@ -306,7 +308,7 @@ test('drain revokes pending ScheduledTask before durable root admission', async undefined, ); assert.equal( - (await fixture.stores.agentRunStore.listSessionRuns(fixture.sessionId)).some( + (await fixture.stores.runtimeEventStore.listSessionInvocations(fixture.sessionId)).some( (run) => run.runId === runId, ), false, @@ -393,7 +395,7 @@ test('Host Goal continuation bridges its exact generation into root authority', if (!resumed) return; const run = await waitForGoalRun(fixture, resumed.id); - assert.equal(run.goalId, resumed.id); + assert.deepEqual(run.opening.root, { kind: 'goal', goalId: resumed.id }); const admission = await fixture.stores.agentRunStore.readRootTurnAdmission( fixture.sessionId, run.turnId, @@ -424,10 +426,10 @@ test('restart closes an admitted Goal without a Run instead of replaying it', as }); await fixture.coordinator.prepareRecovery(); - const run = await fixture.stores.agentRunStore.readRun(fixture.sessionId, runId); - assert.equal(run.goalId, 'goal-restart'); - assert.equal(run.status, 'failed'); - assert.equal(run.failureClass, 'app_restarted'); + const run = await readInvocation(fixture, runId); + assert.deepEqual(run?.opening.root, { kind: 'goal', goalId: 'goal-restart' }); + assert.equal(run && runtimeInvocationOutcome(run), 'failed'); + assert.equal(run && runtimeInvocationFailureClass(run), 'app_restarted'); const user = (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).find( (message) => message.type === 'user' && message.turnId === turnId, ); @@ -491,15 +493,15 @@ test('restart rejects a Goal Run carrying delegated execution lineage', async () sourceMessages: [], admittedAt: 1, }); - await fixture.stores.agentRunStore.createRun( - runHeader({ - sessionId: fixture.sessionId, - turnId, - runId, - goalId, - parentRunId: 'foreign-parent-run', - }), - ); + await seedInvocation(fixture.stores.runtimeEventStore, { + sessionId: fixture.sessionId, + turnId, + runId, + opening: { + root: { kind: 'goal', goalId }, + lineage: { parentRunId: 'foreign-parent-run' }, + }, + }); await assert.rejects( () => fixture.coordinator.prepareRecovery(), @@ -734,13 +736,22 @@ async function createFixture(options: { recoverAdmissions?: boolean } = {}): Pro }, }; } -async function waitForGoalRun( +async function readInvocation( fixture: Fixture, - goalId: string, -): Promise>> { + runId: string, +): Promise { + return (await fixture.stores.runtimeEventStore.listSessionInvocations(fixture.sessionId)).find( + (candidate) => candidate.runId === runId, + ); +} + +async function waitForGoalRun(fixture: Fixture, goalId: string): Promise { for (let attempt = 0; attempt < 100; attempt += 1) { - const run = (await fixture.stores.agentRunStore.listSessionRuns(fixture.sessionId)).find( - (candidate) => candidate.goalId === goalId, + const run = ( + await fixture.stores.runtimeEventStore.listSessionInvocations(fixture.sessionId) + ).find( + (candidate) => + candidate.opening.root.kind === 'goal' && candidate.opening.root.goalId === goalId, ); if (run) return run; await new Promise((resolve) => setImmediate(resolve)); @@ -748,25 +759,6 @@ async function waitForGoalRun( throw new Error('Goal continuation did not reach the root authority'); } -function runHeader(overrides: Partial): AgentRunHeader { - return { - runId: 'run-1', - invocationId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - status: 'created', - backendKind: 'fake', - llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/workspace', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 1, - ...overrides, - }; -} - function operationContext() { return { hostEpoch: 'goal-root-epoch', diff --git a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts index 9d780773b5..553d5d1d4b 100644 --- a/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/root-turn-coordinator.test.ts @@ -19,6 +19,9 @@ import { deferred, withTimeout } from '@maka/core/test-only/async-primitives'; import assert from 'node:assert/strict'; +import { readInvocation, seedInvocation } from '@maka/runtime/test-only/invocation-fixture'; +import { runtimeInvocationOutcome } from '@maka/core/runtime-invocation'; +import { runtimeInvocationFailureClass } from '@maka/runtime/runtime-event-read-model'; import { randomUUID } from 'node:crypto'; import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -520,26 +523,31 @@ test('startup recovery closes a ScheduledTask Run after its pending fire was set text: 'Continue the scheduled work.', origin: { kind: 'scheduled_task', scheduledTaskId: 'task-settled-fire' }, }); - await fixture.stores.agentRunStore.createRun( - { - runId, - invocationId: runId, - sessionId: fixture.sessionId, - turnId, - status: 'created', - backendKind: 'fake', - llmConnectionId: session.llmConnectionId, - llmConnectionSlug: session.llmConnectionSlug, - modelId: session.model, - cwd: session.cwd, - scheduledTaskId: 'task-settled-fire', - permissionMode: session.permissionMode, - collaborationMode: session.collaborationMode, - createdAt: admittedAt, - updatedAt: admittedAt, + await seedInvocation(fixture.stores.runtimeEventStore, { + sessionId: fixture.sessionId, + invocationId: runId, + runId, + turnId, + openedAt: admittedAt, + opening: { + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: session.llmConnectionId!, + llmConnectionSlug: session.llmConnectionSlug, + modelId: session.model, + }, + configuration: { + cwd: session.cwd, + permissionMode: session.permissionMode, + collaborationMode: session.collaborationMode ?? 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + root: { kind: 'scheduled_task', scheduledTaskId: 'task-settled-fire' }, }, - { durable: true }, - ); + }); recovery = fixture.createRecoveryCoordinator(); await recovery.prepareRecovery(); @@ -547,9 +555,9 @@ test('startup recovery closes a ScheduledTask Run after its pending fire was set await fixture.manager.recoverInterruptedSessionsStrict(fixture.stores); await recovery.recover(); - const run = await fixture.stores.agentRunStore.readRun(fixture.sessionId, runId); - assert.equal(run.status, 'failed'); - assert.equal(run.failureClass, 'app_restarted'); + const run = await readInvocation(fixture.stores, fixture.sessionId, runId); + assert.equal(runtimeInvocationOutcome(run), 'failed'); + assert.equal(runtimeInvocationFailureClass(run), 'app_restarted'); assert.deepEqual(recovery.readRootState(fixture.sessionId), { kind: 'idle' }); } finally { await recovery?.close(); @@ -678,7 +686,7 @@ test('a failed exact Capability retry does not poison the parked continuation bi assert.equal(terminal.ok, true); if (terminal.ok) assert.equal(terminal.result.status, 'completed'); assert.equal( - (await fixture.stores.agentRunStore.listSessionRuns(fixture.sessionId)).filter( + (await fixture.stores.runtimeEventStore.listSessionInvocations(fixture.sessionId)).filter( (run) => run.turnId === pending.targetTurnId, ).length, 1, @@ -800,13 +808,10 @@ test('turn.start durably applies one exact per-Turn orchestration override', asy if (!started.ok) return; assertStartedTurn(started); - const run = await fixture.stores.agentRunStore.readRun( - fixture.sessionId, - started.result.turn.runId, - ); - assert.equal(run.orchestrationMode, 'swarm'); - assert.equal(run.orchestrationSource, 'turn_override'); - assert.equal(run.agentSwarmAuthorization, 'turn_override'); + const run = await readInvocation(fixture.stores, fixture.sessionId, started.result.turn.runId); + assert.equal(run.opening.configuration.orchestrationMode, 'swarm'); + assert.equal(run.opening.configuration.orchestrationSource, 'turn_override'); + assert.equal(run.opening.configuration.agentSwarmAuthorization, 'turn_override'); assert.deepEqual( (await fixture.stores.agentRunStore.readRootTurnAdmission(fixture.sessionId, input.turnId)) ?.turnOrchestration, @@ -1657,7 +1662,7 @@ test('linked child Sessions reject public safe-boundary continuation', async () assert.deepEqual(recoveryCoordinator.readRootState(child.id), { kind: 'reserved' }); assert.equal( - (await fixture.stores.agentRunStore.listSessionRuns(child.id)).some( + (await fixture.stores.runtimeEventStore.listSessionInvocations(child.id)).some( (run) => run.turnId === targetTurnId, ), false, @@ -1834,7 +1839,10 @@ test('worktree child Sessions reject roots outside managed child execution', asy (await fixture.stores.agentRunStore.listRootTurnAdmissionsForRecovery(child.id)).length, 1, ); - assert.equal((await fixture.stores.agentRunStore.listSessionRuns(child.id)).length, 1); + assert.equal( + (await fixture.stores.runtimeEventStore.listSessionInvocations(child.id)).length, + 1, + ); backend?.release(); await managed; @@ -1861,7 +1869,10 @@ test('worktree child Sessions reject roots outside managed child execution', asy () => recovery.recover(), /Unable to recover admitted Turn legacy-external-child-turn: operation_unavailable/, ); - assert.equal((await fixture.stores.agentRunStore.listSessionRuns(child.id)).length, 1); + assert.equal( + (await fixture.stores.runtimeEventStore.listSessionInvocations(child.id)).length, + 1, + ); } finally { backend?.release(); await recoveryCoordinator?.close(); @@ -2089,14 +2100,14 @@ test('Agent Graph supervisor wake waits for root idle and binds one durable exec source: 'host_api', }); - const graphRun = await fixture.stores.agentRunStore.readRun( - fixture.sessionId, - graphAdmission!.runId, - ); - assert.equal(graphRun.agentGraphWakeId, wakeId); - assert.equal(graphRun.agentGraphWakeAttemptId, attemptId); - assert.equal(graphRun.orchestrationMode, 'graph'); - assert.equal(graphRun.orchestrationSource, 'turn_override'); + const graphRun = await readInvocation(fixture.stores, fixture.sessionId, graphAdmission!.runId); + assert.deepEqual(graphRun.opening.root, { + kind: 'agent_graph_supervisor_wake', + wakeId, + attemptId, + }); + assert.equal(graphRun.opening.configuration.orchestrationMode, 'graph'); + assert.equal(graphRun.opening.configuration.orchestrationSource, 'turn_override'); const userMessage = (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).find( (message) => message.id === graphAdmission?.userMessageId, ); @@ -2342,7 +2353,7 @@ test('startup recovery replays an admitted context compact with its exact Run id assert.equal(stopped.ok, true); if (stopped.ok) assert.equal(stopped.result.status, 'cancelled'); assert.equal( - (await fixture.stores.agentRunStore.listSessionRuns(fixture.sessionId)).filter( + (await fixture.stores.runtimeEventStore.listSessionInvocations(fixture.sessionId)).filter( (run) => run.turnId === turnId, ).length, 1, @@ -2501,7 +2512,10 @@ test('Agent Graph supervisor wake revalidates freshness before durable root admi await fixture.stores.agentRunStore.listRootTurnAdmissionsForRecovery(fixture.sessionId), [], ); - assert.deepEqual(await fixture.stores.agentRunStore.listSessionRuns(fixture.sessionId), []); + assert.deepEqual( + await fixture.stores.runtimeEventStore.listSessionInvocations(fixture.sessionId), + [], + ); assert.deepEqual(await fixture.stores.sessionStore.readMessages(fixture.sessionId), []); assert.equal(fixture.drainRequested(), false); } finally { @@ -2550,13 +2564,16 @@ test('Agent Graph supervisor recovery closes a durable admission that has no Run await recovery.prepareRecovery(); await recovery.recover(); - const run = await fixture.stores.agentRunStore.readRun(fixture.sessionId, runId); - assert.equal(run.status, 'failed'); - assert.equal(run.failureClass, 'app_restarted'); - assert.equal(run.agentGraphWakeId, wakeId); - assert.equal(run.agentGraphWakeAttemptId, attemptId); - assert.equal(run.orchestrationMode, 'graph'); - assert.equal(run.orchestrationSource, 'turn_override'); + const run = await readInvocation(fixture.stores, fixture.sessionId, runId); + assert.equal(runtimeInvocationOutcome(run), 'failed'); + assert.equal(runtimeInvocationFailureClass(run), 'app_restarted'); + assert.deepEqual(run.opening.root, { + kind: 'agent_graph_supervisor_wake', + wakeId, + attemptId, + }); + assert.equal(run.opening.configuration.orchestrationMode, 'graph'); + assert.equal(run.opening.configuration.orchestrationSource, 'turn_override'); const message = (await fixture.stores.sessionStore.readMessages(fixture.sessionId)).find( (candidate) => candidate.id === userMessageId, ); @@ -3003,7 +3020,8 @@ test('hosted linked child roots share admission, message, terminal, and stop aut const joinedInterrupted = await joinedInitial; assert.equal(interrupted.status, 'cancelled'); assert.deepEqual(joinedInterrupted, interrupted); - const interruptedRun = await stores.agentRunStore.readRun( + const interruptedRun = await readInvocation( + stores, interrupted.childSessionId, interrupted.runId, ); @@ -3351,7 +3369,7 @@ test('shutdown contains a successor backend start rejected by Interaction drain' assert.equal(admissions.length, 2); const successor = admissions[1]; assert.ok(successor); - const run = await fixture.stores.agentRunStore.readRun(fixture.sessionId, successor.runId); + const run = await readInvocation(fixture.stores, fixture.sessionId, successor.runId); const runtimeEvents = await fixture.stores.runtimeEventStore.readImmutableRuntimeEvents( fixture.sessionId, successor.runId, @@ -3881,7 +3899,9 @@ async function assertSessionSuccessorCapabilityDegradation( const followup = admissions[1]; assert.ok(followup); assert.equal( - (await fixture.stores.agentRunStore.readRun(fixture.sessionId, followup.runId)).status, + runtimeInvocationOutcome( + await readInvocation(fixture.stores, fixture.sessionId, followup.runId), + ), 'completed', ); assert.equal(fixture.drainRequested(), false); @@ -4471,10 +4491,7 @@ test('post-start backend failure closes its owner without draining an unrelated runId: unrelatedStarted.result.turn.runId, }); assert.equal(unrelatedBackend.stopCount, 0); - const run = await fixture.stores.agentRunStore.readRun( - fixture.sessionId, - started.result.turn.runId, - ); + const run = await readInvocation(fixture.stores, fixture.sessionId, started.result.turn.runId); const events = await fixture.stores.runtimeEventStore.readImmutableRuntimeEvents( fixture.sessionId, started.result.turn.runId, @@ -4649,10 +4666,7 @@ test('post-start backend AggregateError is contained after its failed terminal t await waitUntil(() => fixture.coordinator.readRootState(fixture.sessionId).kind === 'idle'); assert.equal(fixture.drainRequested(), false); - const run = await fixture.stores.agentRunStore.readRun( - fixture.sessionId, - started.result.turn.runId, - ); + const run = await readInvocation(fixture.stores, fixture.sessionId, started.result.turn.runId); const events = await fixture.stores.runtimeEventStore.readImmutableRuntimeEvents( fixture.sessionId, started.result.turn.runId, @@ -4666,7 +4680,12 @@ test('post-start backend AggregateError is contained after its failed terminal t ); assert.equal(queried.ok, true); if (queried.ok && queried.result.status === 'failed') { - assert.equal(queried.result.failureMessage, run.failureMessage); + assert.equal( + queried.result.failureMessage, + run.terminalEvent?.content?.kind === 'error' + ? run.terminalEvent.content.message + : undefined, + ); assert.ok(queried.result.failureMessage); } @@ -4724,10 +4743,7 @@ test('post-start message owner cleanup failure drains after its failed terminal await waitUntil(() => fixture.drainRequested()); await waitUntil(() => fixture.coordinator.readRootState(fixture.sessionId).kind === 'idle'); - const run = await fixture.stores.agentRunStore.readRun( - fixture.sessionId, - started.result.turn.runId, - ); + const run = await readInvocation(fixture.stores, fixture.sessionId, started.result.turn.runId); const events = await fixture.stores.runtimeEventStore.readImmutableRuntimeEvents( fixture.sessionId, started.result.turn.runId, @@ -4984,32 +5000,37 @@ async function seedPendingSafeBoundaryContinuation( const targetTurnId = `target-turn-${identitySuffix}`; const session = await fixture.stores.sessionStore.readHeaderSnapshot(fixture.sessionId); const createdAt = Date.now(); - const sourceRun = { - runId: sourceRunId, - invocationId: sourceInvocationId, + const sourceRun = await seedInvocation(fixture.stores.runtimeEventStore, { sessionId: fixture.sessionId, + invocationId: sourceInvocationId, + runId: sourceRunId, turnId: sourceTurnId, - status: 'created' as const, - backendKind: 'fake' as const, - llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: session.cwd, - workspaceIdentity, - permissionMode: session.permissionMode, - collaborationMode: session.collaborationMode, - ...(sourceOrchestrationMode - ? { - orchestrationMode: sourceOrchestrationMode, - orchestrationSource: 'session' as const, - agentSwarmAuthorization: - sourceOrchestrationMode === 'swarm' ? ('session_mode' as const) : ('none' as const), - } - : {}), - createdAt, - updatedAt: createdAt, - }; - await fixture.stores.agentRunStore.createRun(sourceRun, { durable: true }); + openedAt: createdAt, + opening: { + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: session.cwd, + workspaceIdentity, + permissionMode: session.permissionMode, + collaborationMode: session.collaborationMode ?? 'agent', + toolMode: 'direct', + ...(sourceOrchestrationMode + ? { + orchestrationMode: sourceOrchestrationMode, + orchestrationSource: 'session' as const, + agentSwarmAuthorization: + sourceOrchestrationMode === 'swarm' ? ('session_mode' as const) : ('none' as const), + } + : { orchestrationMode: 'default' as const, orchestrationSource: 'session' as const }), + }, + }, + }); await fixture.stores.runtimeEventStore.appendRuntimeEvent(fixture.sessionId, sourceRunId, { id: `source-user-${identitySuffix}`, sessionId: fixture.sessionId, @@ -5024,7 +5045,6 @@ async function seedPendingSafeBoundaryContinuation( }); const terminalAt = createdAt + 1; await commitTerminalRunWithRuntimeFact({ - runStore: fixture.stores.agentRunStore, runtimeEventStore: fixture.stores.runtimeEventStore, newId: randomUUID, sessionId: fixture.sessionId, diff --git a/packages/runtime-host/src/__tests__/session-admission-gate.test.ts b/packages/runtime-host/src/__tests__/session-admission-gate.test.ts index 35044a0c1c..84d9e743ed 100644 --- a/packages/runtime-host/src/__tests__/session-admission-gate.test.ts +++ b/packages/runtime-host/src/__tests__/session-admission-gate.test.ts @@ -147,3 +147,28 @@ test('rejects accidental admission re-entry instead of deadlocking', async () => ); }); }); + +test('work detached from an admission takes admissions of its own', async () => { + const gate = new SessionAdmissionGate(); + const release = deferred(); + const order: string[] = []; + let detached!: Promise; + + // The detached work starts inside the admission and admits before the + // admission ends, which is the order a drained Turn reaches its first + // admission in. Inherited context would reject it as re-entry. + await gate.run('session', async () => { + order.push('active:start'); + detached = gate.detach(async () => { + await gate.run('session', () => { + order.push('detached:admitted'); + }); + }); + await Promise.resolve(); + order.push('active:end'); + release.resolve(); + }); + await release.promise; + await detached; + assert.deepEqual(order, ['active:start', 'active:end', 'detached:admitted']); +}); diff --git a/packages/runtime-host/src/__tests__/session-revision-graph-references.test.ts b/packages/runtime-host/src/__tests__/session-revision-graph-references.test.ts index 3d09d00e3b..d42a9b9f59 100644 --- a/packages/runtime-host/src/__tests__/session-revision-graph-references.test.ts +++ b/packages/runtime-host/src/__tests__/session-revision-graph-references.test.ts @@ -19,7 +19,8 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; +import { testInvocationRecord } from '@maka/runtime/test-only/invocation-fixture'; import type { SessionHeader, StoredMessage } from '@maka/core/session'; import { agentGraphIdForRootSession } from '@maka/runtime/stream-graph-coordinator'; import { collectConversationCopyLinkedChildReferences } from '@maka/runtime/conversation-copy'; @@ -199,7 +200,7 @@ test('Agent Graph revision references reject incomplete or mismatched provenance }, { name: 'active child Run', - input: { runs: [agentRun({ status: 'running', completedAt: undefined })] }, + input: { runs: [agentRun({ status: 'running' })] }, code: 'session_busy', }, { @@ -321,7 +322,7 @@ interface PrepareOverrides { readonly messages?: readonly StoredMessage[]; readonly archivedResults?: readonly string[]; readonly sessionHeaders?: readonly SessionHeader[]; - readonly runs?: readonly AgentRunHeader[]; + readonly runs?: readonly RuntimeInvocationRecord[]; readonly sessionGraphState?: 'absent' | 'live' | 'terminal'; readonly graphState?: 'absent' | 'live' | 'terminal'; readonly artifactTurnId?: string; @@ -347,8 +348,8 @@ async function prepare(overrides: PrepareOverrides = {}) { }), }, { - agentRunStore: { - listSessionRuns: async () => overrides.runs ?? [agentRun()], + runtimeEventStore: { + listSessionInvocations: async () => overrides.runs ?? [agentRun()], }, artifacts: { getInSession: async (sessionId, artifactId) => ({ @@ -508,22 +509,30 @@ function childHeader( }; } -function agentRun(overrides: Partial = {}): AgentRunHeader { - return { - runId: CHILD_RUN_ID, - invocationId: 'child-invocation', - sessionId: CHILD_SESSION_ID, - turnId: CHILD_TURN_ID, - status: 'completed', - backendKind: 'fake', - llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/workspace', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 2, - completedAt: 2, - ...overrides, +function agentRun( + overrides: { + runId?: string; + turnId?: string; + status?: 'completed' | 'failed' | 'cancelled' | 'running'; + resumedFromRunId?: string; + retriedFromRunId?: string; + } = {}, +): RuntimeInvocationRecord { + const status = overrides.status ?? 'completed'; + const lineage = { + ...(overrides.resumedFromRunId ? { resumedFromRunId: overrides.resumedFromRunId } : {}), + ...(overrides.retriedFromRunId ? { retriedFromRunId: overrides.retriedFromRunId } : {}), }; + return testInvocationRecord({ + sessionId: CHILD_SESSION_ID, + runId: overrides.runId ?? CHILD_RUN_ID, + turnId: overrides.turnId ?? CHILD_TURN_ID, + invocationId: overrides.runId ?? 'child-invocation', + openedAt: 1, + closedAt: 2, + ...(status === 'running' + ? {} + : { outcome: status === 'cancelled' ? ('aborted' as const) : status }), + ...(Object.keys(lineage).length > 0 ? { opening: { lineage } } : {}), + }); } diff --git a/packages/runtime-host/src/__tests__/session-revision-two-client-uds.test.ts b/packages/runtime-host/src/__tests__/session-revision-two-client-uds.test.ts index bf5f2de201..72e913fc87 100644 --- a/packages/runtime-host/src/__tests__/session-revision-two-client-uds.test.ts +++ b/packages/runtime-host/src/__tests__/session-revision-two-client-uds.test.ts @@ -27,7 +27,10 @@ import { join } from 'node:path'; import { test } from 'node:test'; import { decodeCanonicalToolResultContent } from '@maka/core/tool-result-record-schema'; import { type AgentGraphOperatorProvisionRequest } from '@maka/core/agent-graph-topology'; -import { type AgentRunHeader } from '@maka/core/agent-run'; +import { + seedInvocation, + type SeedInvocationInput, +} from '@maka/runtime/test-only/invocation-fixture'; import { type RuntimeEvent } from '@maka/core/runtime-event'; import { WORKHUB_COORDINATION_SESSION_ID } from '@maka/core/session'; import { @@ -866,28 +869,35 @@ async function seedSource( 'continuation-parent-invocation', 'continuation-parent-turn', ); - const continuationChild: AgentRunHeader = { - ...agentRunHeader( - root, - continuationSource.id, - 'continuation-child-run', - 'continuation-child-invocation', - 'continuation-child-turn', - ), - parentRunId: continuationParent.runId, - agentId: 'child-agent', - agentName: 'Child Agent', - retriedFromRunId: continuationParent.runId, - retriedFromTurnId: continuationParent.turnId, - continuationSource: { - sourceInvocationId: continuationParent.invocationId!, - sourceRunId: continuationParent.runId, - sourceTurnId: continuationParent.turnId, - sourceRuntimeEventHighWater: 1, + const continuationChildBase = agentRunHeader( + root, + continuationSource.id, + 'continuation-child-run', + 'continuation-child-invocation', + 'continuation-child-turn', + ); + const continuationChild: SeedInvocationInput = { + ...continuationChildBase, + opening: { + ...continuationChildBase.opening, + source: { + kind: 'continuation', + sourceInvocationId: continuationParent.invocationId!, + sourceRunId: continuationParent.runId, + sourceTurnId: continuationParent.turnId, + sourceRuntimeEventHighWater: 1, + }, + lineage: { + parentRunId: continuationParent.runId, + agentId: 'child-agent', + agentName: 'Child Agent', + retriedFromRunId: continuationParent.runId, + retriedFromTurnId: continuationParent.turnId, + }, }, }; for (const run of [continuationParent, continuationChild]) { - await execution.agentRunStore.createRun(run); + await seedInvocation(execution.runtimeEventStore, run); if (run.runId === continuationParent.runId) { await execution.runtimeEventStore.appendRuntimeEvent( run.sessionId, @@ -909,17 +919,17 @@ async function seedSource( }), ); } - const persistedContinuationRuns = await execution.agentRunStore.listSessionRuns( + const persistedContinuationRuns = await execution.runtimeEventStore.listSessionInvocations( continuationSource.id, ); const persistedContinuationChild = persistedContinuationRuns.find( (run) => run.runId === continuationChild.runId, ); - assert.equal(persistedContinuationChild?.agentId, continuationChild.agentId); - assert.deepEqual( - persistedContinuationChild?.continuationSource, - continuationChild.continuationSource, + assert.equal( + persistedContinuationChild?.opening.lineage?.agentId, + continuationChild.opening?.lineage?.agentId, ); + assert.deepEqual(persistedContinuationChild?.opening.source, continuationChild.opening?.source); const artifact = await artifacts.create({ id: 'source-artifact', sessionId: source.id, @@ -1002,18 +1012,18 @@ async function seedSource( const sourceRuns = [ agentRunHeader(root, source.id, 'run-turn-1', 'invocation-turn-1', 'turn-1'), agentRunHeader(root, source.id, 'run-turn-2', 'invocation-turn-2', 'turn-2'), - { - ...agentRunHeader( + withParentRun( + agentRunHeader( root, source.id, 'legacy-child-run', 'legacy-child-invocation', 'legacy-child-turn', ), - parentRunId: 'run-turn-1', - }, + 'run-turn-1', + ), ]; - for (const run of sourceRuns) await execution.agentRunStore.createRun(run); + for (const run of sourceRuns) await seedInvocation(execution.runtimeEventStore, run); const sourceRuntimeEvents = [ runtimeEvent(source.id, 'run-turn-1', 'invocation-turn-1', 'turn-1', { id: 'user-1', @@ -1224,7 +1234,8 @@ async function seedSource( source: 'tool_result', now: 3, }); - await execution.agentRunStore.createRun( + await seedInvocation( + execution.runtimeEventStore, agentRunHeader( root, graphChild.header.id, @@ -1328,7 +1339,7 @@ async function seedSource( 'linked-after-turn', ), ]) { - await execution.agentRunStore.createRun(run); + await seedInvocation(execution.runtimeEventStore, run); } const graphRootEvents = [ runtimeEvent(linkedChildSource.id, 'graph-root-run', 'graph-root-invocation', 'linked-turn', { @@ -1486,18 +1497,18 @@ async function seedSource( 'archived-owned-parent-invocation', 'archived-owned-turn', ), - { - ...agentRunHeader( + withParentRun( + agentRunHeader( root, archivedOwnedSource.id, 'archived-owned-child-run', 'archived-owned-child-invocation', 'archived-owned-child-turn', ), - parentRunId: 'archived-owned-parent-run', - }, + 'archived-owned-parent-run', + ), ]; - for (const run of archivedOwnedRuns) await execution.agentRunStore.createRun(run); + for (const run of archivedOwnedRuns) await seedInvocation(execution.runtimeEventStore, run); const archivedOwnedRuntimeEvents = [ runtimeEvent( archivedOwnedSource.id, @@ -1696,7 +1707,12 @@ async function verifyDurableBranch( }); }; const messages = await execution.sessionStore.readMessagesSnapshot(branchSessionId); - assert.equal(messages.length, 5); + // The copied invocation opens on the branch's own spine, so its transcript + // projects the copied turn as ended, exactly as the source reads. + assert.deepEqual( + messages.map((message) => message.type), + ['user', 'assistant', 'tool_call', 'tool_result', 'system_note', 'turn_state'], + ); const user = messages.find((message) => message.type === 'user'); assert.ok(user?.attachments?.[0]); const ref = user?.attachments?.[0]?.ref; @@ -1719,13 +1735,13 @@ async function verifyDurableBranch( .sort(), ['Legacy child task', 'Retained task'], ); - const copiedRuns = await execution.agentRunStore.listSessionRuns(branchSessionId); + const copiedRuns = await execution.runtimeEventStore.listSessionInvocations(branchSessionId); assert.equal(copiedRuns.length, 2); const copiedChild = copiedRuns.find((run) => run.turnId === 'legacy-child-turn'); const copiedParent = copiedRuns.find((run) => run.turnId === 'turn-1'); assert.ok(copiedChild); assert.ok(copiedParent); - assert.equal(copiedChild.parentRunId, copiedParent.runId); + assert.equal(copiedChild.opening.lineage?.parentRunId, copiedParent.runId); const copiedProjectionResult = ( await execution.runtimeEventStore.readRuntimeEvents(branchSessionId, copiedParent.runId) ).find((event) => event.content?.kind === 'function_response'); @@ -1754,8 +1770,10 @@ async function verifyDurableBranch( assert.ok(copiedProjectionArtifact); assert.equal(copiedProjectionPart.ref.relativePath, copiedProjectionArtifact.id); const durableCopiedRuns = - await execution.agentRunStore.listSessionRuns(admittedRevisionTargetId); - const durableCopiedParent = durableCopiedRuns.find((run) => run.turnId === 'turn-1'); + await execution.runtimeEventStore.listSessionInvocations(admittedRevisionTargetId); + const durableCopiedParent = durableCopiedRuns.find( + (invocation) => invocation.turnId === 'turn-1', + ); assert.ok(durableCopiedParent); const copiedParentEvents = ( await execution.runtimeEventStore.readSessionRuntimeEventEntries(admittedRevisionTargetId) @@ -1782,7 +1800,10 @@ async function verifyDurableBranch( ); assert.equal((await artifacts.listPage('revision-target', { offset: 0, limit: 10 })).total, 0); assert.deepEqual(await todos.readOrBootstrap('revision-target'), { items: [] }); - assert.deepEqual(await execution.agentRunStore.listSessionRuns('revision-target'), []); + assert.deepEqual( + await execution.runtimeEventStore.listSessionInvocations('revision-target'), + [], + ); await assert.rejects( () => execution.sessionStore.readHeaderSnapshot('revision-target'), /not found/i, @@ -1833,7 +1854,7 @@ async function verifyDurableBranch( ), ); await assertCopiedUpload(activeSourceSideConversationTargetId); - const sideConversationRuns = await execution.agentRunStore.listSessionRuns( + const sideConversationRuns = await execution.runtimeEventStore.listSessionInvocations( graphSideConversationTargetId, ); const sideConversationRun = sideConversationRuns.find((run) => run.turnId === 'linked-turn'); @@ -1867,7 +1888,7 @@ async function verifyDurableBranch( text: 'graph child result', }, ); - const archivedSideConversationRuns = await execution.agentRunStore.listSessionRuns( + const archivedSideConversationRuns = await execution.runtimeEventStore.listSessionInvocations( archivedSideConversationTargetId, ); const archivedSideConversationChildRun = archivedSideConversationRuns.find( @@ -1919,7 +1940,8 @@ async function verifyDurableBranch( assert.equal(graphResult.content.items[0]?.childSessionId, graphChildSessionId); assert.equal(graphResult.content.items[0]?.runId, 'graph-child-run'); assert.deepEqual(graphResult.content.items[0]?.artifactIds, ['graph-child-artifact']); - const graphRevisionRuns = await execution.agentRunStore.listSessionRuns(graphRevisionTargetId); + const graphRevisionRuns = + await execution.runtimeEventStore.listSessionInvocations(graphRevisionTargetId); const graphRevisionRun = graphRevisionRuns.find((run) => run.turnId === 'linked-turn'); assert.ok(graphRevisionRun); const graphRuntimeResult = ( @@ -2101,28 +2123,41 @@ function operationError(code: RuntimeHostOperationError['code']) { error instanceof RuntimeHostOperationError && error.code === code; } +/** The same seed input, with the lineage edge back to the run that spawned it. */ +function withParentRun(input: SeedInvocationInput, parentRunId: string): SeedInvocationInput { + return { ...input, opening: { ...input.opening, lineage: { parentRunId } } }; +} + function agentRunHeader( cwd: string, sessionId: string, runId: string, invocationId: string, turnId: string, -): AgentRunHeader { +): SeedInvocationInput { return { + sessionId, runId, invocationId, - sessionId, turnId, - status: 'completed', - backendKind: 'fake', - llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd, - permissionMode: 'ask', - createdAt: 1, - updatedAt: 5, - completedAt: 5, + openedAt: 1, + opening: { + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd, + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + }, }; } diff --git a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts index c34c30db73..1da0eab3ce 100644 --- a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts +++ b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts @@ -22,7 +22,8 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import { seedInvocation, testInvocationOpening } from '@maka/runtime/test-only/invocation-fixture'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { type ExecutionStoresWriter, @@ -55,7 +56,12 @@ test('keeps durable history separate from the canonical active overlay', async ( ts: 1, kind: 'session_start', }); - await stores.agentRunStore.createRun(runHeader(session.id)); + await seedInvocation(stores.runtimeEventStore, { + sessionId: session.id, + runId: 'run-1', + turnId: 'turn-1', + openedAt: 1, + }); await stores.runtimeEventStore.appendRuntimeEvent( session.id, 'run-1', @@ -237,8 +243,9 @@ test('stops scanning a control-only ledger at the cumulative immutable event lim ); let scanned = 0; const stores = { - agentRunStore: { readRun: async () => runHeader(sessionId) }, + agentRunStore: {}, runtimeEventStore: { + listSessionInvocations: async () => [testInvocation(sessionId)], readRuntimeEventsBounded: async () => ({ status: 'limit_exceeded' as const }), scanRuntimeEvents: async ( _sessionId: string, @@ -289,8 +296,9 @@ test('stops an oversized active projection before retaining the full RuntimeEven ); let visited = 0; const stores = { - agentRunStore: { readRun: async () => runHeader(sessionId) }, + agentRunStore: {}, runtimeEventStore: { + listSessionInvocations: async () => [testInvocation(sessionId)], scanRuntimeEvents: async ( _sessionId: string, _runId: string, @@ -322,24 +330,6 @@ test('stops an oversized active projection before retaining the full RuntimeEven assert.equal(visited, 8_193); }); -function runHeader(sessionId: string): AgentRunHeader { - return { - runId: 'run-1', - invocationId: 'run-1', - sessionId, - turnId: 'turn-1', - status: 'running', - backendKind: 'fake', - llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 1, - }; -} - function runtimeEvent(sessionId: string, overrides: Partial): RuntimeEvent { return { id: 'event-1', @@ -354,3 +344,14 @@ function runtimeEvent(sessionId: string, overrides: Partial): Runt ...overrides, }; } + +function testInvocation(sessionId: string): RuntimeInvocationRecord { + return { + sessionId, + invocationId: 'run-1', + runId: 'run-1', + turnId: 'turn-1', + openedAt: 1, + opening: testInvocationOpening(), + }; +} diff --git a/packages/runtime-host/src/__tests__/usage-pricing-protocol.test.ts b/packages/runtime-host/src/__tests__/usage-pricing-protocol.test.ts index 2f7efffb7c..37ceb87aec 100644 --- a/packages/runtime-host/src/__tests__/usage-pricing-protocol.test.ts +++ b/packages/runtime-host/src/__tests__/usage-pricing-protocol.test.ts @@ -479,8 +479,8 @@ describe('Usage/Pricing protocol', () => { lease.transaction('write', () => { lease.database .prepare(` - INSERT INTO core_agent_runs(session_id, run_id, created_at, record_json) - VALUES ('session-b', 'run-b', 0, '{}') + INSERT INTO core_agent_runs(session_id, run_id, created_at) + VALUES ('session-b', 'run-b', 0) `) .run(); lease.database diff --git a/packages/runtime-host/src/server/canonical-turn-snapshot.ts b/packages/runtime-host/src/server/canonical-turn-snapshot.ts index bdf7a3ae17..1ba5b1968c 100644 --- a/packages/runtime-host/src/server/canonical-turn-snapshot.ts +++ b/packages/runtime-host/src/server/canonical-turn-snapshot.ts @@ -17,17 +17,18 @@ * under the License. */ -import type { AgentRunHeader } from '@maka/core/agent-run'; import { type ContextCompactionOutcome } from '@maka/core/events'; import { truncateUtf8 } from '@maka/core/diagnostic-log'; import { redactSecrets } from '@maka/core/redaction'; +import { readRunInvocation } from '@maka/core/runtime-event-store'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import { classifyTerminalRuntimeLedger } from '@maka/runtime/terminal-run-commit'; import type { ExecutionStoresWriter } from '@maka/storage/execution-stores'; import { TURN_FAILURE_MESSAGE_MAX_BYTES, type TurnSnapshot } from '../protocol/index.js'; type CanonicalTurnStores = Pick< ExecutionStoresWriter<'interactive'>, - 'agentRunStore' | 'runtimeEventStore' + 'runtimeEventStore' | 'interactionStore' | 'sessionStore' >; export interface CanonicalTurnIdentity { @@ -39,19 +40,16 @@ export interface CanonicalTurnIdentity { export async function readCanonicalTurnSnapshot( stores: CanonicalTurnStores, identity: CanonicalTurnIdentity, - knownRun?: AgentRunHeader, + knownRun?: RuntimeInvocationRecord, ): Promise { const { sessionId, turnId, runId } = identity; - const run = knownRun ?? (await readRunIfPresent(stores, sessionId, runId)); + const run = knownRun ?? (await readInvocationIfPresent(stores, sessionId, runId)); if (!run) return { sessionId, turnId, runId, status: 'admitted' }; if (run.turnId !== turnId) { - throw new Error('Admitted Turn identity does not match its Run header'); + throw new Error('Admitted Turn identity does not match its invocation'); } - const [runEvents, runtimeEvents] = await Promise.all([ - stores.agentRunStore.readEvents(sessionId, runId), - stores.runtimeEventStore.readImmutableRuntimeEvents(sessionId, runId), - ]); + const runtimeEvents = await stores.runtimeEventStore.readImmutableRuntimeEvents(sessionId, runId); const terminal = classifyTerminalRuntimeLedger(run, runtimeEvents); if (terminal.kind === 'fact') { const fact = terminal.fact; @@ -101,13 +99,26 @@ export async function readCanonicalTurnSnapshot( if (terminal.kind !== 'none') { throw new Error('Runtime ledger does not contain one canonical terminal fact'); } - if (run.status === 'completed' || run.status === 'failed' || run.status === 'cancelled') { - throw new Error('Terminal Run header has no canonical terminal RuntimeEvent'); - } - if (run.status !== 'created' && !runEvents.some((event) => event.type === 'run_started')) { - throw new Error('Non-created Run has no durable start fact'); - } - return { sessionId, turnId, runId, status: run.status }; + // No terminal event means the run is still open. Whether it is parked is the + // pending-interaction store's answer, not something the run restates. + const parked = await hasPendingInteraction(stores, sessionId, runId); + return { sessionId, turnId, runId, status: parked ? 'waiting_for_user' : 'running' }; +} + +/** Is this run waiting on a request the user has not answered? */ +async function hasPendingInteraction( + stores: CanonicalTurnStores, + sessionId: string, + runId: string, +): Promise { + const [interactions, boundaries] = await Promise.all([ + stores.interactionStore.listSessionPending(sessionId), + stores.sessionStore.listPendingSandboxBoundaryRequests(sessionId), + ]); + return ( + interactions.some((request) => request.runId === runId) || + boundaries.some((request) => request.runId === runId) + ); } function readContextCompactionOutcome(value: unknown): ContextCompactionOutcome | undefined { @@ -136,13 +147,13 @@ export function worstCaseFailedTurnSnapshot(identity: CanonicalTurnIdentity): Tu }; } -async function readRunIfPresent( +async function readInvocationIfPresent( stores: CanonicalTurnStores, sessionId: string, runId: string, -): Promise { +): Promise { try { - return await stores.agentRunStore.readRun(sessionId, runId); + return await readRunInvocation(stores.runtimeEventStore, sessionId, runId); } catch (error) { if (isMissingFile(error)) return undefined; throw error; diff --git a/packages/runtime-host/src/server/client-capability-coordinator.ts b/packages/runtime-host/src/server/client-capability-coordinator.ts index ba69bda1f6..0a82ffd537 100644 --- a/packages/runtime-host/src/server/client-capability-coordinator.ts +++ b/packages/runtime-host/src/server/client-capability-coordinator.ts @@ -26,7 +26,7 @@ import { type McpToolProvider, } from '@maka/runtime/mcp-tools'; import { type MakaTool } from '@maka/runtime/tool-runtime'; -import type { RootExecutionDescriptor } from '@maka/core/agent-run'; +import type { RootExecutionDescriptor } from '@maka/core/runtime-invocation'; import { clientCapabilityScopeIdentity, type ClientCapabilityGrantTarget, diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index 27d3e69c4b..bb9145e053 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -26,6 +26,10 @@ import type { RuntimeExecutionConnection } from '@maka/core/llm-connections'; import { generalizedErrorMessage } from '@maka/core/redaction'; import { emptyPlanSessionState } from '@maka/core/plan'; import type { PermissionMode } from '@maka/core/permission'; +import { + runtimeInvocationOutcome, + type RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; import { isDeepResearchSession, type SessionHeader, @@ -701,10 +705,18 @@ export async function createExecutionRuntimeHostComposition( stores.runtimeEventStore.readSessionRuntimeEventEntries(sessionId), }, historyCompaction: { - readLatestCheckpoint: (sessionId) => - loadLatestHistoryCompactCheckpointFromRunLedger(stores.agentRunStore, sessionId), - readCheckpoints: (sessionId) => - loadHistoryCompactCheckpointsFromRunLedger(stores.agentRunStore, sessionId), + readLatestCheckpoint: async (sessionId) => + loadLatestHistoryCompactCheckpointFromRunLedger( + stores.agentRunStore, + sessionId, + await sessionRunIds(stores.runtimeEventStore, sessionId), + ), + readCheckpoints: async (sessionId) => + loadHistoryCompactCheckpointsFromRunLedger( + stores.agentRunStore, + sessionId, + await sessionRunIds(stores.runtimeEventStore, sessionId), + ), }, model: createHostMemoryExtractionModel({ runtimePolicy: runtimePolicyStores, @@ -951,7 +963,6 @@ export async function createExecutionRuntimeHostComposition( requestDrain: context.requestDrain, }), readModel: new RuntimeReadModel({ - runStore: stores.agentRunStore, runtimeEventStore: stores.runtimeEventStore, projectionCache: stores.sessionStore, canonicalPermissionOutcomes, @@ -1016,7 +1027,7 @@ export async function createExecutionRuntimeHostComposition( graph.hasLiveSessionState(sessionId), hasLiveLinkedDescendantState( requireSessionManager(manager), - stores.agentRunStore, + stores.runtimeEventStore, sessionId, async (descendantSessionId) => (await runtimeResources!.hasLiveSessionResources(descendantSessionId)) || @@ -1071,7 +1082,6 @@ export async function createExecutionRuntimeHostComposition( }); graphCoordinator = new AgentGraphCoordinator({ sessionStore: stores.sessionStore, - runStore: stores.agentRunStore, runtimeEventStore: stores.runtimeEventStore, controlStore: openedGraphControlStore, epochStore: openedGraphControlStore, @@ -1263,15 +1273,24 @@ export async function createExecutionRuntimeHostComposition( startTurn: (sessionId, input, _activity, abortSignal, isCurrent) => graphExecutions.run(sessionId, input, abortSignal, isCurrent), inspectAttempt: async (rootSessionId, attemptId, turnId) => { - const runs = (await stores.agentRunStore.listSessionRuns(rootSessionId)).filter( - (run) => run.agentGraphWakeAttemptId === attemptId && run.turnId === turnId, + const runs = (await stores.runtimeEventStore.listSessionInvocations(rootSessionId)).filter( + (run) => { + const root = run.opening.root; + return ( + root.kind === 'agent_graph_supervisor_wake' && + root.attemptId === attemptId && + run.turnId === turnId + ); + }, ); if (runs.length > 1) { throw new Error( `Agent graph supervisor wake attempt ${attemptId} has multiple AgentRuns`, ); } - return runs[0]?.status ?? 'missing'; + const attempt = runs[0]; + if (!attempt) return 'missing'; + return runtimeInvocationOutcome(attempt) ?? 'running'; }, recoverContextOverflow: (rootSessionId, { abortSignal }) => graphExecutions.recoverContextOverflow(rootSessionId, randomUUID(), abortSignal), @@ -2236,11 +2255,23 @@ function requireGoal(coordinator: HostGoalCoordinator | undefined): HostGoalCoor return coordinator; } +/** Every run this Session has opened, named by the event spine that defines it. */ +async function sessionRunIds( + runtimeEventStore: SessionInvocationLister, + sessionId: string, +): Promise { + return (await runtimeEventStore.listSessionInvocations(sessionId)).map( + (invocation) => invocation.runId, + ); +} + +interface SessionInvocationLister { + listSessionInvocations(sessionId: string): Promise; +} + async function hasLiveLinkedDescendantState( manager: SessionManager, - runStore: { - listSessionRuns(sessionId: string): Promise; - }, + runtimeEventStore: SessionInvocationLister, rootSessionId: string, hasLiveSessionState: (sessionId: string) => Promise, ): Promise { @@ -2254,20 +2285,12 @@ async function hasLiveLinkedDescendantState( seen.add(child.id); pending.push(child.id); const [runs, liveState] = await Promise.all([ - runStore.listSessionRuns(child.id), + runtimeEventStore.listSessionInvocations(child.id), hasLiveSessionState(child.id), ]); if (liveState) return true; - if ( - runs.some( - (run) => - run.status === 'created' || - run.status === 'running' || - run.status === 'waiting_for_user', - ) - ) { - return true; - } + // A run whose events never closed it is still live. + if (runs.some((run) => runtimeInvocationOutcome(run) === undefined)) return true; } } return false; diff --git a/packages/runtime-host/src/server/execution-inspect-coordinator.ts b/packages/runtime-host/src/server/execution-inspect-coordinator.ts index 3419a17a2c..6dbd39cf6d 100644 --- a/packages/runtime-host/src/server/execution-inspect-coordinator.ts +++ b/packages/runtime-host/src/server/execution-inspect-coordinator.ts @@ -23,6 +23,11 @@ import { type ModelCallAttempt, } from '@maka/core/model-call-attempt'; import type { RuntimeEvent } from '@maka/core/runtime-event'; +import { readRunInvocation } from '@maka/core/runtime-event-store'; +import type { + RuntimeInvocationPageCursor, + RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; import { inspectAgentRunDocument, inspectSessionDocument } from '@maka/runtime/execution-inspect'; import { projectSessionTrace } from '@maka/runtime/session-trace-projection'; import { @@ -53,14 +58,16 @@ interface InspectStores { readonly sessionStore: Pick; readonly agentRunStore: Pick< ExecutionAgentRunReader, - | 'readRun' - | 'listSessionRunsBounded' - | 'listSessionRunsPage' - | 'readEventsBounded' - | 'readEventsByTypeBounded' - | 'readRootTurnAdmission' + 'readEventsBounded' | 'readEventsByTypeBounded' | 'readRootTurnAdmission' + >; + readonly runtimeEventStore: Pick< + ExecutionRuntimeEventReader, + | 'readRuntimeEventsBounded' + | 'listSessionInvocations' + | 'listSessionInvocationsBounded' + | 'listSessionInvocationsPage' + | 'readInvocation' >; - readonly runtimeEventStore: Pick; } /** Host-owned, payload-safe read model for live Interactive execution evidence. */ @@ -118,19 +125,20 @@ export class HostExecutionInspectCoordinator { sessionId: string, agentRunId: string, ): Promise { - let header; + let invocation; try { - header = await this.#stores.agentRunStore.readRun(sessionId, agentRunId); + invocation = await readRunInvocation(this.#stores.runtimeEventStore, sessionId, agentRunId); } catch (error) { if (isMissing(error)) return undefined; throw error; } + if (!invocation) return undefined; const document: AgentRunInspectDocument = await inspectAgentRunDocument( ...this.#budgetedReaders('AgentRun'), { sessionId, agentRunId, - header, + invocation, isFatalReadError: isInspectQueryTooLargeError, }, ); @@ -145,7 +153,7 @@ export class HostExecutionInspectCoordinator { if (isMissing(error)) return undefined; throw error; } - const runPage = await this.#stores.agentRunStore.listSessionRunsBounded( + const runPage = await this.#stores.runtimeEventStore.listSessionInvocationsBounded( sessionId, EXECUTION_INSPECT_SESSION_MAX_RUNS, ); @@ -157,15 +165,12 @@ export class HostExecutionInspectCoordinator { const readers = this.#budgetedReaders('Session'); const document: SessionInspectDocument = await inspectSessionDocument( { readHeader: (id) => this.#stores.sessionStore.readHeaderSnapshot(id) }, - { - ...readers[0], - listSessionRuns: async () => [...runPage.runs], - }, + readers[0], readers[1], sessionId, { header, - runHeaders: runPage.runs, + invocations: runPage.invocations, isFatalReadError: isInspectQueryTooLargeError, }, ); @@ -186,17 +191,20 @@ export class HostExecutionInspectCoordinator { } const before = input.kind === 'session_trace_continue' ? decodeTraceCursor(input.cursor) : undefined; - const runPage = await this.#stores.agentRunStore.listSessionRunsPage(input.sessionId, { - ...(before ? { before } : {}), - limit: EXECUTION_INSPECT_TRACE_PAGE_MAX_TURNS, - }); + const runPage = await this.#stores.runtimeEventStore.listSessionInvocationsPage( + input.sessionId, + { + ...(before ? { before } : {}), + limit: EXECUTION_INSPECT_TRACE_PAGE_MAX_TURNS, + }, + ); const budget = new InspectEvidenceBudget('Session'); const runtimeEvents: RuntimeEvent[] = []; const modelCallAttempts: ModelCallAttempt[] = []; let unreadableRecords = 0; let includedRuns = 0; let acceptedPage: ExecutionInspectQueryResult | undefined; - for (const run of runPage.runs) { + for (const run of runPage.invocations) { let evidence: { readonly runtimeEvents: RuntimeEvent[]; readonly modelCallAttempts: ModelCallAttempt[]; @@ -209,7 +217,7 @@ export class HostExecutionInspectCoordinator { if (includedRuns > 0) break; return oversizedTracePage( input.sessionId, - tracePageCursorAfter(runPage.runs, 1, runPage.nextCursor), + tracePageCursorAfter(runPage.invocations, 1, runPage.nextCursor), ); } const candidateRuntimeEvents = [...runtimeEvents, ...evidence.runtimeEvents]; @@ -227,7 +235,11 @@ export class HostExecutionInspectCoordinator { const candidatePage: ExecutionInspectQueryResult = { kind: 'session_trace_page', ...candidateTrace, - nextCursor: tracePageCursorAfter(runPage.runs, candidateRunCount, runPage.nextCursor), + nextCursor: tracePageCursorAfter( + runPage.invocations, + candidateRunCount, + runPage.nextCursor, + ), }; if ( candidateTrace.turns.length > EXECUTION_INSPECT_TRACE_PAGE_MAX_TURNS || @@ -236,7 +248,7 @@ export class HostExecutionInspectCoordinator { if (includedRuns === 0) { return oversizedTracePage( input.sessionId, - tracePageCursorAfter(runPage.runs, 1, runPage.nextCursor), + tracePageCursorAfter(runPage.invocations, 1, runPage.nextCursor), ); } break; @@ -255,7 +267,7 @@ export class HostExecutionInspectCoordinator { runtimeEvents: [], modelCallAttempts: [], }), - nextCursor: tracePageCursorAfter(runPage.runs, 0, runPage.nextCursor), + nextCursor: tracePageCursorAfter(runPage.invocations, 0, runPage.nextCursor), } ); } @@ -272,15 +284,17 @@ export class HostExecutionInspectCoordinator { } const admission = await this.#stores.agentRunStore.readRootTurnAdmission(sessionId, turnId); if (!admission) return undefined; + let run; try { - const run = await this.#stores.agentRunStore.readRun(sessionId, admission.runId); - if (run.turnId !== turnId) { - throw new InspectQueryInvalidError('Turn trace admission does not match its AgentRun'); - } + run = await readRunInvocation(this.#stores.runtimeEventStore, sessionId, admission.runId); } catch (error) { if (isMissing(error)) return undefined; throw error; } + if (!run) return undefined; + if (run.turnId !== turnId) { + throw new InspectQueryInvalidError('Turn trace admission does not match its AgentRun'); + } const evidence = await this.#readRunTraceEvidence( sessionId, admission.runId, @@ -344,8 +358,6 @@ export class HostExecutionInspectCoordinator { const budget = new InspectEvidenceBudget(label); return [ { - readRun: (sessionId: string, runId: string) => - this.#stores.agentRunStore.readRun(sessionId, runId), readEvents: (sessionId: string, runId: string) => budget.read((remaining) => this.#stores.agentRunStore.readEventsBounded(sessionId, runId, remaining), @@ -356,6 +368,10 @@ export class HostExecutionInspectCoordinator { budget.read((remaining) => this.#stores.runtimeEventStore.readRuntimeEventsBounded(sessionId, runId, remaining), ), + listSessionInvocations: (sessionId: string) => + this.#stores.runtimeEventStore.listSessionInvocations(sessionId), + readInvocation: (sessionId: string, invocationId: string) => + this.#stores.runtimeEventStore.readInvocation(sessionId, invocationId), }, ] as const; } @@ -416,21 +432,23 @@ function oversizedTracePage( } function tracePageCursorAfter( - runs: readonly { readonly runId: string; readonly createdAt: number }[], + invocations: readonly RuntimeInvocationRecord[], includedRuns: number, - sourceNextCursor: { readonly createdAt: number; readonly runId: string } | null, + sourceNextCursor: RuntimeInvocationPageCursor | null, ): string | null { - const last = runs[includedRuns - 1]; + const last = invocations[includedRuns - 1]; if (!last) return null; - const hasMore = includedRuns < runs.length || sourceNextCursor !== null; - return hasMore ? encodeTraceCursor({ createdAt: last.createdAt, runId: last.runId }) : null; + const hasMore = includedRuns < invocations.length || sourceNextCursor !== null; + return hasMore + ? encodeTraceCursor({ openedAt: last.openedAt, invocationId: last.invocationId }) + : null; } -function encodeTraceCursor(cursor: { readonly createdAt: number; readonly runId: string }): string { +function encodeTraceCursor(cursor: RuntimeInvocationPageCursor): string { return Buffer.from(JSON.stringify({ v: 1, ...cursor }), 'utf8').toString('base64url'); } -function decodeTraceCursor(cursor: string): { readonly createdAt: number; readonly runId: string } { +function decodeTraceCursor(cursor: string): RuntimeInvocationPageCursor { try { const value = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8')) as Record< string, @@ -438,15 +456,15 @@ function decodeTraceCursor(cursor: string): { readonly createdAt: number; readon >; if ( value.v !== 1 || - typeof value.createdAt !== 'number' || - !Number.isFinite(value.createdAt) || - typeof value.runId !== 'string' || - !/^[A-Za-z0-9_-]{1,128}$/.test(value.runId) || + typeof value.openedAt !== 'number' || + !Number.isFinite(value.openedAt) || + typeof value.invocationId !== 'string' || + !/^[A-Za-z0-9_-]{1,128}$/.test(value.invocationId) || Object.keys(value).length !== 3 ) { throw new Error('invalid cursor'); } - return { createdAt: value.createdAt, runId: value.runId }; + return { openedAt: value.openedAt, invocationId: value.invocationId }; } catch { throw new InspectQueryInvalidError('Session trace continuation cursor is invalid'); } diff --git a/packages/runtime-host/src/server/host-session-availability.ts b/packages/runtime-host/src/server/host-session-availability.ts index 2fced24a0b..cd69d98945 100644 --- a/packages/runtime-host/src/server/host-session-availability.ts +++ b/packages/runtime-host/src/server/host-session-availability.ts @@ -17,7 +17,7 @@ * under the License. */ -import type { RootExecutionDescriptor } from '@maka/core/agent-run'; +import type { RootExecutionDescriptor } from '@maka/core/runtime-invocation'; import { isWorkHubCoordinationSession, isWorkHubCoordinationSessionId, diff --git a/packages/runtime-host/src/server/hosted-execution-authority.ts b/packages/runtime-host/src/server/hosted-execution-authority.ts index 41c23e5555..c8672e2f24 100644 --- a/packages/runtime-host/src/server/hosted-execution-authority.ts +++ b/packages/runtime-host/src/server/hosted-execution-authority.ts @@ -17,7 +17,8 @@ * under the License. */ -import type { RootExecutionDescriptor } from '@maka/core/agent-run'; +import type { BackendStopMode } from '@maka/core/backend-types'; +import type { RootExecutionDescriptor } from '@maka/core/runtime-invocation'; import type { MessageContent, SessionEvent } from '@maka/core/events'; import type { UserMessageInput } from '@maka/core/runtime-inputs'; import type { StopSessionInput } from '@maka/runtime/session-manager'; diff --git a/packages/runtime-host/src/server/hosted-execution-projection.ts b/packages/runtime-host/src/server/hosted-execution-projection.ts index b5ab57eb2d..209760aac8 100644 --- a/packages/runtime-host/src/server/hosted-execution-projection.ts +++ b/packages/runtime-host/src/server/hosted-execution-projection.ts @@ -18,11 +18,12 @@ */ import { - agentRunMatchesHostedRootExecution, - type AgentRunHeader, + invocationMatchesHostedRootExecution, type RootExecutionDescriptor, -} from '@maka/core/agent-run'; + type RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; import { RuntimeMessageAuthorityInvariantError } from '@maka/runtime/message-authority'; +import { readRunInvocation } from '@maka/core/runtime-event-store'; import type { ExecutionStoresWriter } from '@maka/storage/execution-stores'; import { readCanonicalTurnSnapshot } from './canonical-turn-snapshot.js'; import type { HostedExecutionRef, HostedExecutionSnapshot } from './hosted-execution-authority.js'; @@ -32,7 +33,7 @@ export class HostedExecutionProjectionReader { async read( execution: HostedExecutionRef, - knownRun?: AgentRunHeader, + knownRun?: RuntimeInvocationRecord, ): Promise { const run = knownRun ?? (await this.readRunIfPresent(execution.sessionId, execution.runId)); if (run && run.turnId !== execution.turnId) { @@ -43,21 +44,28 @@ export class HostedExecutionProjectionReader { return readCanonicalTurnSnapshot(this.stores, execution, run); } - async readRunIfPresent(sessionId: string, runId: string): Promise { + async readRunIfPresent( + sessionId: string, + runId: string, + ): Promise { try { - return await this.stores.agentRunStore.readRun(sessionId, runId); + return await readRunInvocation(this.stores.runtimeEventStore, sessionId, runId); } catch (error) { if (isMissingFile(error)) return undefined; throw error; } } - assertRunIdentity(run: AgentRunHeader, turnId: string, execution: RootExecutionDescriptor): void { + assertRunIdentity( + run: RuntimeInvocationRecord, + turnId: string, + execution: RootExecutionDescriptor, + ): void { assertRunMatchesExecution(run, turnId, execution); } async assertRunIdentityAndContinuation( - run: AgentRunHeader, + run: RuntimeInvocationRecord, turnId: string, execution: RootExecutionDescriptor, ): Promise { @@ -89,7 +97,7 @@ export class HostedExecutionProjectionReader { } function assertRunMatchesExecution( - run: AgentRunHeader, + run: RuntimeInvocationRecord, turnId: string, execution: RootExecutionDescriptor, ): void { @@ -98,6 +106,7 @@ function assertRunMatchesExecution( `Admitted Turn ${turnId} does not match Run ${run.runId}`, ); } + const lineage = run.opening.lineage ?? {}; switch (execution.kind) { case 'external_message': case 'workhub_coordination': @@ -109,22 +118,28 @@ function assertRunMatchesExecution( case 'goal': case 'agent_graph_supervisor_wake': case 'safe_boundary_continuation': - if (agentRunMatchesHostedRootExecution(run, execution)) return; + if (invocationMatchesHostedRootExecution(run, execution)) return; break; case 'linked_child_initial': case 'claimed_agent_graph_intent': assertTrustedAgentIdentity(run, turnId, execution); - if (run.resumedFromRunId === undefined && run.retriedFromRunId === undefined) return; + if (lineage.resumedFromRunId === undefined && lineage.retriedFromRunId === undefined) return; break; case 'linked_child_resume': assertTrustedAgentIdentity(run, turnId, execution); - if (run.resumedFromRunId === execution.sourceRunId && run.retriedFromRunId === undefined) { + if ( + lineage.resumedFromRunId === execution.sourceRunId && + lineage.retriedFromRunId === undefined + ) { return; } break; case 'linked_child_provider_retry': assertTrustedAgentIdentity(run, turnId, execution); - if (run.retriedFromRunId === execution.sourceRunId && run.resumedFromRunId === undefined) { + if ( + lineage.retriedFromRunId === execution.sourceRunId && + lineage.resumedFromRunId === undefined + ) { return; } break; @@ -137,7 +152,7 @@ function assertRunMatchesExecution( } function assertTrustedAgentIdentity( - run: AgentRunHeader, + run: RuntimeInvocationRecord, turnId: string, execution: Exclude< RootExecutionDescriptor, @@ -155,7 +170,8 @@ function assertTrustedAgentIdentity( } >, ): void { - if (run.agentId !== execution.agentId || run.agentName !== execution.agentName) { + const lineage = run.opening.lineage; + if (lineage?.agentId !== execution.agentId || lineage.agentName !== execution.agentName) { throw new RuntimeMessageAuthorityInvariantError( `Admitted Turn ${turnId} changed its trusted agent identity`, ); diff --git a/packages/runtime-host/src/server/hosted-execution-recovery.ts b/packages/runtime-host/src/server/hosted-execution-recovery.ts index 1fac7a4c72..82133c7e9d 100644 --- a/packages/runtime-host/src/server/hosted-execution-recovery.ts +++ b/packages/runtime-host/src/server/hosted-execution-recovery.ts @@ -18,7 +18,10 @@ */ import { isDeepStrictEqual } from 'node:util'; -import type { RootExecutionDescriptor } from '@maka/core/agent-run'; +import { + runtimeInvocationOutcome, + type RootExecutionDescriptor, +} from '@maka/core/runtime-invocation'; import { messageContentsEqual, normalizeMessageContent, @@ -60,7 +63,7 @@ export async function prepareHostedExecutionRecovery( for (const session of sessions) { const admissions = await input.rootAdmissions.recoverSession(session.id); const messages = await input.stores.sessionStore.readMessagesForRecovery(session.id); - const runs = await input.stores.agentRunStore.listSessionRunsForRecovery(session.id); + const runs = await input.stores.runtimeEventStore.listSessionInvocations(session.id); const runsById = new Map(runs.map((run) => [run.runId, run])); for (const run of runs) { await input.stores.agentRunStore.readEventsForRecovery(session.id, run.runId); @@ -80,7 +83,10 @@ export async function prepareHostedExecutionRecovery( ? (messageIndex.messagesById.get(admission.userMessageId) ?? []) : []; const executionContract = recoveryExecutionContract(admission.execution); - if (admission.execution.kind === 'scheduled_task' && (!run || !isTerminalRun(run.status))) { + if ( + admission.execution.kind === 'scheduled_task' && + (!run || runtimeInvocationOutcome(run) === undefined) + ) { if (!input.assertScheduledTaskAdmission) { throw new RuntimeMessageAuthorityInvariantError( 'ScheduledTask recovery admission has no canonical authority validator', @@ -443,10 +449,6 @@ function usesHostRecoveryClosure(execution: RootExecutionDescriptor): execution ); } -function isTerminalRun(status: string): boolean { - return status === 'completed' || status === 'failed' || status === 'cancelled'; -} - function indexRecoveryMessages(messages: readonly StoredMessage[]): RecoveryMessageIndex { const index: RecoveryMessageIndex = { userMessagesByTurnId: new Map(), diff --git a/packages/runtime-host/src/server/interactive-turn-coordinator.ts b/packages/runtime-host/src/server/interactive-turn-coordinator.ts index 79a0e4881c..54660dc2d4 100644 --- a/packages/runtime-host/src/server/interactive-turn-coordinator.ts +++ b/packages/runtime-host/src/server/interactive-turn-coordinator.ts @@ -19,7 +19,7 @@ import { createHash } from 'node:crypto'; import { isDeepStrictEqual } from 'node:util'; -import type { RootExecutionDescriptor } from '@maka/core/agent-run'; +import type { RootExecutionDescriptor } from '@maka/core/runtime-invocation'; import { normalizeMessageContent, type MessageContent } from '@maka/core/events'; import type { SkillInvocationResult } from '@maka/core/skill-invocation'; import { RuntimeMessageAuthorityInvariantError } from '@maka/runtime/message-authority'; diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index f9c5f1278f..f5db2de586 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -20,7 +20,10 @@ import { createHash, randomUUID } from 'node:crypto'; import { isDeepStrictEqual } from 'node:util'; import type { BackendStopMode } from '@maka/core/backend-types'; -import type { AgentRunHeader, RootExecutionDescriptor } from '@maka/core/agent-run'; +import type { + RootExecutionDescriptor, + RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; import { INLINE_REFERENCE_MAX_COUNT, messageContentDigest, @@ -44,6 +47,7 @@ import { type RuntimeMessageRunIdentity, } from '@maka/runtime/message-authority'; import { + isShutdownCancelledInteractionAdmission, RuntimeInteractionAdmissionRejectedError, RuntimeInteractionFailStopError, RuntimeInteractionInvariantError, @@ -2289,7 +2293,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { } } this.#executions.activate(entry, replacing); - entry.done = this.drainTurn(input, entry, startSettled); + entry.done = this.sessionAdmission.detach(() => this.drainTurn(input, entry, startSettled)); void entry.done.catch(() => undefined); if (rootReservation) { this.#admissions.activated(rootReservation, entry.done); @@ -2479,7 +2483,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { reason: errorMessage(commandFailure), }); startSettled.reject(commandFailure); - this.requestHostDrain(); + if (!isShutdownCancelledInteractionAdmission(commandFailure)) this.requestHostDrain(); throw commandFailure; } finally { this.observeExecutionCompletion(active, { @@ -2666,7 +2670,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { sessionId: string, turnId: string, runId: string, - knownRun?: AgentRunHeader, + knownRun?: RuntimeInvocationRecord, ): Promise { return this.executionProjection.read({ sessionId, turnId, runId }, knownRun); } @@ -2674,7 +2678,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { private async readRunIfPresent( sessionId: string, runId: string, - ): Promise { + ): Promise { return this.executionProjection.readRunIfPresent(sessionId, runId); } @@ -2685,12 +2689,8 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { if (admission.execution.kind === 'context_compact') return undefined; const mode = admission.execution.kind === 'safe_boundary_continuation' - ? (( - await this.stores.agentRunStore.readRun( - admission.sessionId, - admission.execution.sourceRunId, - ) - ).orchestrationMode ?? + ? ((await this.readRunIfPresent(admission.sessionId, admission.execution.sourceRunId)) + ?.opening.configuration.orchestrationMode ?? resolveEffectiveOrchestration(session.orchestrationMode, undefined).mode) : resolveEffectiveOrchestration(session.orchestrationMode, admission.turnOrchestration) .mode; @@ -2716,7 +2716,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { } private async assertRunMatchesDurableExecution( - run: AgentRunHeader, + run: RuntimeInvocationRecord, turnId: string, execution: RootTurnAdmission['execution'], ): Promise { @@ -2743,7 +2743,8 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { if ( !(error instanceof RuntimeHostedRootConflictError) && !(error instanceof RuntimeHostedRootUnavailableError) && - !(error instanceof HostedRootAdmissionGateError) + !(error instanceof HostedRootAdmissionGateError) && + !isShutdownCancelledInteractionAdmission(error) ) { this.requestHostDrain(); } @@ -3069,22 +3070,6 @@ function isTerminalSnapshot( ); } -function isShutdownCancelledInteractionAdmission(error: unknown): boolean { - // Drain can reach a running question admission before the Turn's stop fence - // closes its Interaction Run, so this expected cancellation is direct. - if ( - error instanceof RuntimeInteractionAdmissionRejectedError && - error.reason === 'authority_draining' - ) { - return true; - } - return ( - error instanceof RuntimeInteractionFailStopError && - error.authorityFailure instanceof RuntimeInteractionAdmissionRejectedError && - error.authorityFailure.reason === 'authority_draining' - ); -} - function isContainableRunFailure(error: unknown): error is Error { return ( error instanceof Error && diff --git a/packages/runtime-host/src/server/session-admission-gate.ts b/packages/runtime-host/src/server/session-admission-gate.ts index 49dc631181..69576581d3 100644 --- a/packages/runtime-host/src/server/session-admission-gate.ts +++ b/packages/runtime-host/src/server/session-admission-gate.ts @@ -81,6 +81,19 @@ export class SessionAdmissionGate { return this.#runQueued([sessionId], operation); } + /** + * Start work that outlives the admission that reserved it. + * + * A drained Turn is not admission work: it runs for as long as the Turn does + * and takes admissions of its own along the way. Started plainly it inherits + * the admission context of the caller, and whether its first admission is + * rejected then comes down to which finishes first — the admission, or the + * Turn reaching its own. Leaving the context here settles that by saying so. + */ + detach(operation: () => T): T { + return this.#context.exit(operation); + } + runAdmitted( sessionId: string, lease: SessionAdmissionLease, diff --git a/packages/runtime-host/src/server/session-revision-coordinator.ts b/packages/runtime-host/src/server/session-revision-coordinator.ts index dff61f5c56..3972b9baf8 100644 --- a/packages/runtime-host/src/server/session-revision-coordinator.ts +++ b/packages/runtime-host/src/server/session-revision-coordinator.ts @@ -445,7 +445,7 @@ export class HostSessionRevisionCoordinator { requests: linkedChildRequests, }, { - agentRunStore: this.#stores.agentRunStore, + runtimeEventStore: this.#stores.runtimeEventStore, artifacts: this.#artifacts, graph: this.options.graph, isSessionActive: this.options.isSessionActive, diff --git a/packages/runtime-host/src/server/session-revision-graph-references.ts b/packages/runtime-host/src/server/session-revision-graph-references.ts index 9cd3070324..d358f992df 100644 --- a/packages/runtime-host/src/server/session-revision-graph-references.ts +++ b/packages/runtime-host/src/server/session-revision-graph-references.ts @@ -17,7 +17,10 @@ * under the License. */ -import type { AgentRunHeader } from '@maka/core/agent-run'; +import { + runtimeInvocationOutcome, + type RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; import { sessionRevisionFamilyId, type SessionHeader } from '@maka/core/session'; import { type AgentGraphCoordinator } from '@maka/runtime/stream-graph-coordinator'; import { @@ -42,8 +45,8 @@ export type AgentGraphRevisionReferencePreparation = type GraphReader = Pick; interface GraphRevisionDependencies { - readonly agentRunStore: { - listSessionRuns(sessionId: string): Promise; + readonly runtimeEventStore: { + listSessionInvocations(sessionId: string): Promise; }; readonly artifacts: Pick; readonly graph: GraphReader; @@ -176,7 +179,7 @@ export async function prepareAgentGraphRevisionReferences( } const references = new Map(); - const runsByChildSession = new Map>(); + const runsByChildSession = new Map>(); for (const request of requests) { const childSessionId = request.childSessionId; const child = headersById.get(childSessionId); @@ -204,16 +207,16 @@ export async function prepareAgentGraphRevisionReferences( let runsById = runsByChildSession.get(childSessionId); if (!runsById) { - let runs: readonly AgentRunHeader[]; + let runs: readonly RuntimeInvocationRecord[]; try { - runs = await dependencies.agentRunStore.listSessionRuns(childSessionId); + runs = await dependencies.runtimeEventStore.listSessionInvocations(childSessionId); } catch { return failure( 'operation_unavailable', 'Retained Agent Graph child lineage is unavailable', ); } - if (runs.some((run) => !isTerminalRunStatus(run.status))) { + if (runs.some((run) => runtimeInvocationOutcome(run) === undefined)) { return failure('session_busy', 'A retained Agent Graph child is not terminal'); } runsById = new Map(runs.map((run) => [run.runId, run])); @@ -296,29 +299,29 @@ function isTerminalRunStatus(status: string): boolean { function linkedResultStatusMatchesRun( request: ConversationCopyLinkedChildReference, - run: AgentRunHeader, + run: RuntimeInvocationRecord, ): boolean { + const outcome = runtimeInvocationOutcome(run); return ( - run.status === request.status || - (request.status === 'failed' && - request.failureClass === 'Timeout' && - run.status === 'cancelled') + outcome === request.status || + (request.status === 'failed' && request.failureClass === 'Timeout' && outcome === 'cancelled') ); } function traceChildRunLineage( - current: AgentRunHeader, - runsById: ReadonlyMap, + current: RuntimeInvocationRecord, + runsById: ReadonlyMap, childSessionId: string, ): { readonly runIds: ReadonlySet; readonly turnIds: ReadonlySet } | undefined { const runIds = new Set(); const turnIds = new Set(); - let cursor: AgentRunHeader | undefined = current; + let cursor: RuntimeInvocationRecord | undefined = current; while (cursor) { if (cursor.sessionId !== childSessionId || runIds.has(cursor.runId)) return undefined; runIds.add(cursor.runId); turnIds.add(cursor.turnId); - const previousRunId = cursor.retriedFromRunId ?? cursor.resumedFromRunId; + const lineage = cursor.opening.lineage; + const previousRunId = lineage?.retriedFromRunId ?? lineage?.resumedFromRunId; if (!previousRunId) break; cursor = runsById.get(previousRunId); if (!cursor) return undefined; diff --git a/packages/runtime-host/src/server/session-transcript-reader.ts b/packages/runtime-host/src/server/session-transcript-reader.ts index 442d3b8851..cc51d9bd4a 100644 --- a/packages/runtime-host/src/server/session-transcript-reader.ts +++ b/packages/runtime-host/src/server/session-transcript-reader.ts @@ -60,14 +60,14 @@ export function createSessionTranscriptReader(input: { readActiveOverlay: async (sessionId, rootTurn) => { if (!rootTurn || isTerminalTurn(rootTurn)) return []; - const run = await input.stores.agentRunStore.readRun(sessionId, rootTurn.runId); + const invocations = await input.stores.runtimeEventStore.listSessionInvocations(sessionId); const events = await readActiveProjectionEvents(input.stores, sessionId, rootTurn.runId); const canonicalPermissionOutcomes = await readCanonicalPermissionOutcomes( events, input.canonicalPermissionOutcomes, ); const projected = projectRuntimeEventsToStoredMessages(activePresentationEvents(events), { - runHeaders: [run], + invocations: invocations.filter((invocation) => invocation.runId === rootTurn.runId), canonicalPermissionOutcomes, }); if (projected.diagnostics.some(isHardRuntimeEventReadModelDiagnostic)) { diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 0a81170fd5..ce454c811e 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -24,6 +24,7 @@ "./session-todo-tools": "./dist/session-todo-tools.js", "./test-only/fake-backend": "./dist/test-only/fake-backend.js", "./test-only/observation-text-reader": "./dist/__tests__/observation-text-reader.js", + "./test-only/invocation-fixture": "./dist/__tests__/invocation-fixture.js", "./filesystem-worker": "./dist/filesystem-worker/index.js", "./sandbox": "./dist/sandbox/index.js", "./network/proxy-test": "./dist/network/proxy-test.js", diff --git a/packages/runtime/src/__tests__/agent-graph-supervisor-wake.test.ts b/packages/runtime/src/__tests__/agent-graph-supervisor-wake.test.ts index 2b79c5c270..04bd4436b2 100644 --- a/packages/runtime/src/__tests__/agent-graph-supervisor-wake.test.ts +++ b/packages/runtime/src/__tests__/agent-graph-supervisor-wake.test.ts @@ -374,7 +374,7 @@ describe('Agent Graph supervisor wake delivery', () => { attempt += 1; return { kind: 'suspended', turnId: input.turnId, reason: 'permission handoff' }; }, - inspectAttempt: async () => 'waiting_for_user', + inspectAttempt: async () => 'running', newId: sequentialIds(), }); try { @@ -408,7 +408,7 @@ describe('Agent Graph supervisor wake delivery', () => { ? { kind: 'suspended', turnId: input.turnId, reason: 'permission handoff' } : { kind: 'completed', turnId: input.turnId }; }, - inspectAttempt: async () => 'waiting_for_user', + inspectAttempt: async () => 'running', newId: sequentialIds(), }); try { @@ -455,7 +455,7 @@ describe('Agent Graph supervisor wake delivery', () => { } return { kind: 'completed', turnId: input.turnId }; }, - inspectAttempt: async () => 'waiting_for_user', + inspectAttempt: async () => 'running', newId: sequentialIds(), }); try { @@ -633,7 +633,7 @@ describe('Agent Graph supervisor wake delivery', () => { delivered += 1; return { kind: 'completed', turnId: input.turnId }; }, - inspectAttempt: async () => 'waiting_for_user', + inspectAttempt: async () => 'running', newId: sequentialIds(), }); try { diff --git a/packages/runtime/src/__tests__/agent-graph-timeline.test.ts b/packages/runtime/src/__tests__/agent-graph-timeline.test.ts index 42e3cb8e7b..bb2997ca2e 100644 --- a/packages/runtime/src/__tests__/agent-graph-timeline.test.ts +++ b/packages/runtime/src/__tests__/agent-graph-timeline.test.ts @@ -21,7 +21,7 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; import { AGENT_GRAPH_SUPERVISOR_WAKE_SCHEMA_VERSION } from '@maka/core/agent-graph-supervisor-wake'; import { type AgentGraphTimelineMetadataSnapshot } from '@maka/core/agent-graph-timeline'; -import { type AgentRunHeader } from '@maka/core/agent-run'; +import { type RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import { type RuntimeEvent } from '@maka/core/runtime-event'; import { createSqliteSessionMetadataStore } from '@maka/storage/sqlite-session-metadata-store'; import { @@ -31,6 +31,7 @@ import { readAgentGraphTimelinePage, } from '../agent-graph-timeline.js'; import { readCommittedAgentGraphProjection } from '../stream-graph-projection.js'; +import { testInvocationRecord } from './invocation-fixture.js'; describe('agent graph replay timeline', () => { test('reconstructs control, child records, parent completion, and supervisor wake chronologically', async () => { @@ -222,7 +223,7 @@ describe('agent graph replay timeline', () => { test('joins a transactionally read SQLite metadata snapshot with immutable run ledgers', async () => { const fixture = await timelineFixture(); - const runsBySession = new Map([ + const runsBySession = new Map([ ['root-session', [...fixture.rootRuns]], ['child-session', [fixture.childRun]], ]); @@ -237,12 +238,10 @@ describe('agent graph replay timeline', () => { return fixture.metadata; }, }, - runStore: { - async listSessionRuns(sessionId) { + runtimeEventStore: { + async listSessionInvocations(sessionId) { return runsBySession.get(sessionId) ?? []; }, - }, - runtimeEventStore: { async readImmutableRuntimeEvents(_sessionId, runId) { return eventsByRun.get(runId) ?? []; }, @@ -268,16 +267,15 @@ describe('agent graph replay timeline', () => { return fixture.metadata; }, }, - runStore: { - async listSessionRuns(sessionId) { + runtimeEventStore: { + async listSessionInvocations(sessionId) { if (sessionId === fixture.rootSessionId) return [...fixture.rootRuns]; if (sessionId === fixture.childRun.sessionId) { - return [{ ...fixture.childRun, status: 'running', completedAt: undefined }]; + const { terminalEvent: _terminalEvent, ...open } = fixture.childRun; + return [open]; } return []; }, - }, - runtimeEventStore: { async readImmutableRuntimeEvents() { return []; }, @@ -287,7 +285,7 @@ describe('agent graph replay timeline', () => { const activation = requireEvent(page.events, 'activation_started'); assert.equal(activation.kind, 'activation_started'); - assert.equal(activation.eventTime, fixture.childRun.createdAt); + assert.equal(activation.eventTime, fixture.childRun.openedAt); assert.deepEqual(activation.activation, { sessionId: 'child-session', runId: 'child-run', @@ -350,29 +348,25 @@ describe('agent graph replay timeline', () => { }); async function timelineFixture() { - const rootRun1 = runHeader({ + const rootRun1 = runInvocation({ sessionId: 'root-session', runId: 'root-run-1', turnId: 'root-turn-1', - status: 'completed', createdAt: 90, completedAt: 115, }); - const rootRun2 = runHeader({ + const rootRun2 = runInvocation({ sessionId: 'root-session', runId: 'root-run-2', turnId: 'root-turn-2', - status: 'completed', createdAt: 122, completedAt: 150, - agentGraphWakeId: 'wake-1', - agentGraphWakeAttemptId: 'attempt-1', + wake: { wakeId: 'wake-1', attemptId: 'attempt-1' }, }); - const childRun = runHeader({ + const childRun = runInvocation({ sessionId: 'child-session', runId: 'child-run', turnId: 'child-turn', - status: 'completed', createdAt: 102, completedAt: 120, }); @@ -425,12 +419,10 @@ async function timelineFixture() { const projection = await readCommittedAgentGraphProjection({ graphId: 'graph-1', operators: [{ operatorId: 'operator-1', sessionId: 'child-session' }], - runStore: { - async listSessionRuns() { + runtimeEventStore: { + async listSessionInvocations() { return [childRun]; }, - }, - runtimeEventStore: { async readImmutableRuntimeEvents() { return childEvents; }, @@ -563,37 +555,51 @@ async function timelineFixture() { }; } -function runHeader( - input: Pick< - AgentRunHeader, - | 'sessionId' - | 'runId' - | 'turnId' - | 'status' - | 'createdAt' - | 'completedAt' - | 'agentGraphWakeId' - | 'agentGraphWakeAttemptId' - >, -): AgentRunHeader { - return { - ...input, +/** + * One invocation as its own events describe it. + * + * A wake-rooted run says so in its opening's root authority, and a finished one + * says so with a terminal event. Neither is a field a writer could set apart + * from the ledger. + */ +function runInvocation(input: { + sessionId: string; + runId: string; + turnId: string; + createdAt: number; + completedAt?: number; + status?: 'completed' | 'failed' | 'aborted'; + wake?: { wakeId: string; attemptId: string }; +}): RuntimeInvocationRecord { + return testInvocationRecord({ + sessionId: input.sessionId, invocationId: `invocation-${input.runId}`, - backendKind: 'ai-sdk', - llmConnectionSlug: 'deepseek', - modelId: 'deepseek-chat', - cwd: '/workspace', - permissionMode: 'explore', - updatedAt: input.completedAt ?? input.createdAt, - }; + runId: input.runId, + turnId: input.turnId, + openedAt: input.createdAt, + ...(input.wake + ? { + opening: { + root: { + kind: 'agent_graph_supervisor_wake', + wakeId: input.wake.wakeId, + attemptId: input.wake.attemptId, + }, + }, + } + : {}), + ...(input.completedAt !== undefined + ? { closedAt: input.completedAt, outcome: input.status ?? 'completed' } + : {}), + }); } function runtimeEvent( - run: AgentRunHeader, + run: RuntimeInvocationRecord, overrides: Partial & Pick, ): RuntimeEvent { return { - invocationId: run.invocationId!, + invocationId: run.invocationId, runId: run.runId, sessionId: run.sessionId, turnId: run.turnId, diff --git a/packages/runtime/src/__tests__/agent-run-inspect.test.ts b/packages/runtime/src/__tests__/agent-run-inspect.test.ts index a1eba13ae2..64a6bf19fc 100644 --- a/packages/runtime/src/__tests__/agent-run-inspect.test.ts +++ b/packages/runtime/src/__tests__/agent-run-inspect.test.ts @@ -19,12 +19,20 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { AgentRunEvent, AgentRunHeader, AgentRunStore } from '@maka/core/agent-run'; -import type { RuntimeEvent } from '@maka/core/runtime-event'; +import type { AgentRunEvent, AgentRunStore } from '@maka/core/agent-run'; +import type { RuntimeEvent, RuntimeEventInvocationOpenedContent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; +import { + buildInvocationOpenedEvent, + runtimeInvocationsFromSessionEvents, +} from '@maka/core/runtime-invocation'; import { inspectAgentRunReadModel } from '../agent-run-inspect.js'; +import { testInvocationOpening } from './invocation-fixture.js'; +import { assertDoubleRunNotSealed } from './runtime-event-store-seal.js'; const sessionId = 'session-1'; +const invocationId = 'inv-1'; const runId = 'run-1'; const turnId = 'turn-1'; const ts = 1_800_000_000_000; @@ -32,14 +40,25 @@ const ts = 1_800_000_000_000; describe('inspectAgentRunReadModel', () => { test('returns consistent diagnostics for a complete run', async () => { const runStore = new MemoryAgentRunStore(); - await runStore.createRun( - makeHeader({ status: 'completed', completedAt: ts + 10, updatedAt: ts + 10 }), + await runStore.appendRuntimeEvent( + sessionId, + runId, + buildInvocationOpenedEvent({ + id: 'rt-open', + run: { sessionId, invocationId, runId, turnId }, + openedAt: ts, + opening: makeOpening(), + }), ); - await runStore.appendEvent(sessionId, runId, makeRunEvent({ type: 'run_started', ts: ts + 1 })); await runStore.appendEvent( sessionId, runId, - makeRunEvent({ type: 'run_completed', ts: ts + 10 }), + makeRunEvent({ type: 'turn_started', ts: ts + 1 }), + ); + await runStore.appendEvent( + sessionId, + runId, + makeRunEvent({ type: 'model_stream_completed', ts: ts + 10 }), ); await runStore.appendRuntimeEvent( sessionId, @@ -81,45 +100,37 @@ describe('inspectAgentRunReadModel', () => { assert.deepStrictEqual(inspected.sourceHealth, { runtimeLedger: 'present', runtimeTerminalPresent: true, - operationalTerminalPresent: true, - statusConsistency: 'consistent', }); assert.strictEqual(inspected.terminalRuntimeFact?.runStatus, 'completed'); - assert.strictEqual(inspected.operationalTerminalEvent?.type, 'run_completed'); assert.deepStrictEqual( inspected.runtimeEvents.map((event) => event.id), - ['rt-user', 'rt-assistant', 'rt-complete'], + ['rt-open', 'rt-user', 'rt-assistant', 'rt-complete'], ); assert.deepStrictEqual( inspected.projection?.messages.map((message) => message.type), ['user', 'assistant', 'turn_state'], ); - assert.strictEqual( - inspected.diagnostics.some((diagnostic) => diagnostic.code === 'status_consistency_mismatch'), - false, - ); }); test('reports missing and corrupt runtime-events without discarding operational facts', async () => { const missingRuntimeStore = new MemoryAgentRunStore(); - await missingRuntimeStore.createRun(makeHeader({ status: 'completed' })); await missingRuntimeStore.appendEvent( sessionId, runId, - makeRunEvent({ type: 'run_completed' }), + makeRunEvent({ type: 'model_stream_completed' }), ); const missing = await inspectAgentRunReadModel(missingRuntimeStore, missingRuntimeStore, { sessionId, runId, + invocation: makeInvocation(), }); assert.deepStrictEqual( missing.events.map((event) => event.type), - ['run_completed'], + ['model_stream_completed'], ); assert.strictEqual(missing.sourceHealth.runtimeLedger, 'missing'); - assert.strictEqual(missing.sourceHealth.operationalTerminalPresent, true); assert.strictEqual(missing.sourceHealth.runtimeTerminalPresent, false); assert.strictEqual( missing.diagnostics.some((diagnostic) => diagnostic.code === 'missing_runtime_ledger'), @@ -127,92 +138,42 @@ describe('inspectAgentRunReadModel', () => { ); const corruptRuntimeStore = new MemoryAgentRunStore({ failRuntimeEventReads: true }); - await corruptRuntimeStore.createRun(makeHeader({ status: 'completed' })); await corruptRuntimeStore.appendEvent( sessionId, runId, - makeRunEvent({ type: 'run_completed' }), + makeRunEvent({ type: 'model_stream_completed' }), ); const corrupt = await inspectAgentRunReadModel(corruptRuntimeStore, corruptRuntimeStore, { sessionId, runId, + invocation: makeInvocation(), }); assert.deepStrictEqual( corrupt.events.map((event) => event.type), - ['run_completed'], + ['model_stream_completed'], ); assert.strictEqual(corrupt.sourceHealth.runtimeLedger, 'read_failed'); - assert.strictEqual(corrupt.sourceHealth.operationalTerminalPresent, true); assert.strictEqual( corrupt.diagnostics.some((diagnostic) => diagnostic.code === 'runtime_ledger_read_failed'), true, ); }); - - test('diagnoses status disagreement between header operational and RuntimeEvent facts', async () => { - const runStore = new MemoryAgentRunStore(); - await runStore.createRun(makeHeader({ status: 'failed', failureClass: 'tool_failed' })); - await runStore.appendEvent(sessionId, runId, makeRunEvent({ type: 'run_failed' })); - await runStore.appendRuntimeEvent( - sessionId, - runId, - makeRuntimeEvent({ - id: 'rt-complete', - role: 'system', - author: 'system', - status: 'completed', - actions: { endInvocation: true }, - }), - ); - - const inspected = await inspectAgentRunReadModel(runStore, runStore, { sessionId, runId }); - - assert.strictEqual(inspected.sourceHealth.statusConsistency, 'inconsistent'); - assert.strictEqual(inspected.terminalRuntimeFact?.runStatus, 'completed'); - assert.strictEqual( - inspected.diagnostics.some((diagnostic) => diagnostic.code === 'status_consistency_mismatch'), - true, - ); - }); }); class MemoryAgentRunStore implements AgentRunStore, RuntimeEventStore { - private headers = new Map(); private events = new Map(); private runtimeEvents = new Map(); private runtimeEventEntries: RuntimeEvent[] = []; constructor(private readonly options: { failRuntimeEventReads?: boolean } = {}) {} - async createRun(header: AgentRunHeader): Promise { - this.headers.set(key(header.sessionId, header.runId), { ...header }); - return { ...header }; - } - - async updateRun( - sessionId: string, - runId: string, - patch: Partial, - ): Promise { - const current = await this.readRun(sessionId, runId); - const next = { ...current, ...patch, sessionId, runId }; - this.headers.set(key(sessionId, runId), next); - return { ...next }; - } - - async readRun(sessionId: string, runId: string): Promise { - const header = this.headers.get(key(sessionId, runId)); - if (!header) throw new Error(`Unknown run ${runId}`); - return { ...header }; - } - - async listSessionRuns(sessionId: string): Promise { - return Array.from(this.headers.values()) - .filter((header) => header.sessionId === sessionId) - .sort((a, b) => a.createdAt - b.createdAt || a.runId.localeCompare(b.runId)) - .map((header) => ({ ...header })); + async listSessionInvocations(sessionId: string): Promise { + return runtimeInvocationsFromSessionEvents( + sessionId, + await this.readSessionRuntimeEvents(sessionId), + ); } async appendEvent(sessionId: string, runId: string, event: AgentRunEvent): Promise { @@ -226,6 +187,7 @@ class MemoryAgentRunStore implements AgentRunStore, RuntimeEventStore { async appendRuntimeEvent(sessionId: string, runId: string, event: RuntimeEvent): Promise { const eventKey = key(sessionId, runId); + assertDoubleRunNotSealed(this.runtimeEvents.get(eventKey) ?? [], event); this.runtimeEvents.set(eventKey, [ ...(this.runtimeEvents.get(eventKey) ?? []), copyRuntimeEvent(event), @@ -283,27 +245,21 @@ class MemoryAgentRunStore implements AgentRunStore, RuntimeEventStore { } } -function makeHeader(overrides: Partial = {}): AgentRunHeader { - return { - runId, - sessionId, - turnId, - status: 'running', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/cwd', - permissionMode: 'ask', - createdAt: ts, - updatedAt: ts, - ...overrides, - }; +function makeOpening(): RuntimeEventInvocationOpenedContent { + return testInvocationOpening({ + configuration: { cwd: '/tmp/cwd' }, + }); +} + +/** The invocation a run is named by, for the cases whose ledger is unreadable. */ +function makeInvocation(): RuntimeInvocationRecord { + return { sessionId, invocationId, runId, turnId, openedAt: ts, opening: makeOpening() }; } function makeRunEvent(overrides: Partial = {}): AgentRunEvent { return { - type: 'run_started', - id: `op-${overrides.type ?? 'run_started'}`, + type: 'turn_started', + id: `op-${overrides.type ?? 'turn_started'}`, runId, sessionId, turnId, diff --git a/packages/runtime/src/__tests__/agent-run-recovery.test.ts b/packages/runtime/src/__tests__/agent-run-recovery.test.ts index d03943d33b..19bca3c7e5 100644 --- a/packages/runtime/src/__tests__/agent-run-recovery.test.ts +++ b/packages/runtime/src/__tests__/agent-run-recovery.test.ts @@ -19,28 +19,41 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import { classifyAgentRunRecovery } from '../agent-run-recovery.js'; +import { testInvocationOpening } from './invocation-fixture.js'; describe('AgentRun startup recovery', () => { test('fails a graph supervisor permission handoff once its live waiter is lost', () => { - const header: AgentRunHeader = { - runId: 'run-1', + const invocation: RuntimeInvocationRecord = { sessionId: 'session-1', + invocationId: 'invocation-1', + runId: 'run-1', turnId: 'turn-1', - status: 'waiting_for_user', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/workspace', - permissionMode: 'ask', - agentGraphWakeId: 'wake-1', - agentGraphWakeAttemptId: 'attempt-1', - createdAt: 1, - updatedAt: 2, + openedAt: 1, + opening: testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'fake-connection', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { cwd: '/tmp/workspace' }, + root: { kind: 'agent_graph_supervisor_wake', wakeId: 'wake-1', attemptId: 'attempt-1' }, + }), }; - const decision = classifyAgentRunRecovery(header, []); + const decision = classifyAgentRunRecovery(invocation, [ + { + type: 'permission_requested', + id: 'op-permission_requested', + sessionId: 'session-1', + runId: 'run-1', + turnId: 'turn-1', + ts: 2, + }, + ]); assert.equal(decision?.status, 'failed'); assert.equal(decision?.failureClass, 'app_restarted'); assert.equal(decision?.diagnostic?.recoveryReason, 'stale_user_wait'); diff --git a/packages/runtime/src/__tests__/agent-run-steering-recovery.test.ts b/packages/runtime/src/__tests__/agent-run-steering-recovery.test.ts index 610c66a335..0c1d48965b 100644 --- a/packages/runtime/src/__tests__/agent-run-steering-recovery.test.ts +++ b/packages/runtime/src/__tests__/agent-run-steering-recovery.test.ts @@ -23,7 +23,6 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import assert from 'node:assert/strict'; import { test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { SessionEvent } from '@maka/core/events'; import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; @@ -33,6 +32,7 @@ import { AgentRun } from '../agent-run.js'; import { RuntimeLedgerRepair } from '../runtime-ledger-repair.js'; import { buildStatusPatch } from '../session-projection-helpers.js'; import { waitFor as pollFor } from '@maka/core/test-only/async-primitives'; +import { seedInvocation } from './invocation-fixture.js'; test('rejects an invalid tool mode before a durable AgentRun can be created', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-tool-mode-')); @@ -70,7 +70,7 @@ test('rejects an invalid tool mode before a durable AgentRun can be created', as }), /invalid tool mode/i, ); - assert.deepEqual(await runStore.listSessionRuns(session.id), []); + assert.deepEqual(await runtimeEventStore.listSessionInvocations(session.id), []); } finally { await rm(root, { recursive: true, force: true }); } @@ -163,7 +163,6 @@ test('acks a steering event whose canonical append preceded proof publication fa const runtimeEventStore = createWorkspaceRuntimeStore(root); const runId = 'run-1'; const turnId = 'turn-1'; - await runStore.createRun(makeRunHeader(session.id, runId, turnId)); const run = new AgentRun({ sessionId: session.id, header: session, @@ -318,7 +317,6 @@ test('recovers a steering transcript message from the committed RuntimeEvent led const turnId = 'turn-steering-crash-cut'; const runStore = createSqliteAgentRunStore(root); const runtimeEventStore = createWorkspaceRuntimeStore(root); - await runStore.createRun(makeRunHeader(session.id, runId, turnId)); const steeringContent = { kind: 'text' as const, text: 'canonical steering envelope', @@ -342,6 +340,13 @@ test('recovers a steering transcript message from the committed RuntimeEvent led ], steering: true as const, }; + await seedInvocation(runtimeEventStore, { + sessionId: session.id, + invocationId: 'invocation-steering-crash-cut', + runId, + turnId, + openedAt: 1, + }); const runtimeEvent: RuntimeEvent = { id: 'runtime-steering-crash-cut', invocationId: 'invocation-steering-crash-cut', @@ -362,11 +367,9 @@ test('recovers a steering transcript message from the committed RuntimeEvent led const recoveredRunStore = createSqliteAgentRunStore(root); const recoveredRuntimeEventStore = createWorkspaceRuntimeStore(root); const repair = new RuntimeLedgerRepair({ - runStore: recoveredRunStore, runtimeEventStore: recoveredRuntimeEventStore, readMessages: (sessionId) => recoveredStore.readMessages(sessionId), appendMessage: (sessionId, message) => recoveredStore.appendMessage(sessionId, message), - appendTurnState: async () => {}, newId: () => 'unused-id', now: () => 10, }); @@ -392,7 +395,7 @@ test('recovers a steering transcript message from the committed RuntimeEvent led } }); -test('awaits canonical Run status persistence before accepting an interaction resume', async () => { +test('awaits the durable settlement fact before accepting an interaction resume', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-status-barrier-')); try { const store = createSessionStore(root); @@ -407,28 +410,26 @@ test('awaits canonical Run status persistence before accepting an interaction re const runId = 'run-status-barrier'; const turnId = 'turn-status-barrier'; await store.updateHeader(session.id, buildStatusPatch('waiting_for_user', 1)); - await runStore.createRun({ - ...makeRunHeader(session.id, runId, turnId), - status: 'waiting_for_user', + const { invocationId } = await seedInvocation(runtimeEventStore, { + sessionId: session.id, + runId, + turnId, + openedAt: 1, }); - const updateStarted = deferred(); - const allowUpdate = deferred(); - const auditStarted = deferred(); - const allowAudit = deferred(); - const delayedRunStore = { - updateRun: async (...args: Parameters) => { - updateStarted.resolve(); - await allowUpdate.promise; - return await runStore.updateRun(...args); - }, - appendEvent: async (...args: Parameters) => { - if (args[2].type === 'run_status_changed') { - auditStarted.resolve(); - await allowAudit.promise; + const appendStarted = deferred(); + const allowAppend = deferred(); + const delayedRuntimeEventStore = { + ...runtimeEventStore, + appendRuntimeEvent: async ( + ...args: Parameters + ) => { + if (args[2].id === 'status-event') { + appendStarted.resolve(); + await allowAppend.promise; } - return await runStore.appendEvent(...args); + return await runtimeEventStore.appendRuntimeEvent(...args); }, - } as typeof runStore; + } as typeof runtimeEventStore; let sessionUpdateStarted = false; const run = new AgentRun({ sessionId: session.id, @@ -437,8 +438,8 @@ test('awaits canonical Run status persistence before accepting an interaction re runId, durability: 'required', store, - runStore: delayedRunStore, - runtimeEventStore, + runStore, + runtimeEventStore: delayedRuntimeEventStore, newId: () => 'status-event', now: () => 10, hooks: { @@ -456,36 +457,45 @@ test('awaits canonical Run status persistence before accepting an interaction re }); let accepted = false; const accepting = run - .recordSessionEvent({ - type: 'user_question_answer_ack', - id: 'answer-ack', - turnId, - ts: 2, - requestId: 'question-1', - toolUseId: 'tool-1', - }) + .acceptMappedEvent( + { + type: 'user_question_answer_ack', + id: 'answer-ack', + turnId, + ts: 2, + requestId: 'question-1', + toolUseId: 'tool-1', + }, + { + id: 'status-event', + invocationId, + runId, + sessionId: session.id, + turnId, + ts: 2, + partial: false, + role: 'system', + author: 'user', + actions: { userQuestionAnswerAccepted: { requestId: 'question-1' } }, + refs: { toolCallId: 'tool-1' }, + } satisfies RuntimeEvent, + ) .then(() => { accepted = true; }); try { - await updateStarted.promise; - assert.equal(accepted, false); - assert.equal((await store.readHeader(session.id)).status, 'waiting_for_user'); - allowUpdate.resolve(); - await auditStarted.promise; + await appendStarted.promise; await Promise.resolve(); assert.equal(accepted, false); assert.equal(sessionUpdateStarted, false); assert.equal((await store.readHeader(session.id)).status, 'waiting_for_user'); - allowAudit.resolve(); + allowAppend.resolve(); await accepting; assert.equal(sessionUpdateStarted, true); - assert.equal((await runStore.readRun(session.id, runId))?.status, 'running'); assert.equal((await store.readHeader(session.id)).status, 'running'); } finally { - allowUpdate.resolve(); - allowAudit.resolve(); + allowAppend.resolve(); await accepting.catch(() => undefined); } } finally { @@ -493,258 +503,6 @@ test('awaits canonical Run status persistence before accepting an interaction re } }); -test('required interaction resume recovers a failed best-effort Run Store latch through terminal commit', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-status-latch-')); - try { - const store = createSessionStore(root); - const session = await store.create({ - cwd: '/tmp/cwd', - llmConnectionSlug: 'fake', - model: 'fake-model', - permissionMode: 'ask', - }); - const runStore = createSqliteAgentRunStore(root); - const runtimeEventStore = createWorkspaceRuntimeStore(root); - const runId = 'run-status-latch'; - const turnId = 'turn-status-latch'; - await store.updateHeader(session.id, buildStatusPatch('waiting_for_user', 1)); - await runStore.createRun({ - ...makeRunHeader(session.id, runId, turnId), - status: 'waiting_for_user', - }); - let failNextAppend = true; - const failingRunStore = { - updateRun: runStore.updateRun.bind(runStore), - appendEvent: async (...args: Parameters) => { - if (failNextAppend) { - failNextAppend = false; - throw new Error('injected trace failure'); - } - return await runStore.appendEvent(...args); - }, - } as typeof runStore; - const run = new AgentRun({ - sessionId: session.id, - header: session, - userInput: { turnId, text: 'fail closed after trace failure' }, - runId, - durability: 'required', - store, - runStore: failingRunStore, - runtimeEventStore, - newId: () => 'status-latch-event', - now: () => 10, - hooks: { - reserveRun: async () => { - throw new Error('reserveRun should not be called'); - }, - unregisterRun: () => {}, - updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), - updateStatus: async (sessionId, status, blockedReason, ts = 0) => { - await store.updateHeader(sessionId, buildStatusPatch(status, ts, blockedReason)); - }, - appendTurnState: async () => {}, - }, - }); - run.recordRunTrace({ - id: 'trace-that-fails', - sessionId: session.id, - turnId, - ts: 1, - phase: 'turn', - type: 'turn_started', - message: 'trip the best-effort trace latch', - }); - await waitFor(async () => - Boolean((await runStore.readRun(session.id, runId))?.traceWriteError), - ); - - await run.recordSessionEvent({ - type: 'user_question_answer_ack', - id: 'answer-after-latch', - turnId, - ts: 2, - requestId: 'question-1', - toolUseId: 'tool-1', - }); - assert.equal((await runStore.readRun(session.id, runId))?.status, 'running'); - assert.equal((await store.readHeader(session.id)).status, 'running'); - - await run.recordRuntimeEvents([ - { - id: 'terminal-after-latch', - invocationId: run.invocationId, - runId, - sessionId: session.id, - turnId, - ts: 3, - partial: false, - role: 'system', - author: 'system', - status: 'completed', - actions: { endInvocation: true }, - }, - ]); - await run.recordSessionEvent({ - type: 'complete', - id: 'complete-after-latch', - turnId, - ts: 3, - stopReason: 'end_turn', - }); - await run.finalize(); - - const completedRun = await runStore.readRun(session.id, runId); - assert.equal(completedRun?.status, 'completed'); - assert.equal(completedRun?.completedAt, 3); - assert.equal( - (await runtimeEventStore.readImmutableRuntimeEvents(session.id, runId)).at(-1)?.status, - 'completed', - ); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('required interaction resume stays fail-closed until a later required write succeeds', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-status-latch-failure-')); - try { - const store = createSessionStore(root); - const session = await store.create({ - cwd: '/tmp/cwd', - llmConnectionSlug: 'fake', - model: 'fake-model', - permissionMode: 'ask', - }); - const runStore = createSqliteAgentRunStore(root); - const runtimeEventStore = createWorkspaceRuntimeStore(root); - const runId = 'run-status-latch-failure'; - const turnId = 'turn-status-latch-failure'; - await store.updateHeader(session.id, buildStatusPatch('waiting_for_user', 1)); - await runStore.createRun({ - ...makeRunHeader(session.id, runId, turnId), - status: 'waiting_for_user', - }); - let failNextAppend = true; - let failRequiredUpdate = false; - const failingRunStore = { - updateRun: async (...args: Parameters) => { - if (failRequiredUpdate) throw new Error('injected required status failure'); - return await runStore.updateRun(...args); - }, - appendEvent: async (...args: Parameters) => { - if (failNextAppend) { - failNextAppend = false; - throw new Error('injected trace failure'); - } - return await runStore.appendEvent(...args); - }, - } as typeof runStore; - const run = new AgentRun({ - sessionId: session.id, - header: session, - userInput: { turnId, text: 'remain waiting after repeated store failure' }, - runId, - durability: 'required', - store, - runStore: failingRunStore, - runtimeEventStore, - newId: () => 'status-latch-failure-event', - now: () => 10, - hooks: { - reserveRun: async () => { - throw new Error('reserveRun should not be called'); - }, - unregisterRun: () => {}, - updateHeader: (sessionId, patch) => store.updateHeader(sessionId, patch), - updateStatus: async (sessionId, status, blockedReason, ts = 0) => { - await store.updateHeader(sessionId, buildStatusPatch(status, ts, blockedReason)); - }, - appendTurnState: async () => {}, - }, - }); - run.recordRunTrace({ - id: 'trace-that-fails-before-required-write', - sessionId: session.id, - turnId, - ts: 1, - phase: 'turn', - type: 'turn_started', - message: 'trip the best-effort trace latch', - }); - await waitFor(async () => - Boolean((await runStore.readRun(session.id, runId))?.traceWriteError), - ); - failRequiredUpdate = true; - - await assert.rejects( - run.recordSessionEvent({ - type: 'user_question_answer_ack', - id: 'answer-after-repeated-failure', - turnId, - ts: 2, - requestId: 'question-1', - toolUseId: 'tool-1', - }), - /injected required status failure/, - ); - assert.equal((await runStore.readRun(session.id, runId))?.status, 'waiting_for_user'); - assert.equal((await store.readHeader(session.id)).status, 'waiting_for_user'); - - failRequiredUpdate = false; - await run.recordSessionEvent({ - type: 'user_question_answer_ack', - id: 'answer-after-required-store-recovers', - turnId, - ts: 3, - requestId: 'question-1', - toolUseId: 'tool-1', - }); - await run.recordRuntimeEvents([ - { - id: 'terminal-after-required-store-recovers', - invocationId: run.invocationId, - runId, - sessionId: session.id, - turnId, - ts: 4, - partial: false, - role: 'system', - author: 'system', - status: 'completed', - actions: { endInvocation: true }, - }, - ]); - await run.recordSessionEvent({ - type: 'complete', - id: 'complete-after-required-store-recovers', - turnId, - ts: 4, - stopReason: 'end_turn', - }); - await run.finalize(); - - assert.equal((await runStore.readRun(session.id, runId))?.status, 'completed'); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -function makeRunHeader(sessionId: string, runId: string, turnId: string): AgentRunHeader { - return { - runId, - sessionId, - turnId, - status: 'running', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/cwd', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 1, - }; -} async function waitFor(predicate: () => Promise): Promise { await pollFor(predicate, { attempts: 100, diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index 926ba7c698..f9693fd817 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -26,7 +26,8 @@ import { describe, test } from 'node:test'; import type { ModelMessage, ModelStreamResult } from '../model-protocol.js'; import { MockLanguageModelV4, simulateReadableStream } from 'ai/test'; import { APICallError, type LanguageModelV4StreamPart } from '@ai-sdk/provider'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRootAuthority } from '@maka/core/runtime-event'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { AttachmentByteReader } from '@maka/core/attachments'; import type { BackendSendInput } from '@maka/core/backend-types'; import type { LlmConnection } from '@maka/core/llm-connections'; @@ -94,6 +95,7 @@ import type { OpenAiResponsesSemanticBaseline } from '../openai-responses-contin import type { OpenAiResponsesTransportState } from '../openai-responses-websocket.js'; import { getAIModel } from '../model-factory.js'; import { waitFor as pollFor } from '@maka/core/test-only/async-primitives'; +import { testInvocationOpening } from './invocation-fixture.js'; describe('AiSdkBackend ApplyPatch routing', () => { test('advertises apply_patch only to supported native OpenAI models', async () => { @@ -4409,8 +4411,8 @@ describe('AiSdkBackend model history', () => { const result = await backend.compactHistory({ turnId: 'turn-compact', runId: 'run-1', - runtimeContextRunHeaders: [ - priorModelRunHeader({ connectionId: 'test-connection-id', modelId: 'mock-model-id' }), + runtimeContextInvocations: [ + priorModelInvocation({ connectionId: 'test-connection-id', modelId: 'mock-model-id' }), ], runtimeContext: [ runtimeTextEvent({ @@ -4885,31 +4887,29 @@ describe('AiSdkBackend model history', () => { text: 'recent', }), ]; - const sourceRunHeader = priorModelRunHeader({ + const sourceRunHeader = priorModelInvocation({ connectionId: 'test-connection-id', modelId: 'mock-model-id', }); - const priorCompactionRunHeader: AgentRunHeader = { - ...priorModelRunHeader({ - connectionId: 'test-connection-id', - modelId: 'mock-model-id', - runId: 'run-1', - }), + const priorCompactionRunHeader = priorModelInvocation({ + connectionId: 'test-connection-id', + modelId: 'mock-model-id', + runId: 'run-1', turnId: 'turn-compact-1', - rootExecutionKind: 'context_compact', - }; + root: { kind: 'context_compact' }, + }); const first = await backend.compactHistory({ turnId: 'turn-compact-1', runId: 'run-1', runtimeContext: history, - runtimeContextRunHeaders: [sourceRunHeader], + runtimeContextInvocations: [sourceRunHeader], }); const repeated = await backend.compactHistory({ turnId: 'turn-compact-2', runId: 'run-2', runtimeContext: history, - runtimeContextRunHeaders: [sourceRunHeader, priorCompactionRunHeader], + runtimeContextInvocations: [sourceRunHeader, priorCompactionRunHeader], }); assert.equal(calls, 1); @@ -4932,7 +4932,7 @@ describe('AiSdkBackend model history', () => { text: 'new source history', }), ], - runtimeContextRunHeaders: [sourceRunHeader, priorCompactionRunHeader], + runtimeContextInvocations: [sourceRunHeader, priorCompactionRunHeader], }); assert.equal(calls, 2, 'changed source fingerprint is eligible again'); }); @@ -6240,8 +6240,8 @@ describe('AiSdkBackend model history', () => { turnId: 'turn-current', text: 'continue', context: [], - runtimeContextRunHeaders: [ - priorModelRunHeader({ connectionId: 'connection-a', modelId: 'claude-a' }), + runtimeContextInvocations: [ + priorModelInvocation({ connectionId: 'connection-a', modelId: 'claude-a' }), ], runtimeContext: [ runtimeTextEvent({ @@ -6325,8 +6325,8 @@ describe('AiSdkBackend model history', () => { turnId: 'turn-current', text: 'continue', context: [], - runtimeContextRunHeaders: [ - priorModelRunHeader({ + runtimeContextInvocations: [ + priorModelInvocation({ connectionId: 'connection-a', modelId: 'claude-a', providerStateIdentity: `sha256:${'a'.repeat(64)}`, @@ -6438,8 +6438,8 @@ describe('AiSdkBackend model history', () => { turnId: 'turn-current', text: 'continue', context: [], - runtimeContextRunHeaders: [ - priorModelRunHeader({ + runtimeContextInvocations: [ + priorModelInvocation({ connectionId: 'connection-copilot', connectionSlug: 'github-copilot', modelId: 'gpt-5.5', @@ -6532,8 +6532,8 @@ describe('AiSdkBackend model history', () => { turnId: 'turn-current', text: 'continue', context: [], - runtimeContextRunHeaders: [ - priorModelRunHeader({ + runtimeContextInvocations: [ + priorModelInvocation({ connectionId: 'connection-openai', connectionSlug: 'openai-main', modelId: 'gpt-5.4', @@ -11935,21 +11935,13 @@ describe('AiSdkBackend thinking persistence', () => { } as unknown as RuntimeEventMapContext; const memory = createSessionEventMapMemory(); const runtimeEvents = events.map((event) => mapSessionEventToRuntimeEvent(event, ctx, memory)); - const runHeader: AgentRunHeader = { + const runHeader = priorModelInvocation({ + modelId: 'mock-model-id', runId: 'run-1', - sessionId: 'session-1', turnId: 'turn-1', - status: 'completed', - backendKind: 'ai-sdk', - llmConnectionSlug: 'anthropic-main', - modelId: 'mock-model-id', - cwd: '/tmp/maka', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 2, - }; + }); const projection = projectRuntimeEventsToStoredMessages(runtimeEvents, { - runHeaders: [runHeader], + invocations: [runHeader], }); const assistant = projection.messages.find((message) => message.type === 'assistant'); assert.ok(assistant && assistant.type === 'assistant'); @@ -12029,21 +12021,13 @@ describe('AiSdkBackend thinking persistence', () => { } as unknown as RuntimeEventMapContext; const memory = createSessionEventMapMemory(); const runtimeEvents = events.map((event) => mapSessionEventToRuntimeEvent(event, ctx, memory)); - const runHeader: AgentRunHeader = { + const runHeader = priorModelInvocation({ + modelId: 'mock-model-id', runId: 'run-1', - sessionId: 'session-1', turnId: 'turn-1', - status: 'completed', - backendKind: 'ai-sdk', - llmConnectionSlug: 'anthropic-main', - modelId: 'mock-model-id', - cwd: '/tmp/maka', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 2, - }; + }); const projection = projectRuntimeEventsToStoredMessages(runtimeEvents, { - runHeaders: [runHeader], + invocations: [runHeader], }); const assistant = projection.messages.find((message) => message.type === 'assistant'); assert.ok(assistant && assistant.type === 'assistant'); @@ -13711,20 +13695,8 @@ describe('AiSdkBackend thinking persistence', () => { const memory = createSessionEventMapMemory(); const runtimeContext = events.map((event) => mapSessionEventToRuntimeEvent(event, ctx, memory)); const projection = projectRuntimeEventsToStoredMessages(runtimeContext, { - runHeaders: [ - { - runId: 'run-prev', - sessionId: 'session-1', - turnId: 'turn-prev', - status: 'completed', - backendKind: 'ai-sdk', - llmConnectionSlug: planConnection.slug, - modelId: 'ark-code-latest', - cwd: '/tmp/maka', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 2, - }, + invocations: [ + priorModelInvocation({ modelId: 'ark-code-latest', connectionSlug: planConnection.slug }), ], }); const projectedAssistant = projection.messages.find( @@ -16321,38 +16293,55 @@ function header(permissionMode: SessionHeader['permissionMode'] = 'ask'): Sessio }; } -function priorModelRunHeader(input: { +function priorModelInvocation(input: { connectionId?: string; modelId: string; connectionSlug?: string; runId?: string; + turnId?: string; + root?: RuntimeInvocationRootAuthority; providerStateIdentity?: `sha256:${string}`; -}): AgentRunHeader { - return { - runId: input.runId ?? 'run-prev', +}): RuntimeInvocationRecord { + const identity = { sessionId: 'session-1', - turnId: 'turn-prev', - status: 'completed', - backendKind: 'ai-sdk', - ...(input.connectionId ? { llmConnectionId: input.connectionId } : {}), - providerStateIdentity: input.providerStateIdentity ?? `sha256:${'1'.repeat(64)}`, - llmConnectionSlug: input.connectionSlug ?? 'anthropic-main', - modelId: input.modelId, - cwd: '/tmp/maka', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 2, - completedAt: 2, + invocationId: input.runId ?? 'run-prev', + runId: input.runId ?? 'run-prev', + turnId: input.turnId ?? 'turn-prev', + }; + return { + ...identity, + openedAt: 1, + opening: testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'ai-sdk', + llmConnectionId: input.connectionId ?? 'anthropic-main-connection', + llmConnectionSlug: input.connectionSlug ?? 'anthropic-main', + modelId: input.modelId, + providerStateIdentity: input.providerStateIdentity ?? `sha256:${'1'.repeat(64)}`, + }, + configuration: { cwd: '/tmp/maka' }, + root: input.root ?? { kind: 'user' }, + }), + terminalEvent: { + id: `${identity.runId}-terminal`, + ...identity, + ts: 2, + partial: false, + role: 'system', + author: 'system', + status: 'completed', + }, }; } function sameRouteReplayProvenance( modelId: string, runId = 'run-prev', -): Pick { +): Pick { return { - runtimeContextRunHeaders: [ - priorModelRunHeader({ connectionId: 'test-connection-id', modelId, runId }), + runtimeContextInvocations: [ + priorModelInvocation({ connectionId: 'test-connection-id', modelId, runId }), ], }; } diff --git a/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts b/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts index 8923a1e74b..916d397b8f 100644 --- a/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts +++ b/packages/runtime/src/__tests__/computer-use-provider-protocol.test.ts @@ -20,7 +20,7 @@ import assert from 'node:assert/strict'; import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; import { after, describe, test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { LlmConnection } from '@maka/core/llm-connections'; @@ -42,6 +42,7 @@ import { backfillRuntimeEventsFromStoredMessages } from '../runtime-event-backfi import { createDurableTurnHarness } from './durable-turn-harness.js'; import { createTestAiSdkBackend } from './execution-boundary-test-helpers.js'; import { latestObservationIn } from './observation-text-reader.js'; +import { testInvocationOpening } from './invocation-fixture.js'; const servers: Array<{ close(): Promise }> = []; const PROVIDER_STATE_IDENTITY = `sha256:${'1'.repeat(64)}` as const; @@ -105,22 +106,18 @@ describe('Anthropic-compatible Computer Use product loops', () => { newId: idGenerator(), now: monotonicClock(), }); - const sourceRun = { - runId: 'run-prev', + const sourceRun = sourceInvocation({ sessionId, + runId: 'run-prev', + invocationId: 'inv-prev', turnId: 'turn-prev', - status: 'completed', - backendKind: 'ai-sdk', llmConnectionId: 'connection-anthropic', llmConnectionSlug: 'anthropic', modelId: 'claude-sonnet-4-5-20250929', - providerStateIdentity: PROVIDER_STATE_IDENTITY, - cwd: '/tmp/maka', permissionMode: 'bypass', - createdAt: 1, - updatedAt: 2, + openedAt: 1, completedAt: 2, - } satisfies AgentRunHeader; + }); for await (const event of createRuntime().send(firstTurn.sendInput())) firstTurn.record(event); assert.deepEqual( firstTurn.ledger @@ -149,7 +146,7 @@ describe('Anthropic-compatible Computer Use product loops', () => { for await (const event of createRuntime().send( secondTurn.sendInput({ runtimeContext: firstTurn.ledger, - runtimeContextRunHeaders: [sourceRun], + runtimeContextInvocations: [sourceRun], }), )) { secondTurn.record(event); @@ -487,22 +484,18 @@ describe('OpenAI-compatible product loops', () => { newId: idGenerator(), now: monotonicClock(), }); - const sourceRun = { - runId: 'run-prev', + const sourceRun = sourceInvocation({ sessionId, + runId: 'run-prev', + invocationId: 'inv-prev', turnId: 'turn-prev', - status: 'completed', - backendKind: 'ai-sdk', llmConnectionId: 'connection-copilot', llmConnectionSlug: 'github-copilot', modelId: 'gpt-5.4', - providerStateIdentity: PROVIDER_STATE_IDENTITY, - cwd: '/tmp/maka', permissionMode: 'bypass', - createdAt: 1, - updatedAt: 2, + openedAt: 1, completedAt: 2, - } satisfies AgentRunHeader; + }); const priorEvents = [ { id: 'rt-user-prev', @@ -551,7 +544,7 @@ describe('OpenAI-compatible product loops', () => { for await (const event of runtime.send( currentTurn.sendInput({ runtimeContext: priorEvents, - runtimeContextRunHeaders: [sourceRun], + runtimeContextInvocations: [sourceRun], }), )) { currentTurn.record(event); @@ -679,23 +672,18 @@ describe('OpenAI-compatible product loops', () => { 'openai-chat', 131_072, ); - const sourceRun = { + const sourceRun = sourceInvocation({ + sessionId, runId: 'run-kimi-openai-recovered-tool-step', invocationId: 'invocation-kimi-openai-recovered-tool-step', - sessionId, turnId: previousTurnId, - status: 'completed', - backendKind: 'ai-sdk', llmConnectionId: 'test-connection-id', llmConnectionSlug: providerConnection.slug, modelId: 'k3', - providerStateIdentity: PROVIDER_STATE_IDENTITY, - cwd: '/tmp/maka', permissionMode: 'ask', - createdAt: 1, - updatedAt: 5, + openedAt: 1, completedAt: 5, - } satisfies AgentRunHeader; + }); const recovered = backfillRuntimeEventsFromStoredMessages({ run: sourceRun, messages: [ @@ -761,7 +749,7 @@ describe('OpenAI-compatible product loops', () => { for await (const event of runtime.send( currentTurn.sendInput({ runtimeContext: recovered.events, - runtimeContextRunHeaders: [sourceRun], + runtimeContextInvocations: [sourceRun], }), )) { currentTurn.record(event); @@ -826,23 +814,18 @@ describe('OpenAI-compatible product loops', () => { 'openai-chat', 131_072, ); - const sourceRun = { + const sourceRun = sourceInvocation({ + sessionId, runId: firstTurn.anchor.runId, invocationId: firstTurn.anchor.invocationId, - sessionId, turnId: firstTurn.anchor.turnId, - status: 'completed', - backendKind: 'ai-sdk', llmConnectionId: 'test-connection-id', llmConnectionSlug: providerConnection.slug, modelId: 'k3', - providerStateIdentity: PROVIDER_STATE_IDENTITY, - cwd: '/tmp/maka', permissionMode: 'ask', - createdAt: firstTurn.anchor.ts, - updatedAt: firstTurn.anchor.ts + 1, + openedAt: firstTurn.anchor.ts, completedAt: firstTurn.anchor.ts + 1, - } satisfies AgentRunHeader; + }); const createRuntime = () => createTestAiSdkBackend({ testProjectionArtifacts: true, @@ -892,7 +875,7 @@ describe('OpenAI-compatible product loops', () => { for await (const event of createRuntime().send( secondTurn.sendInput({ runtimeContext: recovered.events, - runtimeContextRunHeaders: [sourceRun], + runtimeContextInvocations: [sourceRun], }), )) { secondTurn.record(event); @@ -1594,3 +1577,61 @@ function readBody(request: IncomingMessage): Promise { request.on('error', reject); }); } + +/** + * A prior invocation on the same route, as its own events describe it. + * + * The replay path only needs its identity, its route and the fact that it + * ended; none of that is a field a writer sets apart from the ledger. + */ +function sourceInvocation(input: { + sessionId: string; + runId: string; + invocationId: string; + turnId: string; + llmConnectionId: string; + llmConnectionSlug: string; + modelId: string; + permissionMode: 'ask' | 'bypass'; + openedAt: number; + completedAt: number; +}): RuntimeInvocationRecord { + const identity = { + sessionId: input.sessionId, + invocationId: input.invocationId, + runId: input.runId, + turnId: input.turnId, + }; + return { + ...identity, + openedAt: input.openedAt, + opening: testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'ai-sdk', + llmConnectionId: input.llmConnectionId, + llmConnectionSlug: input.llmConnectionSlug, + modelId: input.modelId, + providerStateIdentity: PROVIDER_STATE_IDENTITY, + }, + configuration: { + cwd: '/tmp/maka', + permissionMode: input.permissionMode, + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + }), + terminalEvent: { + ...identity, + id: `${input.runId}-terminal`, + ts: input.completedAt, + partial: false, + role: 'system', + author: 'system', + status: 'completed', + actions: { endInvocation: true }, + }, + }; +} diff --git a/packages/runtime/src/__tests__/context-diagnostics.test.ts b/packages/runtime/src/__tests__/context-diagnostics.test.ts index 9354d762a3..dcdbe9415b 100644 --- a/packages/runtime/src/__tests__/context-diagnostics.test.ts +++ b/packages/runtime/src/__tests__/context-diagnostics.test.ts @@ -23,15 +23,13 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; import { DatabaseSync } from 'node:sqlite'; -import type { - AgentRunEvent, - AgentRunHeader, - AgentRunStore, - EmittedAgentRunEvent, -} from '@maka/core/agent-run'; +import type { AgentRunEvent, AgentRunStore, EmittedAgentRunEvent } from '@maka/core/agent-run'; +import type { RuntimeEventInvocationOpenedContent } from '@maka/core/runtime-event'; import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; +import { createWorkspaceRuntimeStore } from '@maka/storage/runtime-event-persistence'; import { readLatestContextDiagnostics } from '../context-diagnostics.js'; import { readLatestContextSnapshot } from '../latest-context-snapshot.js'; +import { seedInvocation } from './invocation-fixture.js'; test('rejects v2 snapshots that the canonical writer cannot produce', () => { const base = { @@ -78,8 +76,8 @@ test('rejects v2 snapshots that the canonical writer cannot produce', () => { test('serves the sealed snapshot without reading a single run', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { + await openRun(root, 'session-1', 'run-1'); const writer = createSqliteAgentRunStore(root); - await writer.createRun(runHeader('run-1', 1)); await writer.appendEvent( 'session-1', 'run-1', @@ -93,7 +91,7 @@ test('serves the sealed snapshot without reading a single run', async () => { scanned += 1; }); - const diagnostics = await readLatestContextDiagnostics(counted, 'session-1'); + const diagnostics = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(diagnostics.status, 'available'); if (diagnostics.status !== 'available') return; @@ -107,8 +105,8 @@ test('serves the sealed snapshot without reading a single run', async () => { test('does not trust a pre-observation projection over its canonical attempt', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { + await openRun(root, 'session-1', 'run-1'); const writer = createSqliteAgentRunStore(root); - await writer.createRun(runHeader('run-1', 1)); const oldProjection = latestContext('attempt-1', 10); oldProjection.snapshot.schemaVersion = 1; oldProjection.snapshot.composition = { @@ -123,13 +121,11 @@ test('does not trust a pre-observation projection over its canonical attempt', a ); const reader = createSqliteAgentRunStore(root); - const warm = await readLatestContextDiagnostics(reader, 'session-1'); + const warm = await readLatestContextDiagnostics(reader, 'session-1', ['run-1']); const cold = await readLatestContextDiagnostics( - { - listSessionRuns: (sessionId) => reader.listSessionRuns(sessionId), - readEvents: (sessionId, runId) => reader.readEvents(sessionId, runId), - }, + { readEvents: (sessionId, runId) => reader.readEvents(sessionId, runId) }, 'session-1', + ['run-1'], ); assert.equal(warm.status, 'available'); @@ -145,8 +141,8 @@ test('does not trust a pre-observation projection over its canonical attempt', a test('upgrades exact-matched mixed-era composition into the current projection', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { + await openRun(root, 'session-1', 'run-1'); const writer = createSqliteAgentRunStore(root); - await writer.createRun(runHeader('run-1', 1)); const oldProjection = latestContext('attempt-1', 10); oldProjection.snapshot.schemaVersion = 1; await writer.appendEvent( @@ -175,14 +171,14 @@ test('upgrades exact-matched mixed-era composition into the current projection', const counted = countingStore(reader, () => { scanned += 1; }); - const upgraded = await readLatestContextDiagnostics(counted, 'session-1'); + const upgraded = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(upgraded.status, 'available'); if (upgraded.status !== 'available') return; assert.deepEqual(upgraded.composition?.tools, [{ name: 'HistoricalTool', bytes: 700 }]); scanned = 0; - const warm = await readLatestContextDiagnostics(counted, 'session-1'); + const warm = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(warm.status, 'available'); if (warm.status !== 'available') return; assert.deepEqual(warm.composition, upgraded.composition); @@ -195,8 +191,8 @@ test('upgrades exact-matched mixed-era composition into the current projection', test('a failed call does not replace the last good snapshot', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { + await openRun(root, 'session-1', 'run-1'); const writer = createSqliteAgentRunStore(root); - await writer.createRun(runHeader('run-1', 1)); await writer.appendEvent( 'session-1', 'run-1', @@ -218,7 +214,7 @@ test('a failed call does not replace the last good snapshot', async () => { const counted = countingStore(createSqliteAgentRunStore(root), () => { scanned += 1; }); - const diagnostics = await readLatestContextDiagnostics(counted, 'session-1'); + const diagnostics = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(diagnostics.status, 'available'); if (diagnostics.status !== 'available') return; @@ -232,13 +228,11 @@ test('a failed call does not replace the last good snapshot', async () => { test("a subagent's run never becomes the session's context", async () => { const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { - const writer = createSqliteAgentRunStore(root); - await writer.createRun(runHeader('run-parent', 1)); - await writer.createRun({ - ...runHeader('run-child', 2), - parentRunId: 'run-parent', - agentId: 'sub', + await openRun(root, 'session-1', 'run-parent'); + await openRun(root, 'session-1', 'run-child', { + lineage: { parentRunId: 'run-parent', agentId: 'reviewer' }, }); + const writer = createSqliteAgentRunStore(root); await writer.appendEvent( 'session-1', 'run-parent', @@ -252,9 +246,11 @@ test("a subagent's run never becomes the session's context", async () => { { durable: true, latestContext: latestContext('a-child', 20, 'model-child') }, ); + // The caller names the session-inline runs, so the child is never scanned. const diagnostics = await readLatestContextDiagnostics( createSqliteAgentRunStore(root), 'session-1', + ['run-parent'], ); assert.equal(diagnostics.status, 'available'); @@ -271,8 +267,8 @@ test('rebuilds a canonical observation, then repairs it so the next read scans n // two reads. const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { + await openRun(root, 'session-1', 'run-1'); const writer = createSqliteAgentRunStore(root); - await writer.createRun(runHeader('run-1', 1)); await writer.appendEvent( 'session-1', 'run-1', @@ -303,7 +299,7 @@ test('rebuilds a canonical observation, then repairs it so the next read scans n scanned += 1; }); - const cold = await readLatestContextDiagnostics(counted, 'session-1'); + const cold = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(cold.status, 'available'); if (cold.status !== 'available') return; assert.equal(cold.modelId, 'model-new'); @@ -311,7 +307,7 @@ test('rebuilds a canonical observation, then repairs it so the next read scans n assert.ok(scanned > 0, 'the first read falls back to the ledger'); scanned = 0; - const warm = await readLatestContextDiagnostics(counted, 'session-1'); + const warm = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(warm.status, 'available'); if (warm.status !== 'available') return; assert.deepEqual(warm.composition?.tools, [{ name: 'Bash', bytes: 800 }]); @@ -324,7 +320,7 @@ test('rebuilds a canonical observation, then repairs it so the next read scans n test('rebuilds without repairing when the store lacks a ledger revision capability', async () => { const base = runStore([ { - header: runHeader('run-1', 1), + runId: 'run-1', events: [meteringEvent('run-1', 'attempt-1', 20, 'model', 40, 200)], }, ]); @@ -336,7 +332,7 @@ test('rebuilds without repairing when the store lacks a ledger revision capabili }, }; - const diagnostics = await readLatestContextDiagnostics(store, 'session-1'); + const diagnostics = await readLatestContextDiagnostics(store, 'session-1', ['run-1']); assert.equal(diagnostics.status, 'available'); assert.equal(repaired, false); @@ -348,7 +344,7 @@ test('reads a provider-only ledger that predates canonical metering', async () = // lose an answer the ledger still holds. const store = runStore([ { - header: runHeader('run-1', 1), + runId: 'run-1', events: [ attemptEvent('run-1', 'attempt-1', 20, 'completed', 'model-old', 40, 200, [ { kind: 'tool_schema', index: 0, cacheable: true, hash: 't', bytes: 800, label: 'Bash' }, @@ -357,7 +353,7 @@ test('reads a provider-only ledger that predates canonical metering', async () = }, ]); - const diagnostics = await readLatestContextDiagnostics(store, 'session-1'); + const diagnostics = await readLatestContextDiagnostics(store, 'session-1', ['run-1']); assert.equal(diagnostics.status, 'available'); if (diagnostics.status !== 'available') return; @@ -370,7 +366,7 @@ test('a canonical record on the ledger keeps the legacy path out of it', async ( // attempt exists, a newer provider-only attempt is not promoted over it. const store = runStore([ { - header: runHeader('run-1', 1), + runId: 'run-1', events: [ meteringEvent('run-1', 'attempt-1', 10, 'model-canonical', 40, 200), attemptEvent('run-1', 'attempt-2', 30, 'completed', 'model-provider-only', 50, 200), @@ -378,7 +374,7 @@ test('a canonical record on the ledger keeps the legacy path out of it', async ( }, ]); - const diagnostics = await readLatestContextDiagnostics(store, 'session-1'); + const diagnostics = await readLatestContextDiagnostics(store, 'session-1', ['run-1']); assert.equal(diagnostics.status, 'available'); if (diagnostics.status !== 'available') return; @@ -388,7 +384,7 @@ test('a canonical record on the ledger keeps the legacy path out of it', async ( test('cold rebuild takes composition from the canonical attempt observation', async () => { const store = runStore([ { - header: runHeader('run-1', 1), + runId: 'run-1', events: [ meteringEvent('run-1', 'attempt-1', 10, 'model-canonical', 40, 200, { requestObservation: requestObservation([ @@ -417,7 +413,7 @@ test('cold rebuild takes composition from the canonical attempt observation', as }, ]); - const diagnostics = await readLatestContextDiagnostics(store, 'session-1'); + const diagnostics = await readLatestContextDiagnostics(store, 'session-1', ['run-1']); assert.equal(diagnostics.status, 'available'); if (diagnostics.status !== 'available') return; @@ -427,7 +423,7 @@ test('cold rebuild takes composition from the canonical attempt observation', as test('does not enrich a canonical attempt from an identity-mismatched provider row', async () => { const store = runStore([ { - header: runHeader('run-1', 1), + runId: 'run-1', events: [ meteringEvent('run-1', 'attempt-1', 10, 'model-canonical', 40, 200), attemptEvent('run-1', 'attempt-1', 11, 'completed', 'model-other', 40, 200, [ @@ -444,7 +440,7 @@ test('does not enrich a canonical attempt from an identity-mismatched provider r }, ]); - const diagnostics = await readLatestContextDiagnostics(store, 'session-1'); + const diagnostics = await readLatestContextDiagnostics(store, 'session-1', ['run-1']); assert.equal(diagnostics.status, 'available'); if (diagnostics.status !== 'available') return; @@ -455,7 +451,7 @@ test('does not enrich a canonical attempt from an identity-mismatched provider r test('a legacy request whose capture is missing reports no composition, not an older one', async () => { const store = runStore([ { - header: runHeader('run-1', 1), + runId: 'run-1', events: [ meteringEvent('run-1', 'attempt-1', 10, 'model-old', 10, 100), attemptEvent('run-1', 'attempt-1', 10, 'completed', 'model-old', 10, 100, [ @@ -466,7 +462,7 @@ test('a legacy request whose capture is missing reports no composition, not an o }, ]); - const diagnostics = await readLatestContextDiagnostics(store, 'session-1'); + const diagnostics = await readLatestContextDiagnostics(store, 'session-1', ['run-1']); assert.equal(diagnostics.status, 'available'); if (diagnostics.status !== 'available') return; @@ -477,7 +473,7 @@ test('a legacy request whose capture is missing reports no composition, not an o test('a compaction call never becomes the reported context', async () => { const store = runStore([ { - header: runHeader('run-1', 1), + runId: 'run-1', events: [ meteringEvent('run-1', 'attempt-1', 10, 'model-main', 40, 200), meteringEvent('run-1', 'attempt-2', 20, 'model-compact', 5, 200, { @@ -487,7 +483,7 @@ test('a compaction call never becomes the reported context', async () => { }, ]); - const diagnostics = await readLatestContextDiagnostics(store, 'session-1'); + const diagnostics = await readLatestContextDiagnostics(store, 'session-1', ['run-1']); assert.equal(diagnostics.status, 'available'); if (diagnostics.status !== 'available') return; @@ -496,8 +492,9 @@ test('a compaction call never becomes the reported context', async () => { test('reports that no completed request exists instead of inferring session values', async () => { const diagnostics = await readLatestContextDiagnostics( - runStore([{ header: runHeader('run-1', 1), events: [] }]), + runStore([{ runId: 'run-1', events: [] }]), 'session-1', + ['run-1'], ); assert.deepEqual(diagnostics, { status: 'unavailable', reason: 'no_completed_request' }); @@ -511,7 +508,7 @@ test('a rebuilt session reports the fold that was in place when its request star // here describes a different request" rule the sealed row enforces. const store = runStore([ { - header: runHeader('run-1', 1), + runId: 'run-1', events: [ checkpointEvent('run-1', 5, 12, 3, 900), meteringEvent('run-1', 'attempt-1', 20, 'model-new', 40, 200), @@ -520,7 +517,7 @@ test('a rebuilt session reports the fold that was in place when its request star }, ]); - const diagnostics = await readLatestContextDiagnostics(store, 'session-1'); + const diagnostics = await readLatestContextDiagnostics(store, 'session-1', ['run-1']); assert.equal(diagnostics.status, 'available'); if (diagnostics.status !== 'available') return; @@ -541,7 +538,7 @@ test('a canonical ledger with nothing reportable does not fall back to a provide // resurrect exactly the request the canonical rule declined to report. const store = runStore([ { - header: runHeader('run-1', 1), + runId: 'run-1', events: [ meteringEvent('run-1', 'attempt-1', 10, 'model-failed', 40, 200, { status: 'failed' }), meteringEvent('run-1', 'attempt-2', 15, 'model-compact', 5, 200, { @@ -561,7 +558,7 @@ test('a canonical ledger with nothing reportable does not fall back to a provide }, ]); - const diagnostics = await readLatestContextDiagnostics(store, 'session-1'); + const diagnostics = await readLatestContextDiagnostics(store, 'session-1', ['run-1']); assert.deepEqual(diagnostics, { status: 'unavailable', reason: 'no_completed_request' }); }); @@ -574,8 +571,8 @@ test('warm and cold agree on which of two requests that finished together is the // request the panel is describing. const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { + await openRun(root, 'session-1', 'run-1'); const writer = createSqliteAgentRunStore(root); - await writer.createRun(runHeader('run-1', 1)); // Appended greater-id first, so a rule that simply kept the last write // would answer 'model-a' here and disagree with the scan below. await writer.appendEvent( @@ -592,15 +589,13 @@ test('warm and cold agree on which of two requests that finished together is the ); const reader = createSqliteAgentRunStore(root); - const warm = await readLatestContextDiagnostics(reader, 'session-1'); + const warm = await readLatestContextDiagnostics(reader, 'session-1', ['run-1']); // The same ledger read by a session whose projection was never // initialized: the answer has to come out identical. const cold = await readLatestContextDiagnostics( - { - listSessionRuns: (sessionId) => reader.listSessionRuns(sessionId), - readEvents: (sessionId, runId) => reader.readEvents(sessionId, runId), - }, + { readEvents: (sessionId, runId) => reader.readEvents(sessionId, runId) }, 'session-1', + ['run-1'], ); assert.equal(warm.status, 'available'); @@ -620,19 +615,18 @@ test('a session confirmed to have nothing is answered from the projection, not r const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { const writer = createSqliteAgentRunStore(root); - await writer.createRun(runHeader('run-1', 1)); let scanned = 0; const counted = countingStore(createSqliteAgentRunStore(root), () => { scanned += 1; }); - const cold = await readLatestContextDiagnostics(counted, 'session-1'); + const cold = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.deepEqual(cold, { status: 'unavailable', reason: 'no_completed_request' }); assert.ok(scanned > 0, 'an uninitialized projection is not an answer'); scanned = 0; - const warm = await readLatestContextDiagnostics(counted, 'session-1'); + const warm = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.deepEqual(warm, { status: 'unavailable', reason: 'no_completed_request' }); assert.equal(scanned, 0, 'the initialized-empty projection answers on its own'); } finally { @@ -653,12 +647,12 @@ test('names at most the bounded number of tools, and accounts for the rest', asy })); const store = runStore([ { - header: runHeader('run-1', 1), + runId: 'run-1', events: [attemptEvent('run-1', 'attempt-1', 10, 'completed', 'model', 40, 200, segments)], }, ]); - const diagnostics = await readLatestContextDiagnostics(store, 'session-1'); + const diagnostics = await readLatestContextDiagnostics(store, 'session-1', ['run-1']); assert.equal(diagnostics.status, 'available'); if (diagnostics.status !== 'available') return; @@ -679,8 +673,8 @@ test('a request that finished earlier cannot move the answer backwards', async ( // completion, or a late arrival would permanently rewind the panel. const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { + await openRun(root, 'session-1', 'run-1'); const writer = createSqliteAgentRunStore(root); - await writer.createRun(runHeader('run-1', 1)); await writer.appendEvent( 'session-1', 'run-1', @@ -698,6 +692,7 @@ test('a request that finished earlier cannot move the answer backwards', async ( const diagnostics = await readLatestContextDiagnostics( createSqliteAgentRunStore(root), 'session-1', + ['run-1'], ); assert.equal(diagnostics.status, 'available'); @@ -715,8 +710,8 @@ test('a damaged projection is repaired, not preserved forever', async () => { // refresh rescanned the whole session (#2323). const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { + await openRun(root, 'session-1', 'run-1'); const writer = createSqliteAgentRunStore(root); - await writer.createRun(runHeader('run-1', 1)); await writer.appendEvent( 'session-1', 'run-1', @@ -750,14 +745,14 @@ test('a damaged projection is repaired, not preserved forever', async () => { scanned += 1; }); - const first = await readLatestContextDiagnostics(counted, 'session-1'); + const first = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(first.status, 'available'); if (first.status !== 'available') return; assert.equal(first.modelId, 'model', 'the damaged row does not answer'); assert.ok(scanned > 0, 'the first read rebuilds from the ledger'); scanned = 0; - const second = await readLatestContextDiagnostics(counted, 'session-1'); + const second = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(second.status, 'available'); assert.equal(scanned, 0, 'and the rebuild replaced the damaged row'); } finally { @@ -768,8 +763,8 @@ test('a damaged projection is repaired, not preserved forever', async () => { test('repairs malformed projection bytes from the canonical ledger', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { + await openRun(root, 'session-1', 'run-1'); const writer = createSqliteAgentRunStore(root); - await writer.createRun(runHeader('run-1', 1)); await writer.appendEvent( 'session-1', 'run-1', @@ -807,14 +802,14 @@ test('repairs malformed projection bytes from the canonical ledger', async () => const counted = countingStore(createSqliteAgentRunStore(root), () => { scanned += 1; }); - const first = await readLatestContextDiagnostics(counted, 'session-1'); + const first = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(first.status, 'available'); if (first.status !== 'available') return; assert.deepEqual(first.composition?.tools, [{ name: 'Bash', bytes: 800 }]); assert.ok(scanned > 0, 'the malformed bytes force a canonical rebuild'); scanned = 0; - const second = await readLatestContextDiagnostics(counted, 'session-1'); + const second = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(second.status, 'available'); assert.equal(scanned, 0, 'the authority-derived candidate replaced the malformed row'); } finally { @@ -825,8 +820,8 @@ test('repairs malformed projection bytes from the canonical ledger', async () => test('does not persist a cold answer after canonical authority advances', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { + await openRun(root, 'session-1', 'run-1'); const store = createSqliteAgentRunStore(root); - await store.createRun(runHeader('run-1', 1)); await store.appendEvent( 'session-1', 'run-1', @@ -849,7 +844,6 @@ test('does not persist a cold answer after canonical authority advances', async let advanced = false; const racing: Parameters[0] = { - listSessionRuns: (sessionId) => store.listSessionRuns(sessionId), readEvents: async (sessionId, runId) => { const events = await store.readEvents(sessionId, runId); if (!advanced) { @@ -869,12 +863,12 @@ test('does not persist a cold answer after canonical authority advances', async store.repairEventProjection(sessionId, type, event, options), }; - const cold = await readLatestContextDiagnostics(racing, 'session-1'); + const cold = await readLatestContextDiagnostics(racing, 'session-1', ['run-1']); assert.equal(cold.status, 'available'); if (cold.status !== 'available') return; assert.equal(cold.modelId, 'model-1', 'the in-flight read remains a valid earlier snapshot'); - const next = await readLatestContextDiagnostics(store, 'session-1'); + const next = await readLatestContextDiagnostics(store, 'session-1', ['run-1']); assert.equal(next.status, 'available'); if (next.status !== 'available') return; assert.equal(next.modelId, 'model-2', 'the stale scan never becomes the warm projection'); @@ -886,8 +880,8 @@ test('does not persist a cold answer after canonical authority advances', async test('rebuilds a nested-malformed v2 projection from the canonical ledger', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { + await openRun(root, 'session-1', 'run-1'); const writer = createSqliteAgentRunStore(root); - await writer.createRun(runHeader('run-1', 1)); await writer.appendEvent( 'session-1', 'run-1', @@ -931,14 +925,14 @@ test('rebuilds a nested-malformed v2 projection from the canonical ledger', asyn const counted = countingStore(createSqliteAgentRunStore(root), () => { scanned += 1; }); - const first = await readLatestContextDiagnostics(counted, 'session-1'); + const first = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(first.status, 'available'); if (first.status !== 'available') return; assert.deepEqual(first.composition?.tools, [{ name: 'Bash', bytes: 800 }]); assert.ok(scanned > 0, 'the malformed nested value cannot answer the warm read'); scanned = 0; - const second = await readLatestContextDiagnostics(counted, 'session-1'); + const second = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(second.status, 'available'); assert.equal(scanned, 0, 'the canonical rebuild repaired the rejected projection'); } finally { @@ -949,8 +943,8 @@ test('rebuilds a nested-malformed v2 projection from the canonical ledger', asyn test('an old readable-order projection is upgraded after one cold rebuild', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { + await openRun(root, 'session-1', 'run-1'); const writer = createSqliteAgentRunStore(root); - await writer.createRun(runHeader('run-1', 1)); await writer.appendEvent( 'session-1', 'run-1', @@ -983,12 +977,12 @@ test('an old readable-order projection is upgraded after one cold rebuild', asyn const counted = countingStore(createSqliteAgentRunStore(root), () => { scanned += 1; }); - const first = await readLatestContextDiagnostics(counted, 'session-1'); + const first = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(first.status, 'available'); assert.ok(scanned > 0, 'the old schema requires one canonical rebuild'); scanned = 0; - const second = await readLatestContextDiagnostics(counted, 'session-1'); + const second = await readLatestContextDiagnostics(counted, 'session-1', ['run-1']); assert.equal(second.status, 'available'); assert.equal(scanned, 0, 'the rebuilt current schema replaces the old row'); } finally { @@ -996,12 +990,37 @@ test('an old readable-order projection is upgraded after one cold rebuild', asyn } }); +/** + * Open the invocation these ledger writes belong to. + * + * The operational ledger anchors every run on its opening fact, so a test that + * writes rows for a run has to say that the run began — and the opening is also + * where a run says it belongs to a subagent rather than to the session. + */ +async function openRun( + root: string, + sessionId: string, + runId: string, + opening?: Partial, +): Promise { + const runtimeStore = createWorkspaceRuntimeStore(root); + try { + await seedInvocation(runtimeStore, { + sessionId, + runId, + turnId: `turn-${runId}`, + ...(opening ? { opening } : {}), + }); + } finally { + runtimeStore.close(); + } +} + function countingStore( reader: ReturnType, onScan: () => void, ): Parameters[0] { return { - listSessionRuns: (sessionId) => reader.listSessionRuns(sessionId), readEvents: async (sessionId, runId) => { onScan(); return reader.readEvents(sessionId, runId); @@ -1038,28 +1057,10 @@ function latestContext(attemptId: string, completedAt: number, modelId = 'model' } function runStore( - runs: Array<{ header: AgentRunHeader; events: AgentRunEvent[] }>, -): Pick { + runs: Array<{ runId: string; events: AgentRunEvent[] }>, +): Pick { return { - listSessionRuns: async () => runs.map((run) => run.header), - readEvents: async (_sessionId, runId) => - runs.find((run) => run.header.runId === runId)?.events ?? [], - }; -} - -function runHeader(runId: string, createdAt: number): AgentRunHeader { - return { - runId, - sessionId: 'session-1', - turnId: `turn-${runId}`, - status: 'completed', - backendKind: 'ai-sdk', - llmConnectionSlug: 'anthropic-main', - modelId: 'model', - cwd: '/repo', - permissionMode: 'ask', - createdAt, - updatedAt: createdAt, + readEvents: async (_sessionId, runId) => runs.find((run) => run.runId === runId)?.events ?? [], }; } @@ -1074,6 +1075,8 @@ function attemptEvent( segments: Array> = [], ): EmittedAgentRunEvent { const turnId = `turn-${runId}`; + // A row from a retired writer. This build cannot emit the type; the diagnostic + // reader still has to read what older builds persisted. return { type: 'provider_request_attempt_recorded', id: attemptId, @@ -1101,7 +1104,7 @@ function attemptEvent( latencyMs: 1, ...(inputTokens === undefined ? {} : { inputTokens }), }, - }; + } as unknown as EmittedAgentRunEvent; } /** diff --git a/packages/runtime/src/__tests__/continuation-replay.test.ts b/packages/runtime/src/__tests__/continuation-replay.test.ts index eaeff52100..3f305894ec 100644 --- a/packages/runtime/src/__tests__/continuation-replay.test.ts +++ b/packages/runtime/src/__tests__/continuation-replay.test.ts @@ -342,7 +342,7 @@ describe('continuation replay segment', () => { prefixes: [ancestor, source], providerProjectionVersion: PROVIDER_REPLAY_PROJECTION_VERSION, admissionRoute: { - runHeaders: [], + invocations: [], targetProviderStateIdentity: undefined, targetModelId: 'test-model', }, diff --git a/packages/runtime/src/__tests__/conversation-copy.test.ts b/packages/runtime/src/__tests__/conversation-copy.test.ts index e34c6301bf..38821c8462 100644 --- a/packages/runtime/src/__tests__/conversation-copy.test.ts +++ b/packages/runtime/src/__tests__/conversation-copy.test.ts @@ -21,9 +21,10 @@ import assert from 'node:assert/strict'; import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; import { test } from 'node:test'; -import type { AgentRunHeader, AgentRunStore, EmittedAgentRunEvent } from '@maka/core/agent-run'; -import type { RuntimeEvent } from '@maka/core/runtime-event'; +import type { AgentRunStore, EmittedAgentRunEvent } from '@maka/core/agent-run'; +import type { RuntimeEvent, RuntimeEventInvocationOpenedContent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; import type { StoredMessage } from '@maka/core/session'; import { decodeCanonicalToolResultContent } from '@maka/core/tool-result-record-schema'; @@ -34,10 +35,15 @@ import { MODEL_PROJECTION_TRANSITION_EVENT_TYPE, type ModelProjectionTransition, } from '@maka/core/model-projection-transition'; -import { isSessionInlineRun } from '@maka/core/agent-run'; +import { + buildInvocationOpenedEvent, + isSessionInlineInvocation, + type RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; import { createWorkspaceRuntimeStore } from '@maka/storage/runtime-event-persistence'; +import { OPERATIONAL_STATE_DATABASE_NAME } from '@maka/storage/operational-state-store'; import { createSqliteRuntimeStore } from '@maka/storage/sqlite-runtime-store'; import { archivedToolResultContainsLinkedChildReferences, @@ -74,6 +80,7 @@ import { buildArchivedToolResultPlaceholder, isArchivedToolResultPlaceholder, } from '../tool-result-archive.js'; +import { testInvocationOpening, testInvocationRecord } from './invocation-fixture.js'; test('archived tool-result copy preflight detects conversation-owned references', () => { const serialized = (value: unknown): string => JSON.stringify(value); @@ -1083,14 +1090,16 @@ test('conversation copy rewrites owned references without changing opaque tool p }); test('conversation copy rejects continuation authority selected through the child-run closure', async () => { - const parent = agentRunHeader({ runId: 'run-parent', turnId: 'turn-parent' }); - const child = agentRunHeader({ + const parent = invocationRecord({ runId: 'run-parent', turnId: 'turn-parent' }); + const child = invocationRecord({ runId: 'run-child-retry', + invocationId: 'invocation-child-retry', turnId: 'turn-child-retry', parentRunId: 'run-parent', agentId: 'agent-child', - continuationSource: { - sourceInvocationId: parent.invocationId!, + source: { + kind: 'continuation', + sourceInvocationId: parent.invocationId, sourceRunId: parent.runId, sourceTurnId: parent.turnId, sourceRuntimeEventHighWater: 1, @@ -1112,10 +1121,10 @@ test('conversation copy rejects continuation authority selected through the chil }, ], runStore: { - listSessionRuns: async () => runs, readEvents: async () => [], }, runtimeEventStore: { + listSessionInvocations: async () => runs, readRuntimeEvents: async (_sessionId, runId) => { const run = runs.find((candidate) => candidate.runId === runId); assert.ok(run); @@ -1135,157 +1144,19 @@ test('conversation copy rejects continuation authority selected through the chil ); }); -test('conversation copy rejects a retained AgentRun without RuntimeEvent facts', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-conversation-missing-runtime-copy-')); - try { - const runStore = createSqliteAgentRunStore(root); - const runtimeEventStore = createWorkspaceRuntimeStore(root); - const rootRun = agentRunHeader({ - runId: 'run-root', - invocationId: 'invocation-root', - turnId: 'turn-root', - cwd: root, - }); - const childRun = agentRunHeader({ - runId: 'run-child', - invocationId: 'invocation-child', - turnId: 'turn-child', - parentRunId: 'run-root', - agentId: 'researcher', - agentName: 'Researcher', - cwd: root, - }); - await runStore.createRun(rootRun); - await runStore.createRun(childRun); - for (const event of [ - runtimeEvent({ - id: 'event-root-user', - invocationId: 'invocation-root', - runId: 'run-root', - turnId: 'turn-root', - role: 'user', - author: 'user', - content: { kind: 'text', text: 'delegate' }, - }), - runtimeEvent({ - id: 'event-root-terminal', - invocationId: 'invocation-root', - runId: 'run-root', - turnId: 'turn-root', - ts: 2, - status: 'completed', - }), - ]) { - await runtimeEventStore.appendRuntimeEvent(event.sessionId, event.runId, event); - } - const source = await new RuntimeReadModel({ - runStore, - runtimeEventStore, - }).getSessionView('session-source'); - let sequence = 0; - - await assert.rejects( - async () => - cloneConversationRuntimeLedger({ - plan: await prepareTestCopyPlan(source, source.messages, runStore, runtimeEventStore), - copiedMessages: source.messages, - referenceMap: { - mode: 'exact', - linkedChildren: { mode: 'reject' }, - sourceSessionId: 'session-source', - targetSessionId: 'session-target', - artifactIds: new Map(), - relativePaths: new Map(), - }, - runStore, - runtimeEventStore, - newId: () => `target-${++sequence}`, - }), - /Cannot copy AgentRun run-child without RuntimeEvent facts/, - ); - assert.deepEqual(await runStore.listSessionRuns('session-target'), []); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('conversation copy can use RuntimeEvents backfilled by the read model', async () => { - const run = agentRunHeader({ - runId: 'run-backfilled', - invocationId: 'invocation-backfilled', - turnId: 'turn-backfilled', - status: 'completed', - updatedAt: 3, - completedAt: 3, - }); - const legacyMessages: StoredMessage[] = [ - { - type: 'user', - id: 'legacy-user', - turnId: run.turnId, - ts: 1, - text: 'hello', - }, - { - type: 'assistant', - id: 'legacy-assistant', - turnId: run.turnId, - ts: 2, - text: 'world', - modelId: 'fake-model', - }, - { - type: 'turn_state', - id: 'legacy-state', - turnId: run.turnId, - ts: 3, - status: 'completed', - partialOutputRetained: false, - }, - ]; - const runStore = { - listSessionRuns: async () => [run], - readEvents: async () => [], - } as Pick; - const runtimeEventStore = { - readRuntimeEvents: async () => [], - readSessionRuntimeEventEntries: async () => [], - } as Pick; - const source = await new RuntimeReadModel({ - runStore: runStore as AgentRunStore, - runtimeEventStore: runtimeEventStore as RuntimeEventStore, - projectionCache: { readMessages: async () => legacyMessages }, - }).getSessionView(run.sessionId); - - const plan = await prepareConversationRuntimeLedgerCopy({ - sourceSessionId: run.sessionId, - sourceEvents: source.events, - copiedMessages: source.messages, - runStore, - runtimeEventStore, - }); - - assert.deepEqual( - plan.runs[0]?.runtimeEvents.map((event) => event.content?.kind ?? event.status), - ['text', 'text', 'completed'], - ); -}); - test('conversation copy rewrites a complete tool recovery bundle atomically', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-conversation-recovery-copy-')); const runStore = createSqliteAgentRunStore(root); const runtimeEventStore = createSqliteRuntimeStore(join(root, 'runtime.sqlite')); try { await runStore.ready?.(); - await runStore.createRun( - agentRunHeader({ + const sourceEvents: RuntimeEvent[] = [ + invocationOpenedEvent({ runId: 'run-source', invocationId: 'invocation-source', turnId: 'turn-1', cwd: root, }), - ); - const sourceEvents: RuntimeEvent[] = [ runtimeEvent({ id: 'event-user', role: 'user', @@ -1388,7 +1259,7 @@ test('conversation copy rewrites a complete tool recovery bundle atomically', as { runId: 'run-source', events: sourceEvents }, ]); await runStore.appendEvent('session-source', 'run-source', { - type: 'run_completed', + type: 'model_stream_completed', id: 'completed-source', runId: 'run-source', sessionId: 'session-source', @@ -1396,7 +1267,6 @@ test('conversation copy rewrites a complete tool recovery bundle atomically', as ts: 7, }); const source = await new RuntimeReadModel({ - runStore, runtimeEventStore, }).getSessionView('session-source'); await cloneConversationRuntimeLedger({ @@ -1414,7 +1284,7 @@ test('conversation copy rewrites a complete tool recovery bundle atomically', as runtimeEventStore, newId: () => crypto.randomUUID(), }); - const [targetRun] = await runStore.listSessionRuns('session-target'); + const [targetRun] = await runtimeEventStore.listSessionInvocations('session-target'); assert.ok(targetRun); assert.ok(targetRun.invocationId); const targetOperationId = buildToolOperationId({ @@ -1486,15 +1356,13 @@ test('conversation copy rewrites the parent operation id of a nested Code Mode c const runtimeEventStore = createSqliteRuntimeStore(join(root, 'runtime.sqlite')); try { await runStore.ready?.(); - await runStore.createRun( - agentRunHeader({ + const sourceEvents: RuntimeEvent[] = [ + invocationOpenedEvent({ runId: 'run-source', invocationId: 'invocation-source', turnId: 'turn-1', cwd: root, }), - ); - const sourceEvents: RuntimeEvent[] = [ runtimeEvent({ id: 'event-user', role: 'user', @@ -1576,7 +1444,7 @@ test('conversation copy rewrites the parent operation id of a nested Code Mode c { runId: 'run-source', events: sourceEvents }, ]); await runStore.appendEvent('session-source', 'run-source', { - type: 'run_completed', + type: 'model_stream_completed', id: 'completed-source', runId: 'run-source', sessionId: 'session-source', @@ -1584,7 +1452,6 @@ test('conversation copy rewrites the parent operation id of a nested Code Mode c ts: 6, }); const source = await new RuntimeReadModel({ - runStore, runtimeEventStore, }).getSessionView('session-source'); await cloneConversationRuntimeLedger({ @@ -1602,7 +1469,7 @@ test('conversation copy rewrites the parent operation id of a nested Code Mode c runtimeEventStore, newId: () => crypto.randomUUID(), }); - const [targetRun] = await runStore.listSessionRuns('session-target'); + const [targetRun] = await runtimeEventStore.listSessionInvocations('session-target'); assert.ok(targetRun); assert.ok(targetRun.invocationId); const targetOperationId = buildToolOperationId({ @@ -1631,90 +1498,17 @@ test('conversation copy rewrites the parent operation id of a nested Code Mode c } }); -test('conversation copy validates operational events before persisting target ledgers', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-conversation-copy-preflight-')); +test('conversation copy rewrites the nested identity of a model call attempt', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-conversation-model-call-copy-')); try { const runStore = createSqliteAgentRunStore(root); const runtimeEventStore = createWorkspaceRuntimeStore(root); - await runStore.createRun( - agentRunHeader({ - runId: 'run-source', - invocationId: 'invocation-source', - turnId: 'turn-1', - cwd: root, - }), - ); - for (const event of [ - runtimeEvent({ - id: 'event-user', - role: 'user', - author: 'user', - content: { kind: 'text', text: 'copy this turn' }, - }), - runtimeEvent({ - id: 'event-terminal', - ts: 2, - status: 'completed', - }), - ]) { - await runtimeEventStore.appendRuntimeEvent('session-source', 'run-source', event); - } - await runStore.appendEvent('session-source', 'run-source', { - type: 'provider_request_captured', - id: 'capture-source', + await seedRun(runtimeEventStore, { runId: 'run-source', - sessionId: 'session-source', + invocationId: 'invocation-source', turnId: 'turn-1', - ts: 1.5, - data: { - traceId: 'trace-source', - captureId: 'wrong-capture-id', - artifactId: 'artifact-source', - }, + cwd: root, }); - const source = await new RuntimeReadModel({ - runStore, - runtimeEventStore, - }).getSessionView('session-source'); - - await assert.rejects( - async () => - cloneConversationRuntimeLedger({ - plan: await prepareTestCopyPlan(source, source.messages, runStore, runtimeEventStore), - copiedMessages: source.messages, - referenceMap: { - mode: 'exact', - linkedChildren: { mode: 'reject' }, - sourceSessionId: 'session-source', - targetSessionId: 'session-target', - artifactIds: new Map([['artifact-source', 'artifact-target']]), - relativePaths: new Map(), - }, - runStore, - runtimeEventStore, - newId: () => crypto.randomUUID(), - }), - /Cannot copy invalid provider request capture capture-source/, - ); - assert.deepEqual(await runStore.listSessionRuns('session-target'), []); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -test('conversation copy rewrites the nested identity of a model call attempt', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-conversation-model-call-copy-')); - try { - const runStore = createSqliteAgentRunStore(root); - const runtimeEventStore = createWorkspaceRuntimeStore(root); - await runStore.createRun( - agentRunHeader({ - runId: 'run-source', - invocationId: 'invocation-source', - turnId: 'turn-1', - cwd: root, - }), - ); for (const event of [ runtimeEvent({ id: 'event-user', @@ -1761,7 +1555,6 @@ test('conversation copy rewrites the nested identity of a model call attempt', a }, }); const source = await new RuntimeReadModel({ - runStore, runtimeEventStore, }).getSessionView('session-source'); await cloneConversationRuntimeLedger({ @@ -1779,7 +1572,7 @@ test('conversation copy rewrites the nested identity of a model call attempt', a runtimeEventStore, newId: () => crypto.randomUUID(), }); - const [targetRun] = await runStore.listSessionRuns('session-target'); + const [targetRun] = await runtimeEventStore.listSessionInvocations('session-target'); assert.ok(targetRun); const targetEvents = await runStore.readEvents('session-target', targetRun.runId); const attempt = targetEvents.find((event) => event.type === 'model_call_attempt_recorded'); @@ -1814,14 +1607,12 @@ test('conversation copy repairs a model call attempt stranded by a pre-fix copy' try { const runStore = createSqliteAgentRunStore(root); const runtimeEventStore = createWorkspaceRuntimeStore(root); - await runStore.createRun( - agentRunHeader({ - runId: 'run-source', - invocationId: 'invocation-source', - turnId: 'turn-1', - cwd: root, - }), - ); + await seedRun(runtimeEventStore, { + runId: 'run-source', + invocationId: 'invocation-source', + turnId: 'turn-1', + cwd: root, + }); for (const event of [ runtimeEvent({ id: 'event-user', @@ -1869,7 +1660,6 @@ test('conversation copy repairs a model call attempt stranded by a pre-fix copy' }, }); const source = await new RuntimeReadModel({ - runStore, runtimeEventStore, }).getSessionView('session-source'); // The whole copy must not throw `Cannot copy invalid model call attempt`. @@ -1888,7 +1678,7 @@ test('conversation copy repairs a model call attempt stranded by a pre-fix copy' runtimeEventStore, newId: () => crypto.randomUUID(), }); - const [targetRun] = await runStore.listSessionRuns('session-target'); + const [targetRun] = await runtimeEventStore.listSessionInvocations('session-target'); assert.ok(targetRun); const targetEvents = await runStore.readEvents('session-target', targetRun.runId); const attempt = targetEvents.find((event) => event.type === 'model_call_attempt_recorded'); @@ -1916,22 +1706,15 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi try { const runStore = createSqliteAgentRunStore(root); const runtimeEventStore = createWorkspaceRuntimeStore(root); - const sourceRun: AgentRunHeader = { + const sourceRun = runFacts({ runId: 'run-source', invocationId: 'invocation-source', - sessionId: 'session-source', turnId: 'turn-1', - status: 'completed', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'model', cwd: root, - permissionMode: 'ask', - createdAt: 1, - updatedAt: 3, - completedAt: 3, - }; - await runStore.createRun(sourceRun); + openedAt: 1, + closedAt: 3, + }); + await seedRun(runtimeEventStore, sourceRun); const sourceAttachmentText = [ '![chart](maka://runtime/attachments/artifact-source)', 'maka://runtime/attachments/artifact-source?session=other', @@ -2076,7 +1859,7 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi segments: [], artifactId: 'artifact-source', }, - }); + } as unknown as EmittedAgentRunEvent); await runStore.appendEvent('session-source', 'run-source', { type: 'provider_request_attempt_recorded', id: 'attempt-source', @@ -2102,7 +1885,7 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi status: 'completed', latencyMs: 0.5, }, - }); + } as unknown as EmittedAgentRunEvent); await runStore.appendEvent('session-source', 'run-source', { type: 'provider_request_attempt_recorded', id: 'attempt-without-capture-source', @@ -2126,7 +1909,7 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi status: 'completed', latencyMs: 0.05, }, - }); + } as unknown as EmittedAgentRunEvent); // A legacy event from the retired active-full writer is treated like any // other event this build cannot emit and is therefore not copied. await runStore.appendEvent('session-source', 'run-source', { @@ -2178,7 +1961,7 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi }, }); await runStore.appendEvent('session-source', 'run-source', { - type: 'run_completed', + type: 'model_stream_completed', id: 'completed-source', runId: 'run-source', sessionId: 'session-source', @@ -2204,7 +1987,6 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi ), ); const source = await new RuntimeReadModel({ - runStore, runtimeEventStore, }).getSessionView('session-source'); await assert.rejects( @@ -2226,16 +2008,21 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi }), /missing Artifact artifact-deleted/, ); - assert.deepEqual(await runStore.listSessionRuns('session-missing-artifact'), []); + assert.deepEqual( + await runtimeEventStore.listSessionInvocations('session-missing-artifact'), + [], + ); + // A copied run and its copied invocation share one fresh identity, so the + // copy mints one id here rather than two. const ids = [ 'run-target', - 'invocation-target', 'event-target-1', 'event-target-2', 'event-target-3', 'event-target-4', 'event-target-5', 'event-target-6', + 'event-target-7', ]; let nextId = 0; @@ -2268,10 +2055,10 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi : undefined, ['artifact-target-deleted'], ); - const [targetRun] = await runStore.listSessionRuns('session-target'); + const [targetRun] = await runtimeEventStore.listSessionInvocations('session-target'); assert.equal(targetRun?.runId, 'run-target'); - assert.equal(targetRun?.invocationId, 'invocation-target'); - assert.equal(targetRun?.status, 'completed'); + assert.equal(targetRun?.invocationId, 'run-target'); + assert.equal(targetRun?.terminalEvent?.status, 'completed'); const targetEvents = await runtimeEventStore.readRuntimeEvents('session-target', 'run-target'); assert.deepEqual( targetEvents.map((event) => event.id), @@ -2282,6 +2069,7 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi 'event-target-4', 'event-target-5', 'event-target-6', + 'event-target-7', ], ); assert.ok( @@ -2289,34 +2077,37 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi (event) => event.sessionId === 'session-target' && event.runId === 'run-target' && - event.invocationId === 'invocation-target', + event.invocationId === 'run-target', ), ); - assert.equal(targetEvents[0]?.refs?.artifactId, 'artifact-target'); + // The opening fact is the run's first event, so the copied source events + // line up with it one place along. + const copiedEvents = targetEvents.slice(1); + assert.equal(copiedEvents[0]?.refs?.artifactId, 'artifact-target'); assert.equal( - targetEvents[0]?.content?.kind === 'text' ? targetEvents[0].content.text : undefined, + copiedEvents[0]?.content?.kind === 'text' ? copiedEvents[0].content.text : undefined, targetAttachmentText, ); assert.equal( copied.copiedMessages.find((message) => message.type === 'assistant')?.text, targetAttachmentText, ); - assert.equal(targetEvents[1]?.refs?.sourceInvocationId, 'invocation-target'); + assert.equal(copiedEvents[1]?.refs?.sourceInvocationId, 'run-target'); assert.deepEqual( - targetEvents[1]?.content?.kind === 'function_call' ? targetEvents[1].content.args : undefined, + copiedEvents[1]?.content?.kind === 'function_call' ? copiedEvents[1].content.args : undefined, sourceEvents[1]?.content?.kind === 'function_call' ? sourceEvents[1].content.args : undefined, ); assert.deepEqual( - targetEvents[2]?.content?.kind === 'function_response' - ? targetEvents[2].content.result + copiedEvents[2]?.content?.kind === 'function_response' + ? copiedEvents[2].content.result : undefined, sourceEvents[2]?.content?.kind === 'function_response' ? sourceEvents[2].content.result : undefined, ); const typedResultValue = - targetEvents[4]?.content?.kind === 'function_response' - ? targetEvents[4].content.result + copiedEvents[4]?.content?.kind === 'function_response' + ? copiedEvents[4].content.result : undefined; const typedResult = decodeCanonicalToolResultContent(typedResultValue); assert.equal(typedResult.kind === 'subagent' ? typedResult.permissionMode : undefined, 'ask'); @@ -2324,42 +2115,18 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi 'artifact-target-deleted', ]); const targetOperationalEvents = await runStore.readEvents('session-target', 'run-target'); + // The retired provider-request writers are treated like any other type this + // build cannot emit: their rows are not carried into the target. assert.deepEqual( targetOperationalEvents.map((event) => event.type), - [ - 'provider_request_captured', - 'provider_request_attempt_recorded', - 'provider_request_attempt_recorded', - 'history_compact_checkpoint_recorded', - 'run_completed', - ], - ); - const targetCapture = targetOperationalEvents.find( - (event) => event.type === 'provider_request_captured', + ['history_compact_checkpoint_recorded', 'model_stream_completed'], ); - const targetAttempt = targetOperationalEvents.find( - (event) => - event.type === 'provider_request_attempt_recorded' && event.data?.providerId === 'provider', - ); - const targetAttemptWithoutCapture = targetOperationalEvents.find( - (event) => - event.type === 'provider_request_attempt_recorded' && - event.data?.providerId === 'provider-without-capture', - ); - assert.ok(targetCapture); - assert.ok(targetAttempt); - assert.ok(targetAttemptWithoutCapture); - assert.equal(targetCapture.data?.captureId, targetCapture.id); - assert.equal(targetCapture.data?.artifactId, 'artifact-target'); - assert.notEqual(targetCapture.data?.traceId, 'provider-trace-source'); - assert.equal(targetAttempt.data?.attemptId, targetAttempt.id); - assert.equal(targetAttempt.data?.captureId, targetCapture.id); - assert.equal(targetAttempt.data?.captureArtifactId, 'artifact-target'); - assert.equal(targetAttempt.data?.traceId, targetCapture.data?.traceId); - assert.equal(targetAttemptWithoutCapture.data?.captureId, undefined); - assert.equal(targetAttemptWithoutCapture.data?.captureArtifactId, undefined); - assert.equal(targetEvents[1]?.refs?.providerRequestTraceId, targetCapture.data?.traceId); - assert.equal(targetEvents[1]?.refs?.traceEventId, targetCapture.id); + // A copied RuntimeEvent still points somewhere new, though. Carrying the + // source's trace identity into the target is the thing the copy exists to + // prevent, whether or not the record naming that trace came along. + assert.notEqual(copiedEvents[1]?.refs?.providerRequestTraceId, 'provider-trace-source'); + assert.ok(copiedEvents[1]?.refs?.providerRequestTraceId); + assert.equal(targetEvents[1]?.refs?.traceEventId, undefined); assert.doesNotMatch(JSON.stringify(targetOperationalEvents), /OPAQUE_SOURCE_COMPACTION_STATE/); const projectedCheckpoint = await runStore.readEventProjection?.( 'session-target', @@ -2375,7 +2142,8 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi ).reason, undefined, ); - assert.equal((await runStore.readRun('session-source', 'run-source')).status, 'completed'); + const [sourceInvocation] = await runtimeEventStore.listSessionInvocations('session-source'); + assert.equal(sourceInvocation?.terminalEvent?.status, 'completed'); } finally { await rm(root, { recursive: true, force: true }); } @@ -2386,34 +2154,32 @@ test('conversation copy rebuilds an inline checkpoint without legacy child event try { const runStore = createSqliteAgentRunStore(root); const runtimeEventStore = createWorkspaceRuntimeStore(root); - const firstRun = agentRunHeader({ + const firstRun = runFacts({ runId: 'run-1', invocationId: 'invocation-1', turnId: 'turn-1', cwd: root, }); - const secondRun = agentRunHeader({ + const secondRun = runFacts({ runId: 'run-2', invocationId: 'invocation-2', turnId: 'turn-2', cwd: root, - createdAt: 3, - updatedAt: 5, - completedAt: 5, + openedAt: 3, + closedAt: 5, }); - const childRun = agentRunHeader({ + const childRun = runFacts({ runId: 'run-child', invocationId: 'invocation-child', turnId: 'turn-child', parentRunId: 'run-1', cwd: root, - createdAt: 2.1, - updatedAt: 2.9, - completedAt: 2.9, + openedAt: 2.1, + closedAt: 2.9, }); - await runStore.createRun(firstRun); - await runStore.createRun(childRun); - await runStore.createRun(secondRun); + await seedRun(runtimeEventStore, firstRun); + await seedRun(runtimeEventStore, childRun); + await seedRun(runtimeEventStore, secondRun); const firstEvents = [ runtimeEvent({ id: 'event-1-user', @@ -2542,7 +2308,6 @@ test('conversation copy rebuilds an inline checkpoint without legacy child event }, }); const source = await new RuntimeReadModel({ - runStore, runtimeEventStore, }).getSessionView('session-source'); let sequence = 0; @@ -2563,14 +2328,14 @@ test('conversation copy rebuilds an inline checkpoint without legacy child event newId: () => `target-${++sequence}`, }); - const targetRuns = await runStore.listSessionRuns('session-target'); + const targetRuns = await runtimeEventStore.listSessionInvocations('session-target'); const targetInlineRunIds = new Set( - targetRuns.filter(isSessionInlineRun).map((run) => run.runId), + targetRuns.filter((run) => isSessionInlineInvocation(run.opening)).map((run) => run.runId), ); const targetEvents = (await runtimeEventStore.readSessionRuntimeEventEntries('session-target')) .map(({ event }) => event) .filter((event) => targetInlineRunIds.has(event.runId)); - assert.ok(targetRuns.some((run) => !isSessionInlineRun(run))); + assert.ok(targetRuns.some((run) => !isSessionInlineInvocation(run.opening))); const projectedCheckpoint = await runStore.readEventProjection?.( 'session-target', 'history_compact_checkpoint_recorded', @@ -2603,14 +2368,14 @@ test('conversation copy drops a checkpoint from a superseded source policy inste try { const runStore = createSqliteAgentRunStore(root); const runtimeEventStore = createWorkspaceRuntimeStore(root); - const run = agentRunHeader({ + const run = runFacts({ runId: 'run-source', invocationId: 'invocation-1', turnId: 'turn-1', cwd: root, - completedAt: 3, + closedAt: 3, }); - await runStore.createRun(run); + await seedRun(runtimeEventStore, run); const sourceEvents = [ runtimeEvent({ id: 'event-user', @@ -2665,7 +2430,6 @@ test('conversation copy drops a checkpoint from a superseded source policy inste }, }); const source = await new RuntimeReadModel({ - runStore, runtimeEventStore, }).getSessionView('session-source'); let sequence = 0; @@ -2686,7 +2450,7 @@ test('conversation copy drops a checkpoint from a superseded source policy inste newId: () => `target-${++sequence}`, }); - const targetRuns = await runStore.listSessionRuns('session-target'); + const targetRuns = await runtimeEventStore.listSessionInvocations('session-target'); assert.ok(targetRuns.length > 0); const targetOperationalEvents = ( await Promise.all(targetRuns.map((run) => runStore.readEvents('session-target', run.runId))) @@ -2700,7 +2464,8 @@ test('conversation copy drops a checkpoint from a superseded source policy inste targetRuns.map((run) => runtimeEventStore.readRuntimeEvents('session-target', run.runId)), ) ).flat(); - assert.equal(targetEvents.length, sourceEvents.length); + // The opening fact is one of the run's events, so the copy carries it too. + assert.equal(targetEvents.length, sourceEvents.length + 1); } finally { await rm(root, { recursive: true, force: true }); } @@ -2711,13 +2476,13 @@ test('conversation copy rebuilds a resumed child checkpoint over its child run c try { const runStore = createSqliteAgentRunStore(root); const runtimeEventStore = createWorkspaceRuntimeStore(root); - const rootRun = agentRunHeader({ + const rootRun = runFacts({ runId: 'run-root', invocationId: 'invocation-root', turnId: 'turn-root', cwd: root, }); - const firstChild = agentRunHeader({ + const firstChild = runFacts({ runId: 'run-child-1', invocationId: 'invocation-child-1', turnId: 'turn-child-1', @@ -2725,11 +2490,10 @@ test('conversation copy rebuilds a resumed child checkpoint over its child run c agentId: 'researcher', agentName: 'Researcher', cwd: root, - createdAt: 3, - updatedAt: 5, - completedAt: 5, + openedAt: 3, + closedAt: 5, }); - const resumedChild = agentRunHeader({ + const resumedChild = runFacts({ runId: 'run-child-2', invocationId: 'invocation-child-2', turnId: 'turn-child-2', @@ -2738,11 +2502,10 @@ test('conversation copy rebuilds a resumed child checkpoint over its child run c agentId: 'researcher', agentName: 'Researcher', cwd: root, - createdAt: 6, - updatedAt: 8, - completedAt: 8, + openedAt: 6, + closedAt: 8, }); - for (const run of [rootRun, firstChild, resumedChild]) await runStore.createRun(run); + for (const run of [rootRun, firstChild, resumedChild]) await seedRun(runtimeEventStore, run); const rootEvents = [ runtimeEvent({ @@ -2832,7 +2595,6 @@ test('conversation copy rebuilds a resumed child checkpoint over its child run c }, }); const source = await new RuntimeReadModel({ - runStore, runtimeEventStore, }).getSessionView('session-source'); let sequence = 0; @@ -2856,8 +2618,10 @@ test('conversation copy rebuilds a resumed child checkpoint over its child run c const runIds = new Map( copied.runIdMap.map(({ sourceRunId, targetRunId }) => [sourceRunId, targetRunId]), ); - const targetResumedChild = await runStore.readRun('session-target', runIds.get('run-child-2')!); - assert.equal(targetResumedChild.resumedFromRunId, runIds.get('run-child-1')); + const targetResumedChild = ( + await runtimeEventStore.listSessionInvocations('session-target') + ).find((run) => run.runId === runIds.get('run-child-2')); + assert.ok(targetResumedChild); const targetChildEvents = ( await Promise.all( ['run-child-1', 'run-child-2'].map((sourceRunId) => @@ -2925,14 +2689,12 @@ test('conversation copy rebuilds projection transitions against the copied event try { const runStore = createSqliteAgentRunStore(root); const runtimeEventStore = createWorkspaceRuntimeStore(root); - await runStore.createRun( - agentRunHeader({ - runId: 'run-source', - invocationId: 'invocation-source', - turnId: 'turn-1', - cwd: root, - }), - ); + await seedRun(runtimeEventStore, { + runId: 'run-source', + invocationId: 'invocation-source', + turnId: 'turn-1', + cwd: root, + }); const resultEvent = runtimeEvent({ id: 'event-result', ts: 2, @@ -2996,7 +2758,7 @@ test('conversation copy rebuilds projection transitions against the copied event }); } await runStore.appendEvent('session-source', 'run-source', { - type: 'run_completed', + type: 'model_stream_completed', id: 'completed-source', runId: 'run-source', sessionId: 'session-source', @@ -3004,7 +2766,6 @@ test('conversation copy rebuilds projection transitions against the copied event ts: 4, }); const source = await new RuntimeReadModel({ - runStore, runtimeEventStore, }).getSessionView('session-source'); @@ -3027,7 +2788,7 @@ test('conversation copy rebuilds projection transitions against the copied event newId: () => crypto.randomUUID(), }); - const [targetRun] = await runStore.listSessionRuns('session-target'); + const [targetRun] = await runtimeEventStore.listSessionInvocations('session-target'); assert.ok(targetRun); const targetEvents = await runtimeEventStore.readRuntimeEvents( 'session-target', @@ -3039,6 +2800,7 @@ test('conversation copy rebuilds projection transitions against the copied event const copiedTransitions = await loadModelProjectionTransitionsFromRunLedger( runStore, 'session-target', + (await runtimeEventStore.listSessionInvocations('session-target')).map((run) => run.runId), ); assert.equal(copiedTransitions.transitions.length, 2); const copiedFirst = copiedTransitions.transitions.find( @@ -3093,8 +2855,9 @@ test('conversation copy carries a transition recorded by a later, uncopied run', ['run-first', 'turn-1'], ['run-second', 'turn-2'], ]) { - await runStore.createRun( - agentRunHeader({ + await seedRun( + runtimeEventStore, + runFacts({ runId, invocationId: `invocation-${runId}`, turnId, @@ -3193,7 +2956,7 @@ test('conversation copy carries a transition recorded by a later, uncopied run', ['run-second', 'turn-2', 'completed-second'], ]) { await runStore.appendEvent('session-source', runId, { - type: 'run_completed', + type: 'model_stream_completed', id, runId, sessionId: 'session-source', @@ -3202,7 +2965,6 @@ test('conversation copy carries a transition recorded by a later, uncopied run', }); } const source = await new RuntimeReadModel({ - runStore, runtimeEventStore, }).getSessionView('session-source'); const firstTurnMessages = source.messages.filter( @@ -3225,13 +2987,15 @@ test('conversation copy carries a transition recorded by a later, uncopied run', newId: () => crypto.randomUUID(), }); - const targetRuns = await runStore.listSessionRuns('session-target'); + const targetRuns = await runtimeEventStore.listSessionInvocations('session-target'); assert.equal(targetRuns.length, 1); const targetEvents = await runtimeEventStore.readRuntimeEvents( 'session-target', targetRuns[0]!.runId, ); - const copied = await loadModelProjectionTransitionsFromRunLedger(runStore, 'session-target'); + const copied = await loadModelProjectionTransitionsFromRunLedger(runStore, 'session-target', [ + targetRuns[0]!.runId, + ]); assert.equal(copied.transitions.length, 1); assert.equal( copied.transitions[0]?.target.runtimeEventId, @@ -3260,8 +3024,9 @@ test('conversation copy reproduces the source fold rather than re-deciding it', ['run-first', 'turn-1'], ['run-second', 'turn-2'], ]) { - await runStore.createRun( - agentRunHeader({ runId, invocationId: `invocation-${runId}`, turnId, cwd: root }), + await seedRun( + runtimeEventStore, + runFacts({ runId, invocationId: `invocation-${runId}`, turnId, cwd: root }), ); } const resultEvent = runtimeEvent({ @@ -3366,7 +3131,7 @@ test('conversation copy reproduces the source fold rather than re-deciding it', ['run-second', 'turn-2', 'completed-second'], ]) { await runStore.appendEvent('session-source', runId, { - type: 'run_completed', + type: 'model_stream_completed', id, runId, sessionId: 'session-source', @@ -3375,7 +3140,6 @@ test('conversation copy reproduces the source fold rather than re-deciding it', }); } const source = await new RuntimeReadModel({ - runStore, runtimeEventStore, }).getSessionView('session-source'); const firstTurnMessages = source.messages.filter( @@ -3401,13 +3165,15 @@ test('conversation copy reproduces the source fold rather than re-deciding it', newId: () => crypto.randomUUID(), }); - const [targetRun] = await runStore.listSessionRuns('session-target'); + const [targetRun] = await runtimeEventStore.listSessionInvocations('session-target'); assert.ok(targetRun); const targetEvents = await runtimeEventStore.readRuntimeEvents( 'session-target', targetRun.runId, ); - const copied = await loadModelProjectionTransitionsFromRunLedger(runStore, 'session-target'); + const copied = await loadModelProjectionTransitionsFromRunLedger(runStore, 'session-target', [ + targetRun.runId, + ]); // Only the transition the source fold applied is rebuilt. Carrying the // rejected rival would let the copy re-decide and show a placeholder the // source never showed. @@ -3430,11 +3196,116 @@ test('conversation copy reproduces the source fold rather than re-deciding it', } }); +test('conversation copy gives a run whose opening the migration shelved its opening back', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-conversation-shelved-copy-')); + try { + const runStore = createSqliteAgentRunStore(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); + await seedRun( + runtimeEventStore, + runFacts({ runId: 'run-source', invocationId: 'run-source', turnId: 'turn-1', cwd: root }), + ); + const sourceEvents: RuntimeEvent[] = [ + runtimeEvent({ + id: 'event-user', + invocationId: 'run-source', + role: 'user', + author: 'user', + content: { kind: 'text', text: 'hello' }, + }), + runtimeEvent({ + id: 'event-terminal', + invocationId: 'run-source', + ts: 3, + status: 'completed', + }), + ]; + for (const event of sourceEvents) { + await runtimeEventStore.appendRuntimeEvent('session-source', 'run-source', event); + } + // Leave the run the way the migration leaves one that already owned an + // immutable sequence: its opening on the legacy shelf, not among its events. + const db = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const opening = db + .prepare( + "SELECT event_id, payload_json, committed_at FROM runtime_events WHERE run_id = 'run-source' AND event_kind = 'invocation_opened'", + ) + .get() as { event_id: string; payload_json: string; committed_at: number }; + db.prepare('DELETE FROM runtime_session_event_ordinals WHERE event_id = ?').run( + opening.event_id, + ); + db.prepare('DELETE FROM runtime_events WHERE event_id = ?').run(opening.event_id); + const anchor = db + .prepare( + "SELECT event_id FROM runtime_events WHERE run_id = 'run-source' ORDER BY event_seq ASC LIMIT 1", + ) + .get() as { event_id: string }; + db.prepare(` + INSERT INTO runtime_legacy_invocation_openings ( + invocation_id, session_id, run_id, turn_id, opened_at, opening_json, + anchor_event_id + ) VALUES ('run-source', 'session-source', 'run-source', 'turn-1', ?, ?, ?) + `).run( + opening.committed_at, + JSON.stringify((JSON.parse(opening.payload_json) as { content: unknown }).content), + anchor.event_id, + ); + } finally { + db.close(); + } + const [sourceRun] = await runtimeEventStore.listSessionInvocations('session-source'); + assert.equal(sourceRun?.runId, 'run-source', 'the shelved opening still names the run'); + assert.equal( + (await runtimeEventStore.readRuntimeEvents('session-source', 'run-source')).some( + (event) => event.content?.kind === 'invocation_opened', + ), + false, + 'but its events do not carry it', + ); + + const source = await new RuntimeReadModel({ runtimeEventStore }).getSessionView( + 'session-source', + ); + const copied = await cloneConversationRuntimeLedger({ + plan: await prepareTestCopyPlan(source, source.messages, runStore, runtimeEventStore), + copiedMessages: source.messages, + referenceMap: { + mode: 'exact', + linkedChildren: { mode: 'reject' }, + sourceSessionId: 'session-source', + targetSessionId: 'session-target', + artifactIds: new Map(), + relativePaths: new Map(), + }, + runStore, + runtimeEventStore, + newId: () => crypto.randomUUID(), + }); + + const [targetRun] = await runtimeEventStore.listSessionInvocations('session-target'); + assert.equal(targetRun?.runId, copied.runIdMap[0]?.targetRunId); + assert.equal(targetRun?.terminalEvent?.status, 'completed'); + assert.deepEqual(targetRun?.opening.configuration.cwd, root); + const targetEvents = await runtimeEventStore.readRuntimeEvents( + 'session-target', + targetRun!.runId, + ); + assert.deepEqual( + targetEvents.map((event) => event.content?.kind ?? event.status), + ['invocation_opened', 'text', 'completed'], + 'the copy is a fresh sequence, so the opening is its first event', + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + function prepareTestCopyPlan( source: RuntimeReadModelSessionView, copiedMessages: readonly StoredMessage[], - runStore: Pick, - runtimeEventStore: Pick, + runStore: Pick, + runtimeEventStore: Pick, ) { return prepareConversationRuntimeLedgerCopy({ sourceSessionId: 'session-source', @@ -3460,21 +3331,93 @@ function runtimeEvent(overrides: Partial): RuntimeEvent { }; } -function agentRunHeader(overrides: Partial): AgentRunHeader { - return { - runId: 'run', - invocationId: 'invocation', - sessionId: 'session-source', - turnId: 'turn', - status: 'completed', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'model', - cwd: '/tmp', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 2, - completedAt: 2, - ...overrides, +interface SeededRun { + runId?: string; + invocationId?: string; + sessionId?: string; + turnId?: string; + cwd?: string; + parentRunId?: string; + resumedFromRunId?: string; + agentId?: string; + agentName?: string; + openedAt?: number; + closedAt?: number; + /** How the run ended. `open` leaves it with no terminal event. */ + outcome?: 'completed' | 'failed' | 'aborted' | 'open'; +} + +/** The facts a test states about a run it seeds. Nothing keeps this shape after the seed. */ +function runFacts(overrides: SeededRun): SeededRun { + return { sessionId: 'session-source', ...overrides }; +} + +/** One invocation as a reader sees it, for tests that stub the store instead of writing to it. */ +function invocationRecord( + run: SeededRun & { source?: RuntimeEventInvocationOpenedContent['source'] } = {}, +): RuntimeInvocationRecord { + const openedAt = run.openedAt ?? 1; + return testInvocationRecord({ + sessionId: run.sessionId ?? 'session-source', + invocationId: run.invocationId ?? 'invocation', + runId: run.runId ?? 'run', + turnId: run.turnId ?? 'turn', + openedAt, + closedAt: run.closedAt ?? openedAt + 1, + ...(run.outcome === 'open' ? {} : { outcome: run.outcome ?? 'completed' }), + opening: invocationOpening(run), + }); +} + +function invocationOpening( + run: SeededRun & { source?: RuntimeEventInvocationOpenedContent['source'] }, +): RuntimeEventInvocationOpenedContent { + const lineage = { + ...(run.parentRunId ? { parentRunId: run.parentRunId } : {}), + ...(run.resumedFromRunId ? { resumedFromRunId: run.resumedFromRunId } : {}), + ...(run.agentId ? { agentId: run.agentId } : {}), + ...(run.agentName ? { agentName: run.agentName } : {}), }; + return testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'fake-connection', + llmConnectionSlug: 'fake', + modelId: 'model', + }, + configuration: { cwd: run.cwd ?? '/tmp' }, + ...(run.source ? { source: run.source } : {}), + ...(Object.keys(lineage).length > 0 ? { lineage } : {}), + }); +} + +/** + * Open one invocation on the spine. + * + * Every test here writes the run's own events afterwards, ending included, so + * the seed states only that the run began and what it was routed to. + */ +async function seedRun( + runtimeEventStore: Pick, + run: SeededRun = {}, +): Promise { + const event = invocationOpenedEvent(run); + await runtimeEventStore.appendRuntimeEvent(event.sessionId, event.runId, event); +} + +/** The opening event of one seeded run, for a test that writes its ledger in one batch. */ +function invocationOpenedEvent(run: SeededRun = {}): RuntimeEvent { + const identity = { + sessionId: run.sessionId ?? 'session-source', + invocationId: run.invocationId ?? 'invocation', + runId: run.runId ?? 'run', + turnId: run.turnId ?? 'turn', + }; + return buildInvocationOpenedEvent({ + id: `${identity.runId}-invocation-opened`, + run: identity, + openedAt: run.openedAt ?? 1, + opening: invocationOpening(run), + }); } diff --git a/packages/runtime/src/__tests__/execution-inspect.test.ts b/packages/runtime/src/__tests__/execution-inspect.test.ts index 9c1d4d46eb..46d75bcf25 100644 --- a/packages/runtime/src/__tests__/execution-inspect.test.ts +++ b/packages/runtime/src/__tests__/execution-inspect.test.ts @@ -22,17 +22,14 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, test } from 'node:test'; -import type { - AgentRunEvent, - AgentRunEventType, - AgentRunHeader, - EmittedAgentRunEvent, -} from '@maka/core/agent-run'; +import type { AgentRunEvent, AgentRunEventType, EmittedAgentRunEvent } from '@maka/core/agent-run'; +import { buildInvocationOpenedEvent } from '@maka/core/runtime-invocation'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { createSessionStore } from '@maka/storage/session-store'; import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; import { createWorkspaceRuntimeStore } from '@maka/storage/runtime-event-persistence'; import { inspectAgentRunDocument, renderAgentRunInspectTree } from '../execution-inspect.js'; +import { testInvocationOpening } from './invocation-fixture.js'; describe('versioned execution inspect documents', () => { test('reports unknown tool outcomes without copying Runtime payloads', async () => { @@ -46,9 +43,12 @@ describe('versioned execution inspect documents', () => { model: 'fake-model', permissionMode: 'ask', }); - const header = runHeader(session.id); - await runStore.createRun(header); - await runStore.appendEvent(session.id, RUN_ID, runEvent(session.id, 'run_completed')); + await runtimeStore.appendRuntimeEvent(session.id, RUN_ID, openingEvent(session.id)); + await runStore.appendEvent( + session.id, + RUN_ID, + runEvent(session.id, 'model_stream_completed'), + ); await runtimeStore.appendRuntimeEvent( session.id, RUN_ID, @@ -87,7 +87,9 @@ describe('versioned execution inspect documents', () => { eventId: 'call', }, ]); - assert.equal(document.sources.runtimeCoverage?.highWater.sequence, 1); + // The opening fact is the run's first runtime event, so the call and the + // terminal event that follow it sit at sequences 1 and 2. + assert.equal(document.sources.runtimeCoverage?.highWater.sequence, 2); assert.equal( document.diagnostics.some((item) => item.code === 'tool_response_missing'), true, @@ -107,22 +109,22 @@ const RUN_ID = 'run-1'; const TURN_ID = 'turn-1'; const TS = 1_800_000_000_000; -function runHeader(sessionId: string): AgentRunHeader { - return { - runId: RUN_ID, - invocationId: 'invocation-1', - sessionId, - turnId: TURN_ID, - status: 'completed', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/workspace', - permissionMode: 'ask', - createdAt: TS, - updatedAt: TS + 1, - completedAt: TS + 1, - }; +function openingEvent(sessionId: string) { + return buildInvocationOpenedEvent({ + id: 'rt-open', + run: { sessionId, invocationId: 'invocation-1', runId: RUN_ID, turnId: TURN_ID }, + openedAt: TS, + opening: testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'fake-connection', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { cwd: '/tmp/workspace' }, + }), + }); } function runEvent(sessionId: string, type: AgentRunEventType): EmittedAgentRunEvent { diff --git a/packages/runtime/src/__tests__/history-compact-checkpoint.test.ts b/packages/runtime/src/__tests__/history-compact-checkpoint.test.ts index 30fba8c08c..ef9100d561 100644 --- a/packages/runtime/src/__tests__/history-compact-checkpoint.test.ts +++ b/packages/runtime/src/__tests__/history-compact-checkpoint.test.ts @@ -19,7 +19,7 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { AgentRunEvent, AgentRunHeader, AgentRunStore } from '@maka/core/agent-run'; +import type { AgentRunEvent, AgentRunStore } from '@maka/core/agent-run'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { buildHistoryCompactCheckpoint, @@ -389,8 +389,8 @@ describe('history compact checkpoint', () => { previousCheckpointId: first.checkpointId, now: 20, }); + const runIds = ['run-1', 'run-2', 'run-3']; const store = new StubAgentRunStore( - [run('run-1', 10), run('run-2', 20), run('run-3', 30)], new Map([ ['run-1', [checkpointEvent('ledger-1', 'run-1', first, 10)]], ['run-2', [checkpointEvent('ledger-2', 'run-2', latest, 20)]], @@ -406,11 +406,15 @@ describe('history compact checkpoint', () => { ]), ); - const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger( + store, + 'session-1', + runIds, + ); assert.equal(loaded?.checkpointId, latest.checkpointId); assert.deepEqual( - (await loadHistoryCompactCheckpointsFromRunLedger(store, 'session-1')).map( + (await loadHistoryCompactCheckpointsFromRunLedger(store, 'session-1', runIds)).map( (checkpoint) => checkpoint.checkpointId, ), [first.checkpointId, latest.checkpointId], @@ -494,12 +498,16 @@ describe('history compact checkpoint', () => { }, now: 20, }); + const runIds = ['run-1']; const store = new StubAgentRunStore( - [run('run-1', 20)], new Map([['run-1', [checkpointEvent('ledger-v3', 'run-1', checkpoint, 20)]]]), ); - const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger( + store, + 'session-1', + runIds, + ); assert.deepEqual(loaded, checkpoint); assert.equal( @@ -527,15 +535,19 @@ describe('history compact checkpoint', () => { previousCheckpointId: valid.checkpointId, now: 20, }); + const runIds = ['run-valid', 'run-poisoned']; const store = new StubAgentRunStore( - [run('run-valid', 10), run('run-poisoned', 20)], new Map([ ['run-valid', [checkpointEvent('ledger-valid', 'run-valid', valid, 10)]], ['run-poisoned', [checkpointEvent('ledger-poisoned', 'run-poisoned', poisoned, 20)]], ]), ); - const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger( + store, + 'session-1', + runIds, + ); assert.equal(loaded?.checkpointId, valid.checkpointId); }); @@ -556,15 +568,19 @@ describe('history compact checkpoint', () => { previousCheckpointId: valid.checkpointId, now: 20, }); + const runIds = ['run-valid', 'run-poisoned']; const store = new StubAgentRunStore( - [run('run-valid', 10), run('run-poisoned', 20)], new Map([ ['run-valid', [checkpointEvent('ledger-valid', 'run-valid', valid, 10)]], ['run-poisoned', [checkpointEvent('ledger-poisoned', 'run-poisoned', poisoned, 20)]], ]), ); - const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger( + store, + 'session-1', + runIds, + ); assert.equal(loaded?.checkpointId, valid.checkpointId); }); @@ -641,15 +657,19 @@ describe('history compact checkpoint', () => { }), summaryFormat: 'sections_v1' as const, }; + const runIds = ['run-valid', 'run-marked']; const store = new StubAgentRunStore( - [run('run-valid', 10), run('run-marked', 20)], new Map([ ['run-valid', [checkpointEvent('ledger-valid', 'run-valid', valid, 10)]], ['run-marked', [checkpointEvent('ledger-marked', 'run-marked', markedMalformed, 20)]], ]), ); - const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger( + store, + 'session-1', + runIds, + ); assert.equal(loaded?.checkpointId, valid.checkpointId); }); @@ -682,11 +702,12 @@ describe('history compact checkpoint', () => { assert.equal(options.ifLedgerRevision, 'ledger-revision'); replacedEventIds.push(options?.replaceEventId); }, - listSessionRuns: async () => [run('run-canonical', 10)], readEvents: async () => [canonicalEvent], }; - const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1', [ + 'run-canonical', + ]); assert.equal(loaded?.checkpointId, valid.checkpointId); assert.deepEqual(replacedEventIds, [poisonedProjection.id]); @@ -705,15 +726,19 @@ describe('history compact checkpoint', () => { summary: 'stale coverage', summaryFormat: 'legacy_freeform', }); + const runIds = ['run-furthest', 'run-stale']; const store = new StubAgentRunStore( - [run('run-furthest', 10), run('run-stale', 20)], new Map([ ['run-furthest', [checkpointEvent('ledger-furthest', 'run-furthest', furthest, 30)]], ['run-stale', [checkpointEvent('ledger-stale', 'run-stale', stale, 40)]], ]), ); - const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger( + store, + 'session-1', + runIds, + ); assert.equal(loaded?.checkpointId, furthest.checkpointId); }); @@ -743,8 +768,8 @@ describe('history compact checkpoint', () => { previousCheckpointId: second.checkpointId, now: 30, }); + const runIds = ['parent-created-first', 'child-created-later']; const store = new StubAgentRunStore( - [run('parent-created-first', 10), run('child-created-later', 20)], new Map([ [ 'parent-created-first', @@ -759,7 +784,11 @@ describe('history compact checkpoint', () => { ], ]), ); - const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger( + store, + 'session-1', + runIds, + ); assert.equal(loaded?.checkpointId, tip.checkpointId); }); @@ -774,15 +803,14 @@ describe('history compact checkpoint', () => { const projectedEvent = checkpointEvent('projection-event', 'run-projection', checkpoint, 20); const store = { readEventProjection: async () => projectedEvent, - listSessionRuns: async () => { - throw new Error('run enumeration must stay cold'); - }, readEvents: async () => { throw new Error('run ledger reads must stay cold'); }, }; - const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1', [ + 'run-canonical', + ]); assert.equal(loaded?.checkpointId, checkpoint.checkpointId); }); @@ -790,15 +818,14 @@ describe('history compact checkpoint', () => { test('uses an empty bounded projection without enumerating run ledgers', async () => { const store = { readEventProjection: async () => null, - listSessionRuns: async () => { - throw new Error('run enumeration must stay cold'); - }, readEvents: async () => { throw new Error('run ledger reads must stay cold'); }, }; - const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1', [ + 'run-canonical', + ]); assert.equal(loaded, undefined); }); @@ -824,11 +851,12 @@ describe('history compact checkpoint', () => { assert.equal(options.ifLedgerRevision, 'ledger-revision'); repaired.push(repairedEvent); }, - listSessionRuns: async () => [run('run-recovered', 10)], readEvents: async () => [event], }; - const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1', [ + 'run-canonical', + ]); assert.equal(loaded?.checkpointId, checkpoint.checkpointId); assert.deepEqual(repaired, [event]); @@ -848,11 +876,12 @@ describe('history compact checkpoint', () => { repairEventProjection: async () => { repaired = true; }, - listSessionRuns: async () => [run('run-recovered', 10)], readEvents: async () => [event], }; - const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1', [ + 'run-canonical', + ]); assert.equal(loaded?.checkpointId, checkpoint.checkpointId); assert.equal(repaired, false); @@ -884,11 +913,12 @@ describe('history compact checkpoint', () => { assert.equal(options.ifLedgerRevision, 'ledger-revision'); replacedEventIds.push(options?.replaceEventId); }, - listSessionRuns: async () => [run('run-canonical', 10)], readEvents: async () => [canonicalEvent], }; - const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'); + const loaded = await loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1', [ + 'run-canonical', + ]); assert.equal(loaded?.checkpointId, checkpoint.checkpointId); assert.deepEqual(replacedEventIds, [invalidProjection.id]); @@ -899,14 +929,13 @@ describe('history compact checkpoint', () => { readEventProjection: async () => { throw new Error('damaged projection'); }, - listSessionRuns: async () => { + readEvents: async () => { throw new Error('ledger recovery failed'); }, - readEvents: async () => [], }; await assert.rejects( - loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1'), + loadLatestHistoryCompactCheckpointFromRunLedger(store, 'session-1', ['run-canonical']), /ledger recovery failed/, ); }); @@ -988,22 +1017,6 @@ function textEvent(index: number): RuntimeEvent { }; } -function run(runId: string, createdAt: number): AgentRunHeader { - return { - runId, - sessionId: 'session-1', - turnId: `turn-${runId}`, - status: 'completed', - backendKind: 'ai-sdk', - llmConnectionSlug: 'test', - modelId: 'test', - cwd: '/tmp', - permissionMode: 'ask', - createdAt, - updatedAt: createdAt, - }; -} - function checkpointEvent( id: string, runId: string, @@ -1022,28 +1035,12 @@ function checkpointEvent( } class StubAgentRunStore implements AgentRunStore { - constructor( - private readonly runs: AgentRunHeader[], - private readonly events: Map, - ) {} - - async listSessionRuns(): Promise { - return this.runs; - } + constructor(private readonly events: Map) {} async readEvents(_sessionId: string, runId: string): Promise { return this.events.get(runId) ?? []; } - async createRun(): Promise { - throw new Error('not implemented'); - } - async updateRun(): Promise { - throw new Error('not implemented'); - } - async readRun(): Promise { - throw new Error('not implemented'); - } async appendEvent(): Promise { throw new Error('not implemented'); } diff --git a/packages/runtime/src/__tests__/history-compaction.test.ts b/packages/runtime/src/__tests__/history-compaction.test.ts index 64432af88e..e4d66c4f10 100644 --- a/packages/runtime/src/__tests__/history-compaction.test.ts +++ b/packages/runtime/src/__tests__/history-compaction.test.ts @@ -19,7 +19,6 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { applyRuntimeEventHistoryCompact, @@ -28,6 +27,7 @@ import { type PlanHistoryCompactionInput, } from '../history-compaction.js'; import { HistoryCompactSummarizerError } from '../history-compact-summarizer.js'; +import { testInvocationRecord } from './invocation-fixture.js'; import { matchHistoryCompactCheckpointPrefix } from '../history-compact-checkpoint.js'; describe('safe compaction prefix selection', () => { @@ -192,7 +192,7 @@ describe('plan context compaction', () => { planInput({ phase: 'standalone', orderedEvents: events, - runHeaders: HEADERS_A, + invocations: RUNS_A, acceptedRoute: ROUTE_A, reserveTailEvents: 0, summarize: ({ coveredRuntimeEvents }) => { @@ -233,7 +233,7 @@ describe('plan context compaction', () => { planInput({ phase: 'standalone', orderedEvents: first, - runHeaders: HEADERS_A, + invocations: RUNS_A, acceptedRoute: ROUTE_A, reserveTailEvents: 0, summarize: () => { @@ -257,7 +257,7 @@ describe('plan context compaction', () => { planInput({ phase: 'standalone', orderedEvents: later, - runHeaders: HEADERS_A, + invocations: RUNS_A, acceptedRoute: ROUTE_A, reserveTailEvents: 0, previousCheckpoint: retreated.checkpoint, @@ -286,7 +286,7 @@ describe('plan context compaction', () => { planInput({ phase: 'standalone', orderedEvents: events, - runHeaders: HEADERS_A, + invocations: RUNS_A, acceptedRoute: ROUTE_A, reserveTailEvents: 0, summarize: () => { @@ -305,7 +305,7 @@ describe('plan context compaction', () => { test("a mixed-route session retreats to this route's own newest reply", async () => { // History can span runs on several routes. A span another model accepted // proves nothing about this summarizer's window, so the retreat targets the - // newest reply THIS route produced, found through the run headers. + // newest reply THIS route produced, found through each run's opening. const events = [ user('old-user', 'old-turn'), modelOnRun('mine', 'old-turn', 'run-1', 'accepted by this route'), @@ -317,10 +317,7 @@ describe('plan context compaction', () => { planInput({ phase: 'standalone', orderedEvents: events, - runHeaders: [ - runHeader('run-1', 'model-a', 'conn-a'), - runHeader('run-2', 'model-b', 'conn-b'), - ], + invocations: [runOn('run-1', 'model-a', 'conn-a'), runOn('run-2', 'model-b', 'conn-b')], acceptedRoute: ROUTE_A, reserveTailEvents: 0, summarize: ({ coveredRuntimeEvents }) => { @@ -348,7 +345,7 @@ describe('plan context compaction', () => { planInput({ phase: 'standalone', orderedEvents: [user('u1', 't1'), modelOnRun('theirs', 't1', 'run-2')], - runHeaders: [runHeader('run-2', 'model-b', 'conn-b')], + invocations: [runOn('run-2', 'model-b', 'conn-b')], acceptedRoute: ROUTE_A, reserveTailEvents: 0, summarize: () => { @@ -370,7 +367,7 @@ describe('plan context compaction', () => { planInput({ phase: 'standalone', orderedEvents: [user('u1', 't1'), user('u2', 't1'), user('u3', 't2')], - runHeaders: HEADERS_A, + invocations: RUNS_A, acceptedRoute: ROUTE_A, reserveTailEvents: 0, summarize: () => { @@ -560,24 +557,26 @@ function model(id: string, turnId: string, text: string = id): RuntimeEvent { function modelOnRun(id: string, turnId: string, runId: string, text: string = id): RuntimeEvent { return { ...model(id, turnId, text), runId, invocationId: runId }; } -function runHeader(runId: string, modelId: string, llmConnectionId: string): AgentRunHeader { - return { - runId, +/** A completed run opened on the named route. */ +function runOn(runId: string, modelId: string, llmConnectionId: string) { + return testInvocationRecord({ sessionId: 'session-1', + runId, turnId: 'turn-1', - status: 'completed', - backendKind: 'ai-sdk', - llmConnectionId, - llmConnectionSlug: llmConnectionId, - modelId, - cwd: '/tmp/maka', - permissionMode: 'ask', - createdAt: 1_800_000_000_000, - updatedAt: 1_800_000_000_000, - }; + outcome: 'completed', + opening: { + route: { + provenance: 'runtime', + backendKind: 'ai-sdk', + llmConnectionId, + llmConnectionSlug: llmConnectionId, + modelId, + }, + }, + }); } const ROUTE_A = { modelId: 'model-a', connectionId: 'conn-a' }; -const HEADERS_A = [runHeader('run-1', 'model-a', 'conn-a')]; +const RUNS_A = [runOn('run-1', 'model-a', 'conn-a')]; function call(id: string, callId: string, turnId: string): RuntimeEvent { return { diff --git a/packages/runtime/src/__tests__/invocation-fixture.ts b/packages/runtime/src/__tests__/invocation-fixture.ts new file mode 100644 index 0000000000..821c48818c --- /dev/null +++ b/packages/runtime/src/__tests__/invocation-fixture.ts @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { randomUUID } from 'node:crypto'; +import type { RuntimeEvent, RuntimeEventInvocationOpenedContent } from '@maka/core/runtime-event'; +import { + buildInvocationOpenedEvent, + type RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; + +export interface SeededInvocationIdentity { + readonly sessionId: string; + readonly invocationId: string; + readonly runId: string; + readonly turnId: string; +} + +export type TestInvocationOpeningOverrides = Omit< + Partial, + 'configuration' +> & { configuration?: Partial }; + +export interface SeedInvocationInput { + readonly sessionId: string; + readonly runId: string; + readonly turnId: string; + readonly invocationId?: string; + readonly openedAt?: number; + readonly opening?: TestInvocationOpeningOverrides; +} + +/** + * The opening a test gets when it does not care what the run was routed to. + * + * `configuration` merges field by field, so a test states only the setting it + * is about. `route` replaces whole: which fields it carries depends on where + * the route came from, and merging halves of two routes makes neither. + */ +export function testInvocationOpening( + overrides: TestInvocationOpeningOverrides = {}, +): RuntimeEventInvocationOpenedContent { + const { configuration, ...rest } = overrides; + return { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'fake-connection', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + ...rest, + configuration: { + cwd: '/tmp', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + ...configuration, + }, + root: overrides.root ?? { kind: 'user' }, + source: overrides.source ?? { kind: 'fresh' }, + }; +} + +/** + * One invocation as a reader sees it, without a store. + * + * `outcome` writes the terminal event that decides it; leaving it out leaves the + * invocation running, which is what "no terminal event" means everywhere else. + */ +export function testInvocationRecord(input: { + sessionId: string; + runId: string; + turnId: string; + invocationId?: string; + openedAt?: number; + closedAt?: number; + outcome?: 'completed' | 'failed' | 'aborted'; + failureClass?: string; + opening?: TestInvocationOpeningOverrides; +}): RuntimeInvocationRecord { + const invocationId = input.invocationId ?? input.runId; + const openedAt = input.openedAt ?? 1; + const identity = { + sessionId: input.sessionId, + invocationId, + runId: input.runId, + turnId: input.turnId, + }; + return { + ...identity, + openedAt, + opening: testInvocationOpening(input.opening), + ...(input.outcome + ? { + terminalEvent: { + id: `${invocationId}-terminal`, + ...identity, + ts: input.closedAt ?? openedAt + 1, + partial: false, + role: 'system', + author: 'system', + status: input.outcome, + actions: { + endInvocation: true, + ...(input.failureClass ? { stateDelta: { failureClass: input.failureClass } } : {}), + }, + }, + } + : {}), + }; +} + +/** The event that opens one invocation, ready to append. */ +export function testInvocationOpenedEvent(input: SeedInvocationInput): RuntimeEvent { + return buildInvocationOpenedEvent({ + id: randomUUID(), + run: { + sessionId: input.sessionId, + invocationId: input.invocationId ?? input.runId, + runId: input.runId, + turnId: input.turnId, + }, + openedAt: input.openedAt ?? Date.now(), + opening: testInvocationOpening(input.opening), + }); +} + +/** The one invocation that opened this run, or a failure naming what is missing. */ +export async function readInvocation( + stores: { + runtimeEventStore: { + listSessionInvocations(sessionId: string): Promise; + }; + }, + sessionId: string, + runId: string, +): Promise { + const found = (await stores.runtimeEventStore.listSessionInvocations(sessionId)).find( + (candidate) => candidate.runId === runId, + ); + if (!found) throw new Error(`Session ${sessionId} has no invocation for run ${runId}`); + return found; +} + +/** Open one invocation on the spine, the way the runtime would. */ +export async function seedInvocation( + runtimeEventStore: { + appendRuntimeEvent(sessionId: string, runId: string, event: RuntimeEvent): Promise; + }, + input: SeedInvocationInput, +): Promise { + const event = testInvocationOpenedEvent(input); + await runtimeEventStore.appendRuntimeEvent(input.sessionId, input.runId, event); + return { + sessionId: event.sessionId, + invocationId: event.invocationId, + runId: event.runId, + turnId: event.turnId, + }; +} diff --git a/packages/runtime/src/__tests__/latest-context-commit.test.ts b/packages/runtime/src/__tests__/latest-context-commit.test.ts index 5075d6cbb9..c04e4afc71 100644 --- a/packages/runtime/src/__tests__/latest-context-commit.test.ts +++ b/packages/runtime/src/__tests__/latest-context-commit.test.ts @@ -110,10 +110,12 @@ test('a real send seals its observation into SQLite and reconstructs it after re } let scanned = 0; + const sessionRunIds = (await runtimeEventStore.listSessionInvocations(session.id)).map( + (invocation) => invocation.runId, + ); const diagnostics = await readLatestContextDiagnostics( { - listSessionRuns: (sessionId) => runStore.listSessionRuns(sessionId), - readEvents: async (sessionId, runId) => { + readEvents: async (sessionId: string, runId: string) => { scanned += 1; return runStore.readEvents(sessionId, runId); }, @@ -122,6 +124,7 @@ test('a real send seals its observation into SQLite and reconstructs it after re runStore.repairEventProjection(sessionId, type, event, options), }, session.id, + sessionRunIds, ); assert.equal(diagnostics.status, 'available'); @@ -140,11 +143,10 @@ test('a real send seals its observation into SQLite and reconstructs it after re const reopened = createSqliteAgentRunStore(root); try { - const runs = await reopened.listSessionRuns(session.id); const canonicalAttempts = ( await Promise.all( - runs.map(async (run) => { - const events = await reopened.readEvents(session.id, run.runId); + sessionRunIds.map(async (runId) => { + const events = await reopened.readEvents(session.id, runId); return events .filter((event) => event.type === 'model_call_attempt_recorded') .map((event) => decodeModelCallAttempt(event.data)); @@ -161,8 +163,7 @@ test('a real send seals its observation into SQLite and reconstructs it after re let coldScans = 0; const cold = await readLatestContextDiagnostics( { - listSessionRuns: (sessionId) => reopened.listSessionRuns(sessionId), - readEvents: async (sessionId, runId) => { + readEvents: async (sessionId: string, runId: string) => { coldScans += 1; return reopened.readEvents(sessionId, runId); }, @@ -170,6 +171,7 @@ test('a real send seals its observation into SQLite and reconstructs it after re reopened.repairEventProjection(sessionId, type, event, options), }, session.id, + sessionRunIds, ); assert.ok(coldScans > 0, 'omitting the projection reader forces a restart-safe ledger fold'); @@ -252,14 +254,16 @@ test('an artifact captured before abort does not create a canonical sent attempt // Drain the aborted turn through the real AgentRun store. } - const runs = await runStore.listSessionRuns(session.id); + const runIds = (await runtimeEventStore.listSessionInvocations(session.id)).map( + (invocation) => invocation.runId, + ); const events = ( - await Promise.all(runs.map((run) => runStore.readEvents(session.id, run.runId))) + await Promise.all(runIds.map((runId) => runStore.readEvents(session.id, runId))) ).flat(); assert.equal(artifactWrites, 1); assert.equal(providerCalls, 0); assert.equal(events.filter((event) => event.type === 'model_call_attempt_recorded').length, 0); - assert.deepEqual(await readLatestContextDiagnostics(runStore, session.id), { + assert.deepEqual(await readLatestContextDiagnostics(runStore, session.id, runIds), { status: 'unavailable', reason: 'no_completed_request', }); diff --git a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts index 059acb4c08..b3e6e99556 100644 --- a/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts +++ b/packages/runtime/src/__tests__/mid-turn-capacity-backend.test.ts @@ -17,7 +17,8 @@ * under the License. */ -import type { AgentRunHeader, ModelCallCommit } from '@maka/core/agent-run'; +import type { ModelCallCommit } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; import { setImmediate as flushMacrotask } from 'node:timers/promises'; @@ -54,6 +55,7 @@ import { createTestAiSdkBackend, testToolResultArchive, } from './execution-boundary-test-helpers.js'; +import { testInvocationOpening } from './invocation-fixture.js'; const RAW_SPAN_ONE = 'RAW_SPAN_ONE_'.repeat(24); const RAW_SPAN_TWO = 'RAW_SPAN_TWO_'.repeat(160); @@ -71,7 +73,7 @@ interface MidTurnFixture { toolExecutions: string[]; summarizerCalls: number; priorEvents: RuntimeEvent[]; - priorRunHeaders: AgentRunHeader[]; + priorInvocations: RuntimeInvocationRecord[]; anchor: RuntimeEvent; /** The fixture's durable RuntimeEvent ledger for the current turn/run. */ ledger: RuntimeEvent[]; @@ -164,7 +166,7 @@ interface MidTurnFixtureOptions { /** Prior-turn RuntimeEvents appended after the shaped priors (e.g. a persisted usage anchor). */ extraPriorEvents?: readonly RuntimeEvent[]; /** Run headers for the prior turns, so a persisted anchor can be identity-gated. */ - priorRunHeaders?: readonly AgentRunHeader[]; + priorInvocations?: readonly RuntimeInvocationRecord[]; /** System prompt size sent through the provider's separate system field. */ systemPromptChars?: number; /** An always-active tool whose schema dominates the request payload. */ @@ -660,7 +662,7 @@ function buildFixture(options: MidTurnFixtureOptions = {}): MidTurnFixture { return fixture.ledgerReads; }, priorEvents, - priorRunHeaders: [...(options.priorRunHeaders ?? [])], + priorInvocations: [...(options.priorInvocations ?? [])], anchor, ledger, modelCalls, @@ -685,7 +687,7 @@ async function runFixtureTurn( text: ANCHOR_TEXT, context: [], runtimeContext: [...fixture.priorEvents], - runtimeContextRunHeaders: [...fixture.priorRunHeaders], + runtimeContextInvocations: [...fixture.priorInvocations], })) { if (consumer === 'slow') { // Scheduling perturbation: hold the durable write back across several @@ -1233,7 +1235,7 @@ function defineMidTurnSuite(consumer: ConsumerMode): void { finalAtSecondCall: true, firstStepUsage: { input: 150, output: 40 }, extraPriorEvents: [priorUsageEvent({ inputTokens: 300, outputTokens: 20 })], - priorRunHeaders: [priorRunHeader()], + priorInvocations: [priorRunInvocation()], }); await runFixtureTurn(fixture, consumer); @@ -1305,7 +1307,7 @@ function defineMidTurnSuite(consumer: ConsumerMode): void { singleRequest: true, finalStepUsage: { input: 3_716, output: 10 }, extraPriorEvents: [priorUsageEvent({ inputTokens: 3_716, outputTokens: 12 })], - priorRunHeaders: [priorRunHeader()], + priorInvocations: [priorRunInvocation()], }); await runFixtureTurn(fixture, consumer); @@ -1322,7 +1324,7 @@ function defineMidTurnSuite(consumer: ConsumerMode): void { singleRequest: true, finalStepUsage: { input: 4_000, output: 10 }, extraPriorEvents: [priorUsageEvent({ inputTokens: 3_716, outputTokens: 12 })], - priorRunHeaders: [priorRunHeader()], + priorInvocations: [priorRunInvocation()], }); await runFixtureTurn(fixture, consumer); @@ -1344,7 +1346,7 @@ function defineMidTurnSuite(consumer: ConsumerMode): void { singleRequest: true, finalStepUsage: { input: 900, output: 10 }, extraPriorEvents: [priorUsageEvent({ inputTokens: 3_716, outputTokens: 12 })], - priorRunHeaders: [priorRunHeader()], + priorInvocations: [priorRunInvocation()], }); await runFixtureTurn(fixture, consumer); @@ -1363,7 +1365,7 @@ function defineMidTurnSuite(consumer: ConsumerMode): void { singleRequest: true, finalStepUsage: { input: 3_716, output: 10 }, extraPriorEvents: [priorUsageEvent({ inputTokens: 3_716, outputTokens: 12 })], - priorRunHeaders: [priorRunHeader()], + priorInvocations: [priorRunInvocation()], }); await runFixtureTurn(fixture, consumer); @@ -1769,15 +1771,27 @@ describe('the shipped runtime default drives the proactive long-turn journey (is assert.equal(anchor?.outputTokens, 10); }); - test('an anchor is discarded unless a run header proves it came from this model', async () => { + test('an anchor is discarded unless its invocation proves it came from this model', async () => { // Input tokens are a count in one model's tokenizer; nothing converts them. - // The anchor sits ABOVE the declared window, so it is the header check - // alone that decides: a matching header folds at step 0, while a header - // naming another model and no header at all leave the request alone. + // The anchor sits ABOVE the declared window, so it is the opening's route + // alone that decides: a matching route folds at step 0, while a route + // naming another model and no invocation at all leave the request alone. const anchor = priorUsageEvent({ inputTokens: 30_000, outputTokens: 10 }); - for (const [priorRunHeaders, folds] of [ - [[priorRunHeader()], true], - [[{ ...priorRunHeader(), modelId: 'some-other-model' }], false], + const otherModel = priorRunInvocation(); + for (const [priorInvocations, folds] of [ + [[priorRunInvocation()], true], + [ + [ + { + ...otherModel, + opening: { + ...otherModel.opening, + route: { ...otherModel.opening.route, modelId: 'some-other-model' }, + }, + }, + ], + false, + ], [[], false], ] as const) { const fixture = buildFixture({ @@ -1785,7 +1799,7 @@ describe('the shipped runtime default drives the proactive long-turn journey (is contextWindow: 20_000, finalAtSecondCall: true, extraPriorEvents: [anchor], - priorRunHeaders: [...priorRunHeaders], + priorInvocations: [...priorInvocations], }); await runFixtureTurn(fixture); @@ -1814,7 +1828,7 @@ describe('the shipped runtime default drives the proactive long-turn journey (is actions: { tokenUsage: { input: 0, output: 0 } }, }, ], - priorRunHeaders: [priorRunHeader()], + priorInvocations: [priorRunInvocation()], }); await runFixtureTurn(fixture); @@ -1843,7 +1857,7 @@ describe('the shipped runtime default drives the proactive long-turn journey (is finalAtSecondCall: true, modelMaxOutputTokens: 600, extraPriorEvents: [priorUsageEvent({ inputTokens: 900, outputTokens: anchorOutput })], - priorRunHeaders: [priorRunHeader()], + priorInvocations: [priorRunInvocation()], }); await runFixtureTurn(fixture); // 960 + 120 crosses 1,000; 905 + 10 does not. The 600-token output limit @@ -1893,22 +1907,37 @@ function priorUsageEvent(lastRequestAnchor: { }; } -function priorRunHeader(): AgentRunHeader { - return { - runId: 'run-0', - invocationId: 'run-0', +/** The prior invocation on this route, as its own events describe it. */ +function priorRunInvocation(): RuntimeInvocationRecord { + const identity = { sessionId: 'session-1', + invocationId: 'run-0', + runId: 'run-0', turnId: 'turn-0', - status: 'completed', - backendKind: 'ai-sdk', - llmConnectionId: 'test-connection-id', - llmConnectionSlug: 'anthropic-main', - modelId: 'mock-model-id', - cwd: '/tmp/maka', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 2, - completedAt: 2, + }; + return { + ...identity, + openedAt: 1, + opening: testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'ai-sdk', + llmConnectionId: 'test-connection-id', + llmConnectionSlug: 'anthropic-main', + modelId: 'mock-model-id', + }, + configuration: { cwd: '/tmp/maka' }, + }), + terminalEvent: { + ...identity, + id: `${identity.runId}-terminal`, + ts: 2, + partial: false, + role: 'system', + author: 'system', + status: 'completed', + actions: { endInvocation: true }, + }, }; } diff --git a/packages/runtime/src/__tests__/model-projection-transition-ledger.test.ts b/packages/runtime/src/__tests__/model-projection-transition-ledger.test.ts index 9af813c7ed..fb54ca3e65 100644 --- a/packages/runtime/src/__tests__/model-projection-transition-ledger.test.ts +++ b/packages/runtime/src/__tests__/model-projection-transition-ledger.test.ts @@ -20,7 +20,7 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { AgentRunEvent, AgentRunHeader } from '@maka/core/agent-run'; +import type { AgentRunEvent } from '@maka/core/agent-run'; import { buildModelProjectionTransition, durableToolResultProjectionDigest, @@ -437,8 +437,6 @@ describe('transition ledger reads', () => { data, }); const runStore = { - listSessionRuns: async () => - [{ runId: 'run-1' }, { runId: 'run-2' }] as unknown as AgentRunHeader[], readEvents: async (_sessionId: string, runId: string): Promise => runId === 'run-1' ? [ @@ -453,7 +451,10 @@ describe('transition ledger reads', () => { : [ledgerEvent(`${transition.transitionId}-replay`, { transition })], }; - const loaded = await loadModelProjectionTransitionsFromRunLedger(runStore, 'session-1'); + const loaded = await loadModelProjectionTransitionsFromRunLedger(runStore, 'session-1', [ + 'run-1', + 'run-2', + ]); assert.deepEqual( loaded.transitions.map((entry) => entry.transitionId), diff --git a/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts b/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts index 8166923445..a778ba50c6 100644 --- a/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts +++ b/packages/runtime/src/__tests__/overflow-reactive-recovery.test.ts @@ -27,7 +27,8 @@ import type { SessionHeader } from '@maka/core/session'; import type { SessionEvent } from '@maka/core/events'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { z } from 'zod'; -import type { AgentRunHeader, ModelCallCommit } from '@maka/core/agent-run'; +import type { ModelCallCommit } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import { decodeModelCallAttempt, type ModelCallAttempt } from '@maka/core/model-call-attempt'; import { AiSdkBackend } from '../ai-sdk-backend.js'; import { @@ -49,6 +50,7 @@ import { createTestAiSdkBackend, testToolResultArchive, } from './execution-boundary-test-helpers.js'; +import { testInvocationOpening } from './invocation-fixture.js'; // The checkpoint write gate validates summary structure and floors the size // for large folds (#3029), so the stub summary is shaped like a real @@ -204,7 +206,7 @@ interface ReactiveFixture { summarizerCalls: () => number; anchor: RuntimeEvent; priorEvents: RuntimeEvent[]; - priorRunHeaders: AgentRunHeader[]; + priorInvocations: RuntimeInvocationRecord[]; events: SessionEvent[]; messages: unknown[]; llmCalls: ReactiveLlmCall[]; @@ -495,10 +497,10 @@ function buildReactiveFixture(options: ReactiveFixtureOptions): ReactiveFixture ] : []), ]; - const priorRunHeaders: AgentRunHeader[] = options.reasoningReplayTail + const priorInvocations: RuntimeInvocationRecord[] = options.reasoningReplayTail ? [ - priorRunHeader('same-route-prior-run', 'test-connection-id', 'mock-model-id'), - priorRunHeader('prior-run', 'source-connection-id', 'source-model-id'), + priorRunInvocation('same-route-prior-run', 'test-connection-id', 'mock-model-id'), + priorRunInvocation('prior-run', 'source-connection-id', 'source-model-id'), ] : []; const anchor: RuntimeEvent = { @@ -713,7 +715,7 @@ function buildReactiveFixture(options: ReactiveFixtureOptions): ReactiveFixture summarizerCalls: () => counters.summarizerCalls, anchor, priorEvents, - priorRunHeaders, + priorInvocations, events, messages, llmCalls, @@ -736,7 +738,7 @@ async function runTurn( text: ANCHOR_TEXT, context: [], runtimeContext: [...fixture.priorEvents], - runtimeContextRunHeaders: fixture.priorRunHeaders, + runtimeContextInvocations: [...fixture.priorInvocations], ...(pullSteering ? { pullSteering } : {}), })) { if (consumer === 'slow') { @@ -1988,23 +1990,43 @@ function header(): SessionHeader { }; } -function priorRunHeader(runId: string, llmConnectionId: string, modelId: string): AgentRunHeader { - return { - runId, +/** One prior invocation, as its own opening fact and terminal event describe it. */ +function priorRunInvocation( + runId: string, + llmConnectionId: string, + modelId: string, +): RuntimeInvocationRecord { + const identity = { sessionId: 'session-1', + invocationId: `invocation-${runId}`, + runId, turnId: 'turn-0', - status: 'completed', - backendKind: 'ai-sdk', - llmConnectionId, - llmConnectionSlug: 'anthropic-source', - modelId, - providerStateIdentity: - runId === 'same-route-prior-run' ? PROVIDER_STATE_IDENTITY : `sha256:${'2'.repeat(64)}`, - cwd: '/tmp/maka', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 2, - completedAt: 2, + }; + return { + ...identity, + openedAt: 1, + opening: testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'ai-sdk', + llmConnectionId: llmConnectionId, + llmConnectionSlug: 'anthropic-source', + modelId: modelId, + providerStateIdentity: + runId === 'same-route-prior-run' ? PROVIDER_STATE_IDENTITY : `sha256:${'2'.repeat(64)}`, + }, + configuration: { cwd: '/tmp/maka' }, + }), + terminalEvent: { + ...identity, + id: `${identity.runId}-terminal`, + ts: 2, + partial: false, + role: 'system', + author: 'system', + status: 'completed', + actions: { endInvocation: true }, + }, }; } diff --git a/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts b/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts index 62c815dae5..a44272cda5 100644 --- a/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts +++ b/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts @@ -26,9 +26,12 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; - import type { RuntimeEvent } from '@maka/core/runtime-event'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; +import { + buildInvocationOpenedEvent, + runtimeInvocationOutcome, +} from '@maka/core/runtime-invocation'; import { createSessionStore } from '@maka/storage/session-store'; import { createSqliteRuntimeStore } from '@maka/storage/sqlite-runtime-store'; import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; @@ -37,16 +40,15 @@ import { type RuntimeContinuationFailpoint } from '../agent-run.js'; import { BackendRegistry, SessionManager } from '../session-manager.js'; import { FakeBackend } from '../test-only/fake-backend.js'; import { terminateChildProcessTree } from '../process-tree-terminator.js'; +import { testInvocationOpening } from './invocation-fixture.js'; const CRASH_CHILD_ENV = 'MAKA_RUNTIME_CONTINUATION_CRASH_CHILD'; const CRASH_CHILD_READY_TIMEOUT_MS = process.platform === 'win32' ? 30_000 : 10_000; const CRASH_HARNESS_TIMEOUT_MS = process.platform === 'win32' ? 180_000 : 60_000; const FAILPOINTS: readonly RuntimeContinuationFailpoint[] = [ 'after_continuation_claim_committed', - 'after_run_created', 'after_continuation_start_committed', 'after_terminal_event_committed', - 'after_terminal_header_committed', ]; if (process.env[CRASH_CHILD_ENV] === '1') { @@ -71,9 +73,11 @@ if (process.env[CRASH_CHILD_ENV] === '1') { session.id, ); assert.ok(claimState, `${failpoint} did not persist the continuation claim`); - const runsBeforeRecovery = await runStore.listSessionRuns(session.id); - const continuation = runsBeforeRecovery.find( - (run) => run.runId === claimState.claim.target.runId, + const invocationsBeforeRecovery = await runtimeEventStore.listSessionInvocations( + session.id, + ); + const continuation = invocationsBeforeRecovery.find( + (invocation) => invocation.runId === claimState.claim.target.runId, ); const prefix = await runtimeEventStore.readRuntimeEvents( session.id, @@ -93,10 +97,10 @@ if (process.env[CRASH_CHILD_ENV] === '1') { sourceRunId: 'source-run', }); assert.equal(repeatedPlan.disposition, 'park'); + // A crash after the terminal event is not an unfinished claim: the + // event is the continuation's ending, so the boundary already has one. assert.deepEqual(repeatedPlan.rejectionReasons, [ - failpoint === 'after_continuation_claim_committed' || - failpoint === 'after_run_created' || - failpoint === 'after_terminal_event_committed' + failpoint === 'after_continuation_claim_committed' ? 'continuation_claim_repair_required' : failpoint === 'after_continuation_start_committed' ? 'continuation_started_indeterminate' @@ -104,7 +108,11 @@ if (process.env[CRASH_CHILD_ENV] === '1') { ]); await manager.recoverInterruptedSessions(); - const repaired = await runStore.readRun(session.id, claimState.claim.target.runId); + const repaired = await readInvocation( + recoveryRuntimeStore, + session.id, + claimState.claim.target.runId, + ); const repairedEvents = await recoveryRuntimeStore.readRuntimeEvents( session.id, claimState.claim.target.runId, @@ -114,7 +122,7 @@ if (process.env[CRASH_CHILD_ENV] === '1') { ); if (failpoint === 'after_continuation_start_committed') { assert.equal(terminalEvents.length, 0); - assert.equal(['created', 'running'].includes(repaired.status), true); + assert.equal(runtimeInvocationOutcome(repaired), undefined); const parked = await manager.planAuthoritativeSafeBoundaryContinuation(session.id, { sourceRunId: 'source-run', }); @@ -122,9 +130,7 @@ if (process.env[CRASH_CHILD_ENV] === '1') { } else { assert.equal(terminalEvents.length, 1, `${failpoint} must recover one terminal fact`); assert.ok( - repaired.status === 'completed' || - repaired.status === 'failed' || - repaired.status === 'cancelled', + runtimeInvocationOutcome(repaired), `${failpoint} left the continuation non-terminal`, ); } @@ -181,7 +187,7 @@ async function runCrashChild(): Promise { safeBoundaryResumeEnabled: true, inspectContinuationSafety: async () => stableSafetyObservation(), continuationFailpoint: async (point) => { - if (point !== failpoint || point === 'after_terminal_header_committed') return; + if (point !== failpoint) return; await suspendCrashChild(point, resolveSelectedFailpoint); }, newId: () => `id-${++id}`, @@ -197,8 +203,7 @@ async function runCrashChild(): Promise { permissionMode: 'ask', name: 'continuation crash child', }); - await runStore.createRun(sourceHeader(session.id, workspaceRoot)); - for (const event of sourceEvents(session.id)) { + for (const event of sourceEvents(session.id, workspaceRoot)) { await runtimeEventStore.appendRuntimeEvent(session.id, 'source-run', event); } const plan = await manager.planAuthoritativeSafeBoundaryContinuation(session.id, { @@ -209,13 +214,6 @@ async function runCrashChild(): Promise { for await (const _event of manager.resumeSafeBoundaryContinuation(plan.continuation)) { // drain until the selected failpoint suspends the child } - if (failpoint === 'after_terminal_header_committed') { - const continuation = await runStore.readRun(session.id, plan.continuation.runId); - if (continuation.status !== 'completed') { - throw new Error(`continuation terminal header did not settle: ${continuation.status}`); - } - await suspendCrashChild(failpoint, resolveSelectedFailpoint); - } // Terminal projection finalization may continue after the public event stream // closes. Wait for the selected durable boundary instead of racing that // background finalizer and reporting a false negative. @@ -329,24 +327,40 @@ function killCrashChild(child: ReturnType): Promise { return Promise.resolve(child.kill('SIGKILL')); } +/** The one invocation that opened this run, once its ledger says it opened. */ +async function readInvocation( + runtimeEventStore: ReturnType, + sessionId: string, + runId: string, +): Promise { + const found = (await runtimeEventStore.listSessionInvocations(sessionId)).find( + (invocation) => invocation.runId === runId, + ); + if (!found) throw new Error(`Runtime invocation not found: ${runId}`); + return found; +} + +/** + * What a crash at each boundary left durable. + * + * A continuation's opening fact rides its continuation-start event, so a crash + * before that commit leaves the target invocation unopened. There is no separate + * run record left over to disagree with the ledger. + */ function assertPrefix( failpoint: RuntimeContinuationFailpoint, - header: AgentRunHeader | undefined, + invocation: RuntimeInvocationRecord | undefined, events: readonly RuntimeEvent[], ): void { if (failpoint === 'after_continuation_claim_committed') { - assert.equal(header, undefined); - assert.deepEqual(events, []); - return; - } - assert.ok(header); - if (failpoint === 'after_run_created') { - assert.equal(header.status, 'created'); + assert.equal(invocation, undefined); assert.deepEqual(events, []); return; } + assert.ok(invocation); assert.equal(events[0]?.actions?.continuationStart?.protocol, 'continuation_start_v2'); if (failpoint === 'after_continuation_start_committed') { + assert.equal(runtimeInvocationOutcome(invocation), undefined); assert.equal( events.some((event) => event.actions?.endInvocation === true), false, @@ -354,34 +368,10 @@ function assertPrefix( return; } assert.equal(events.filter((event) => event.actions?.endInvocation === true).length, 1); - if (failpoint === 'after_terminal_event_committed') { - assert.equal(['created', 'running'].includes(header.status), true); - return; - } - assert.equal(header.status, 'completed'); + assert.ok(runtimeInvocationOutcome(invocation)); } -function sourceHeader(sessionId: string, cwd: string): AgentRunHeader { - return { - runId: 'source-run', - invocationId: 'source-invocation', - sessionId, - turnId: 'source-turn', - status: 'failed', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd, - workspaceIdentity: 'workspace-1', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 2, - completedAt: 2, - failureClass: 'app_restarted', - }; -} - -function sourceEvents(sessionId: string): RuntimeEvent[] { +function sourceEvents(sessionId: string, cwd: string): RuntimeEvent[] { const identity = { sessionId, invocationId: 'source-invocation', @@ -389,6 +379,29 @@ function sourceEvents(sessionId: string): RuntimeEvent[] { turnId: 'source-turn', }; return [ + buildInvocationOpenedEvent({ + id: 'source-open', + run: identity, + openedAt: 1, + opening: testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'fake-connection', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd, + workspaceIdentity: 'workspace-1', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: 'direct', + }, + }), + }), { ...identity, id: 'source-user', diff --git a/packages/runtime/src/__tests__/runtime-continuation.test.ts b/packages/runtime/src/__tests__/runtime-continuation.test.ts index 678a0f96fe..07cb84cf98 100644 --- a/packages/runtime/src/__tests__/runtime-continuation.test.ts +++ b/packages/runtime/src/__tests__/runtime-continuation.test.ts @@ -26,8 +26,8 @@ import { runtimePrefixSegment, type ImmutableRuntimePrefixV1, } from '@maka/core/runtime-boundary'; -import type { RuntimeEvent } from '@maka/core/runtime-event'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeEvent, RuntimeEventInvocationOpenedContent } from '@maka/core/runtime-event'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import { createLocalContinuationSafetyInspector } from '../continuation-safety.js'; import { buildContinuationReplayPlan } from '../continuation-replay.js'; @@ -40,6 +40,7 @@ import { buildSafeBoundaryContinuationPlan, type RuntimeContinuation, } from '../runtime-resume.js'; +import { testInvocationRecord } from './invocation-fixture.js'; test('local continuation safety inspector returns current authoritative workspace facts', async () => { const inspect = createLocalContinuationSafetyInspector({ @@ -77,9 +78,10 @@ test('RuntimeContinuationPlanner reads the durable source boundary and allocates }), ]; const sourcePrefix = immutablePrefix(sourceEvents); - const ids = ['invocation-2', 'run-2', 'turn-2', 'claim-2']; + // Run and invocation are one identity, so the planner mints three ids, not four. + const ids = ['invocation-2', 'turn-2', 'claim-2']; const planner = new RuntimeContinuationPlanner({ - readSourceRun: async () => runHeader('run-1'), + readSourceInvocation: async () => runInvocation('run-1'), readImmutableRuntimePrefix: async () => sourcePrefix, newId: () => ids.shift() ?? 'unexpected-id', }); @@ -99,7 +101,7 @@ test('RuntimeContinuationPlanner reads the durable source boundary and allocates assert.deepEqual(plan.continuation, { sessionId: 'session-1', invocationId: 'invocation-2', - runId: 'run-2', + runId: 'invocation-2', turnId: 'turn-2', sourceInvocationId: 'invocation-1', sourceRunId: 'run-1', @@ -131,7 +133,7 @@ test('RuntimeContinuationPlanner reads the durable source boundary and allocates test('RuntimeContinuationPlanner parks with a stable reason when the ledger cannot be read', async () => { const planner = new RuntimeContinuationPlanner({ - readSourceRun: async () => runHeader('run-1'), + readSourceInvocation: async () => runInvocation('run-1'), readImmutableRuntimePrefix: async () => { throw new Error('corrupt ledger'); }, @@ -155,7 +157,7 @@ test('RuntimeContinuationPlanner parks with a stable reason when the ledger cann test('RuntimeContinuationPlanner derives terminal repair from durable run and event facts', async () => { const planner = new RuntimeContinuationPlanner({ - readSourceRun: async () => runHeader('run-1', { status: 'running' }), + readSourceInvocation: async () => runInvocation('run-1', { outcome: 'open' }), readImmutableRuntimePrefix: async () => immutablePrefix([ event({ @@ -183,17 +185,11 @@ test('RuntimeContinuationPlanner derives terminal repair from durable run and ev assert.deepEqual(plan.rejectionReasons, ['terminal_repair_failed']); }); -test('RuntimeContinuationPlanner parks when the terminal run header disagrees with the ledger fact', async () => { +test('RuntimeContinuationPlanner parks when the source ledger does not end on its terminal fact', async () => { const planner = new RuntimeContinuationPlanner({ - readSourceRun: async () => runHeader('run-1', { status: 'completed' }), + readSourceInvocation: async () => runInvocation('run-1', { outcome: 'completed' }), readImmutableRuntimePrefix: async () => immutablePrefix([ - event({ - id: 'source-user', - role: 'user', - author: 'user', - content: { kind: 'text', text: 'continue' }, - }), event({ id: 'source-terminal', role: 'system', @@ -201,6 +197,12 @@ test('RuntimeContinuationPlanner parks when the terminal run header disagrees wi status: 'failed', actions: { endInvocation: true }, }), + event({ + id: 'source-user', + role: 'user', + author: 'user', + content: { kind: 'text', text: 'continue' }, + }), ]), newId: () => 'fresh-id', }); @@ -222,7 +224,7 @@ test('RuntimeContinuationPlanner parks when the terminal run header disagrees wi test('RuntimeContinuationPlanner rejects immutable output after the source terminal fact', async () => { const planner = new RuntimeContinuationPlanner({ - readSourceRun: async () => runHeader('run-1'), + readSourceInvocation: async () => runInvocation('run-1'), readImmutableRuntimePrefix: async () => immutablePrefix([ event({ @@ -267,7 +269,7 @@ test('RuntimeContinuationPlanner rejects immutable output after the source termi test('RuntimeContinuationPlanner uses canonical provider items for composite head and tail gates', async () => { let nextId = 0; const planner = new RuntimeContinuationPlanner({ - readSourceRun: async () => runHeader('run-1'), + readSourceInvocation: async () => runInvocation('run-1'), readImmutableRuntimePrefix: async () => immutablePrefix([ event({ @@ -313,7 +315,7 @@ test('RuntimeContinuationPlanner uses canonical provider items for composite hea test('RuntimeContinuationPlanner rejects a ledger returned for another source run', async () => { const planner = new RuntimeContinuationPlanner({ - readSourceRun: async () => runHeader('run-1'), + readSourceInvocation: async () => runInvocation('run-1'), readImmutableRuntimePrefix: async () => immutablePrefix([ event({ @@ -352,16 +354,18 @@ test('RuntimeContinuationPlanner rejects a ledger returned for another source ru test('RuntimeContinuationPlanner fails a cyclic continuation lineage closed', async () => { const runs = { - 'run-1': runHeader('run-1', { - continuationSource: { + 'run-1': runInvocation('run-1', { + source: { + kind: 'continuation' as const, sourceInvocationId: 'invocation-2', sourceRunId: 'run-2', sourceTurnId: 'turn-2', sourceRuntimeEventHighWater: 1, }, }), - 'run-2': runHeader('run-2', { - continuationSource: { + 'run-2': runInvocation('run-2', { + source: { + kind: 'continuation' as const, sourceInvocationId: 'invocation-1', sourceRunId: 'run-1', sourceTurnId: 'turn-1', @@ -374,7 +378,7 @@ test('RuntimeContinuationPlanner fails a cyclic continuation lineage closed', as ['run-2', prefixForIdentity('invocation-2', 'run-2', 'turn-2')], ]); const planner = new RuntimeContinuationPlanner({ - readSourceRun: async (_sessionId, runId) => runs[runId as keyof typeof runs], + readSourceInvocation: async (_sessionId, runId) => runs[runId as keyof typeof runs], readImmutableRuntimePrefix: async ({ runId }) => prefixes.get(runId)!, newId: () => 'unused', }); @@ -397,10 +401,11 @@ test('RuntimeContinuationPlanner fails a cyclic continuation lineage closed', as test('RuntimeContinuationPlanner parks when a continuation ancestor is unavailable', async () => { const source = prefixForIdentity('invocation-2', 'run-2', 'turn-2'); const planner = new RuntimeContinuationPlanner({ - readSourceRun: async (_sessionId, runId) => { + readSourceInvocation: async (_sessionId, runId) => { if (runId === 'run-2') { - return runHeader('run-2', { - continuationSource: { + return runInvocation('run-2', { + source: { + kind: 'continuation' as const, sourceInvocationId: 'invocation-1', sourceRunId: 'run-missing', sourceTurnId: 'turn-1', @@ -433,14 +438,15 @@ test('RuntimeContinuationPlanner parks when a continuation ancestor is unavailab }); test('RuntimeContinuationPlanner caps continuation lineage at 64 segments', async () => { - const runs = new Map(); + const runs = new Map(); const prefixes = new Map(); for (let index = 1; index <= 64; index += 1) { const runId = `run-${index}`; runs.set( runId, - runHeader(runId, { - continuationSource: { + runInvocation(runId, { + source: { + kind: 'continuation' as const, sourceInvocationId: `invocation-${index + 1}`, sourceRunId: `run-${index + 1}`, sourceTurnId: `turn-${index + 1}`, @@ -451,7 +457,7 @@ test('RuntimeContinuationPlanner caps continuation lineage at 64 segments', asyn prefixes.set(runId, prefixForIdentity(`invocation-${index}`, runId, `turn-${index}`)); } const planner = new RuntimeContinuationPlanner({ - readSourceRun: async (_sessionId, runId) => { + readSourceInvocation: async (_sessionId, runId) => { const run = runs.get(runId); if (!run) throw new Error('unexpected lineage read'); return run; @@ -503,21 +509,21 @@ test('RuntimeContinuationPlanner verifies a v2 lineage edge prefix digest', asyn highWater: 1, prefixDigest: 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', }, - replayManifestDigest: - 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', providerProjectionVersion: 1, providerReplayDigest: 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + replayManifestDigest: + 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', }, }, }), ]); const planner = new RuntimeContinuationPlanner({ - readSourceRun: async (_sessionId, runId) => + readSourceInvocation: async (_sessionId, runId) => runId === 'run-2' - ? runHeader('run-2', { - continuationSource: { - protocol: 'continuation_source_v2', + ? runInvocation('run-2', { + source: { + kind: 'continuation' as const, claimId: 'claim-1', boundaryDigest: 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', @@ -525,13 +531,9 @@ test('RuntimeContinuationPlanner verifies a v2 lineage edge prefix digest', asyn sourceRunId: 'run-1', sourceTurnId: 'turn-1', sourceRuntimeEventHighWater: 1, - sourcePrefixDigest: - 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', - replayManifestDigest: - 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', }, }) - : runHeader('run-1'), + : runInvocation('run-1'), readImmutableRuntimePrefix: async ({ runId }) => (runId === 'run-2' ? source : ancestor), newId: () => 'unused', }); @@ -594,22 +596,20 @@ test('RuntimeContinuationPlanner binds every v2 lineage edge to its continuation }), ]); const planner = new RuntimeContinuationPlanner({ - readSourceRun: async (_sessionId, runId) => + readSourceInvocation: async (_sessionId, runId) => runId === 'run-2' - ? runHeader('run-2', { - continuationSource: { - protocol: 'continuation_source_v2', + ? runInvocation('run-2', { + source: { + kind: 'continuation' as const, claimId: 'claim-expected', boundaryDigest: ancestorBoundary.manifestDigest, sourceInvocationId: ancestor.identity.invocationId, sourceRunId: ancestor.identity.runId, sourceTurnId: ancestor.identity.turnId, sourceRuntimeEventHighWater: ancestor.position.lastEventSeq, - sourcePrefixDigest: ancestor.prefixDigest, - replayManifestDigest: ancestorBoundary.manifestDigest, }, }) - : runHeader('run-1'), + : runInvocation('run-1'), readImmutableRuntimePrefix: async ({ runId }) => (runId === 'run-2' ? source : ancestor), newId: () => 'unused', }); @@ -678,17 +678,18 @@ test('RuntimeContinuationPlanner rejects downgrading a canonical v2 start to leg }), ]); const planner = new RuntimeContinuationPlanner({ - readSourceRun: async (_sessionId, runId) => + readSourceInvocation: async (_sessionId, runId) => runId === sourceIdentity.runId - ? runHeader(sourceIdentity.runId, { - continuationSource: { + ? runInvocation(sourceIdentity.runId, { + source: { + kind: 'continuation' as const, sourceInvocationId: ancestor.identity.invocationId, sourceRunId: ancestor.identity.runId, sourceTurnId: ancestor.identity.turnId, sourceRuntimeEventHighWater: ancestor.position.lastEventSeq, }, }) - : runHeader('run-1'), + : runInvocation('run-1'), readImmutableRuntimePrefix: async ({ runId }) => runId === sourceIdentity.runId ? source : ancestor, newId: () => 'unused', @@ -758,22 +759,20 @@ test('RuntimeContinuationPlanner requires a durable target before authenticating }), ]); const planner = new RuntimeContinuationPlanner({ - readSourceRun: async (_sessionId, runId) => + readSourceInvocation: async (_sessionId, runId) => runId === sourceIdentity.runId - ? runHeader(sourceIdentity.runId, { - continuationSource: { - protocol: 'continuation_source_v2', + ? runInvocation(sourceIdentity.runId, { + source: { + kind: 'continuation' as const, claimId: 'claim-1', boundaryDigest: ancestorReplay.plan.boundary.manifestDigest, sourceInvocationId: ancestor.identity.invocationId, sourceRunId: ancestor.identity.runId, sourceTurnId: ancestor.identity.turnId, sourceRuntimeEventHighWater: ancestor.position.lastEventSeq, - sourcePrefixDigest: ancestor.prefixDigest, - replayManifestDigest: ancestorReplay.plan.boundary.manifestDigest, }, }) - : runHeader('run-1'), + : runInvocation('run-1'), readImmutableRuntimePrefix: async ({ runId }) => runId === sourceIdentity.runId ? source : ancestor, newId: () => 'unused', @@ -842,22 +841,20 @@ test('RuntimeContinuationPlanner rejects a v2 lineage edge whose durable claim i actions: { endInvocation: true, stateDelta: { failureClass: 'test_failure' } }, }), ]); - const sourceRun = runHeader(sourceIdentity.runId, { - continuationSource: { - protocol: 'continuation_source_v2', + const sourceRun = runInvocation(sourceIdentity.runId, { + source: { + kind: 'continuation' as const, claimId: 'claim-1', boundaryDigest: ancestorReplay.plan.boundary.manifestDigest, sourceInvocationId: ancestor.identity.invocationId, sourceRunId: ancestor.identity.runId, sourceTurnId: ancestor.identity.turnId, sourceRuntimeEventHighWater: ancestor.position.lastEventSeq, - sourcePrefixDigest: ancestor.prefixDigest, - replayManifestDigest: ancestorReplay.plan.boundary.manifestDigest, }, }); const planner = new RuntimeContinuationPlanner({ - readSourceRun: async (_sessionId, runId) => - runId === sourceIdentity.runId ? sourceRun : runHeader('run-1'), + readSourceInvocation: async (_sessionId, runId) => + runId === sourceIdentity.runId ? sourceRun : runInvocation('run-1'), readImmutableRuntimePrefix: async ({ runId }) => runId === sourceIdentity.runId ? source : ancestor, readContinuationClaimStateByBoundary: async () => undefined, @@ -881,33 +878,50 @@ test('RuntimeContinuationPlanner rejects a v2 lineage edge whose durable claim i function sameRouteAdmission() { return { - runHeaders: ['run-1', 'run-2', 'run-3'].map((runId) => - runHeader(runId, { llmConnectionId: 'connection-1' }), - ), + invocations: ['run-1', 'run-2', 'run-3'].map((runId) => runInvocation(runId)), targetProviderStateIdentity: undefined, targetModelId: 'test-model', }; } -function runHeader(runId: string, overrides: Partial = {}): AgentRunHeader { +interface RunFacts { + source?: RuntimeEventInvocationOpenedContent['source']; + outcome?: 'completed' | 'failed' | 'aborted' | 'open'; + failureClass?: string; + providerStateIdentity?: `sha256:${string}`; + modelId?: string; + cwd?: string; +} + +/** One source invocation as the planner reads it back off the spine. */ +function runInvocation(runId: string, facts: RunFacts = {}): RuntimeInvocationRecord { const ordinal = runId.match(/(\d+)$/)?.[1] ?? '1'; - const status = overrides.status ?? 'failed'; - return { - runId, - invocationId: `invocation-${ordinal}`, + const outcome = facts.outcome ?? 'failed'; + const failureClass = outcome === 'failed' ? (facts.failureClass ?? 'test_failure') : undefined; + return testInvocationRecord({ sessionId: 'session-1', + invocationId: `invocation-${ordinal}`, + runId, turnId: `turn-${ordinal}`, - status, - backendKind: 'fake', - llmConnectionSlug: 'test', - modelId: 'test-model', - cwd: '/workspace/repo', - permissionMode: 'ask', - ...(status === 'failed' ? { failureClass: 'test_failure' } : {}), - createdAt: 1, - updatedAt: 1, - ...overrides, - }; + openedAt: 1, + closedAt: 1, + ...(outcome === 'open' ? {} : { outcome }), + ...(failureClass ? { failureClass } : {}), + opening: { + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'connection-1', + llmConnectionSlug: 'test', + modelId: facts.modelId ?? 'test-model', + ...(facts.providerStateIdentity + ? { providerStateIdentity: facts.providerStateIdentity } + : {}), + }, + configuration: { cwd: facts.cwd ?? '/workspace/repo' }, + ...(facts.source ? { source: facts.source } : {}), + }, + }); } function event(overrides: Partial): RuntimeEvent { diff --git a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts index c914c32745..c47760680f 100644 --- a/packages/runtime/src/__tests__/runtime-event-read-model.test.ts +++ b/packages/runtime/src/__tests__/runtime-event-read-model.test.ts @@ -19,7 +19,7 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { CreateSessionInput, SessionListFilter } from '@maka/core/runtime-inputs'; import type { RuntimeEvent, RuntimeEventActions } from '@maka/core/runtime-event'; import type { SessionHeader, SessionSummary, StoredMessage, TurnRecord } from '@maka/core/session'; @@ -34,6 +34,7 @@ import { import { buildRuntimeEventModelReplayPlan } from '../model-history.js'; import { backfillRuntimeEventsFromStoredMessages } from '../runtime-event-backfill.js'; import { BackendRegistry, SessionManager, type SessionStore } from '../session-manager.js'; +import { testInvocationOpening } from './invocation-fixture.js'; const ts = 1_800_000_000_000; const sessionId = 'sess-1'; @@ -42,20 +43,50 @@ const turnId = 'turn-1'; const invocationId = 'inv-1'; let eventSeq = 0; -const header: AgentRunHeader = { - runId, +/** The same invocation, ended a different way. */ +function endedAs( + status: 'completed' | 'failed' | 'aborted', + failureClass?: string, +): RuntimeInvocationRecord { + return { + ...invocation, + terminalEvent: { + ...invocation.terminalEvent!, + status, + ...(failureClass ? { actions: { endInvocation: true, stateDelta: { failureClass } } } : {}), + }, + }; +} + +const invocation: RuntimeInvocationRecord = { sessionId, + invocationId, + runId, turnId, - status: 'completed', - backendKind: 'ai-sdk', - llmConnectionSlug: 'anthropic', - modelId: 'claude-sonnet-4-5', - cwd: '/tmp/work', - permissionMode: 'ask', - createdAt: ts, - updatedAt: ts + 20, - completedAt: ts + 20, - parentTurnId: 'parent-turn', + openedAt: ts, + opening: testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'ai-sdk', + llmConnectionId: 'anthropic-connection', + llmConnectionSlug: 'anthropic', + modelId: 'claude-sonnet-4-5', + }, + configuration: { cwd: '/tmp/work' }, + lineage: { parentTurnId: 'parent-turn' }, + }), + terminalEvent: { + id: `${runId}-terminal`, + sessionId, + invocationId, + runId, + turnId, + ts: ts + 20, + partial: false, + role: 'system', + author: 'system', + status: 'completed', + }, }; function ev(overrides: Partial): RuntimeEvent { @@ -305,7 +336,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { refs: { storedMessageId: 'user-skill' }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); assert.deepStrictEqual(out.messages, [ { @@ -331,7 +362,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { }); test('full RuntimeEvent turn projects legacy-compatible rows', () => { - const out = projectRuntimeEventsToStoredMessages(baseEvents(), { runHeaders: [header] }); + const out = projectRuntimeEventsToStoredMessages(baseEvents(), { invocations: [invocation] }); assert.deepStrictEqual( out.messages.map((message) => message.type), @@ -452,7 +483,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { }), ]; - const projected = projectRuntimeEventsToStoredMessages(events, { runHeaders: [header] }); + const projected = projectRuntimeEventsToStoredMessages(events, { invocations: [invocation] }); assert.deepStrictEqual(projected.diagnostics, []); assert.partialDeepStrictEqual(projected.messages[0], { type: 'assistant' }); assert.deepStrictEqual( @@ -552,7 +583,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { actions: { endInvocation: true }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); assert.deepStrictEqual( @@ -682,7 +713,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { refs: { toolCallId: 'tool-subagent' }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); const projected = out.messages.find((message) => message.type === 'tool_result'); @@ -791,7 +822,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { refs: { toolCallId: 'tool-agent-swarm' }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); const projected = out.messages.find((message) => message.type === 'tool_result'); @@ -827,7 +858,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { refs: { providerEventId: 'message-1' }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); const assistant = out.messages.find((message) => message.type === 'assistant'); @@ -855,7 +886,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { reason: 'stale_tool_result_pruned_before_compact', }; - const out = projectRuntimeEventsToStoredMessages(events, { runHeaders: [header] }); + const out = projectRuntimeEventsToStoredMessages(events, { invocations: [invocation] }); const projected = out.messages.find((message) => message.type === 'tool_result'); assert.partialDeepStrictEqual(projected, { type: 'tool_result', toolUseId: 'tool-1' }); @@ -926,13 +957,13 @@ describe('projectRuntimeEventsToStoredMessages', () => { reason: 'stale_tool_result_pruned_before_compact', }; - const defaultOut = projectRuntimeEventsToStoredMessages(events, { runHeaders: [header] }); + const defaultOut = projectRuntimeEventsToStoredMessages(events, { invocations: [invocation] }); const defaultProjected = defaultOut.messages.find((message) => message.type === 'tool_result'); assert.partialDeepStrictEqual(defaultProjected, { type: 'tool_result' }); assert.strictEqual(archivedStatus(defaultProjected), 'not_loaded'); const missingOut = projectRuntimeEventsToStoredMessagesWithArchiveStatuses(events, { - runHeaders: [header], + invocations: [invocation], archiveStatuses: { 'evt-tool-result': 'missing' }, }); const missingProjected = missingOut.messages.find((message) => message.type === 'tool_result'); @@ -940,7 +971,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { assert.strictEqual(archivedStatus(missingProjected), 'missing'); const corruptOut = projectRuntimeEventsToStoredMessagesWithArchiveStatuses(events, { - runHeaders: [header], + invocations: [invocation], archiveStatuses: [{ runtimeEventId: 'evt-tool-result', status: 'corrupt' }], }); const corruptProjected = corruptOut.messages.find((message) => message.type === 'tool_result'); @@ -965,7 +996,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { content: { kind: 'text', text: 'final' }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); assert.strictEqual(out.messages.length, 1); @@ -996,7 +1027,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { refs: { toolCallId: 'tool-1', operationId: 'toolop-1' }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); assert.deepStrictEqual(out.messages, []); @@ -1014,7 +1045,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { refs: { toolCallId: 'tool-1' }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); assert.deepStrictEqual(out.messages, []); @@ -1064,7 +1095,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { refs: { toolCallId: 'tool-1', operationId: 'toolop-1' }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); assert.deepStrictEqual(out.messages, []); @@ -1103,7 +1134,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); assert.deepStrictEqual(out.messages, []); @@ -1152,7 +1183,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { refs: { toolCallId: 'tool-1' }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); assert.deepStrictEqual(out.messages, []); @@ -1297,7 +1328,9 @@ describe('projectRuntimeEventsToStoredMessages', () => { // owns no chat row, so a broken one costs a reader nothing the session view // would otherwise show. test(`a sandbox boundary ${name} stays unclaimed`, () => { - const out = projectRuntimeEventsToStoredMessages([makeEvent()], { runHeaders: [header] }); + const out = projectRuntimeEventsToStoredMessages([makeEvent()], { + invocations: [invocation], + }); assert.deepStrictEqual(out.messages, []); assert.deepStrictEqual( @@ -1334,7 +1367,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { refs: { storedMessageId: 'legacy-assistant' }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); const legacy: StoredMessage[] = [ { @@ -1397,7 +1430,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { refs: { providerEventId: 'step-2' }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); const assistants = out.messages.filter((message) => message.type === 'assistant'); @@ -1454,7 +1487,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); assert.deepStrictEqual(out.messages, []); @@ -1492,7 +1525,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { content: { kind: 'text', text: 'hello' }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); assert.deepStrictEqual( @@ -1520,7 +1553,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { content: { kind: 'not_yet_projected', text: 'a reader would have seen this' } as never, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); assert.deepStrictEqual(out.messages, []); @@ -1531,18 +1564,18 @@ describe('projectRuntimeEventsToStoredMessages', () => { assert.strictEqual(out.diagnostics.every(isHardRuntimeEventReadModelDiagnostic), true); }); - test('failed terminal RuntimeEvent maps to failed turn state when run header carries failure class', () => { + test('failed terminal RuntimeEvent maps to failed turn state with the class it states', () => { const out = projectRuntimeEventsToStoredMessages( [ ev({ id: 'evt-failed', ts: ts + 9, status: 'failed', - actions: { endInvocation: true }, + actions: { endInvocation: true, stateDelta: { failureClass: 'tool_failed' } }, }), ], { - runHeaders: [{ ...header, status: 'failed', failureClass: 'tool_failed' }], + invocations: [endedAs('failed', 'tool_failed')], }, ); @@ -1583,7 +1616,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { }), ], { - runHeaders: [{ ...header, status: 'failed', failureClass: 'context_budget_exhausted' }], + invocations: [endedAs('failed', 'context_budget_exhausted')], }, ); @@ -1617,7 +1650,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { }), ], { - runHeaders: [{ ...header, status: 'failed', failureClass: 'tool_step_cap_reached' }], + invocations: [endedAs('failed', 'tool_step_cap_reached')], }, ); @@ -1644,7 +1677,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { }), ], { - runHeaders: [{ ...header, status: 'cancelled' }], + invocations: [endedAs('aborted')], }, ); @@ -1664,7 +1697,9 @@ describe('projectRuntimeEventsToStoredMessages', () => { assert.deepStrictEqual(out.diagnostics, []); }); - test('aborted terminal RuntimeEvent keeps an explicit diagnostic when abort source is unavailable', () => { + // The omission is `classifyRuntimeEventTerminalFact`'s to report. Repeating it + // here would turn a transcript row that reads fine into an unreadable Session. + test('aborted terminal RuntimeEvent that states no source still projects its turn state', () => { const out = projectRuntimeEventsToStoredMessages( [ ev({ @@ -1675,7 +1710,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { }), ], { - runHeaders: [{ ...header, status: 'cancelled' }], + invocations: [endedAs('aborted')], }, ); @@ -1684,10 +1719,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { status: 'aborted', abortedAt: ts + 9, }); - assert.deepStrictEqual( - out.diagnostics.map((diag) => diag.code), - ['incomplete_event'], - ); + assert.deepStrictEqual(out.diagnostics, []); }); test('projects tool_call stepId from refs so the UI timeline keeps step pairing', () => { @@ -1706,7 +1738,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { }); const withStep = projectRuntimeEventsToStoredMessages([stepCall('tool-step', 'step-1')], { - runHeaders: [header], + invocations: [invocation], }); assert.partialDeepStrictEqual(withStep.messages[0], { type: 'tool_call', @@ -1717,7 +1749,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { // Legacy events without refs.stepId must not grow a stepId key: the UI // uses its absence to pick the backward-compatible tools-first ordering. const withoutStep = projectRuntimeEventsToStoredMessages([stepCall('tool-legacy')], { - runHeaders: [header], + invocations: [invocation], }); const legacyCall = withoutStep.messages[0]; assert.partialDeepStrictEqual(legacyCall, { type: 'tool_call', id: 'tool-legacy' }); @@ -1754,7 +1786,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); assert.strictEqual(out.messages.length, 2); @@ -1785,7 +1817,7 @@ describe('projectRuntimeEventsToStoredMessages', () => { refs: { toolCallId: 'tool-kind' }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); assert.partialDeepStrictEqual(out.messages[0], { @@ -1909,8 +1941,9 @@ const ACTION_COVERAGE_SAMPLES: ActionCoverageSamples = { event: { author: 'user', refs: { toolCallId: 'coverage-form-tool' } }, }, transferToAgent: { action: 'agent-b' }, - // The terminal fact is one of the actions that does own a row. - endInvocation: { action: true }, + // The terminal fact is one of the actions that does own a row, and the event + // states the outcome it ends on. + endInvocation: { action: true, event: { status: 'completed' } }, tokenUsage: { action: { input: 10, output: 5 } }, toolDispatch: { action: { @@ -1969,7 +2002,7 @@ describe('RuntimeEventActions projection coverage', () => { // Guards an entry that names a field but leaves it absent at runtime. assert.strictEqual(field in actions, true); const out = projectRuntimeEventsToStoredMessages([ev({ ...sample.event, actions })], { - runHeaders: [header], + invocations: [invocation], }); assert.deepStrictEqual(out.diagnostics.filter(isUnclaimedRuntimeEventDiagnostic), []); @@ -2004,7 +2037,7 @@ describe('compareRuntimeReadModelMessages', () => { }, }), ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); const legacy: StoredMessage[] = [ { @@ -2072,13 +2105,13 @@ describe('compareRuntimeReadModelMessages', () => { }), anchored, ], - { runHeaders: [header] }, + { invocations: [invocation] }, ); const usage = projected.messages.find((message) => message.type === 'token_usage'); assert.partialDeepStrictEqual(usage, { type: 'token_usage', input: 370, lastRequestAnchor }); const backfilled = backfillRuntimeEventsFromStoredMessages({ - run: header, + run: { sessionId, invocationId, runId, turnId }, messages: projected.messages, now: () => ts, }); @@ -2133,7 +2166,9 @@ describe('compareRuntimeReadModelMessages', () => { }); test('rejects missing tool result and assistant text cases', () => { - const projected = projectRuntimeEventsToStoredMessages(baseEvents(), { runHeaders: [header] }); + const projected = projectRuntimeEventsToStoredMessages(baseEvents(), { + invocations: [invocation], + }); const missing = projected.messages.filter( (message) => message.type !== 'tool_result' && message.type !== 'assistant', ); diff --git a/packages/runtime/src/__tests__/runtime-event-store-seal.ts b/packages/runtime/src/__tests__/runtime-event-store-seal.ts new file mode 100644 index 0000000000..500feecf62 --- /dev/null +++ b/packages/runtime/src/__tests__/runtime-event-store-seal.ts @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { isTerminalRuntimeEvent } from '@maka/core/runtime-event'; +import type { RuntimeEvent } from '@maka/core/runtime-event'; +import { RunSealedError } from '@maka/core/runtime-event-store'; + +/** + * The seal every `RuntimeEventStore` owes its callers, for the doubles. + * + * `RuntimeEventStore` requires an implementation to refuse any new event on a + * run that already holds a terminal one. A double that skips it manufactures a + * ledger no supported store can produce, and a test built on that ledger proves + * nothing about production. Stated here once so the doubles cannot drift apart + * from each other or from the SQLite store. + * + * A test that genuinely needs a corrupt ledger should assemble it underneath the + * store rather than appending through it. + */ +export function assertDoubleRunNotSealed( + storedEvents: readonly RuntimeEvent[], + incoming: RuntimeEvent, +): void { + // An exact-id replay is idempotent: the event is already inside the seal. + if (storedEvents.some((event) => event.id === incoming.id)) return; + if (storedEvents.some(isTerminalRuntimeEvent)) throw new RunSealedError(incoming.runId); +} diff --git a/packages/runtime/src/__tests__/runtime-invocation-index.test.ts b/packages/runtime/src/__tests__/runtime-invocation-index.test.ts new file mode 100644 index 0000000000..dd76eccef4 --- /dev/null +++ b/packages/runtime/src/__tests__/runtime-invocation-index.test.ts @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * The invocation index is a query over the canonical events, not a second + * record. This test writes real turns through the production seams, then checks + * that what the index answers is exactly what rebuilding from those events + * alone produces. + */ +import assert from 'node:assert/strict'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { test } from 'node:test'; +import { encodeCanonicalRuntimeEvent } from '@maka/core/canonical-runtime-event'; +import { runtimeInvocationsFromSessionEvents } from '@maka/core/runtime-invocation'; +import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; +import { OPERATIONAL_STATE_DATABASE_NAME } from '@maka/storage/operational-state-store'; +import { createWorkspaceRuntimeStore } from '@maka/storage/runtime-event-persistence'; +import { createSessionStore } from '@maka/storage/session-store'; +import type { SessionEvent } from '@maka/core/events'; +import type { BackendSendInput } from '@maka/core/backend-types'; +import { BackendRegistry, SessionManager } from '../session-manager.js'; + +test('the invocation index returns the same inventory as a rebuild from events alone', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-invocation-index-')); + try { + const sessionStore = createSessionStore(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); + const backends = new BackendRegistry(); + backends.register('ai-sdk', (ctx) => ({ + kind: 'ai-sdk' as const, + sessionId: ctx.sessionId, + async *send(input: BackendSendInput): AsyncIterable { + yield { + type: 'complete', + id: `${input.turnId}-complete`, + turnId: input.turnId, + ts: 2, + stopReason: 'end_turn', + }; + }, + async stop() {}, + async respondToSandboxBoundary() {}, + async dispose() {}, + })); + let ids = 0; + let clock = 1_000; + const manager = new SessionManager({ + store: sessionStore, + runStore, + runtimeEventStore, + backends, + newId: () => `index-${++ids}`, + now: () => (clock += 1), + }); + const session = await manager.createSession({ + cwd: root, + llmConnectionSlug: 'fake', + permissionMode: 'bypass', + }); + + for (const turnId of ['turn-1', 'turn-2', 'turn-3']) { + for await (const _event of manager.sendMessage(session.id, { turnId, text: turnId })) { + // Drain the turn so its run reaches the durable ledger. + } + } + + const invocations = await runtimeEventStore.listSessionInvocations(session.id); + const rebuilt = runtimeInvocationsFromSessionEvents( + session.id, + await runtimeEventStore.readSessionRuntimeEvents(session.id), + ); + assert.equal(invocations.length, 3); + + assert.deepStrictEqual( + invocations, + rebuilt, + 'the index must return exactly what a rebuild from events alone produces', + ); + + for (const invocation of invocations) { + assert.equal( + invocation.terminalEvent?.status, + 'completed', + 'a finished invocation must expose its terminal event through the index', + ); + } + + // A ledger written before the store sealed runs can carry a straggler after + // the terminal event: stop sealed the run while the stream was still + // draining, and nothing has ever removed those. Recovery, the read model and + // continuation resume all read such a run as ended, so the index must too — + // reading it as active is what makes one reader disagree with the rest. + const straggler = invocations[0]!; + runtimeEventStore.close(); + const db = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const { json } = encodeCanonicalRuntimeEvent({ + id: 'post-terminal-straggler', + invocationId: straggler.invocationId, + runId: straggler.runId, + sessionId: session.id, + turnId: straggler.turnId, + ts: 9_999, + partial: false, + role: 'model', + author: 'agent', + modelVisibility: 'visible', + content: { kind: 'text', text: 'arrived after the run was sealed' }, + }); + db.prepare(` + INSERT INTO runtime_events ( + event_id, session_id, invocation_id, run_id, turn_id, event_seq, + event_kind, payload_json, committed_at + ) VALUES (?, ?, ?, ?, ?, ( + SELECT MAX(event_seq) + 1 FROM runtime_events WHERE invocation_id = ? + ), 'text', ?, 9999) + `).run( + 'post-terminal-straggler', + session.id, + straggler.invocationId, + straggler.runId, + straggler.turnId, + straggler.invocationId, + json, + ); + } finally { + db.close(); + } + + const reopened = createWorkspaceRuntimeStore(root); + try { + const afterStraggler = await reopened.listSessionInvocations(session.id); + assert.deepStrictEqual( + afterStraggler, + runtimeInvocationsFromSessionEvents( + session.id, + await reopened.readSessionRuntimeEvents(session.id), + ), + 'the index and a rebuild must still agree once a straggler follows the terminal', + ); + assert.equal( + afterStraggler.find((invocation) => invocation.invocationId === straggler.invocationId) + ?.terminalEvent?.status, + 'completed', + 'a run that ended stays ended when an unsealed-era straggler follows it', + ); + } finally { + reopened.close(); + } + + runStore.close?.(); + sessionStore.close?.(); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts index 066c6fedd8..3e42ad0777 100644 --- a/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts +++ b/packages/runtime/src/__tests__/runtime-ledger-repair.test.ts @@ -29,6 +29,11 @@ import { createSessionStore } from '@maka/storage/session-store'; import { createSqliteRuntimeStore } from '@maka/storage/sqlite-runtime-store'; import { buildRuntimeEventModelReplayPlan } from '../model-history.js'; import { buildPriorRuntimeContext } from '../prior-run-context.js'; +import { + buildInvocationOpenedEvent, + runtimeInvocationOutcome, +} from '@maka/core/runtime-invocation'; +import { runtimeInvocationFailureClass } from '../runtime-event-read-model.js'; import { backfillRuntimeEventsFromStoredMessages } from '../runtime-event-backfill.js'; import { RuntimeLedgerRepair } from '../runtime-ledger-repair.js'; @@ -97,11 +102,9 @@ test('repairs imported transcript turns into provider-neutral canonical history' ); assert.equal(session.transcriptLedgerVersion, 0); const repair = new RuntimeLedgerRepair({ - runStore: runs, runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), appendMessage: (sessionId, message) => sessions.appendMessage(sessionId, message), - appendTurnState: async () => undefined, newId, now: () => 100, }); @@ -109,11 +112,11 @@ test('repairs imported transcript turns into provider-neutral canonical history' await repair.materializeTranscriptLedger(session); await repair.materializeTranscriptLedger(session); - const [importedRun] = await runs.listSessionRuns(session.id); + const [importedRun] = await runtimeEvents.listSessionInvocations(session.id); assert.ok(importedRun); assert.equal(importedRun.turnId, 'turn-1'); - assert.equal(importedRun.status, 'completed'); - assert.ok(importedRun.createdAt < session.createdAt); + assert.equal(runtimeInvocationOutcome(importedRun), 'completed'); + assert.ok(importedRun.openedAt < session.createdAt); const importedEvents = await runtimeEvents.readRuntimeEvents(session.id, importedRun.runId); assert.deepEqual( @@ -151,16 +154,22 @@ test('repairs imported transcript turns into provider-neutral canonical history' partialOutputRetained: true, }, ]; - const continuedRun = await runs.createRun({ - ...importedRun, + const continuedRun = { + sessionId: session.id, runId: 'continued-run', invocationId: 'continued-invocation', turnId: 'turn-2', - status: 'completed', - createdAt: session.createdAt + 1, - updatedAt: session.createdAt + 3, - completedAt: session.createdAt + 3, - }); + }; + await runtimeEvents.appendRuntimeEvent( + session.id, + continuedRun.runId, + buildInvocationOpenedEvent({ + id: newId(), + run: continuedRun, + openedAt: session.createdAt + 1, + opening: importedRun.opening, + }), + ); for (const event of backfillRuntimeEventsFromStoredMessages({ run: continuedRun, messages: continuedMessages, @@ -171,25 +180,27 @@ test('repairs imported transcript turns into provider-neutral canonical history' } const currentRunId = 'current-run'; - await runs.createRun({ - ...continuedRun, - runId: currentRunId, - invocationId: 'current-invocation', - turnId: 'turn-3', - status: 'running', - createdAt: session.createdAt + 4, - updatedAt: session.createdAt + 4, - completedAt: undefined, - }); + await runtimeEvents.appendRuntimeEvent( + session.id, + currentRunId, + buildInvocationOpenedEvent({ + id: newId(), + run: { + sessionId: session.id, + runId: currentRunId, + invocationId: 'current-invocation', + turnId: 'turn-3', + }, + openedAt: session.createdAt + 4, + opening: importedRun.opening, + }), + ); const prior = await buildPriorRuntimeContext({ sessionId: session.id, currentRunId, currentTurnId: 'turn-3', - runStore: runs, runtimeEventStore: runtimeEvents, - runStoreAvailable: true, runtimeEventStoreAvailable: true, - readMessages: () => sessions.readMessages(session.id), }); assert.deepEqual( buildRuntimeEventModelReplayPlan(prior?.events ?? []).items.map((item) => @@ -273,24 +284,22 @@ test('an imported snapshot cutoff survives materialization as aborted', async () { adapterId: 'claude-code', sourceSessionId: 'cut-1' }, ); const repair = new RuntimeLedgerRepair({ - runStore: runs, runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), appendMessage: (sessionId, message) => sessions.appendMessage(sessionId, message), - appendTurnState: async () => undefined, newId, now: () => 100, }); await repair.materializeTranscriptLedger(session); - const [run] = await runs.listSessionRuns(session.id); + const [run] = await runtimeEvents.listSessionInvocations(session.id); assert.ok(run); // `cancelled`, not `failed`: the Ledger accepted the recorded abort. Before // the adapter emitted one, this same transcript materialized as // `failed / missing_terminal_event`. - assert.equal(run.status, 'cancelled'); - assert.notEqual(run.failureClass, 'missing_terminal_event'); + assert.equal(runtimeInvocationOutcome(run), 'cancelled'); + assert.notEqual(runtimeInvocationFailureClass(run), 'missing_terminal_event'); } finally { await runtimeEvents.close?.(); await rm(root, { recursive: true, force: true }); @@ -332,18 +341,16 @@ test('does not import Host-handed-off transcript messages as synthetic runs', as { adapterId: 'test', sourceSessionId: 'host-session' }, ); const repair = new RuntimeLedgerRepair({ - runStore: runs, runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), appendMessage: (sessionId, message) => sessions.appendMessage(sessionId, message), - appendTurnState: async () => undefined, newId: () => `host-repair-${++sequence}`, now: () => 100, }); await repair.materializeTranscriptLedger(session); - assert.deepEqual(await runs.listSessionRuns(session.id), []); + assert.deepEqual(await runtimeEvents.listSessionInvocations(session.id), []); } finally { runtimeEvents.close(); runs.close?.(); @@ -386,21 +393,19 @@ test('an imported turn with no terminal state is repaired to failed', async () = { adapterId: 'claude-code', sourceSessionId: 'missing-1' }, ); const repair = new RuntimeLedgerRepair({ - runStore: runs, runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), appendMessage: (sessionId, message) => sessions.appendMessage(sessionId, message), - appendTurnState: async () => undefined, newId, now: () => 100, }); await repair.materializeTranscriptLedger(session); - const [run] = await runs.listSessionRuns(session.id); + const [run] = await runtimeEvents.listSessionInvocations(session.id); assert.ok(run); - assert.equal(run.status, 'failed'); - assert.equal(run.failureClass, 'missing_terminal_event'); + assert.equal(runtimeInvocationOutcome(run), 'failed'); + assert.equal(runtimeInvocationFailureClass(run), 'missing_terminal_event'); } finally { await runtimeEvents.close?.(); await rm(root, { recursive: true, force: true }); @@ -554,17 +559,15 @@ test('a resolved Claude transcript replays as the conversation the user kept', a { adapterId: 'claude-code', sourceSessionId: SOURCE_SESSION_ID }, ); const repair = new RuntimeLedgerRepair({ - runStore: runs, runtimeEventStore: runtimeEvents, readMessages: (sessionId) => sessions.readMessages(sessionId), appendMessage: (sessionId, message) => sessions.appendMessage(sessionId, message), - appendTurnState: async () => undefined, newId, now: () => 100, }); await repair.materializeTranscriptLedger(session); - const [run] = await runs.listSessionRuns(session.id); + const [run] = await runtimeEvents.listSessionInvocations(session.id); assert.ok(run); const events = await runtimeEvents.readRuntimeEvents(session.id, run.runId); const replay = buildRuntimeEventModelReplayPlan(events).items; @@ -613,8 +616,8 @@ test('a resolved Claude transcript replays as the conversation the user kept', a assert.deepEqual(shape, ['call:toolu_a', 'call:toolu_b', 'result:toolu_a', 'result:toolu_b']); // And the turn is terminal on its own evidence, not repaired into one. - assert.equal(run.status, 'completed'); - assert.notEqual(run.failureClass, 'missing_terminal_event'); + assert.equal(runtimeInvocationOutcome(run), 'completed'); + assert.notEqual(runtimeInvocationFailureClass(run), 'missing_terminal_event'); } finally { runtimeEvents.close(); runs.close?.(); diff --git a/packages/runtime/src/__tests__/runtime-resume-crash.test.ts b/packages/runtime/src/__tests__/runtime-resume-crash.test.ts index 13d188f9b1..8cd38d0970 100644 --- a/packages/runtime/src/__tests__/runtime-resume-crash.test.ts +++ b/packages/runtime/src/__tests__/runtime-resume-crash.test.ts @@ -27,7 +27,6 @@ import { spawn } from 'node:child_process'; import { describe, test } from 'node:test'; import type { RuntimeEvent } from '@maka/core/runtime-event'; -import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; import { createWorkspaceRuntimeStore } from '@maka/storage/runtime-event-persistence'; import { @@ -60,26 +59,6 @@ if (process.env[CRASH_CHILD_ENV] === '1') { committedEventCount(failpoint.committedPrefix), ); - // Production creates the run header before any RuntimeEvent append. Keep the - // crash boundary focused on the child event writer while preserving the - // storage identity contract used when the ledger is reopened. - const runStore = createSqliteAgentRunStore(workspaceRoot); - await runStore.createRun({ - runId, - invocationId: `invocation-${runId}`, - sessionId, - turnId: `turn-${runId}`, - status: 'running', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: workspaceRoot, - permissionMode: 'ask', - createdAt: 1, - updatedAt: 1, - }); - runStore.close?.(); - await crashWriterAfterCommit({ workspaceRoot, sessionId, diff --git a/packages/runtime/src/__tests__/runtime-resume.test.ts b/packages/runtime/src/__tests__/runtime-resume.test.ts index a751d8d894..7d9a7c698b 100644 --- a/packages/runtime/src/__tests__/runtime-resume.test.ts +++ b/packages/runtime/src/__tests__/runtime-resume.test.ts @@ -26,7 +26,8 @@ import { } from '@maka/core/runtime-boundary'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; import type { RuntimeEvent } from '@maka/core/runtime-event'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeEventInvocationOpenedContent } from '@maka/core/runtime-event'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import { buildContinuationReplayPlan } from '../continuation-replay.js'; import { PROVIDER_REPLAY_PROJECTION_VERSION } from '../model-history.js'; @@ -38,6 +39,7 @@ import { buildResumeReplayRuntimeEvents, projectToolOperationsFromRuntimeEvents, } from '../runtime-resume.js'; +import { testInvocationRecord } from './invocation-fixture.js'; describe('runtime resume phase 0 projection', () => { test('publishes the stable P0-P11 crash failpoint catalog', () => { @@ -292,17 +294,18 @@ describe('runtime resume phase 1 safe-boundary continuation', () => { }, ]; const planner = new RuntimeContinuationPlanner({ - readSourceRun: async (_sessionId, runId) => + readSourceInvocation: async (_sessionId, runId) => runId === 'run-2' - ? runHeader('run-2', { - continuationSource: { + ? runInvocation('run-2', { + source: { + kind: 'continuation', sourceInvocationId: 'invocation-1', sourceRunId: 'run-1', sourceTurnId: 'turn-1', sourceRuntimeEventHighWater: rootEvents.length, }, }) - : runHeader('run-1'), + : runInvocation('run-1'), readImmutableRuntimePrefix: async ({ runId, upToEventSeq }) => { const events = runId === 'run-2' ? childEvents : rootEvents; return immutablePrefix(upToEventSeq === undefined ? events : events.slice(0, upToEventSeq)); @@ -731,33 +734,39 @@ function safeBoundaryFacts() { function sameRouteAdmission() { return { - runHeaders: ['run-1', 'run-2', 'run-3'].map((runId) => - runHeader(runId, { llmConnectionId: 'connection-1' }), - ), + invocations: ['run-1', 'run-2', 'run-3'].map((runId) => runInvocation(runId)), targetProviderStateIdentity: undefined, targetModelId: 'test-model', }; } -function runHeader(runId: string, overrides: Partial = {}): AgentRunHeader { +/** One failed invocation on the shared route, as its own events describe it. */ +function runInvocation( + runId: string, + facts: { source?: RuntimeEventInvocationOpenedContent['source'] } = {}, +): RuntimeInvocationRecord { const ordinal = runId.match(/(\d+)$/)?.[1] ?? '1'; - const status = overrides.status ?? 'failed'; - return { - runId, - invocationId: `invocation-${ordinal}`, + return testInvocationRecord({ sessionId: 'session-1', + invocationId: `invocation-${ordinal}`, + runId, turnId: `turn-${ordinal}`, - status, - backendKind: 'fake', - llmConnectionSlug: 'test', - modelId: 'test-model', - cwd: '/workspace/repo', - permissionMode: 'ask', - ...(status === 'failed' ? { failureClass: 'test_failure' } : {}), - createdAt: 1, - updatedAt: 1, - ...overrides, - }; + openedAt: 1, + closedAt: 1, + outcome: 'failed', + failureClass: 'test_failure', + opening: { + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'connection-1', + llmConnectionSlug: 'test', + modelId: 'test-model', + }, + configuration: { cwd: '/workspace/repo' }, + ...(facts.source ? { source: facts.source } : {}), + }, + }); } function base(overrides: Partial): RuntimeEvent { diff --git a/packages/runtime/src/__tests__/sandbox-boundary-restart-recovery.test.ts b/packages/runtime/src/__tests__/sandbox-boundary-restart-recovery.test.ts index 3174c5b12f..1ad13b4d54 100644 --- a/packages/runtime/src/__tests__/sandbox-boundary-restart-recovery.test.ts +++ b/packages/runtime/src/__tests__/sandbox-boundary-restart-recovery.test.ts @@ -23,7 +23,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, it } from 'node:test'; -import type { AgentRunEvent, EmittedAgentRunEvent, AgentRunHeader } from '@maka/core/agent-run'; +import type { AgentRunEvent, EmittedAgentRunEvent } from '@maka/core/agent-run'; +import { buildInvocationOpenedEvent } from '@maka/core/runtime-invocation'; +import { runtimeInvocationFailureClass } from '../runtime-event-read-model.js'; import type { CreateSessionInput } from '@maka/core/runtime-inputs'; import type { SessionHeader, StoredMessage } from '@maka/core/session'; import { @@ -34,6 +36,7 @@ import { createSessionStore, type SessionAuthorityStore } from '@maka/storage/se import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; import { createWorkspaceRuntimeStore } from '@maka/storage/runtime-event-persistence'; import { BackendRegistry, SessionManager } from '../session-manager.js'; +import { testInvocationOpening } from './invocation-fixture.js'; /** * Restart behaviour against the canonical SQLite stores. Memory stores can @@ -45,7 +48,7 @@ describe('sandbox boundary restart recovery on durable stores', () => { it('attributes a closure whose RuntimeEvent never reached the ledger', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-boundary-restart-')); try { - const session = await withStores(root, async ({ sessions, runs }) => { + const session = await withStores(root, async ({ sessions, runs, runtimeEvents }) => { const header = await sessions.create(sessionInput(root)); await sessions.createSandboxBoundaryRequest({ sessionId: header.id, @@ -55,7 +58,7 @@ describe('sandbox boundary restart recovery on durable stores', () => { expansion: { network: { enabled: true } }, justification: 'Fetch a dependency.', }); - await seedInterruptedTurn(sessions, runs, header.id); + await seedInterruptedTurn(sessions, runs, runtimeEvents, header.id); // Deliberately no boundary RuntimeEvent: the process died in the // fail-open window between the row commit and the event append. return header; @@ -65,14 +68,17 @@ describe('sandbox boundary restart recovery on durable stores', () => { await manager(stores).recoverInterruptedSessions(); }); - await withStores(root, async ({ sessions, runs }) => { + await withStores(root, async ({ sessions, runtimeEvents }) => { assert.deepEqual(await sessions.listPendingSandboxBoundaryRequests(session.id), []); const [turn] = await sessions.listTurns(session.id); assert.equal(turn?.status, 'failed'); assert.equal(turn?.errorClass, 'sandbox_boundary_closed_by_restart'); - const [run] = await runs.listSessionRuns(session.id); - assert.equal(run?.status, 'failed'); - assert.equal(run?.failureClass, 'sandbox_boundary_closed_by_restart'); + const [invocation] = await runtimeEvents.listSessionInvocations(session.id); + assert.equal(invocation?.terminalEvent?.status, 'failed'); + assert.equal( + invocation && runtimeInvocationFailureClass(invocation), + 'sandbox_boundary_closed_by_restart', + ); }); } finally { await rm(root, { recursive: true, force: true }); @@ -82,7 +88,7 @@ describe('sandbox boundary restart recovery on durable stores', () => { it('re-reads a closure across a recovery interrupted before the terminal commit', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-boundary-restart-twice-')); try { - const session = await withStores(root, async ({ sessions, runs }) => { + const session = await withStores(root, async ({ sessions, runs, runtimeEvents }) => { const header = await sessions.create(sessionInput(root)); await sessions.createSandboxBoundaryRequest({ sessionId: header.id, @@ -100,7 +106,7 @@ describe('sandbox boundary restart recovery on durable stores', () => { decision: 'deny', closureReason: 'host_restarted', }); - await seedInterruptedTurn(sessions, runs, header.id); + await seedInterruptedTurn(sessions, runs, runtimeEvents, header.id); assert.deepEqual(await sessions.listPendingSandboxBoundaryRequests(header.id), []); return header; }); @@ -109,11 +115,14 @@ describe('sandbox boundary restart recovery on durable stores', () => { await manager(stores).recoverInterruptedSessions(); }); - const failedStatesAfterFirst = await withStores(root, async ({ sessions, runs }) => { + const failedStatesAfterFirst = await withStores(root, async ({ sessions, runtimeEvents }) => { const [turn] = await sessions.listTurns(session.id); assert.equal(turn?.errorClass, 'sandbox_boundary_closed_by_restart'); - const [run] = await runs.listSessionRuns(session.id); - assert.equal(run?.failureClass, 'sandbox_boundary_closed_by_restart'); + const [invocation] = await runtimeEvents.listSessionInvocations(session.id); + assert.equal( + invocation && runtimeInvocationFailureClass(invocation), + 'sandbox_boundary_closed_by_restart', + ); return countFailedTurnStates(await sessions.readMessages(session.id)); }); @@ -123,15 +132,18 @@ describe('sandbox boundary restart recovery on durable stores', () => { await manager(stores).recoverInterruptedSessions(); }); - await withStores(root, async ({ sessions, runs }) => { + await withStores(root, async ({ sessions, runtimeEvents }) => { const [turn] = await sessions.listTurns(session.id); assert.equal(turn?.errorClass, 'sandbox_boundary_closed_by_restart'); assert.equal( countFailedTurnStates(await sessions.readMessages(session.id)), failedStatesAfterFirst, ); - const [run] = await runs.listSessionRuns(session.id); - assert.equal(run?.failureClass, 'sandbox_boundary_closed_by_restart'); + const [invocation] = await runtimeEvents.listSessionInvocations(session.id); + assert.equal( + invocation && runtimeInvocationFailureClass(invocation), + 'sandbox_boundary_closed_by_restart', + ); const closures = await sessions.listSandboxBoundaryRestartClosures(session.id); assert.deepEqual( closures.map((closure) => [closure.requestId, closure.turnId, closure.runId]), @@ -199,6 +211,7 @@ function manager(stores: DurableStores): SessionManager { async function seedInterruptedTurn( sessions: SessionAuthorityStore, runs: DurableAgentRunStore, + runtimeEvents: DurableRuntimeEventStore, sessionId: string, ): Promise { await sessions.appendMessages(sessionId, [ @@ -213,7 +226,7 @@ async function seedInterruptedTurn( }, ]); await sessions.updateHeader(sessionId, { status: 'waiting_for_user' }); - await runs.createRun(runHeader(sessionId)); + await runtimeEvents.appendRuntimeEvent(sessionId, 'run-1', openingEvent(sessionId)); await runs.appendEvent(sessionId, 'run-1', runEvent(sessionId)); } @@ -222,26 +235,28 @@ function countFailedTurnStates(messages: readonly StoredMessage[]): number { .length; } -function runHeader(sessionId: string): AgentRunHeader { - return { - runId: 'run-1', - sessionId, - turnId: 'turn-1', - status: 'waiting_for_user', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/cwd', - permissionMode: 'ask', - createdAt: 10, - updatedAt: 10, - }; +function openingEvent(sessionId: string) { + return buildInvocationOpenedEvent({ + id: 'run-1-open', + run: { sessionId, invocationId: 'run-1', runId: 'run-1', turnId: 'turn-1' }, + openedAt: 10, + opening: testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'fake-connection', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { cwd: '/tmp/cwd' }, + }), + }); } function runEvent(sessionId: string): EmittedAgentRunEvent { return { - type: 'run_started', - id: 'run-1-run_started-11', + type: 'turn_started', + id: 'run-1-turn_started-11', runId: 'run-1', sessionId, turnId: 'turn-1', diff --git a/packages/runtime/src/__tests__/session-event-runtime-mapper.test.ts b/packages/runtime/src/__tests__/session-event-runtime-mapper.test.ts index e46de3ae4a..65632860cb 100644 --- a/packages/runtime/src/__tests__/session-event-runtime-mapper.test.ts +++ b/packages/runtime/src/__tests__/session-event-runtime-mapper.test.ts @@ -20,7 +20,7 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { SessionEvent } from '@maka/core/events'; import type { BackendSessionEvent } from '@maka/core/backend-types'; import type { RuntimeEvent } from '@maka/core/runtime-event'; @@ -43,6 +43,7 @@ import { } from '../runtime-event-read-model.js'; import { isNonTerminalErrorRuntimeEvent } from '../agent-run.js'; import { backfillRuntimeEventsFromStoredMessages } from '../runtime-event-backfill.js'; +import { testInvocationOpening } from './invocation-fixture.js'; // ============================================================================ // Event builders @@ -653,19 +654,22 @@ const PROJECTION_SAMPLES: ProjectionSamples = { abort: { subject: { type: 'abort', id: 'e', turnId: 'turn-1', ts: 1, reason: 'user_stop' } }, }; -const projectionRunHeader: AgentRunHeader = { - runId: 'run-1', +const projectionInvocation: RuntimeInvocationRecord = { sessionId: 'session-1', + invocationId: 'invocation-1', + runId: 'run-1', turnId: 'turn-1', - status: 'completed', - backendKind: 'ai-sdk', - llmConnectionSlug: 'anthropic', - modelId: 'model-1', - cwd: '/tmp', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 2, - completedAt: 2, + openedAt: 1, + opening: testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'ai-sdk', + llmConnectionId: 'anthropic-connection', + llmConnectionSlug: 'anthropic', + modelId: 'model-1', + }, + configuration: { cwd: '/tmp' }, + }), }; describe('SessionEvent projection coverage', () => { @@ -730,7 +734,7 @@ describe('SessionEvent projection coverage', () => { .filter((event) => !isNonTerminalErrorRuntimeEvent(event)); const projected = projectRuntimeEventsToStoredMessages(runtimeEvents, { - runHeaders: [projectionRunHeader], + invocations: [projectionInvocation], }); assert.deepEqual(projected.diagnostics.filter(isUnclaimedRuntimeEventDiagnostic), []); @@ -754,7 +758,7 @@ describe('SessionEvent projection coverage', () => { assert.equal(runtimeEvent.actions?.stateDelta?.unmappedSessionEventType, 'not_yet_mapped'); const projected = projectRuntimeEventsToStoredMessages([runtimeEvent], { - runHeaders: [projectionRunHeader], + invocations: [projectionInvocation], }); assert.deepEqual(projected.messages, []); // Filtered through the predicate the contract above uses, not just compared diff --git a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts index 9c343b10b4..3c0f693608 100644 --- a/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts +++ b/packages/runtime/src/__tests__/session-manager-terminal-ledger.test.ts @@ -29,7 +29,15 @@ import { ToolLedgerRejectionError, } from '@maka/core/tool-ledger-scanner'; import type { SandboxBoundaryResponse } from '@maka/core/sandbox-boundary'; -import type { AgentRunEvent, AgentRunHeader, AgentRunStore } from '@maka/core/agent-run'; +import type { AgentRunEvent, AgentRunStore } from '@maka/core/agent-run'; +import { runtimeInvocationFailureClass } from '../runtime-event-read-model.js'; +import { + buildInvocationOpenedEvent, + buildSyntheticTerminalRuntimeEvent, + runtimeInvocationOutcome, + runtimeInvocationsFromSessionEvents, + type RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; import type { CreateSessionInput, SessionListFilter } from '@maka/core/runtime-inputs'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; @@ -47,7 +55,6 @@ import { import type { AgentBackend } from '@maka/core/backend-types'; import { buildRecoveredTerminalRuntimeEvent, - buildSyntheticTerminalRuntimeEvent, classifyTerminalRuntimeLedger, commitOrCreateTerminalRunFact, commitTerminalRunWithRuntimeFact, @@ -55,6 +62,8 @@ import { import { RuntimeReadModel } from '../runtime-read-model.js'; import { RuntimeKernel } from '../runtime-kernel.js'; import type { RuntimeInteractionAuthority } from '../interaction-authority.js'; +import { testInvocationOpening } from './invocation-fixture.js'; +import { assertDoubleRunNotSealed } from './runtime-event-store-seal.js'; describe('SessionManager terminal ledger invariants', () => { test('coalesces one partial stream and flushes it before the final model event', async () => { @@ -214,10 +223,10 @@ describe('SessionManager terminal ledger invariants', () => { await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' })); - const [run] = await runStore.listSessionRuns(session.id); + const [run] = await runStore.listSessionInvocations(session.id); if (!run) throw new Error('run was not recorded'); - assert.strictEqual(run.status, 'failed'); - assert.strictEqual(run.failureClass, 'tool_failed'); + assert.strictEqual(runtimeInvocationOutcome(run), 'failed'); + assert.strictEqual(runtimeInvocationFailureClass(run), 'tool_failed'); const runtimeEvents = await runStore.readRuntimeEvents(session.id, run.runId); assert.strictEqual( runtimeEvents.some( @@ -271,10 +280,10 @@ describe('SessionManager terminal ledger invariants', () => { await stopPromise; while (!(await iterator.next()).done) {} - const [run] = await runStore.listSessionRuns(session.id); + const [run] = await runStore.listSessionInvocations(session.id); if (!run) throw new Error('run was not recorded'); - assert.strictEqual(run.status, 'cancelled'); - assert.strictEqual(run.abortSource, 'renderer.stop_button'); + assert.strictEqual(runtimeInvocationOutcome(run), 'cancelled'); + assert.strictEqual(run.terminalEvent?.actions?.stateDelta?.abortSource, 'renderer.stop_button'); const terminalEvents = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( isTerminalRuntimeEvent, ); @@ -308,10 +317,9 @@ describe('SessionManager terminal ledger invariants', () => { }); const expected = workHubDirectStopAbortSource('workhub-stop-action'); - const [run] = await runStore.listSessionRuns(session.id); + const [run] = await runStore.listSessionInvocations(session.id); if (!run) throw new Error('run was not recorded'); - assert.strictEqual(run.status, 'cancelled'); - assert.strictEqual(run.abortSource, expected); + assert.strictEqual(runtimeInvocationOutcome(run), 'cancelled'); const [terminal] = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( isTerminalRuntimeEvent, ); @@ -339,10 +347,10 @@ describe('SessionManager terminal ledger invariants', () => { assert.strictEqual((await iterator.next()).value?.type, 'text_delta'); await manager.stopSession(session.id, { source: 'stop_button' }); - const [run] = await runStore.listSessionRuns(session.id); + const [run] = await runStore.listSessionInvocations(session.id); if (!run) throw new Error('run was not recorded'); - assert.strictEqual(run.status, 'cancelled'); - assert.strictEqual(run.abortSource, 'renderer.stop_button'); + assert.strictEqual(runtimeInvocationOutcome(run), 'cancelled'); + assert.strictEqual(run.terminalEvent?.actions?.stateDelta?.abortSource, 'renderer.stop_button'); const terminalEvents = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( isTerminalRuntimeEvent, ); @@ -381,9 +389,9 @@ describe('SessionManager terminal ledger invariants', () => { await assert.rejects(() => manager.stopSession(session.id, { source: 'stop_button' })); await manager.stopSession(session.id, { source: 'stop_button' }); - const [run] = await runStore.listSessionRuns(session.id); + const [run] = await runStore.listSessionInvocations(session.id); if (!run) throw new Error('run was not recorded'); - assert.strictEqual(run.status, 'cancelled'); + assert.strictEqual(runtimeInvocationOutcome(run), 'cancelled'); const terminalEvents = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( isTerminalRuntimeEvent, ); @@ -413,7 +421,7 @@ describe('SessionManager terminal ledger invariants', () => { assert.strictEqual((await iterator.next()).value?.type, 'text_delta'); await manager.stopSession(session.id, { source: 'stop_button' }); - const [run] = await runStore.listSessionRuns(session.id); + const [run] = await runStore.listSessionInvocations(session.id); if (!run) throw new Error('run was not recorded'); const terminalEvents = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( isTerminalRuntimeEvent, @@ -453,9 +461,9 @@ describe('SessionManager terminal ledger invariants', () => { await sendPromise; assert.strictEqual((await store.readHeader(session.id)).status, 'active'); - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.status, 'failed'); - assert.strictEqual(run?.failureClass, 'tool_step_cap_reached'); + const [run] = await runStore.listSessionInvocations(session.id); + assert.strictEqual(run && runtimeInvocationOutcome(run), 'failed'); + assert.strictEqual(run && runtimeInvocationFailureClass(run), 'tool_step_cap_reached'); const terminalEvents = (await runStore.readRuntimeEvents(session.id, run!.runId)).filter( isTerminalRuntimeEvent, ); @@ -485,13 +493,6 @@ describe('SessionManager terminal ledger invariants', () => { now: nextNow(22_000), hooks: inertAgentRunHooks(store), }); - await runStore.createRun( - makeRunHeader({ - sessionId: session.id, - runId: run.runId, - turnId: run.turnId, - }), - ); const first = run.recordRuntimeEvents([ runtimeEvent({ id: 'terminal-one', @@ -552,9 +553,6 @@ describe('SessionManager terminal ledger invariants', () => { now: nextNow(23_000), hooks: inertAgentRunHooks(store), }); - await runStore.createRun( - makeRunHeader({ sessionId: session.id, runId: run.runId, turnId: run.turnId }), - ); // The rejection still fails the caller — a producer bug must not pass // quietly — and it is recorded on the run. @@ -570,7 +568,7 @@ describe('SessionManager terminal ledger invariants', () => { (error: unknown) => error instanceof ToolLedgerRejectionError, ); assert.match( - String((await runStore.readRun(session.id, run.runId)).traceWriteError), + String(await traceWriteFailure(runStore, session.id, run.runId)), /Tool ledger transition rejected: orphan_response/, ); @@ -625,9 +623,9 @@ describe('SessionManager terminal ledger invariants', () => { now: nextNow(24_000), hooks: inertAgentRunHooks(store), }); - await runStore.createRun( - makeRunHeader({ sessionId: session.id, runId: run.runId, turnId: run.turnId }), - ); + // The run is driven past its start here, so open its invocation the way + // starting it would have. + await seedOpening(runStore, { sessionId: session.id, runId: run.runId, turnId: run.turnId }); // A tool fact is what a damaged ledger refuses. await assert.rejects( @@ -643,7 +641,7 @@ describe('SessionManager terminal ledger invariants', () => { (error: unknown) => error instanceof ToolLedgerCorruptionError, ); assert.match( - String((await runStore.readRun(session.id, run.runId)).traceWriteError), + String(await traceWriteFailure(runStore, session.id, run.runId)), /Tool ledger is corrupt: duplicate_call/, ); @@ -672,7 +670,7 @@ describe('SessionManager terminal ledger invariants', () => { isTerminalRuntimeEvent, ); assert.strictEqual(terminalEvents.length, 1); - assert.strictEqual((await runStore.readRun(session.id, run.runId)).status, 'failed'); + assert.strictEqual(await runOutcome(runStore, session.id, run.runId), 'failed'); }); test('finalization keeps the silent skip when even the terminal barrier is refused', async () => { @@ -697,9 +695,6 @@ describe('SessionManager terminal ledger invariants', () => { now: nextNow(24_100), hooks: inertAgentRunHooks(store), }); - await runStore.createRun( - makeRunHeader({ sessionId: session.id, runId: run.runId, turnId: run.turnId }), - ); await run .recordRuntimeEvents([ runtimeEvent({ @@ -718,7 +713,7 @@ describe('SessionManager terminal ledger invariants', () => { (await runStore.readRuntimeEvents(session.id, run.runId)).some(isTerminalRuntimeEvent), false, ); - assert.strictEqual((await runStore.readRun(session.id, run.runId)).status, 'running'); + assert.strictEqual(await runOutcome(runStore, session.id, run.runId), undefined); }); test('a sealed-run refusal neither latches the store nor stamps a trace failure', async () => { @@ -742,9 +737,6 @@ describe('SessionManager terminal ledger invariants', () => { now: nextNow(24_200), hooks: inertAgentRunHooks(store), }); - await runStore.createRun( - makeRunHeader({ sessionId: session.id, runId: run.runId, turnId: run.turnId }), - ); run.stop('stop_button'); await run.settleStopTerminal(); assert.strictEqual( @@ -767,7 +759,7 @@ describe('SessionManager terminal ledger invariants', () => { (error: unknown) => error instanceof RunSealedError, ); - assert.strictEqual((await runStore.readRun(session.id, run.runId)).traceWriteError, undefined); + assert.strictEqual(await traceWriteFailure(runStore, session.id, run.runId), undefined); // The seal is per run and permanent, the way SqliteRuntimeStore keeps // refusing; the store stays healthy for everything else, so a second // run on the same store still writes. @@ -782,9 +774,6 @@ describe('SessionManager terminal ledger invariants', () => { now: nextNow(24_300), hooks: inertAgentRunHooks(store), }); - await runStore.createRun( - makeRunHeader({ sessionId: session.id, runId: second.runId, turnId: second.turnId }), - ); await second.recordRuntimeEvents([ runtimeEvent({ id: 'post-seal-probe', @@ -802,32 +791,18 @@ describe('SessionManager terminal ledger invariants', () => { ); }); - test('the continuation boundary hook fires between the terminal barrier and the header', async () => { + test('the continuation boundary hook fires only after the terminal barrier', async () => { // The #2313 recovery path defers 'after_terminal_event_committed' into // this hook because the claimed event's own write never ran; a crash at // the boundary must always find the terminal fact durable first. const order: string[] = []; - class OrderRecordingStore extends TinyAgentRunStore { - override async updateRun( - sessionId: string, - runId: string, - patch: Partial, - ): Promise { - order.push('header'); - return super.updateRun(sessionId, runId, patch); - } - } - const runStore = new OrderRecordingStore({ + const runStore = new TinyAgentRunStore({ beforeTerminalRuntimeEventAppend: async () => { order.push('barrier'); }, }); - await runStore.createRun( - makeRunHeader({ sessionId: 'session-1', runId: 'run-1', turnId: 'turn-1' }), - ); await commitOrCreateTerminalRunFact({ - runStore, runtimeEventStore: runStore, newId: nextId(), sessionId: 'session-1', @@ -836,13 +811,12 @@ describe('SessionManager terminal ledger invariants', () => { ts: 24_400, fallbackStatus: 'cancelled', fallbackInvocationId: 'run-1', - allowHeaderCommitFailure: false, afterTerminalDurable: async () => { order.push('boundary'); }, }); - assert.deepStrictEqual(order.slice(0, 3), ['barrier', 'boundary', 'header']); + assert.deepStrictEqual(order, ['barrier', 'boundary']); }); test('synthetic finalization claims its terminal outcome before its first await', async () => { @@ -870,13 +844,6 @@ describe('SessionManager terminal ledger invariants', () => { }, }, }); - await runStore.createRun( - makeRunHeader({ - sessionId: session.id, - runId: run.runId, - turnId: run.turnId, - }), - ); const finalization = run.finalize(); await headerUpdateStarted.promise; @@ -884,9 +851,7 @@ describe('SessionManager terminal ledger invariants', () => { releaseHeaderUpdate.resolve(); await finalization; - const header = await runStore.readRun(session.id, run.runId); - assert.strictEqual(header.status, 'failed'); - assert.strictEqual(header.failureClass, 'missing_terminal_event'); + assert.strictEqual(await runOutcome(runStore, session.id, run.runId), 'failed'); const terminals = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( isTerminalRuntimeEvent, ); @@ -896,18 +861,16 @@ describe('SessionManager terminal ledger invariants', () => { test('terminal run commits reject mismatched terminal RuntimeEvent statuses', async () => { const runStore = new TinyAgentRunStore(); - const run = makeRunHeader({ status: 'running' }); + const run = makeRunIdentity(); const completedTerminal = runtimeEvent({ id: 'rt-completed', status: 'completed', actions: { endInvocation: true }, }); - await runStore.createRun(run); await runStore.appendRuntimeEvent(run.sessionId, run.runId, completedTerminal); await assert.rejects( commitTerminalRunWithRuntimeFact({ - runStore, runtimeEventStore: runStore, newId: nextId(), sessionId: run.sessionId, @@ -918,25 +881,23 @@ describe('SessionManager terminal ledger invariants', () => { terminalEvent: completedTerminal, failureClass: 'tool_failed', }), - /terminal RuntimeEvent status completed cannot commit failed run header/, + /terminal RuntimeEvent status completed cannot commit a failed run/, ); - assert.strictEqual((await runStore.readRun(run.sessionId, run.runId)).status, 'running'); + assert.strictEqual(await runOutcome(runStore, run.sessionId, run.runId), undefined); }); test('terminal run commits reject terminal RuntimeEvents from another run', async () => { const runStore = new TinyAgentRunStore(); - const run = makeRunHeader({ status: 'running' }); + const run = makeRunIdentity(); const foreignTerminal = runtimeEvent({ id: 'rt-foreign-completed', runId: 'another-run', status: 'completed', actions: { endInvocation: true }, }); - await runStore.createRun(run); await assert.rejects( commitTerminalRunWithRuntimeFact({ - runStore, runtimeEventStore: runStore, newId: nextId(), sessionId: run.sessionId, @@ -946,25 +907,23 @@ describe('SessionManager terminal ledger invariants', () => { ts: 3, terminalEvent: foreignTerminal, }), - /terminal RuntimeEvent identity does not match run header commit/, + /terminal RuntimeEvent identity does not match the run it ends/, ); - assert.strictEqual((await runStore.readRun(run.sessionId, run.runId)).status, 'running'); + assert.strictEqual(await runOutcome(runStore, run.sessionId, run.runId), undefined); }); test('terminal run commits reject partial terminal RuntimeEvents', async () => { const runStore = new TinyAgentRunStore(); - const run = makeRunHeader({ status: 'running' }); + const run = makeRunIdentity(); const partialTerminal = runtimeEvent({ id: 'rt-partial-completed', status: 'completed', partial: true, actions: { endInvocation: true }, }); - await runStore.createRun(run); await assert.rejects( commitTerminalRunWithRuntimeFact({ - runStore, runtimeEventStore: runStore, newId: nextId(), sessionId: run.sessionId, @@ -974,18 +933,16 @@ describe('SessionManager terminal ledger invariants', () => { ts: 3, terminalEvent: partialTerminal, }), - /terminal RuntimeEvent must be final before terminal run header/, + /terminal RuntimeEvent must be final before it is committed/, ); - assert.strictEqual((await runStore.readRun(run.sessionId, run.runId)).status, 'running'); + assert.strictEqual(await runOutcome(runStore, run.sessionId, run.runId), undefined); }); - test('synthetic cancelled terminal commits the fallback abortSource to the run header', async () => { + test('a synthetic cancelled terminal carries the fallback abortSource', async () => { const runStore = new TinyAgentRunStore(); - const run = makeRunHeader({ status: 'running' }); - await runStore.createRun(run); + const run = await seedOpening(runStore, makeRunIdentity()); await commitOrCreateTerminalRunFact({ - runStore, runtimeEventStore: runStore, newId: nextId(), sessionId: run.sessionId, @@ -996,9 +953,7 @@ describe('SessionManager terminal ledger invariants', () => { fallbackInvocationId: run.runId, }); - const header = await runStore.readRun(run.sessionId, run.runId); - assert.strictEqual(header.status, 'cancelled'); - assert.strictEqual(header.abortSource, 'user_stop'); + assert.strictEqual(await runOutcome(runStore, run.sessionId, run.runId), 'cancelled'); const terminalEvents = (await runStore.readRuntimeEvents(run.sessionId, run.runId)).filter( isTerminalRuntimeEvent, ); @@ -1012,12 +967,10 @@ describe('SessionManager terminal ledger invariants', () => { const runStore = new TinyAgentRunStore({ failTerminalRuntimeEventDurabilityAfterAppend: true, }); - const run = makeRunHeader({ status: 'running' }); - await runStore.createRun(run); + const run = makeRunIdentity(); await assert.rejects( commitOrCreateTerminalRunFact({ - runStore, runtimeEventStore: runStore, newId: nextId(), sessionId: run.sessionId, @@ -1027,18 +980,17 @@ describe('SessionManager terminal ledger invariants', () => { fallbackStatus: 'failed', fallbackInvocationId: run.runId, fallbackFailureClass: 'missing_terminal_event', - allowHeaderCommitFailure: true, }), DurableStoreWriteError, ); - assert.strictEqual((await runStore.readRun(run.sessionId, run.runId)).status, 'running'); + assert.strictEqual(await runOutcome(runStore, run.sessionId, run.runId), undefined); assert.strictEqual((await runStore.readRuntimeEvents(run.sessionId, run.runId)).length, 1); assert.strictEqual((await runStore.readEvents(run.sessionId, run.runId)).length, 0); }); test('synthetic terminal builder keeps live and recovered metadata distinct', () => { - const run = makeRunHeader({ status: 'running' }); + const run = makeRunIdentity(); const live = buildSyntheticTerminalRuntimeEvent({ id: 'live-terminal', invocationId: run.runId, @@ -1067,7 +1019,7 @@ describe('SessionManager terminal ledger invariants', () => { }); test('terminal ledger classification rejects multiple terminal RuntimeEvent signals', () => { - const run = makeRunHeader({ status: 'running' }); + const run = makeRunIdentity(); const result = classifyTerminalRuntimeLedger(run, [ runtimeEvent({ @@ -1091,7 +1043,7 @@ describe('SessionManager terminal ledger invariants', () => { }), ]); - assert.strictEqual(result.kind, 'ambiguous'); + assert.strictEqual(result.kind, 'corrupt'); assert.deepStrictEqual( result.terminalEvents.map((event) => event.id), ['rt-completed', 'rt-failed'], @@ -1191,14 +1143,6 @@ describe('SessionManager terminal ledger invariants', () => { appendTurnState: async () => {}, }, }); - await runStore.createRun( - makeRunHeader({ - sessionId: session.id, - runId: run.runId, - turnId: run.turnId, - status: 'running', - }), - ); const terminalEvent = runtimeEvent({ id: 'rt-completed', sessionId: session.id, @@ -1221,7 +1165,7 @@ describe('SessionManager terminal ledger invariants', () => { }); await run.finalize(); - assert.strictEqual((await runStore.readRun(session.id, run.runId)).status, 'running'); + assert.strictEqual(await runOutcome(runStore, session.id, run.runId), undefined); assert.strictEqual( (await runStore.readRuntimeEvents(session.id, run.runId)).some(isTerminalRuntimeEvent), false, @@ -1251,20 +1195,10 @@ describe('SessionManager terminal ledger invariants', () => { appendTurnState: async () => {}, }, }); - await runStore.createRun( - makeRunHeader({ - sessionId: session.id, - runId: run.runId, - turnId: run.turnId, - status: 'running', - }), - ); await run.finalize(); - const header = await runStore.readRun(session.id, run.runId); - assert.strictEqual(header.status, 'failed'); - assert.strictEqual(header.failureClass, 'missing_terminal_event'); + assert.strictEqual(await runOutcome(runStore, session.id, run.runId), 'failed'); const terminalEvents = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( isTerminalRuntimeEvent, ); @@ -1276,9 +1210,7 @@ describe('SessionManager terminal ledger invariants', () => { 'missing_terminal_event', ); assert.strictEqual(terminalEvents[0]?.actions?.stateDelta?.recovered, undefined); - await new RuntimeReadModel({ runStore, runtimeEventStore: runStore }).getSessionView( - session.id, - ); + await new RuntimeReadModel({ runtimeEventStore: runStore }).getSessionView(session.id); }); test('direct AgentRun stop synthesizes a cancelled terminal fact when no terminal event was recorded', async () => { @@ -1323,10 +1255,7 @@ describe('SessionManager terminal ledger invariants', () => { run.stop('stop_button'); await run.finalize(); - const header = await runStore.readRun(session.id, run.runId); - assert.strictEqual(header.status, 'cancelled'); - assert.strictEqual(header.failureClass, undefined); - assert.strictEqual(header.abortSource, 'renderer.stop_button'); + assert.strictEqual(await runOutcome(runStore, session.id, run.runId), 'cancelled'); const terminalEvents = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( isTerminalRuntimeEvent, ); @@ -1336,9 +1265,7 @@ describe('SessionManager terminal ledger invariants', () => { assert.strictEqual(terminalEvents[0]?.actions?.stateDelta?.abortSource, 'renderer.stop_button'); assert.strictEqual(terminalEvents[0]?.actions?.stateDelta?.failureClass, undefined); assert.strictEqual(terminalEvents[0]?.actions?.stateDelta?.recovered, undefined); - await new RuntimeReadModel({ runStore, runtimeEventStore: runStore }).getSessionView( - session.id, - ); + await new RuntimeReadModel({ runtimeEventStore: runStore }).getSessionView(session.id); }); test('a stop settlement racing finalize commits exactly one terminal run event', async () => { @@ -1398,10 +1325,6 @@ describe('SessionManager terminal ledger invariants', () => { releaseTerminalAppend.resolve(); await Promise.all([settled, finalized]); - const runEvents = (await runStore.readEvents(session.id, run.runId)).filter( - (event) => event.type === 'run_cancelled', - ); - assert.strictEqual(runEvents.length, 1); const terminalEvents = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( isTerminalRuntimeEvent, ); @@ -1628,7 +1551,7 @@ describe('SessionManager terminal ledger invariants', () => { assert.strictEqual(terminalEvents[0]?.actions?.stateDelta?.abortSource, 'renderer.stop_button'); }); - test('stop settlement probes a latched run store instead of skipping the header commit', async () => { + test('stop settlement probes a latched run store instead of skipping the terminal commit', async () => { const store = new TinySessionStore(); const runStore = new TinyAgentRunStore(); const session = await store.create(makeInput()); @@ -1668,7 +1591,7 @@ describe('SessionManager terminal ledger invariants', () => { await run.begin(); // One best-effort trace append failure latches the Run store. Nothing // surfaces to the user, which is what made the pre-fix behaviour a - // silent stop success: commitTerminalRun skips under the latch and the + // silent stop success: the terminal commit skips under the latch and the // run stays non-terminal with no error to retry on. runStore.failNextRunEventAppends = 1; run.recordRunTrace({ @@ -1691,10 +1614,6 @@ describe('SessionManager terminal ledger invariants', () => { ); assert.strictEqual(terminalEvents.length, 1); assert.strictEqual(terminalEvents[0]?.status, 'aborted'); - const cancelled = (await runStore.readEvents(session.id, run.runId)).filter( - (event) => event.type === 'run_cancelled', - ); - assert.strictEqual(cancelled.length, 1); }); test('Runtime execution still commits failed terminal facts when failed turn projection fails', async () => { @@ -1706,10 +1625,10 @@ describe('SessionManager terminal ledger invariants', () => { await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' })); - const [header] = await runStore.listSessionRuns(session.id); + const [header] = await runStore.listSessionInvocations(session.id); if (!header) throw new Error('run was not recorded'); - assert.strictEqual(header.status, 'failed'); - assert.strictEqual(header.failureClass, 'tool_failed'); + assert.strictEqual(runtimeInvocationOutcome(header), 'failed'); + assert.strictEqual(runtimeInvocationFailureClass(header), 'tool_failed'); const terminalEvents = (await runStore.readRuntimeEvents(session.id, header.runId)).filter( isTerminalRuntimeEvent, ); @@ -1718,7 +1637,7 @@ describe('SessionManager terminal ledger invariants', () => { assert.strictEqual(terminalEvents[0]?.actions?.stateDelta?.failureClass, 'tool_failed'); }); - test('startup recovery reuses an incomplete existing terminal RuntimeEvent instead of appending another', async () => { + test('startup recovery leaves a failed terminal RuntimeEvent that states no failure class alone', async () => { const store = new TinySessionStore(); const runStore = new TinyAgentRunStore(); const manager = new SessionManager({ @@ -1730,16 +1649,16 @@ describe('SessionManager terminal ledger invariants', () => { now: nextNow(50_000), }); const session = await store.create(makeInput({ status: 'active' })); - const run = await runStore.createRun( - makeRunHeader({ + const run = await seedOpening( + runStore, + makeRunIdentity({ sessionId: session.id, runId: 'run-incomplete-terminal', turnId: 'turn-incomplete-terminal', - status: 'running', }), ); await runStore.appendEvent(session.id, run.runId, { - type: 'run_started', + type: 'turn_started', id: 'run-started', sessionId: session.id, runId: run.runId, @@ -1761,23 +1680,24 @@ describe('SessionManager terminal ledger invariants', () => { await manager.recoverInterruptedSessions(); - const header = await runStore.readRun(session.id, run.runId); - assert.strictEqual(header.status, 'failed'); - assert.strictEqual(header.failureClass, 'app_restarted'); + const invocation = await readInvocation(runStore, session.id, run.runId); + assert.strictEqual(runtimeInvocationOutcome(invocation), 'failed'); + // The run already ended, and its ending is immutable, so recovery has + // nothing to attribute and no second record to attribute it to. + assert.strictEqual(runtimeInvocationFailureClass(invocation), undefined); const terminalEvents = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( isTerminalRuntimeEvent, ); assert.strictEqual(terminalEvents.length, 1); assert.strictEqual(terminalEvents[0]?.id, 'rt-failed-without-class'); const view = await new RuntimeReadModel({ - runStore, runtimeEventStore: runStore, }).getSessionView(session.id); assert.strictEqual(view.terminalFacts.length, 1); - assert.strictEqual(view.terminalFacts[0]?.failureClass, 'app_restarted'); + assert.strictEqual(view.terminalFacts[0]?.failureClass, 'unknown'); }); - test('startup recovery completes an existing aborted terminal RuntimeEvent without appending another', async () => { + test('startup recovery leaves an aborted terminal RuntimeEvent that states no source alone', async () => { const store = new TinySessionStore(); const runStore = new TinyAgentRunStore(); const manager = new SessionManager({ @@ -1789,16 +1709,16 @@ describe('SessionManager terminal ledger invariants', () => { now: nextNow(60_000), }); const session = await store.create(makeInput({ status: 'active' })); - const run = await runStore.createRun( - makeRunHeader({ + const run = await seedOpening( + runStore, + makeRunIdentity({ sessionId: session.id, runId: 'run-incomplete-abort', turnId: 'turn-incomplete-abort', - status: 'running', }), ); await runStore.appendEvent(session.id, run.runId, { - type: 'run_started', + type: 'turn_started', id: 'run-started', sessionId: session.id, runId: run.runId, @@ -1820,31 +1740,29 @@ describe('SessionManager terminal ledger invariants', () => { await manager.recoverInterruptedSessions(); - const header = await runStore.readRun(session.id, run.runId); - assert.strictEqual(header.status, 'cancelled'); - assert.strictEqual(header.abortSource, 'unknown'); + assert.strictEqual(await runOutcome(runStore, session.id, run.runId), 'cancelled'); const terminalEvents = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( isTerminalRuntimeEvent, ); assert.strictEqual(terminalEvents.length, 1); assert.strictEqual(terminalEvents[0]?.id, 'rt-aborted-without-source'); const view = await new RuntimeReadModel({ - runStore, runtimeEventStore: runStore, }).getSessionView(session.id); assert.strictEqual(view.terminalFacts.length, 1); assert.strictEqual(view.terminalFacts[0]?.abortSource, 'unknown'); }); - test('RuntimeReadModel reads a non-terminal header when a terminal RuntimeEvent fact exists', async () => { + test('RuntimeReadModel reads a run outcome off its terminal RuntimeEvent fact', async () => { const runStore = new TinyAgentRunStore(); - const run = makeRunHeader({ - sessionId: 'session-read-model', - runId: 'run-read-model', - turnId: 'turn-read-model', - status: 'running', - }); - await runStore.createRun(run); + const run = await seedOpening( + runStore, + makeRunIdentity({ + sessionId: 'session-read-model', + runId: 'run-read-model', + turnId: 'turn-read-model', + }), + ); await runStore.appendRuntimeEvent( run.sessionId, run.runId, @@ -1868,12 +1786,17 @@ describe('SessionManager terminal ledger invariants', () => { ); const view = await new RuntimeReadModel({ - runStore, runtimeEventStore: runStore, }).getSessionView(run.sessionId); - assert.strictEqual(view.runs[0]?.status, 'failed'); - assert.strictEqual(view.runs[0]?.failureClass, 'tool_failed'); + assert.strictEqual( + view.invocations[0] && runtimeInvocationOutcome(view.invocations[0]), + 'failed', + ); + assert.strictEqual( + view.invocations[0] && runtimeInvocationFailureClass(view.invocations[0]), + 'tool_failed', + ); assert.strictEqual(view.terminalFacts.length, 1); assert.strictEqual(view.terminalFacts[0]?.failureClass, 'tool_failed'); const turnState = view.messages.find((message) => message.type === 'turn_state'); @@ -1882,89 +1805,23 @@ describe('SessionManager terminal ledger invariants', () => { assert.strictEqual(turnState.errorClass, 'tool_failed'); }); - test('RuntimeReadModel treats the terminal RuntimeEvent as the failure fact when the header is stale', async () => { + test('RuntimeReadModel preserves per-run event order when timestamps disagree', async () => { const runStore = new TinyAgentRunStore(); - const run = makeRunHeader({ - sessionId: 'session-stale-failure-class', - runId: 'run-stale-failure-class', - turnId: 'turn-stale-failure-class', - status: 'failed', - completedAt: 10, - failureClass: 'stale_header_failure', - }); - await runStore.createRun(run); - await runStore.appendRuntimeEvent( - run.sessionId, - run.runId, - runtimeEvent({ - id: 'rt-user-stale-failure', - sessionId: run.sessionId, - runId: run.runId, - turnId: run.turnId, - ts: 8, - role: 'user', - author: 'user', - content: { kind: 'text', text: 'hello' }, - }), - ); - await runStore.appendRuntimeEvent( - run.sessionId, - run.runId, - runtimeEvent({ - id: 'rt-failed-runtime-fact', - sessionId: run.sessionId, - runId: run.runId, - turnId: run.turnId, - ts: 10, - status: 'failed', - content: { - kind: 'error', - code: 'runtime_failure', - reason: 'runtime_failure', - message: 'Runtime failed', - }, - actions: { - endInvocation: true, - stateDelta: { failureClass: 'runtime_failure' }, - }, - }), - ); - - const view = await new RuntimeReadModel({ + const run = await seedOpening( runStore, - runtimeEventStore: runStore, - }).getSessionView(run.sessionId); - - assert.strictEqual(view.terminalFacts[0]?.failureClass, 'runtime_failure'); - assert.strictEqual(view.runs[0]?.failureClass, 'runtime_failure'); - const turnState = view.messages.find((message) => message.type === 'turn_state'); - if (turnState?.type !== 'turn_state') throw new Error('turn_state was not projected'); - assert.strictEqual(turnState.errorClass, 'runtime_failure'); - assert.strictEqual( - view.diagnostics.some( - (diagnostic) => - diagnostic.message === 'terminal run header does not match RuntimeEvent terminal fact', - ), - true, + makeRunIdentity({ + sessionId: 'session-durable-order', + runId: 'run-durable-order', + turnId: 'turn-durable-order', + }), ); - }); - - test('RuntimeReadModel preserves per-run event order when timestamps disagree', async () => { - const runStore = new TinyAgentRunStore(); - const run = makeRunHeader({ - sessionId: 'session-durable-order', - runId: 'run-durable-order', - turnId: 'turn-durable-order', - status: 'completed', - }); - await runStore.createRun(run); for (const event of [ runtimeEvent({ id: 'rt-user-durable-order', sessionId: run.sessionId, runId: run.runId, turnId: run.turnId, - ts: 2, + ts: 3, role: 'user', author: 'user', content: { kind: 'text', text: 'hello' }, @@ -1974,7 +1831,7 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: run.sessionId, runId: run.runId, turnId: run.turnId, - ts: 1, + ts: 2, role: 'model', author: 'agent', content: { kind: 'text', text: 'world' }, @@ -1984,7 +1841,7 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: run.sessionId, runId: run.runId, turnId: run.turnId, - ts: 3, + ts: 4, status: 'completed', actions: { endInvocation: true }, }), @@ -1993,104 +1850,37 @@ describe('SessionManager terminal ledger invariants', () => { } const view = await new RuntimeReadModel({ - runStore, runtimeEventStore: runStore, }).getSessionView(run.sessionId); assert.deepStrictEqual( view.events.map((event) => event.id), - ['rt-user-durable-order', 'rt-assistant-durable-order', 'rt-terminal-durable-order'], - ); - }); - - test('RuntimeReadModel places backfilled events after durable session order', async () => { - const sessionId = 'session-mixed-durable-order'; - const firstRun = makeRunHeader({ - sessionId, - runId: 'run-first-durable', - turnId: 'turn-first-durable', - status: 'running', - createdAt: 1, - }); - const backfilledRun = makeRunHeader({ - sessionId, - runId: 'run-backfilled', - turnId: 'turn-backfilled', - status: 'completed', - createdAt: 2, - }); - const lastRun = makeRunHeader({ - sessionId, - runId: 'run-last-durable', - turnId: 'turn-last-durable', - status: 'completed', - createdAt: 3, - }); - const runStore = new TinyAgentRunStore(); - for (const run of [firstRun, backfilledRun, lastRun]) await runStore.createRun(run); - - const firstEvent = runtimeEvent({ - id: 'rt-first-durable', - invocationId: 'inv-first-durable', - sessionId, - runId: firstRun.runId, - turnId: firstRun.turnId, - ts: 100, - status: 'completed', - actions: { endInvocation: true }, - }); - const lastEvent = runtimeEvent({ - id: 'rt-last-durable', - invocationId: 'inv-last-durable', - sessionId, - runId: lastRun.runId, - turnId: lastRun.turnId, - ts: 1, - status: 'completed', - actions: { endInvocation: true }, - }); - await runStore.appendRuntimeEvent(sessionId, firstRun.runId, firstEvent); - await runStore.appendRuntimeEvent(sessionId, lastRun.runId, lastEvent); - - const runtimeEventStore = Object.assign(runStore, { - readSessionRuntimeEventEntries: async () => [ - { ordinal: 1, event: firstEvent }, - { ordinal: 2, event: lastEvent }, + [ + 'run-durable-order-invocation-opened', + 'rt-user-durable-order', + 'rt-assistant-durable-order', + 'rt-terminal-durable-order', ], - }); - const legacyMessages: StoredMessage[] = [ - { - type: 'turn_state', - id: 'legacy-state', - turnId: backfilledRun.turnId, - ts: 50, - status: 'completed', - partialOutputRetained: false, - }, - ]; - - const view = await new RuntimeReadModel({ - runStore, - runtimeEventStore, - projectionCache: { readMessages: async () => legacyMessages }, - }).getSessionView(sessionId); - - assert.deepStrictEqual( - view.events.map((event) => event.runId), - [firstRun.runId, lastRun.runId, backfilledRun.runId], ); }); test('RuntimeReadModel retains terminal partial snapshots alongside durable events', async () => { const runStore = new TinyAgentRunStore(); - const run = makeRunHeader({ status: 'cancelled', abortSource: 'user' }); - await runStore.createRun(run); - const opening = runtimeEvent({ - id: 'rt-partial-opening', + const run = await seedOpening( + runStore, + makeRunIdentity({ + sessionId: 'session-partial-order', + runId: 'run-partial-order', + turnId: 'turn-partial-order', + }), + ); + const [opened] = await runStore.readRuntimeEvents(run.sessionId, run.runId); + const prompt = runtimeEvent({ + id: 'rt-partial-prompt', sessionId: run.sessionId, runId: run.runId, turnId: run.turnId, - ts: 1, + ts: 2, role: 'user', author: 'user', content: { kind: 'text', text: 'hello' }, @@ -2100,7 +1890,7 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: run.sessionId, runId: run.runId, turnId: run.turnId, - ts: 2, + ts: 3, partial: true, role: 'model', author: 'agent', @@ -2111,33 +1901,36 @@ describe('SessionManager terminal ledger invariants', () => { sessionId: run.sessionId, runId: run.runId, turnId: run.turnId, - ts: 3, + ts: 4, status: 'cancelled', - actions: { endInvocation: true }, + actions: { endInvocation: true, stateDelta: { abortSource: 'user' } }, }); + await runStore.appendRuntimeEvent(run.sessionId, run.runId, prompt); + await runStore.appendRuntimeEvent(run.sessionId, run.runId, terminal); + // The partial never reached durable session order, which is exactly the + // event the run read has to keep. const runtimeEventStore = Object.assign(runStore, { - readRuntimeEvents: async () => [opening, partial, terminal], - readSessionRuntimeEventEntries: async () => [ - { ordinal: 1, event: opening }, - { ordinal: 2, event: terminal }, - ], + readRuntimeEvents: async () => [opened!, prompt, partial, terminal], }); - const view = await new RuntimeReadModel({ - runStore, - runtimeEventStore, - }).getSessionView(run.sessionId); + const view = await new RuntimeReadModel({ runtimeEventStore }).getSessionView(run.sessionId); assert.deepStrictEqual( view.events.map((event) => event.id), - [opening.id, partial.id, terminal.id], + [opened!.id, prompt.id, partial.id, terminal.id], ); }); test('RuntimeReadModel rejects a failing durable-order reader', async () => { const runStore = new TinyAgentRunStore(); - const run = makeRunHeader({ status: 'completed' }); - await runStore.createRun(run); + const run = await seedOpening( + runStore, + makeRunIdentity({ + sessionId: 'session-durable-order-read', + runId: 'run-durable-order-read', + turnId: 'turn-durable-order-read', + }), + ); const runtimeEventStore = Object.assign(runStore, { readSessionRuntimeEventEntries: async () => { throw new Error('ordinal read rejected'); @@ -2145,21 +1938,21 @@ describe('SessionManager terminal ledger invariants', () => { }); await assert.rejects( - new RuntimeReadModel({ runStore, runtimeEventStore }).getSessionView(run.sessionId), + new RuntimeReadModel({ runtimeEventStore }).getSessionView(run.sessionId), /RuntimeEvent session order read failed/, ); }); - test('RuntimeReadModel rejects terminal headers when the ledger has no valid terminal fact', async () => { + test('RuntimeReadModel rejects a run whose ledger has no valid terminal fact', async () => { const runStore = new TinyAgentRunStore(); - const run = makeRunHeader({ - sessionId: 'session-ambiguous-terminal-read', - runId: 'run-ambiguous-terminal-read', - turnId: 'turn-ambiguous-terminal-read', - status: 'completed', - completedAt: 10, - }); - await runStore.createRun(run); + const run = await seedOpening( + runStore, + makeRunIdentity({ + sessionId: 'session-ambiguous-terminal-read', + runId: 'run-ambiguous-terminal-read', + turnId: 'turn-ambiguous-terminal-read', + }), + ); await runStore.appendRuntimeEvent( run.sessionId, run.runId, @@ -2202,203 +1995,10 @@ describe('SessionManager terminal ledger invariants', () => { ); await assert.rejects( - new RuntimeReadModel({ runStore, runtimeEventStore: runStore }).getSessionView(run.sessionId), + new RuntimeReadModel({ runtimeEventStore: runStore }).getSessionView(run.sessionId), /valid terminal fact/, ); }); - - test('startup recovery does not append another terminal RuntimeEvent when the ledger is ambiguous', async () => { - const store = new TinySessionStore(); - const runStore = new TinyAgentRunStore(); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends: new BackendRegistry(), - newId: nextId(), - now: nextNow(70_000), - }); - const session = await store.create(makeInput({ status: 'active' })); - const run = await runStore.createRun( - makeRunHeader({ - sessionId: session.id, - runId: 'run-ambiguous-terminal', - turnId: 'turn-ambiguous-terminal', - status: 'running', - }), - ); - await runStore.appendEvent(session.id, run.runId, { - type: 'run_started', - id: 'run-started', - sessionId: session.id, - runId: run.runId, - turnId: run.turnId, - ts: 2, - }); - await runStore.appendRuntimeEvent( - session.id, - run.runId, - runtimeEvent({ - id: 'rt-completed', - sessionId: session.id, - runId: run.runId, - turnId: run.turnId, - status: 'completed', - actions: { endInvocation: true }, - }), - ); - await runStore.appendRuntimeEvent( - session.id, - run.runId, - runtimeEvent({ - id: 'rt-failed', - sessionId: session.id, - runId: run.runId, - turnId: run.turnId, - status: 'failed', - content: { - kind: 'error', - code: 'tool_failed', - reason: 'tool_failed', - message: 'Tool failed', - }, - actions: { - endInvocation: true, - stateDelta: { failureClass: 'tool_failed' }, - }, - }), - ); - - const recovered = await manager.recoverInterruptedSessions(); - - assert.deepStrictEqual(recovered, []); - assert.strictEqual((await runStore.readRun(session.id, run.runId)).status, 'running'); - const terminalEvents = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( - isTerminalRuntimeEvent, - ); - assert.deepStrictEqual( - terminalEvents.map((event) => event.id), - ['rt-completed', 'rt-failed'], - ); - }); - - test('startup recovery treats terminal headers without ledger facts as missing terminal events', async () => { - const store = new TinySessionStore(); - const runStore = new TinyAgentRunStore(); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends: new BackendRegistry(), - newId: nextId(), - now: nextNow(80_000), - }); - const completedSession = await store.create(makeInput({ status: 'active' })); - const failedSession = await store.create(makeInput({ status: 'active' })); - const cancelledSession = await store.create(makeInput({ status: 'active' })); - await runStore.createRun( - makeRunHeader({ - sessionId: completedSession.id, - runId: 'run-completed-empty-ledger', - turnId: 'turn-completed-empty-ledger', - status: 'completed', - completedAt: 20, - }), - ); - await runStore.appendEvent(completedSession.id, 'run-completed-empty-ledger', { - type: 'run_completed', - id: 'run-completed-event', - sessionId: completedSession.id, - runId: 'run-completed-empty-ledger', - turnId: 'turn-completed-empty-ledger', - ts: 20, - }); - await runStore.createRun( - makeRunHeader({ - sessionId: failedSession.id, - runId: 'run-failed-empty-ledger', - turnId: 'turn-failed-empty-ledger', - status: 'failed', - failureClass: 'tool_failed', - completedAt: 21, - }), - ); - await runStore.appendEvent(failedSession.id, 'run-failed-empty-ledger', { - type: 'run_failed', - id: 'run-failed-event', - sessionId: failedSession.id, - runId: 'run-failed-empty-ledger', - turnId: 'turn-failed-empty-ledger', - ts: 21, - data: { failureClass: 'tool_failed' }, - }); - await runStore.createRun( - makeRunHeader({ - sessionId: cancelledSession.id, - runId: 'run-cancelled-empty-ledger', - turnId: 'turn-cancelled-empty-ledger', - status: 'cancelled', - abortSource: 'user_stop', - completedAt: 22, - }), - ); - await runStore.appendEvent(cancelledSession.id, 'run-cancelled-empty-ledger', { - type: 'run_cancelled', - id: 'run-cancelled-event', - sessionId: cancelledSession.id, - runId: 'run-cancelled-empty-ledger', - turnId: 'turn-cancelled-empty-ledger', - ts: 22, - }); - - const recovered = await manager.recoverInterruptedSessions(); - - assert.deepStrictEqual(recovered, [completedSession.id, failedSession.id, cancelledSession.id]); - const completedEvents = ( - await runStore.readRuntimeEvents(completedSession.id, 'run-completed-empty-ledger') - ).filter(isTerminalRuntimeEvent); - assert.strictEqual(completedEvents.length, 1); - assert.strictEqual(completedEvents[0]?.status, 'failed'); - assert.strictEqual( - completedEvents[0]?.actions?.stateDelta?.failureClass, - 'missing_terminal_event', - ); - const failedEvents = ( - await runStore.readRuntimeEvents(failedSession.id, 'run-failed-empty-ledger') - ).filter(isTerminalRuntimeEvent); - assert.strictEqual(failedEvents.length, 1); - assert.strictEqual(failedEvents[0]?.status, 'failed'); - assert.strictEqual( - failedEvents[0]?.actions?.stateDelta?.failureClass, - 'missing_terminal_event', - ); - const cancelledEvents = ( - await runStore.readRuntimeEvents(cancelledSession.id, 'run-cancelled-empty-ledger') - ).filter(isTerminalRuntimeEvent); - assert.strictEqual(cancelledEvents.length, 1); - assert.strictEqual(cancelledEvents[0]?.status, 'failed'); - assert.strictEqual( - cancelledEvents[0]?.actions?.stateDelta?.failureClass, - 'missing_terminal_event', - ); - - const completedView = await new RuntimeReadModel({ - runStore, - runtimeEventStore: runStore, - }).getSessionView(completedSession.id); - assert.strictEqual(completedView.terminalFacts[0]?.runStatus, 'failed'); - assert.strictEqual(completedView.terminalFacts[0]?.failureClass, 'missing_terminal_event'); - const failedView = await new RuntimeReadModel({ - runStore, - runtimeEventStore: runStore, - }).getSessionView(failedSession.id); - assert.strictEqual(failedView.terminalFacts[0]?.failureClass, 'missing_terminal_event'); - const cancelledView = await new RuntimeReadModel({ - runStore, - runtimeEventStore: runStore, - }).getSessionView(cancelledSession.id); - assert.strictEqual(cancelledView.terminalFacts[0]?.failureClass, 'missing_terminal_event'); - }); }); type ScriptEvent = @@ -2657,7 +2257,6 @@ class TinySessionStore implements SessionStore { } class TinyAgentRunStore implements AgentRunStore, RuntimeEventStore { - private headers = new Map(); private events = new Map(); private runtimeEvents = new Map(); private runtimeEventEntries: RuntimeEvent[] = []; @@ -2688,35 +2287,6 @@ class TinyAgentRunStore implements AgentRunStore, RuntimeEventStore { return this.options.durability; } - async createRun(header: AgentRunHeader): Promise { - this.headers.set(key(header.sessionId, header.runId), clone(header)); - return clone(header); - } - - async updateRun( - sessionId: string, - runId: string, - patch: Partial, - ): Promise { - const current = await this.readRun(sessionId, runId); - const next = { ...current, ...patch, sessionId, runId }; - this.headers.set(key(sessionId, runId), clone(next)); - return clone(next); - } - - async readRun(sessionId: string, runId: string): Promise { - const header = this.headers.get(key(sessionId, runId)); - if (!header) throw new Error(`Unknown run ${runId}`); - return clone(header); - } - - async listSessionRuns(sessionId: string): Promise { - return Array.from(this.headers.values()) - .filter((header) => header.sessionId === sessionId) - .sort((a, b) => a.createdAt - b.createdAt || a.runId.localeCompare(b.runId)) - .map(clone); - } - async appendEvent(sessionId: string, runId: string, event: AgentRunEvent): Promise { if (this.failNextRunEventAppends > 0) { this.failNextRunEventAppends -= 1; @@ -2755,9 +2325,7 @@ class TinyAgentRunStore implements AgentRunStore, RuntimeEventStore { if (isTerminalRuntimeEvent(event)) await this.options.beforeTerminalRuntimeEventAppend?.(); const eventKey = key(sessionId, runId); this.runtimeEvents.set(eventKey, [...(this.runtimeEvents.get(eventKey) ?? []), clone(event)]); - if (event.partial !== true && !this.runtimeEventEntries.some(({ id }) => id === event.id)) { - this.runtimeEventEntries.push(clone(event)); - } + if (event.partial !== true) this.runtimeEventEntries.push(clone(event)); } async ensureTerminalRuntimeEventDurable( @@ -2810,6 +2378,13 @@ class TinyAgentRunStore implements AgentRunStore, RuntimeEventStore { ); return ordered.map((item) => item.event); } + + async listSessionInvocations(sessionId: string): Promise { + return runtimeInvocationsFromSessionEvents( + sessionId, + await this.readSessionRuntimeEvents(sessionId), + ); + } } class BatchingRuntimeEventStore implements RuntimeEventStore { @@ -2820,6 +2395,7 @@ class BatchingRuntimeEventStore implements RuntimeEventStore { constructor(private readonly failPartialBatch = false) {} async appendRuntimeEvent(_sessionId: string, _runId: string, event: RuntimeEvent): Promise { + assertDoubleRunNotSealed(this.events, event); this.order.push(`append:${event.id}`); this.events.push(clone(event)); } @@ -2848,14 +2424,18 @@ class BatchingRuntimeEventStore implements RuntimeEventStore { return clone(this.events); } - async readSessionRuntimeEventEntries(sessionId: string) { + async readSessionRuntimeEvents(): Promise { + return clone(this.events); + } + + async readSessionRuntimeEventEntries() { return this.events - .filter((event) => event.sessionId === sessionId && event.partial !== true) + .filter((event) => event.partial !== true) .map((event, index) => ({ ordinal: index + 1, event: clone(event) })); } - async readSessionRuntimeEvents(): Promise { - return clone(this.events); + async listSessionInvocations(sessionId: string): Promise { + return runtimeInvocationsFromSessionEvents(sessionId, clone(this.events)); } } @@ -2871,21 +2451,76 @@ function makeInput(overrides: Partial = {}): CreateSessionIn }; } -function makeRunHeader(overrides: Partial = {}): AgentRunHeader { - return { - runId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - status: 'running', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/cwd', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 1, - ...overrides, - }; +/** The identity a run is named by. Everything else about it lives on its events. */ +function makeRunIdentity( + overrides: Partial<{ sessionId: string; runId: string; turnId: string }> = {}, +): { sessionId: string; runId: string; turnId: string } { + return { sessionId: 'session-1', runId: 'run-1', turnId: 'turn-1', ...overrides }; +} + +/** Open one invocation on the spine, the way the runtime would. */ +async function seedOpening( + runtimeEventStore: Pick, + run: { sessionId: string; runId: string; turnId: string }, + openedAt = 1, +): Promise<{ sessionId: string; runId: string; turnId: string }> { + await runtimeEventStore.appendRuntimeEvent( + run.sessionId, + run.runId, + buildInvocationOpenedEvent({ + id: `${run.runId}-invocation-opened`, + run: { ...run, invocationId: run.runId }, + openedAt, + opening: testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'fake-connection', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { cwd: '/tmp/cwd' }, + }), + }), + ); + return run; +} + +/** The one invocation that opened this run. */ +async function readInvocation( + runtimeEventStore: Pick, + sessionId: string, + runId: string, +): Promise { + const found = (await runtimeEventStore.listSessionInvocations(sessionId)).find( + (candidate) => candidate.runId === runId, + ); + if (!found) throw new Error(`Session ${sessionId} has no invocation for run ${runId}`); + return found; +} + +/** What the run's own operational ledger says went wrong writing its trace. */ +async function traceWriteFailure( + runStore: Pick, + sessionId: string, + runId: string, +): Promise { + const failure = (await runStore.readEvents(sessionId, runId)).find( + (event) => event.type === 'trace_write_failed', + ); + return failure ? String(failure.message) : undefined; +} + +/** What the run's events say it ended as, or `undefined` while it is still open. */ +async function runOutcome( + runtimeEventStore: Pick, + sessionId: string, + runId: string, +): Promise<'completed' | 'failed' | 'cancelled' | undefined> { + const invocation = (await runtimeEventStore.listSessionInvocations(sessionId)).find( + (candidate) => candidate.runId === runId, + ); + return invocation ? runtimeInvocationOutcome(invocation) : undefined; } /** Mirrors the private predicate in `sqlite-runtime-store.ts` that gates the @@ -2901,10 +2536,13 @@ function isToolLedgerBearingEvent(event: RuntimeEvent): boolean { } function runtimeEvent(overrides: Partial): RuntimeEvent { + const runId = overrides.runId ?? 'run-1'; return { id: 'rt-event', - invocationId: 'inv-1', - runId: 'run-1', + // One invocation per run here, named by it, exactly as `seedOpening` opens + // it and as a run with no explicit invocation id names its own. + invocationId: runId, + runId, sessionId: 'session-1', turnId: 'turn-1', ts: 2, diff --git a/packages/runtime/src/__tests__/session-manager.test.ts b/packages/runtime/src/__tests__/session-manager.test.ts index a0babc9ecb..3acbdb5837 100644 --- a/packages/runtime/src/__tests__/session-manager.test.ts +++ b/packages/runtime/src/__tests__/session-manager.test.ts @@ -18,8 +18,26 @@ */ import { nextId } from '@maka/core/test-only/async-primitives'; +import { runtimeInvocationFailureClass } from '../runtime-event-read-model.js'; import { describe, test } from 'node:test'; import assert from 'node:assert/strict'; +import { DEFAULT_TOOL_MODE } from '@maka/core/tool-mode'; +import { + buildInvocationOpenedEvent, + isSessionInlineInvocation, + runtimeInvocationOutcome, + runtimeInvocationsFromSessionEvents, + type RootExecutionDescriptor, + type RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; +import type { + RuntimeEventInvocationOpenedContent, + RuntimeInvocationLineage, + RuntimeInvocationRootAuthority, +} from '@maka/core/runtime-event'; +import type { PermissionMode } from '@maka/core/permission'; +import type { PersistedBackendKind } from '@maka/core/session'; +import type { ToolMode } from '@maka/core/tool-mode'; import { setTimeout as timerDelay } from 'node:timers/promises'; import { createHash } from 'node:crypto'; import { @@ -31,7 +49,6 @@ import { createWorkspaceWritePermissionProfile } from '@maka/core/permission-pro import { DEEP_RESEARCH_SESSION_LABEL } from '@maka/core/deep-research'; import { RUNTIME_CONTINUATION_AUTHORITY_V1 } from '@maka/core/runtime-event-store'; import { deriveTurnRecords } from '@maka/core/session'; -import { isSessionInlineRun } from '@maka/core/agent-run'; import { isTerminalRuntimeEvent } from '@maka/core/runtime-event'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; import { buildImmutableRuntimePrefix, decodeContinuationClaim } from '@maka/core/runtime-boundary'; @@ -57,13 +74,7 @@ import type { AgentGraphOperatorProvisionRequest, AgentGraphOperatorProvisionResult, } from '@maka/core/agent-graph-topology'; -import type { - AgentRunEvent, - EmittedAgentRunEvent, - AgentRunHeader, - AgentRunStore, - RootExecutionDescriptor, -} from '@maka/core/agent-run'; +import type { AgentRunEvent, AgentRunStore, EmittedAgentRunEvent } from '@maka/core/agent-run'; import type { ArtifactRecord } from '@maka/core/artifacts'; import type { ContinuationClaimV1, RuntimeBoundaryDigest } from '@maka/core/runtime-boundary'; import type { @@ -80,6 +91,7 @@ import type { import { PlanConflictError, emptyPlanSessionState, type PlanStore } from '@maka/core/plan'; import { MockLanguageModelV4, simulateReadableStream } from 'ai/test'; import { createTestAiSdkBackend } from './execution-boundary-test-helpers.js'; +import { assertDoubleRunNotSealed } from './runtime-event-store-seal.js'; import type { LanguageModelV4StreamPart } from '@ai-sdk/provider'; import { z } from 'zod'; import { AiSdkBackend } from '../ai-sdk-backend.js'; @@ -192,7 +204,7 @@ test('sendMessage rejects removed child AgentRun lineage as a live trigger', asy ), /removed child AgentRun lineage/, ); - assert.deepStrictEqual(await runStore.listSessionRuns(session.id), []); + assert.deepStrictEqual(await runStore.listSessionInvocations(session.id), []); } }); @@ -529,7 +541,7 @@ describe('SessionManager graph operator provisioning', () => { .sendMessage(parent.id, { turnId: 'supervisor-turn', text: 'schedule graph work' }) [Symbol.asyncIterator](); await parentTurn.next(); - const sourceRun = (await runStore.listSessionRuns(parent.id))[0]; + const sourceRun = (await runStore.listSessionInvocations(parent.id))[0]; if (!sourceRun) throw new Error('Supervisor Run was not recorded'); let provisionSettled = false; @@ -587,7 +599,8 @@ describe('SessionManager graph operator provisioning', () => { } as never, }); const parent = await manager.createSession(makeInput({ permissionMode: 'bypass' })); - await runStore.createRun( + await seedInvocationFromHeader( + runStore, makeRunHeader({ sessionId: parent.id, runId: 'supervisor-run', @@ -676,7 +689,8 @@ describe('SessionManager graph operator provisioning', () => { permissionMode: 'ask', }), ); - await runStore.createRun( + await seedInvocationFromHeader( + runStore, makeRunHeader({ sessionId: parent.id, runId: 'supervisor-run', @@ -716,7 +730,7 @@ describe('SessionManager graph operator provisioning', () => { assert.strictEqual(result.header.permissionMode, 'explore'); assert.strictEqual(result.provision.initialTurnId, result.header.subagentSpawn?.initialTurnId); assert.strictEqual(result.provision.initialRunId, result.header.subagentSpawn?.initialRunId); - assert.deepStrictEqual(await runStore.listSessionRuns(result.header.id), []); + assert.deepStrictEqual(await runStore.listSessionInvocations(result.header.id), []); }); test('keeps four large graph branches and a replacement off the supervisor data plane', async () => { @@ -732,7 +746,8 @@ describe('SessionManager graph operator provisioning', () => { now: nextNow(90), }); const parent = await manager.createSession(makeInput({ permissionMode: 'ask' })); - await runStore.createRun( + await seedInvocationFromHeader( + runStore, makeRunHeader({ sessionId: parent.id, runId: 'large-supervisor-run', @@ -813,7 +828,10 @@ describe('SessionManager graph operator provisioning', () => { role: 'system', author: 'system', status: failed ? 'failed' : 'completed', - actions: { endInvocation: true }, + actions: { + endInvocation: true, + ...(failed ? { stateDelta: { failureClass: 'branch_failed' } } : {}), + }, }), ]); outputs.push( @@ -946,7 +964,8 @@ describe('SessionManager graph operator provisioning', () => { permissionMode: 'ask', }), ); - await runStore.createRun( + await seedInvocationFromHeader( + runStore, makeRunHeader({ sessionId: parent.id, runId: 'supervisor-run', @@ -1200,7 +1219,7 @@ describe('SessionManager claimed graph intent execution', () => { /requires its trusted graph execution capability/, ); - assert.deepStrictEqual(await runStore.listSessionRuns(child.id), []); + assert.deepStrictEqual(await runStore.listSessionInvocations(child.id), []); assert.deepStrictEqual(await store.readMessages(child.id), []); assert.strictEqual(backendBuilds, 0); }); @@ -1372,7 +1391,7 @@ describe('SessionManager claimed graph intent execution', () => { assert.strictEqual(hostedExecutions, 0); assert.strictEqual(backendBuilds, 0); - assert.deepStrictEqual(await runStore.listSessionRuns(child.id), []); + assert.deepStrictEqual(await runStore.listSessionInvocations(child.id), []); assert.deepStrictEqual(await store.readMessages(child.id), []); assert.strictEqual( await runStore.readRootTurnAdmission(child.id, proposedClaim.targetTurnId), @@ -1471,7 +1490,8 @@ describe('SessionManager claimed graph intent execution', () => { }, 'must not be backfilled', ); - await runStore.createRun( + await seedInvocationFromHeader( + runStore, makeRunHeader({ sessionId: child.id, runId: claim.targetRunId, @@ -1511,7 +1531,10 @@ describe('SessionManager claimed graph intent execution', () => { assert.strictEqual(hostedExecutions, 0); assert.strictEqual(backendBuilds, 0); assert.deepStrictEqual(await store.readMessages(child.id), []); - assert.strictEqual((await runStore.readRun(child.id, claim.targetRunId)).status, 'completed'); + assert.strictEqual( + runtimeInvocationOutcome(await readInvocation(runStore, child.id, claim.targetRunId)), + 'completed', + ); }); test('hosted explicit abort stops only the exact claimed root identity', async () => { @@ -1608,16 +1631,16 @@ describe('SessionManager claimed graph intent execution', () => { }, }); - const run = await runStore.readRun(child.id, claim.targetRunId); - assert.partialDeepStrictEqual(run, { - status: 'failed', - failureClass: 'app_restarted', + const run = await readInvocation(runStore, child.id, claim.targetRunId); + assert.strictEqual(runtimeInvocationOutcome(run), 'failed'); + assert.strictEqual(runtimeInvocationFailureClass(run), 'app_restarted'); + assert.partialDeepStrictEqual(run.opening.lineage, { agentId: LOCAL_READ_AGENT_ID, agentName: LOCAL_READ_AGENT_DEFINITION.name, }); - assert.strictEqual(run.workspaceIdentity, undefined); - assert.strictEqual(run.resumedFromRunId, undefined); - assert.strictEqual(run.retriedFromRunId, undefined); + assert.strictEqual(run.opening.configuration.workspaceIdentity, undefined); + assert.strictEqual(run.opening.lineage?.resumedFromRunId, undefined); + assert.strictEqual(run.opening.lineage?.retriedFromRunId, undefined); const terminalEvents = (await runStore.readRuntimeEvents(child.id, claim.targetRunId)).filter( (event) => event.status === 'failed', ); @@ -1694,9 +1717,9 @@ describe('SessionManager claimed graph intent execution', () => { agentName: LOCAL_READ_AGENT_DEFINITION.name, }, ]); - const run = await runStore.readRun(child.id, 'graph-run'); - assert.strictEqual(isSessionInlineRun(run), true); - assert.strictEqual(run.parentRunId, undefined); + const run = await readInvocation(runStore, child.id, 'graph-run'); + assert.strictEqual(isSessionInlineInvocation(run.opening), true); + assert.strictEqual(run.opening.lineage?.parentRunId, undefined); assert.strictEqual(run.turnId, 'graph-turn'); assert.partialDeepStrictEqual( (await store.readMessages(child.id)).find( @@ -1717,7 +1740,7 @@ describe('SessionManager claimed graph intent execution', () => { status: 'completed', summary: 'ok', }); - assert.strictEqual((await runStore.listSessionRuns(child.id)).length, 1); + assert.strictEqual((await runStore.listSessionInvocations(child.id)).length, 1); assert.strictEqual(backendsBySession.get(child.id)?.sendInputs.length, 1); await expectRejects( manager.runClaimedAgentGraphIntent({ @@ -1770,7 +1793,7 @@ describe('SessionManager claimed graph intent execution', () => { const [firstResult, joinedResult] = await Promise.all([first, joined]); assert.deepStrictEqual(joinedResult, firstResult); assert.strictEqual(childBackend?.sendInputs.length, 1); - assert.strictEqual((await runStore.listSessionRuns(child.id)).length, 1); + assert.strictEqual((await runStore.listSessionInvocations(child.id)).length, 1); }); test('serializes different claims per child session without letting a queued abort stop active work', async () => { @@ -1794,7 +1817,7 @@ describe('SessionManager claimed graph intent execution', () => { queuedAbort.abort(); await new Promise((resolve) => setImmediate(resolve)); assert.strictEqual(backend?.stopCalls, 0); - assert.strictEqual((await runStore.listSessionRuns(child.id)).length, 1); + assert.strictEqual((await runStore.listSessionInvocations(child.id)).length, 1); activeGate.release(); assert.strictEqual((await first).status, 'completed'); @@ -1808,7 +1831,7 @@ describe('SessionManager claimed graph intent execution', () => { ); assert.strictEqual(third.status, 'completed'); assert.strictEqual(backend?.sendInputs.length, 2); - assert.strictEqual((await runStore.listSessionRuns(child.id)).length, 2); + assert.strictEqual((await runStore.listSessionInvocations(child.id)).length, 2); }); test('evaluates execution admission only after a claimed child-session slot is available', async () => { @@ -1866,7 +1889,7 @@ describe('SessionManager claimed graph intent execution', () => { await expectRejects(queued, /cancelled before runtime admission/); assert.strictEqual(admissionChecks, 1); assert.strictEqual(backend?.sendInputs.length, 1); - assert.strictEqual((await runStore.listSessionRuns(child.id)).length, 1); + assert.strictEqual((await runStore.listSessionInvocations(child.id)).length, 1); }); test('keeps a stop pending across graph admission with an idle cached backend', async () => { @@ -1927,7 +1950,10 @@ describe('SessionManager claimed graph intent execution', () => { assert.strictEqual(result.status, 'cancelled'); assert.strictEqual(backend?.stopCalls, 1); assert.strictEqual(backend?.sendInputs?.length, 1); - assert.strictEqual((await runStore.readRun(child.id, claim.targetRunId)).status, 'cancelled'); + assert.strictEqual( + runtimeInvocationOutcome(await readInvocation(runStore, child.id, claim.targetRunId)), + 'cancelled', + ); }); test('runtime stop settles queued graph claims without letting their slots pass the active claim', async () => { @@ -1974,7 +2000,7 @@ describe('SessionManager claimed graph intent execution', () => { [firstClaim.targetTurnId], ); assert.deepStrictEqual( - (await runStore.listSessionRuns(child.id)).map((run) => run.turnId), + (await runStore.listSessionInvocations(child.id)).map((run) => run.turnId), [firstClaim.targetTurnId], ); assert.deepStrictEqual( @@ -2026,7 +2052,7 @@ describe('SessionManager claimed graph intent execution', () => { sessionId: child.id, runId: claim.targetRunId, turnId: claim.targetTurnId, - type: 'run_started', + type: 'turn_started', ts: 81, }), ], @@ -2039,7 +2065,7 @@ describe('SessionManager claimed graph intent execution', () => { assert.strictEqual(recovered.status, 'failed'); assert.strictEqual(recovered.failureClass, 'app_restarted'); assert.strictEqual(backendBuilds, 0); - assert.strictEqual((await runStore.listSessionRuns(child.id)).length, 1); + assert.strictEqual((await runStore.listSessionInvocations(child.id)).length, 1); }); test('target Session stop owns a claimed graph execution before its first runtime preflight', async () => { @@ -2089,7 +2115,10 @@ describe('SessionManager claimed graph intent execution', () => { const [result] = await Promise.all([executing, stopping]); assert.strictEqual(result.status, 'cancelled'); assert.deepStrictEqual(backend?.sendInputs, []); - assert.strictEqual((await runStore.readRun(child.id, claim.targetRunId)).status, 'cancelled'); + assert.strictEqual( + runtimeInvocationOutcome(await readInvocation(runStore, child.id, claim.targetRunId)), + 'cancelled', + ); assert.strictEqual((await store.readHeader(child.id)).status === 'blocked', false); }); @@ -2146,7 +2175,8 @@ describe('SessionManager claimed graph intent execution', () => { const child = await createGraphOperatorSession(store, parent.id); const claim = graphIntentClaim({ targetSessionId: child.id }, 'must not run'); - await runStore.createRun( + await seedInvocationFromHeader( + runStore, makeRunHeader({ sessionId: child.id, runId: 'different-run', @@ -2250,7 +2280,7 @@ describe('SessionManager child-session runtime primitive', () => { .sendMessage(parent.id, { turnId: 'parent-turn', text: 'private parent history' }) [Symbol.asyncIterator](); await parentTurn.next(); - const [parentRun] = await runStore.listSessionRuns(parent.id); + const [parentRun] = await runStore.listSessionInvocations(parent.id); if (!parentRun) throw new Error('parent run was not recorded'); const result = await manager.spawnChildSession(parent.id, { @@ -2304,12 +2334,12 @@ describe('SessionManager child-session runtime primitive', () => { assert.strictEqual(childHeader.subagentSpawn?.initialTurnId, result.turnId); assert.strictEqual(childHeader.subagentSpawn?.initialRunId, result.runId); - const [childRun] = await runStore.listSessionRuns(result.childSessionId); + const [childRun] = await runStore.listSessionInvocations(result.childSessionId); if (!childRun) throw new Error('child run was not recorded'); assert.strictEqual(childRun.runId, result.runId); - assert.strictEqual(childRun.parentRunId, undefined); - assert.strictEqual(childRun.agentId, LOCAL_READ_AGENT_ID); - assert.strictEqual(isSessionInlineRun(childRun), true); + assert.strictEqual(childRun.opening.lineage?.parentRunId, undefined); + assert.strictEqual(childRun.opening.lineage?.agentId, LOCAL_READ_AGENT_ID); + assert.strictEqual(isSessionInlineInvocation(childRun.opening), true); assert.strictEqual(result.status, 'completed'); assert.deepStrictEqual(backendActivationSessions, [parent.id, result.childSessionId]); assert.strictEqual( @@ -2375,8 +2405,8 @@ describe('SessionManager child-session runtime primitive', () => { sessionId: result.childSessionId, currentRunId: result.runId, }); - assert.strictEqual(output.header.sessionId, result.childSessionId); - assert.strictEqual(output.header.runId, result.runId); + assert.strictEqual(output.invocation.sessionId, result.childSessionId); + assert.strictEqual(output.invocation.runId, result.runId); const unrelatedParent = await manager.createSession(makeInput({ name: 'Unrelated parent' })); await expectRejects( manager.readChildAgentOutput(unrelatedParent.id, { @@ -2450,7 +2480,7 @@ describe('SessionManager child-session runtime primitive', () => { .sendMessage(parent.id, { turnId: 'parent-turn-preset', text: 'delegate' }) [Symbol.asyncIterator](); await parentTurn.next(); - const [parentRun] = await runStore.listSessionRuns(parent.id); + const [parentRun] = await runStore.listSessionInvocations(parent.id); if (!parentRun) throw new Error('parent run was not recorded'); const result = await manager.spawnChildSession(parent.id, { @@ -2501,7 +2531,7 @@ describe('SessionManager child-session runtime primitive', () => { .sendMessage(parent.id, { turnId: 'parent-turn', text: 'keep the parent active' }) [Symbol.asyncIterator](); await parentTurn.next(); - const [parentRun] = await runStore.listSessionRuns(parent.id); + const [parentRun] = await runStore.listSessionInvocations(parent.id); if (!parentRun) throw new Error('parent run was not recorded'); const child = await manager.spawnChildSession(parent.id, { @@ -2546,7 +2576,7 @@ describe('SessionManager child-session runtime primitive', () => { .sendMessage(parent.id, { turnId: 'parent-turn', text: 'keep parent active' }) [Symbol.asyncIterator](); await parentTurn.next(); - const [parentRun] = await runStore.listSessionRuns(parent.id); + const [parentRun] = await runStore.listSessionInvocations(parent.id); if (!parentRun) throw new Error('parent run was not recorded'); const spawnInput = { spawnedBy: { @@ -2574,7 +2604,10 @@ describe('SessionManager child-session runtime primitive', () => { const [firstResult, joinedResult] = await Promise.all([first, joined]); assert.strictEqual(joinedResult.childSessionId, firstResult.childSessionId); assert.strictEqual(joinedResult.runId, firstResult.runId); - assert.strictEqual((await runStore.listSessionRuns(firstResult.childSessionId)).length, 1); + assert.strictEqual( + (await runStore.listSessionInvocations(firstResult.childSessionId)).length, + 1, + ); const durableRetry = await manager.spawnChildSession(parent.id, spawnInput); assert.strictEqual(durableRetry.childSessionId, firstResult.childSessionId); @@ -2591,10 +2624,8 @@ describe('SessionManager child-session runtime primitive', () => { const store = new MemorySessionStore(); const abortController = new AbortController(); const runStore = new MemoryAgentRunStore({ - beforeRunRead: (sessionId, runId) => { - if (sessionId === 'session-3' && runId === 'cancelled-child-run') { - abortController.abort(); - } + beforeListSessionRuns: (sessionId) => { + if (sessionId === 'session-3') abortController.abort(); }, }); const backends = new BackendRegistry(); @@ -2617,7 +2648,7 @@ describe('SessionManager child-session runtime primitive', () => { .sendMessage(parent.id, { turnId: 'parent-turn', text: 'keep parent active' }) [Symbol.asyncIterator](); await parentTurn.next(); - const [parentRun] = await runStore.listSessionRuns(parent.id); + const [parentRun] = await runStore.listSessionInvocations(parent.id); if (!parentRun) throw new Error('parent run was not recorded'); const seedMetadataOnlyChild = async ( @@ -2700,7 +2731,7 @@ describe('SessionManager child-session runtime primitive', () => { assert.strictEqual(resumed.childSessionId, metadataOnly.id); assert.strictEqual(resumed.runId, 'metadata-only-run'); assert.strictEqual(readyCalls, 1); - assert.strictEqual((await runStore.listSessionRuns(metadataOnly.id)).length, 1); + assert.strictEqual((await runStore.listSessionInvocations(metadataOnly.id)).length, 1); const cancelled = await seedMetadataOnlyChild( 'cancelled-metadata-tool', @@ -2726,7 +2757,7 @@ describe('SessionManager child-session runtime primitive', () => { /cancelled before its first run/, ); assert.strictEqual(cancelledReadyCalls, 0); - assert.deepStrictEqual(await runStore.listSessionRuns(cancelled.id), []); + assert.deepStrictEqual(await runStore.listSessionInvocations(cancelled.id), []); parentGate.release(); while (!(await parentTurn.next()).done) {} @@ -2756,7 +2787,7 @@ describe('SessionManager child-session runtime primitive', () => { .sendMessage(parent.id, { turnId: 'parent-turn', text: 'keep parent active' }) [Symbol.asyncIterator](); await parentTurn.next(); - const [parentRun] = await runStore.listSessionRuns(parent.id); + const [parentRun] = await runStore.listSessionInvocations(parent.id); if (!parentRun) throw new Error('parent run was not recorded'); const child = await manager.spawnChildSession(parent.id, { spawnedBy: { @@ -2855,7 +2886,7 @@ describe('SessionManager child-session runtime primitive', () => { .sendMessage(parent.id, { turnId: 'parent-turn', text: 'private parent context' }) [Symbol.asyncIterator](); await parentTurn.next(); - const [parentRun] = await runStore.listSessionRuns(parent.id); + const [parentRun] = await runStore.listSessionInvocations(parent.id); if (!parentRun) throw new Error('parent run was not recorded'); const child = await manager.spawnChildSession(parent.id, { @@ -2960,7 +2991,7 @@ describe('SessionManager child-session runtime primitive', () => { .sendMessage(parent.id, { turnId: 'parent-turn', text: 'keep parent active' }) [Symbol.asyncIterator](); await parentTurn.next(); - const [parentRun] = await runStore.listSessionRuns(parent.id); + const [parentRun] = await runStore.listSessionInvocations(parent.id); if (!parentRun) throw new Error('parent run was not recorded'); const toolCallId = 'recovery-tool-call'; const prompt = 'recover this exact request'; @@ -3030,7 +3061,7 @@ describe('SessionManager child-session runtime primitive', () => { sessionId: child.id, runId: 'stale-child-run', turnId: 'stale-child-turn', - type: 'run_started', + type: 'turn_started', ts: 191, }), ], @@ -3073,7 +3104,7 @@ describe('SessionManager child-session runtime primitive', () => { await drain( manager.sendMessage(parent.id, { turnId: 'parent-turn', text: 'already complete' }), ); - const [parentRun] = await runStore.listSessionRuns(parent.id); + const [parentRun] = await runStore.listSessionInvocations(parent.id); if (!parentRun) throw new Error('parent run was not recorded'); await expectRejects( @@ -3119,7 +3150,7 @@ describe('SessionManager child-session runtime primitive', () => { }); const parent = await manager.createSession(makeInput()); await drain(manager.sendMessage(parent.id, { turnId: 'parent-turn', text: 'parent' })); - const [parentRun] = await runStore.listSessionRuns(parent.id); + const [parentRun] = await runStore.listSessionInvocations(parent.id); if (!parentRun) throw new Error('parent run was not recorded'); externalParent = { sessionId: parent.id, @@ -3169,7 +3200,7 @@ describe('SessionManager child-session runtime primitive', () => { .sendMessage(parent.id, { turnId: 'parent-turn', text: 'coordinate children' }) [Symbol.asyncIterator](); await parentTurn.next(); - const [parentRun] = await runStore.listSessionRuns(parent.id); + const [parentRun] = await runStore.listSessionInvocations(parent.id); if (!parentRun) throw new Error('parent run was not recorded'); const childOneStarted = makeGate(); @@ -3216,7 +3247,10 @@ describe('SessionManager child-session runtime primitive', () => { assert.strictEqual(backendsBySession.get(childOneId)?.stopCalls, 1); assert.strictEqual(backendsBySession.get(childTwoId)?.stopCalls, 0); assert.strictEqual(backendsBySession.get(parent.id)?.stopCalls, 0); - assert.strictEqual((await runStore.readRun(parent.id, parentRun.runId)).status, 'running'); + assert.strictEqual( + runtimeInvocationOutcome(await readInvocation(runStore, parent.id, parentRun.runId)), + undefined, + ); await manager.stopSession(parent.id, { source: 'stop_button' }); assert.strictEqual(backendsBySession.get(parent.id)?.stopCalls, 1); @@ -3302,7 +3336,7 @@ describe('SessionManager child-session runtime primitive', () => { sessionId: child.id, runId: 'child-run', turnId: 'child-turn', - type: 'run_started', + type: 'turn_started', ts: 11, }), makeRunEvent({ @@ -3319,11 +3353,11 @@ describe('SessionManager child-session runtime primitive', () => { const recovered = await manager.recoverInterruptedSessions(); assert.deepStrictEqual(recovered, [child.id]); - const recoveredRun = await runStore.readRun(child.id, 'child-run'); - assert.strictEqual(recoveredRun.parentRunId, undefined); - assert.strictEqual(isSessionInlineRun(recoveredRun), true); - assert.strictEqual(recoveredRun.status, 'failed'); - assert.strictEqual(recoveredRun.failureClass, 'app_restarted'); + const recoveredRun = await readInvocation(runStore, child.id, 'child-run'); + assert.strictEqual(recoveredRun.opening.lineage?.parentRunId, undefined); + assert.strictEqual(isSessionInlineInvocation(recoveredRun.opening), true); + assert.strictEqual(runtimeInvocationOutcome(recoveredRun), 'failed'); + assert.strictEqual(runtimeInvocationFailureClass(recoveredRun), 'app_restarted'); assert.strictEqual( (await store.readMessages(child.id)).some( (message) => @@ -3357,10 +3391,12 @@ describe('SessionManager manual compaction and quiescent session changes', () => ); await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' })); - const sourceRun = (await runStore.listSessionRuns(session.id)).find( + const sourceRun = (await runStore.listSessionInvocations(session.id)).find( (run) => run.turnId === 'turn-1', ); assert.ok(sourceRun); + const sourceRoute = sourceRun.opening.route; + assert.equal(sourceRoute.provenance, 'runtime'); runStore.operations = []; const events = await collectSessionEvents( manager.compactSession(session.id, { turnId: 'turn-compact' }), @@ -3369,12 +3405,14 @@ describe('SessionManager manual compaction and quiescent session changes', () => assert.deepStrictEqual(compactCalls, [ { turnId: 'turn-compact', - runtimeContextCount: 3, + // Opening fact, prompt, answer, terminal. + runtimeContextCount: 4, sourceRoutes: [ { runId: sourceRun.runId, - connectionId: sourceRun.llmConnectionId, - modelId: sourceRun.modelId, + connectionId: + sourceRoute.provenance === 'runtime' ? sourceRoute.llmConnectionId : undefined, + modelId: sourceRoute.modelId, }, ], }, @@ -3411,11 +3449,11 @@ describe('SessionManager manual compaction and quiescent session changes', () => true, ); - const compactRun = (await runStore.listSessionRuns(session.id)).find( + const compactRun = (await runStore.listSessionInvocations(session.id)).find( (run) => run.turnId === 'turn-compact', ); - assert.strictEqual(compactRun?.status, 'completed'); - assert.deepStrictEqual(runStore.operations, ['terminalRuntimeEvent', 'completedRunHeader']); + assert.strictEqual(compactRun && runtimeInvocationOutcome(compactRun), 'completed'); + assert.deepStrictEqual(runStore.operations, ['terminalRuntimeEvent']); assert.strictEqual( (await runStore.readRuntimeEvents(session.id, compactRun!.runId)).some( (event) => event.actions?.stateDelta?.contextCompactionOutcome, @@ -3518,7 +3556,7 @@ describe('SessionManager manual compaction and quiescent session changes', () => await drain(manager.sendMessage(session.id, { turnId: 'turn-2', text: 'second' })); await drain(manager.compactSession(session.id, { turnId: 'turn-compact' })); - const compactRun = (await runStore.listSessionRuns(session.id)).find( + const compactRun = (await runStore.listSessionInvocations(session.id)).find( (run) => run.turnId === 'turn-compact', ); assert.ok(compactRun, 'the kernel opens a run for a manual compaction'); @@ -3613,10 +3651,10 @@ describe('SessionManager manual compaction and quiescent session changes', () => compactEvents.some((event) => event.type === 'token_usage'), false, ); - const compactRun = (await runStore.listSessionRuns(session.id)).find( + const compactRun = (await runStore.listSessionInvocations(session.id)).find( (run) => run.turnId === 'turn-compact', ); - assert.strictEqual(compactRun?.status, 'cancelled'); + assert.strictEqual(compactRun && runtimeInvocationOutcome(compactRun), 'cancelled'); }); test('cold manual compaction normalizes only its execution cancellation reason', async () => { @@ -3681,10 +3719,10 @@ describe('SessionManager manual compaction and quiescent session changes', () => }); await Promise.all([abortErrorRejection, abortErrorStop]); - const [cancelledRun] = await runStore.listSessionRuns(cancelledSession.id); - const [abortErrorRun] = await runStore.listSessionRuns(abortErrorSession.id); - assert.strictEqual(cancelledRun?.status, 'cancelled'); - assert.strictEqual(abortErrorRun?.status, 'cancelled'); + const [cancelledRun] = await runStore.listSessionInvocations(cancelledSession.id); + const [abortErrorRun] = await runStore.listSessionInvocations(abortErrorSession.id); + assert.strictEqual(cancelledRun && runtimeInvocationOutcome(cancelledRun), 'cancelled'); + assert.strictEqual(abortErrorRun && runtimeInvocationOutcome(abortErrorRun), 'cancelled'); }); test('stopSession waits for compaction blocked before Run reservation', async () => { @@ -3724,10 +3762,10 @@ describe('SessionManager manual compaction and quiescent session changes', () => await compact.catch(() => []); assert.deepStrictEqual(compactCalls, []); - const compactRun = (await runStore.listSessionRuns(session.id)).find( + const compactRun = (await runStore.listSessionInvocations(session.id)).find( (run) => run.turnId === 'turn-compact-pending', ); - assert.strictEqual(compactRun?.status, 'cancelled'); + assert.strictEqual(compactRun && runtimeInvocationOutcome(compactRun), 'cancelled'); }); test('manual compaction is stopped through the active runtime run lifecycle', async () => { @@ -3772,10 +3810,10 @@ describe('SessionManager manual compaction and quiescent session changes', () => compactEvents.some((event) => event.type === 'token_usage'), false, ); - const compactRun = (await runStore.listSessionRuns(session.id)).find( + const compactRun = (await runStore.listSessionInvocations(session.id)).find( (run) => run.turnId === 'turn-compact', ); - assert.strictEqual(compactRun?.status, 'cancelled'); + assert.strictEqual(compactRun && runtimeInvocationOutcome(compactRun), 'cancelled'); }); test('compactSession rejects while a turn is running and writes no compact artifacts', async () => { @@ -3821,7 +3859,7 @@ describe('SessionManager manual compaction and quiescent session changes', () => messages.some((message) => message.turnId === 'turn-compact'), false, ); - const compactRun = (await runStore.listSessionRuns(session.id)).find( + const compactRun = (await runStore.listSessionInvocations(session.id)).find( (run) => run.turnId === 'turn-compact', ); assert.strictEqual(compactRun, undefined); @@ -4483,9 +4521,14 @@ describe('SessionManager permission mode updates', () => { await first.next(); await first.next(); assert.strictEqual((await store.readHeader(session.id)).status, 'running'); - const afterFirstRuns = await runStore.listSessionRuns(session.id); - assert.strictEqual(afterFirstRuns.find((run) => run.turnId === 'turn-1')?.status, 'completed'); - assert.strictEqual(afterFirstRuns.find((run) => run.turnId === 'turn-2')?.status, 'running'); + const afterFirstRuns = await runStore.listSessionInvocations(session.id); + assert.deepStrictEqual( + afterFirstRuns.map((run) => [run.turnId, runtimeInvocationOutcome(run)]), + [ + ['turn-1', 'completed'], + ['turn-2', undefined], + ], + ); await expectRejects(manager.setPermissionMode(session.id, 'bypass'), /当前任务正在运行/); @@ -4493,18 +4536,14 @@ describe('SessionManager permission mode updates', () => { await second.next(); await second.next(); assert.strictEqual((await store.readHeader(session.id)).status, 'active'); - const finalRuns = await runStore.listSessionRuns(session.id); + const finalRuns = await runStore.listSessionInvocations(session.id); assert.deepStrictEqual( - finalRuns.map((run) => [run.turnId, run.status]), + finalRuns.map((run) => [run.turnId, runtimeInvocationOutcome(run)]), [ ['turn-1', 'completed'], ['turn-2', 'completed'], ], ); - const firstEvents = await runStore.readEvents(session.id, finalRuns[0]!.runId); - assert.ok(firstEvents.map((event) => event.type).includes('run_created')); - assert.ok(firstEvents.map((event) => event.type).includes('run_started')); - assert.ok(firstEvents.map((event) => event.type).includes('run_completed')); const summary = await manager.setPermissionMode(session.id, 'bypass'); assert.strictEqual(summary.permissionMode, 'bypass'); @@ -4568,9 +4607,12 @@ describe('SessionManager permission mode updates', () => { events.map((event) => event.type), ['text_complete', 'complete'], ); - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.llmConnectionId, '11111111-1111-4111-8111-111111111111'); - assert.strictEqual(run?.workspaceIdentity, undefined); + const [run] = await runStore.listSessionInvocations(session.id); + assert.partialDeepStrictEqual(run?.opening.route, { + provenance: 'runtime', + llmConnectionId: '11111111-1111-4111-8111-111111111111', + }); + assert.strictEqual(run?.opening.configuration.workspaceIdentity, undefined); }); test('does not inspect continuation safety on normal turns while resume is disabled', async () => { @@ -4605,8 +4647,8 @@ describe('SessionManager permission mode updates', () => { ); assert.strictEqual(inspectionCalls, 0); - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.workspaceIdentity, undefined); + const [run] = await runStore.listSessionInvocations(session.id); + assert.strictEqual(run?.opening.configuration.workspaceIdentity, undefined); }); test('declares the T1 protocol for an AiSdk run when the host wires the durable boundary', async () => { @@ -4632,7 +4674,7 @@ describe('SessionManager permission mode updates', () => { }), ); - const [run] = await runStore.listSessionRuns(session.id); + const [run] = await runStore.listSessionInvocations(session.id); if (!run) throw new Error('expected run'); const events = await runStore.readRuntimeEvents(session.id, run.runId); assert.deepStrictEqual(events[0]?.actions?.runtimeProtocol, { @@ -4734,7 +4776,8 @@ describe('SessionManager permission mode updates', () => { }); const session = await manager.createSession(makeInput()); const header = await store.readHeader(session.id); - await runStore.createRun( + await seedInvocationFromHeader( + runStore, makeRunHeader({ runId: 'source-run-safety-failure', sessionId: session.id, @@ -4891,29 +4934,117 @@ describe('SessionManager permission mode updates', () => { ); assert.strictEqual(backend?.sendInputs[0]?.toolMode, 'code_mode'); - const [run] = await runStore.listSessionRuns(session.id); - if (!run) throw new Error('AgentRunStore run was not created'); + const [run] = await runtimeEventStore.listSessionInvocations(session.id); + if (!run) throw new Error('the run opened no invocation'); const runtimeEvents = await runtimeEventStore.readRuntimeEvents(session.id, run.runId); - assert.deepStrictEqual(backend?.sendInputs[0]?.headAnchorRuntimeEvent, runtimeEvents[0]); + assert.deepStrictEqual(backend?.sendInputs[0]?.headAnchorRuntimeEvent, runtimeEvents[1]); assert.deepStrictEqual( runtimeEvents.map((event) => event.runId), - [run.runId, run.runId, run.runId], + [run.runId, run.runId, run.runId, run.runId], ); assert.deepStrictEqual( runtimeEvents.map((event) => event.sessionId), - [session.id, session.id, session.id], + [session.id, session.id, session.id, session.id], ); assert.deepStrictEqual( runtimeEvents.map((event) => event.turnId), - ['turn-1', 'turn-1', 'turn-1'], + ['turn-1', 'turn-1', 'turn-1', 'turn-1'], ); assert.deepStrictEqual( runtimeEvents.map((event) => event.role), - ['user', 'model', 'system'], + ['system', 'user', 'model', 'system'], + ); + assert.strictEqual(runtimeEvents[0]?.content?.kind, 'invocation_opened'); + assert.deepStrictEqual(runtimeEvents[1]?.content, { kind: 'text', text: 'hello' }); + assert.deepStrictEqual(runtimeEvents[2]?.content, { kind: 'text', text: 'ok' }); + assert.strictEqual(runtimeEvents[3]?.status, 'completed'); + }); + + test('the invocation opening fact is durable before any dispatch', async () => { + const store = new MemorySessionStore(); + const trace: string[] = []; + const runStore = new MemoryAgentRunStore({ + beforeRuntimeEventAppend: (_sessionId, _runId, event, options) => { + trace.push( + `runtime:${event.content?.kind ?? event.status ?? 'fact'}:durable=${options?.durable === true}`, + ); + }, + beforeAgentRunEventAppend: (_sessionId, _runId, event) => { + trace.push(`ledger:${event.type}`); + }, + }); + const backends = new BackendRegistry(); + backends.register('ai-sdk', (ctx) => { + trace.push('backend:activated'); + return new FinalTextTestBackend(ctx); + }); + const manager = new SessionManager({ + store, + runStore, + runtimeEventStore: runStore, + backends, + newId: nextId(), + now: nextNow(7_100), + }); + const session = await manager.createSession(makeInput()); + await collectSessionEvents( + manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' }), + ); + + const openingIndex = trace.findIndex((entry) => entry.startsWith('runtime:invocation_opened')); + assert.notStrictEqual(openingIndex, -1, 'the invocation must commit an opening fact'); + assert.strictEqual( + trace.slice(0, openingIndex).some((entry) => entry.startsWith('runtime:')), + false, + 'the opening fact must be the first RuntimeEvent of the invocation', + ); + assert.ok( + openingIndex < trace.findIndex((entry) => entry.startsWith('runtime:text')), + 'the opening fact must precede the first model-visible event of the turn', ); - assert.deepStrictEqual(runtimeEvents[0]?.content, { kind: 'text', text: 'hello' }); - assert.deepStrictEqual(runtimeEvents[1]?.content, { kind: 'text', text: 'ok' }); - assert.strictEqual(runtimeEvents[2]?.status, 'completed'); + }); + + test('a rejected opening fact stops the turn before the backend can dispatch', async () => { + const store = new MemorySessionStore(); + const sends: string[] = []; + const runStore = new MemoryAgentRunStore({ + beforeRuntimeEventAppend: (_sessionId, _runId, event) => { + if (event.content?.kind === 'invocation_opened') { + throw new Error('opening fact store is unavailable'); + } + }, + }); + const canonicalRuntimeEventStore: RuntimeEventStore = Object.assign( + Object.create(Object.getPrototypeOf(runStore) as object) as MemoryAgentRunStore, + runStore, + { durability: 'canonical' as const }, + ); + const backends = new BackendRegistry(); + let backend: FinalTextTestBackend | undefined; + backends.register('ai-sdk', (ctx) => { + backend = new FinalTextTestBackend(ctx); + return backend; + }); + const manager = new SessionManager({ + store, + runStore, + runtimeEventStore: canonicalRuntimeEventStore, + backends, + newId: nextId(), + now: nextNow(7_150), + }); + const session = await manager.createSession(makeInput()); + await assert.rejects( + collectSessionEvents(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' })), + /opening fact store is unavailable/, + ); + + assert.deepStrictEqual( + backend?.sendInputs ?? [], + [], + 'no provider dispatch may happen without a durable opening fact', + ); + assert.deepStrictEqual(sends, []); }); test('snapshots mutable turn content before durable commit and backend dispatch', async () => { @@ -4938,6 +5069,7 @@ describe('SessionManager permission mode updates', () => { readSessionRuntimeEventEntries: (sessionId) => durableEvents.readSessionRuntimeEventEntries(sessionId), readSessionRuntimeEvents: (sessionId) => durableEvents.readSessionRuntimeEvents(sessionId), + listSessionInvocations: (sessionId) => durableEvents.listSessionInvocations(sessionId), }; const backends = new BackendRegistry(); let providerInput: BackendSendInput | undefined; @@ -5037,9 +5169,13 @@ describe('SessionManager permission mode updates', () => { assert.strictEqual(Object.isFrozen(providerInput?.headAnchorRuntimeEvent), true); assert.strictEqual(Object.isFrozen(headContent), true); - const [run] = await runStore.listSessionRuns(session.id); - if (!run) throw new Error('AgentRunStore run was not created'); - const [storedUserEvent] = await durableEvents.readRuntimeEvents(session.id, run.runId); + const [run] = await runtimeEventStore.listSessionInvocations(session.id); + if (!run) throw new Error('the run opened no invocation'); + const [openingFact, storedUserEvent] = await durableEvents.readRuntimeEvents( + session.id, + run.runId, + ); + assert.strictEqual(openingFact?.content?.kind, 'invocation_opened'); assert.deepStrictEqual(storedUserEvent?.content, { kind: 'text', text: 'inspect the attachment', @@ -5193,7 +5329,9 @@ describe('SessionManager permission mode updates', () => { const sourceRunId = 'source-run'; const sourceTurnId = 'source-turn'; const sourceInvocationId = 'source-invocation'; - await runStore.createRun({ + // The source run states its own terminal event below, so only its opening + // is seeded here: a second ending would leave the invocation ambiguous. + await seedInvocationOpening(runStore, { runId: sourceRunId, invocationId: sourceInvocationId, sessionId: session.id, @@ -5319,15 +5457,18 @@ describe('SessionManager permission mode updates', () => { sessionEvents.map((event) => event.type), ['text_complete', 'complete'], ); - const continuationRun = await runStore.readRun(session.id, plan.continuation.runId); + const continuationRun = await readInvocation(runStore, session.id, plan.continuation.runId); assert.strictEqual(continuationRun.invocationId, plan.continuation.invocationId); assert.strictEqual(continuationRun.turnId, plan.continuation.turnId); - assert.strictEqual(continuationRun.parentRunId, sourceRunId); - assert.strictEqual(continuationRun.parentTurnId, sourceTurnId); - assert.strictEqual(continuationRun.cwd, movedCwd); - assert.strictEqual(continuationRun.status, 'completed'); - assert.strictEqual(continuationRun.providerStateIdentity, providerStateIdentity); - assert.partialDeepStrictEqual(continuationRun, { + assert.strictEqual(continuationRun.opening.lineage?.parentRunId, sourceRunId); + assert.strictEqual(continuationRun.opening.lineage?.parentTurnId, sourceTurnId); + assert.strictEqual(continuationRun.opening.configuration.cwd, movedCwd); + assert.strictEqual(runtimeInvocationOutcome(continuationRun), 'completed'); + assert.partialDeepStrictEqual(continuationRun.opening.route, { + provenance: 'runtime', + providerStateIdentity, + }); + assert.partialDeepStrictEqual(continuationRun.opening.configuration, { orchestrationMode: 'swarm', orchestrationSource: 'turn_override', agentSwarmAuthorization: 'turn_override', @@ -5335,18 +5476,21 @@ describe('SessionManager permission mode updates', () => { }); assert.strictEqual(backend?.sendInputs[0]?.toolMode, 'code_mode'); assert.deepStrictEqual( - backend?.sendInputs[0]?.runtimeContextRunHeaders?.map((runHeader) => ({ - runId: runHeader.runId, - llmConnectionId: runHeader.llmConnectionId, - modelId: runHeader.modelId, - providerStateIdentity: runHeader.providerStateIdentity, + backend?.sendInputs[0]?.runtimeContextInvocations?.map((invocation) => ({ + runId: invocation.runId, + route: invocation.opening.route, })), [ { runId: sourceRunId, - llmConnectionId: header.llmConnectionId, - modelId: header.model, - providerStateIdentity, + route: { + provenance: 'runtime', + backendKind: 'ai-sdk', + llmConnectionId: header.llmConnectionId, + llmConnectionSlug: header.llmConnectionSlug, + modelId: header.model, + providerStateIdentity, + }, }, ], ); @@ -5363,7 +5507,8 @@ describe('SessionManager permission mode updates', () => { invocationId: sourceInvocationId, runId: sourceRunId, turnId: sourceTurnId, - highWater: sourceEvents.length, + // The opening fact is event 1 of the source invocation. + highWater: sourceEvents.length + 1, prefixDigest: plan.continuation.boundary?.segments.at(-1)?.prefixDigest, }, replayManifestDigest: plan.continuation.boundary?.manifestDigest, @@ -5382,7 +5527,10 @@ describe('SessionManager permission mode updates', () => { (await store.readMessages(session.id)).some((message) => message.type === 'user'), false, ); - assert.deepStrictEqual(await runStore.readRuntimeEvents(session.id, sourceRunId), sourceEvents); + assert.deepStrictEqual( + (await runStore.readRuntimeEvents(session.id, sourceRunId)).slice(1), + sourceEvents, + ); assert.deepStrictEqual( lifecycleEvents.map((event) => event.type), ['plan_approved', 'execution_started', 'execution_completed'], @@ -5399,10 +5547,13 @@ describe('SessionManager permission mode updates', () => { followUpContext.some((event) => event.runId === plan.continuation?.runId), true, ); - const followUpRun = (await runStore.listSessionRuns(session.id)).find( + const followUpRun = (await runStore.listSessionInvocations(session.id)).find( (runHeader) => runHeader.turnId === 'turn-after-continuation', ); - assert.strictEqual(followUpRun?.providerStateIdentity, providerStateIdentity); + assert.partialDeepStrictEqual(followUpRun?.opening.route, { + provenance: 'runtime', + providerStateIdentity, + }); }); test('authenticates the exact target-aware continuation projection that reaches the provider', async () => { @@ -5490,7 +5641,8 @@ describe('SessionManager permission mode updates', () => { const sourceRunId = 'source-run-cross-route'; const sourceInvocationId = 'source-invocation-cross-route'; const sourceTurnId = 'source-turn-cross-route'; - await runStore.createRun({ + // The source run states its own terminal event below. + await seedInvocationOpening(runStore, { runId: sourceRunId, invocationId: sourceInvocationId, sessionId: session.id, @@ -5859,57 +6011,17 @@ describe('SessionManager permission mode updates', () => { assert.deepStrictEqual(repeatedPlan.rejectionReasons, ['continuation_already_exists']); const targetRunId = firstPlan.continuation.runId; - const targetRun = await runStore.readRun(session.id, targetRunId); - await runStore.updateRun(session.id, targetRunId, { - modelId: 'tampered-model', - updatedAt: targetRun.updatedAt + 1, - }); - const targetIdentityMismatch = await manager.planSafeBoundaryContinuation(session.id, { - sourceRunId, - currentCwd: header.cwd, - sourceWorkspaceIdentity: 'workspace-1', - currentWorkspaceIdentity: 'workspace-1', - backgroundOperationsSettled: true, - availableToolNames: [], - }); - assert.deepStrictEqual(targetIdentityMismatch.rejectionReasons, [ - 'continuation_claim_repair_required', - ]); - - await runStore.updateRun(session.id, targetRunId, { - modelId: targetRun.modelId, - status: 'failed', - failureClass: 'tampered_terminal_state', - updatedAt: targetRun.updatedAt + 2, - }); - const targetTerminalMismatch = await manager.planSafeBoundaryContinuation(session.id, { - sourceRunId, - currentCwd: header.cwd, - sourceWorkspaceIdentity: 'workspace-1', - currentWorkspaceIdentity: 'workspace-1', - backgroundOperationsSettled: true, - availableToolNames: [], - }); - assert.deepStrictEqual(targetTerminalMismatch.rejectionReasons, [ - 'continuation_claim_repair_required', - ]); - - await runStore.updateRun(session.id, targetRunId, { - status: targetRun.status, - failureClass: targetRun.failureClass, - completedAt: targetRun.completedAt, - updatedAt: targetRun.updatedAt, - }); - await runStore.appendRuntimeEvent( + const targetRun = await readInvocation(runStore, session.id, targetRunId); + runStore.seedRuntimeEvent( session.id, targetRunId, runtimeEvent({ id: 'post-terminal-continuation-output', - invocationId: targetRun.invocationId ?? targetRun.runId, + invocationId: targetRun.invocationId, runId: targetRun.runId, sessionId: targetRun.sessionId, turnId: targetRun.turnId, - ts: targetRun.updatedAt + 1, + ts: (targetRun.terminalEvent?.ts ?? targetRun.openedAt) + 1, role: 'model', author: 'agent', content: { kind: 'text', text: 'must not follow a terminal fact' }, @@ -6221,7 +6333,9 @@ describe('SessionManager permission mode updates', () => { test('does not call the backend and claim recovery closes a continuation-start persistence failure', async () => { const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore({ failRuntimeEventAppendAfter: 2 }); + // The source invocation's opening and its two events are the three appends + // that must succeed; the continuation-start is the one that fails. + const runStore = new MemoryAgentRunStore({ failRuntimeEventAppendAfter: 3 }); const backends = new BackendRegistry(); let backendCalls = 0; backends.register( @@ -6245,7 +6359,9 @@ describe('SessionManager permission mode updates', () => { const sourceRunId = 'source-run-write-failure'; const sourceTurnId = 'source-turn-write-failure'; const sourceInvocationId = 'source-invocation-write-failure'; - await runStore.createRun({ + // The source states its own terminal event below. + await seedInvocationOpening(runStore, { + invocationId: sourceInvocationId, runId: sourceRunId, sessionId: session.id, turnId: sourceTurnId, @@ -6306,9 +6422,9 @@ describe('SessionManager permission mode updates', () => { ); assert.strictEqual(backendCalls, 0); - const targetRun = await runStore.readRun(session.id, plan.continuation.runId); - assert.strictEqual(['created', 'running'].includes(targetRun.status), true); - assert.strictEqual(targetRun.completedAt, undefined); + // The continuation-start never reached the ledger, so nothing opened the + // target invocation and the inventory does not know it. + await assert.rejects(readInvocation(runStore, session.id, plan.continuation.runId)); assert.deepStrictEqual( await runStore.readRuntimeEvents(session.id, plan.continuation.runId), [], @@ -6316,11 +6432,11 @@ describe('SessionManager permission mode updates', () => { await manager.recoverInterruptedSessions(); - const recoveredRun = await runStore.readRun(session.id, plan.continuation.runId); + const recoveredRun = await readInvocation(runStore, session.id, plan.continuation.runId); const recoveredEvents = await runStore.readRuntimeEvents(session.id, plan.continuation.runId); - assert.strictEqual(recoveredRun.status, 'failed'); + assert.strictEqual(runtimeInvocationOutcome(recoveredRun), 'failed'); assert.strictEqual( - recoveredRun.failureClass, + runtimeInvocationFailureClass(recoveredRun), 'continuation_abandoned_before_provider_dispatch', ); assert.strictEqual(recoveredEvents.length, 2); @@ -6426,8 +6542,8 @@ describe('SessionManager permission mode updates', () => { await execution.catch(() => []); assert.strictEqual(backendCalls, 0); - const targetRun = await runStore.readRun(session.id, plan.continuation.runId); - assert.strictEqual(targetRun.status, 'cancelled'); + const targetRun = await readInvocation(runStore, session.id, plan.continuation.runId); + assert.strictEqual(runtimeInvocationOutcome(targetRun), 'cancelled'); const targetEvents = await runStore.readRuntimeEvents(session.id, plan.continuation.runId); assert.strictEqual( targetEvents.filter((event) => event.actions?.continuationStart !== undefined).length, @@ -6514,10 +6630,13 @@ describe('SessionManager permission mode updates', () => { ); assert.strictEqual(backendCalls, 0); - await expectRejects(runStore.readRun(session.id, plan.continuation.runId), /unknown run/i); + await expectRejects( + readInvocation(runStore, session.id, plan.continuation.runId), + /unknown run/i, + ); }); - test('revalidates terminal ledger consistency before executing a planned continuation', async () => { + test('refuses a planned continuation whose source ledger changed after planning', async () => { const store = new MemorySessionStore(); const runStore = new MemoryAgentRunStore(); const backends = new BackendRegistry(); @@ -6590,22 +6709,33 @@ describe('SessionManager permission mode updates', () => { }); if (!plan.continuation) throw new Error('expected continuation'); - await runStore.updateRun(session.id, sourceRunId, { - status: 'completed', - updatedAt: 3, - completedAt: 3, - }); + runStore.seedRuntimeEvent( + session.id, + sourceRunId, + runtimeEvent({ + id: 'source-second-terminal-race', + invocationId: sourceInvocationId, + runId: sourceRunId, + sessionId: session.id, + turnId: sourceTurnId, + ts: 3, + status: 'completed', + actions: { endInvocation: true }, + }), + ); + // The second ending moved the source boundary the plan was cut against, so + // the plan no longer describes the ledger it would continue from. await expectRejects( collectSessionEvents(manager.resumeSafeBoundaryContinuation(plan.continuation)), - /terminal/i, + /continuation boundary changed/i, ); assert.strictEqual(backendCalls, 0); }); test('startup recovery retries claim-only terminal projection without dispatching the provider', async () => { const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore({ failUpdateRunStatusOnce: 'failed' }); + const runStore = new MemoryAgentRunStore(); const backends = new BackendRegistry(); let backendCalls = 0; let failOnce = true; @@ -6689,37 +6819,27 @@ describe('SessionManager permission mode updates', () => { collectSessionEvents(manager.resumeSafeBoundaryContinuation(plan.continuation)), /simulated claim-only crash/, ); - await expectRejects(runStore.readRun(session.id, plan.continuation.runId), /unknown run/i); - assert.strictEqual(backendCalls, 0); - - assert.ok(!(await manager.recoverInterruptedSessions()).includes(session.id)); - const durableRepairEvents = await runStore.readRuntimeEvents( - session.id, - plan.continuation.runId, - ); - assert.strictEqual(durableRepairEvents.length, 2); - assert.strictEqual(durableRepairEvents.filter(isTerminalRuntimeEvent).length, 1); - assert.strictEqual( - (await runStore.readRun(session.id, plan.continuation.runId)).status, - 'created', + await expectRejects( + readInvocation(runStore, session.id, plan.continuation.runId), + /unknown run/i, ); + assert.strictEqual(backendCalls, 0); + // One pass finishes the claim: the continuation-start the crash never + // committed, and the terminal event that ends a run nothing will dispatch. + // There is no second record left to settle afterwards. assert.ok((await manager.recoverInterruptedSessions()).includes(session.id)); - const repairedRun = await runStore.readRun(session.id, plan.continuation.runId); - assert.strictEqual(repairedRun.status, 'failed'); - assert.strictEqual(repairedRun.failureClass, 'continuation_abandoned_before_provider_dispatch'); - assert.strictEqual( - repairedRun.continuationSource && 'protocol' in repairedRun.continuationSource - ? repairedRun.continuationSource.protocol - : undefined, - 'continuation_source_v2', - ); + const repairedRun = await readInvocation(runStore, session.id, plan.continuation.runId); + assert.strictEqual(runtimeInvocationOutcome(repairedRun), 'failed'); assert.strictEqual( - repairedRun.continuationSource && 'claimId' in repairedRun.continuationSource - ? repairedRun.continuationSource.claimId - : undefined, - plan.continuation.claimId, + runtimeInvocationFailureClass(repairedRun), + 'continuation_abandoned_before_provider_dispatch', ); + assert.partialDeepStrictEqual(repairedRun.opening.source, { + kind: 'continuation', + sourceRunId, + claimId: plan.continuation.claimId, + }); const repairedEvents = await runStore.readRuntimeEvents(session.id, plan.continuation.runId); assert.strictEqual(repairedEvents.length, 2); assert.strictEqual( @@ -6736,7 +6856,7 @@ describe('SessionManager permission mode updates', () => { await manager.recoverInterruptedSessions(); assert.strictEqual( JSON.stringify({ - run: await runStore.readRun(session.id, plan.continuation.runId), + run: await readInvocation(runStore, session.id, plan.continuation.runId), events: await runStore.readRuntimeEvents(session.id, plan.continuation.runId), }), snapshot, @@ -7049,7 +7169,10 @@ describe('SessionManager permission mode updates', () => { ); assert.strictEqual(backendCalls, 0); - await expectRejects(runStore.readRun(session.id, plan.continuation.runId), /Unknown run/); + await expectRejects( + readInvocation(runStore, session.id, plan.continuation.runId), + /Unknown run/, + ); }); test('fails closed when continuation execution has no authoritative safety inspector', async () => { @@ -7130,112 +7253,6 @@ describe('SessionManager permission mode updates', () => { assert.strictEqual(backendCalls, 0); }); - test('sendMessage preserves token usage fields while resuming from an empty prior runtime ledger', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - let backend: TestBackend | undefined; - backends.register('ai-sdk', (ctx) => { - backend = new TestBackend(ctx); - return backend; - }); - const newId = nextId(); - const now = nextNow(7_000); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId, - now, - }); - const session = await manager.createSession(makeInput()); - await store.appendMessages(session.id, [ - { type: 'user', id: 'legacy-user', turnId: 'turn-1', ts: 101, text: 'prior question' }, - { - type: 'assistant', - id: 'legacy-assistant', - turnId: 'turn-1', - ts: 102, - text: 'prior answer', - modelId: 'fake-model', - }, - { - type: 'token_usage', - id: 'legacy-usage', - turnId: 'turn-1', - ts: 103, - input: 100, - output: 25, - runtimeSteps: 3, - contextRemaining: 9000, - providerRequestTraceId: 'provider-trace-1', - }, - { - type: 'turn_state', - id: 'legacy-state', - turnId: 'turn-1', - ts: 104, - status: 'completed', - partialOutputRetained: true, - }, - ]); - await runStore.createRun( - makeRunHeader({ - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - status: 'completed', - createdAt: 100, - updatedAt: 104, - completedAt: 104, - }), - ); - - const restarted = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId, - now, - }); - const sessionEvents = await collectSessionEvents( - restarted.sendMessage(session.id, { turnId: 'turn-2', text: 'follow up' }), - ); - - assert.deepStrictEqual( - sessionEvents.map((event) => event.type), - ['text_delta', 'complete'], - ); - assert.deepStrictEqual( - backend?.sendInputs[0]?.context.map((message) => message.type), - ['user', 'assistant', 'token_usage', 'turn_state'], - ); - assert.deepStrictEqual( - backend?.sendInputs[0]?.context.map((message) => - 'text' in message ? message.text : message.type, - ), - ['prior question', 'prior answer', 'token_usage', 'turn_state'], - ); - assert.deepStrictEqual( - backend?.sendInputs[0]?.runtimeContext?.map((event) => event.runId), - ['run-1', 'run-1', 'run-1', 'run-1'], - ); - const resumedUsage = backend?.sendInputs[0]?.runtimeContext?.find( - (event) => event.actions?.tokenUsage, - ); - assert.strictEqual(resumedUsage?.actions?.tokenUsage?.runtimeSteps, 3); - assert.strictEqual(resumedUsage?.actions?.tokenUsage?.contextRemaining, 9000); - assert.strictEqual(resumedUsage?.refs?.providerRequestTraceId, 'provider-trace-1'); - const repairedRuntimeEvents = await runStore.readRuntimeEvents(session.id, 'run-1'); - assert.deepStrictEqual( - repairedRuntimeEvents.map((event) => event.refs?.storedMessageId), - ['legacy-user', 'legacy-assistant', 'legacy-usage', 'legacy-state'], - ); - assert.strictEqual(repairedRuntimeEvents.at(-1)?.status, 'completed'); - }); - test('sendMessage completes interrupted imported history beside native runs', async () => { const store = new MemorySessionStore(); const runStore = new MemoryAgentRunStore({ failRuntimeEventAppendAfter: 3 }); @@ -7359,7 +7376,7 @@ describe('SessionManager permission mode updates', () => { ], ); assert.strictEqual((await store.readHeader(session.id)).transcriptLedgerVersion, 1); - const repairedRuns = await runStore.listSessionRuns(session.id); + const repairedRuns = await runStore.listSessionInvocations(session.id); assert.strictEqual(repairedRuns.filter((run) => run.turnId === 'turn-1').length, 1); assert.strictEqual(repairedRuns.filter((run) => run.turnId === 'turn-2').length, 1); }); @@ -7385,10 +7402,10 @@ describe('SessionManager permission mode updates', () => { /history is still being prepared/, ); - assert.strictEqual((await runStore.listSessionRuns(session.id)).length, 0); + assert.strictEqual((await runStore.listSessionInvocations(session.id)).length, 0); }); - test('sendMessage rejects prior runtime context without a valid terminal fact', async () => { + test('sendMessage replays a prior run left non-terminal by an unanswered interaction', async () => { const store = new MemorySessionStore(); const runStore = new MemoryAgentRunStore(); const backends = new BackendRegistry(); @@ -7403,114 +7420,21 @@ describe('SessionManager permission mode updates', () => { runtimeEventStore: runStore, backends, newId: nextId(), - now: nextNow(7_050), + now: nextNow(7_075), }); const session = await manager.createSession(makeInput()); + // A run parked on AskUserQuestion and then stopped: the header never left + // `waiting_for_user` and the ledger never received a terminal fact. Its + // turn is still conversation the model must see. await seedRuntimeRun( runStore, makeRunHeader({ sessionId: session.id, runId: 'run-1', turnId: 'turn-1', - status: 'completed', + status: 'waiting_for_user', createdAt: 100, - updatedAt: 103, - completedAt: 103, - }), - [ - runtimeEvent({ - id: 'rt-user', - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - ts: 101, - role: 'user', - author: 'user', - content: { kind: 'text', text: 'prior question' }, - }), - runtimeEvent({ - id: 'rt-assistant', - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - ts: 102, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'prior answer' }, - }), - runtimeEvent({ - id: 'rt-completed-a', - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - ts: 103, - status: 'completed', - actions: { endInvocation: true }, - }), - runtimeEvent({ - id: 'rt-completed-b', - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - ts: 104, - status: 'completed', - actions: { endInvocation: true }, - }), - ], - ); - - await expectRejects( - drain(manager.sendMessage(session.id, { turnId: 'turn-2', text: 'follow up' })), - /valid terminal fact/, - ); - assert.strictEqual(backend?.sendInputs.length ?? 0, 0); - const currentRun = (await runStore.listSessionRuns(session.id)).find( - (run) => run.turnId === 'turn-2', - ); - if (!currentRun) throw new Error('current AgentRunStore run was not created'); - assert.strictEqual(currentRun.status, 'failed'); - assert.strictEqual(currentRun.failureClass, 'missing_terminal_event'); - const currentTerminalEvents = ( - await runStore.readRuntimeEvents(session.id, currentRun.runId) - ).filter(isTerminalRuntimeEvent); - assert.strictEqual(currentTerminalEvents.length, 1); - assert.strictEqual(currentTerminalEvents[0]?.status, 'failed'); - assert.strictEqual( - currentTerminalEvents[0]?.actions?.stateDelta?.failureClass, - 'missing_terminal_event', - ); - }); - - test('sendMessage replays a prior run left non-terminal by an unanswered interaction', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const backends = new BackendRegistry(); - let backend: TestBackend | undefined; - backends.register('ai-sdk', (ctx) => { - backend = new TestBackend(ctx); - return backend; - }); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(7_075), - }); - const session = await manager.createSession(makeInput()); - // A run parked on AskUserQuestion and then stopped: the header never left - // `waiting_for_user` and the ledger never received a terminal fact. Its - // turn is still conversation the model must see. - await seedRuntimeRun( - runStore, - makeRunHeader({ - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - status: 'waiting_for_user', - createdAt: 100, - updatedAt: 102, + updatedAt: 102, }), [ runtimeEvent({ @@ -7540,7 +7464,7 @@ describe('SessionManager permission mode updates', () => { assert.deepStrictEqual( backend?.sendInputs[0]?.runtimeContext?.map((event) => event.id), - ['rt-user', 'rt-assistant'], + ['run-1-invocation-opened', 'rt-user', 'rt-assistant'], ); }); @@ -7561,7 +7485,6 @@ describe('SessionManager permission mode updates', () => { store.failReadMessagesFor.add(session.id); const view = await new RuntimeReadModel({ - runStore, runtimeEventStore: runStore, }).getSessionView(session.id); @@ -7643,7 +7566,6 @@ describe('SessionManager permission mode updates', () => { ); const cachedView = await new RuntimeReadModel({ - runStore, runtimeEventStore: runStore, projectionCache: { readMessages: async () => @@ -7729,7 +7651,6 @@ describe('SessionManager permission mode updates', () => { await seedCanonicalPermissionRun(runStore, header); await assert.rejects( new RuntimeReadModel({ - runStore, runtimeEventStore: runStore, canonicalPermissionOutcomes: { readPermissionOutcome: async () => outcome, @@ -7795,7 +7716,6 @@ describe('SessionManager permission mode updates', () => { initialWorkersStarted = resolve; }); const viewPromise = new RuntimeReadModel({ - runStore, runtimeEventStore: runStore, canonicalPermissionOutcomes: { readPermissionOutcome: async (requestId) => { @@ -7862,12 +7782,11 @@ describe('SessionManager permission mode updates', () => { }); const view = await new RuntimeReadModel({ - runStore, runtimeEventStore: runStore, }).getSessionView(session.id); assert.deepStrictEqual( - view.runs.map((run) => run.runId), + view.invocations.map((run) => run.runId), ['parent-run'], ); assert.deepStrictEqual( @@ -7937,249 +7856,21 @@ describe('SessionManager permission mode updates', () => { author: 'system', status: 'completed', actions: { endInvocation: true }, - }), - ], - ); - - assert.deepStrictEqual(await manager.getMessages(session.id), [ - { type: 'user', id: 'rt-user', turnId: 'turn-1', ts: 101, text: 'question' }, - { - type: 'turn_state', - id: 'rt-complete', - turnId: 'turn-1', - ts: 103, - status: 'completed', - partialOutputRetained: false, - }, - ]); - }); - - test('getMessages repairs a non-empty RuntimeEvent ledger that is missing only the terminal fact', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const manager = makeManagerForReadCutover(store, runStore); - const session = await manager.createSession(makeInput()); - await store.appendMessages(session.id, [ - { type: 'user', id: 'legacy-user', turnId: 'turn-1', ts: 101, text: 'question' }, - { - type: 'assistant', - id: 'legacy-assistant', - turnId: 'turn-1', - ts: 102, - text: 'answer', - modelId: 'fake-model', - }, - { - type: 'turn_state', - id: 'legacy-state', - turnId: 'turn-1', - ts: 103, - status: 'completed', - partialOutputRetained: true, - }, - ]); - await seedRuntimeRun( - runStore, - makeRunHeader({ - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - status: 'completed', - createdAt: 100, - updatedAt: 103, - completedAt: 103, - }), - [ - runtimeEvent({ - id: 'rt-user', - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - ts: 101, - role: 'user', - author: 'user', - content: { kind: 'text', text: 'question' }, - }), - runtimeEvent({ - id: 'rt-assistant', - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - ts: 102, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'answer' }, - }), - ], - ); - - const messages = await manager.getMessages(session.id); - const runtimeEvents = await runStore.readRuntimeEvents(session.id, 'run-1'); - - assert.deepStrictEqual( - messages.map((message) => message.type), - ['user', 'assistant', 'turn_state'], - ); - assert.deepStrictEqual(messages.at(-1), { - type: 'turn_state', - id: 'legacy-state', - turnId: 'turn-1', - ts: 103, - status: 'completed', - partialOutputRetained: true, - }); - assert.deepStrictEqual( - runtimeEvents.slice(0, 2).map((event) => event.id), - ['rt-user', 'rt-assistant'], - ); - assert.strictEqual(runtimeEvents.at(-1)?.status, 'completed'); - assert.strictEqual(runtimeEvents.at(-1)?.refs?.storedMessageId, 'legacy-state'); - }); - - test('getMessages repair writes terminal turn_state for a continuation run', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore(); - const manager = makeManagerForReadCutover(store, runStore); - const session = await manager.createSession(makeInput()); - const sourceRunId = 'repair-source-run'; - const sourceTurnId = 'repair-source-turn'; - const sourceInvocationId = 'repair-source-invocation'; - await seedRuntimeRun( - runStore, - makeRunHeader({ - sessionId: session.id, - runId: sourceRunId, - turnId: sourceTurnId, - status: 'completed', - createdAt: 100, - updatedAt: 101, - completedAt: 101, - }), - [ - runtimeEvent({ - id: 'repair-source-complete', - invocationId: sourceInvocationId, - sessionId: session.id, - runId: sourceRunId, - turnId: sourceTurnId, - ts: 101, - status: 'completed', - actions: { endInvocation: true }, - }), - ], - ); - await seedRuntimeRun( - runStore, - makeRunHeader({ - sessionId: session.id, - runId: 'repair-continuation-run', - turnId: 'repair-continuation-turn', - status: 'completed', - parentRunId: sourceRunId, - parentTurnId: sourceTurnId, - continuationSource: { - sourceInvocationId, - sourceRunId, - sourceTurnId, - sourceRuntimeEventHighWater: 1, - }, - createdAt: 102, - updatedAt: 104, - completedAt: 104, - }), - [ - runtimeEvent({ - id: 'repair-continuation-text', - invocationId: 'repair-continuation-invocation', - sessionId: session.id, - runId: 'repair-continuation-run', - turnId: 'repair-continuation-turn', - ts: 103, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'retained continuation output' }, - }), - ], - ); - - await manager.getMessages(session.id); - - const cachedMessages = await store.readMessages(session.id); - assert.partialDeepStrictEqual( - cachedMessages.find( - (message) => message.type === 'turn_state' && message.turnId === 'repair-continuation-turn', - ), - { - type: 'turn_state', - status: 'failed', - errorClass: 'missing_terminal_event', - parentTurnId: sourceTurnId, - partialOutputRetained: false, - }, - ); - }); - - test('getMessages can retry repair when the failed header update is interrupted', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore({ failUpdateRunOnce: true }); - const manager = makeManagerForReadCutover(store, runStore); - const session = await manager.createSession(makeInput()); - await store.appendMessages(session.id, [ - { type: 'user', id: 'legacy-user', turnId: 'turn-1', ts: 101, text: 'question' }, - { - type: 'assistant', - id: 'legacy-assistant', - turnId: 'turn-1', - ts: 102, - text: 'answer', - modelId: 'fake-model', - }, - ]); - await seedRuntimeRun( - runStore, - makeRunHeader({ - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - status: 'completed', - createdAt: 100, - updatedAt: 103, - completedAt: 103, - }), - [ - runtimeEvent({ - id: 'rt-user', - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - ts: 101, - role: 'user', - author: 'user', - content: { kind: 'text', text: 'question' }, - }), - runtimeEvent({ - id: 'rt-assistant', - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - ts: 102, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'answer' }, - }), - ], - ); - - await expectRejects(manager.getMessages(session.id), /update run failed/); - - const messages = await manager.getMessages(session.id); - const repairedRun = await runStore.readRun(session.id, 'run-1'); - const runtimeEvents = await runStore.readRuntimeEvents(session.id, 'run-1'); + }), + ], + ); - assert.strictEqual(repairedRun.status, 'failed'); - assert.strictEqual(repairedRun.failureClass, 'missing_terminal_event'); - assert.strictEqual(messages.at(-1)?.type, 'turn_state'); - assert.strictEqual(runtimeEvents.filter((event) => event.status === 'failed').length, 1); + assert.deepStrictEqual(await manager.getMessages(session.id), [ + { type: 'user', id: 'rt-user', turnId: 'turn-1', ts: 101, text: 'question' }, + { + type: 'turn_state', + id: 'rt-complete', + turnId: 'turn-1', + ts: 103, + status: 'completed', + partialOutputRetained: false, + }, + ]); }); test('getMessages repairs missing failed header class from an existing terminal RuntimeEvent', async () => { @@ -8253,10 +7944,10 @@ describe('SessionManager permission mode updates', () => { const messages = await manager.getMessages(session.id); await manager.getMessages(session.id); - const repairedRun = await runStore.readRun(session.id, 'run-1'); + const repairedRun = await readInvocation(runStore, session.id, 'run-1'); const runtimeEvents = await runStore.readRuntimeEvents(session.id, 'run-1'); - assert.strictEqual(repairedRun.failureClass, 'tool_failed'); + assert.strictEqual(runtimeInvocationFailureClass(repairedRun), 'tool_failed'); assert.deepStrictEqual(messages.at(-1), { type: 'turn_state', id: 'rt-failed', @@ -8269,7 +7960,7 @@ describe('SessionManager permission mode updates', () => { assert.strictEqual(runtimeEvents.filter((event) => event.status === 'failed').length, 1); }); - test('getMessages uses fallback failed header class when an existing terminal RuntimeEvent has no class', async () => { + test('getMessages leaves a failed terminal RuntimeEvent that states no class alone', async () => { const store = new MemorySessionStore(); const runStore = new MemoryAgentRunStore(); const manager = makeManagerForReadCutover(store, runStore); @@ -8333,76 +8024,16 @@ describe('SessionManager permission mode updates', () => { ); await manager.getMessages(session.id); - await manager.getMessages(session.id); - const repairedRun = await runStore.readRun(session.id, 'run-1'); + const messages = await manager.getMessages(session.id); + const repairedRun = await readInvocation(runStore, session.id, 'run-1'); const runtimeEvents = await runStore.readRuntimeEvents(session.id, 'run-1'); - assert.strictEqual(repairedRun.failureClass, 'missing_terminal_event'); - assert.strictEqual(runtimeEvents.filter((event) => event.status === 'failed').length, 1); - }); - - test('getMessages serializes concurrent terminal repairs for the same run', async () => { - const store = new MemorySessionStore(); - let repairReads = 0; - const runStore = new MemoryAgentRunStore({ - beforeRuntimeEventRead: async (_sessionId, runId) => { - if (runId !== 'run-1' || repairReads >= 2) return; - repairReads += 1; - await Promise.resolve(); - }, - }); - const manager = makeManagerForReadCutover(store, runStore); - const session = await manager.createSession(makeInput()); - await store.appendMessages(session.id, [ - { type: 'user', id: 'legacy-user', turnId: 'turn-1', ts: 101, text: 'question' }, - { - type: 'assistant', - id: 'legacy-assistant', - turnId: 'turn-1', - ts: 102, - text: 'answer', - modelId: 'fake-model', - }, - ]); - await seedRuntimeRun( - runStore, - makeRunHeader({ - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - status: 'completed', - createdAt: 100, - updatedAt: 103, - completedAt: 103, - }), - [ - runtimeEvent({ - id: 'rt-user', - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - ts: 101, - role: 'user', - author: 'user', - content: { kind: 'text', text: 'question' }, - }), - runtimeEvent({ - id: 'rt-assistant', - sessionId: session.id, - runId: 'run-1', - turnId: 'turn-1', - ts: 102, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'answer' }, - }), - ], - ); - - await Promise.all([manager.getMessages(session.id), manager.getMessages(session.id)]); - - const runtimeEvents = await runStore.readRuntimeEvents(session.id, 'run-1'); + // The run already ended. Its ending is immutable, so the class it never + // stated stays unstated, and no read appends a second ending to supply one. + assert.strictEqual(runtimeInvocationFailureClass(repairedRun), undefined); assert.strictEqual(runtimeEvents.filter((event) => event.status === 'failed').length, 1); + const [turnState] = messages.filter((message) => message.type === 'turn_state'); + assert.partialDeepStrictEqual(turnState, { status: 'failed', errorClass: 'unknown' }); }); test('getMessages includes continuation output without inlining child agent output', async () => { @@ -8600,7 +8231,8 @@ describe('SessionManager permission mode updates', () => { }, ]; await store.appendMessages(session.id, activeMessages); - await runStore.createRun( + await seedInvocationFromHeader( + runStore, makeRunHeader({ sessionId: session.id, runId: 'run-2', @@ -8629,7 +8261,6 @@ describe('SessionManager permission mode updates', () => { ]); const view = await new RuntimeReadModel({ - runStore, runtimeEventStore: runStore, projectionCache: store, }).getSessionView(session.id); @@ -8655,7 +8286,7 @@ describe('SessionManager permission mode updates', () => { createdAt: 100, updatedAt: 125, }); - await runStore.createRun(header); + await seedInvocationFromHeader(runStore, header); await store.appendMessages(session.id, [ { type: 'user', @@ -8764,7 +8395,7 @@ describe('SessionManager permission mode updates', () => { createdAt: 100, updatedAt: 125, }); - await runStore.createRun(header); + await seedInvocationFromHeader(runStore, header); await store.appendMessages(session.id, [ { type: 'user', id: 'active-user', turnId: header.turnId, ts: 100, text: 'build it' }, { @@ -8823,7 +8454,6 @@ describe('SessionManager permission mode updates', () => { ); const view = await new RuntimeReadModel({ - runStore, runtimeEventStore: runStore, projectionCache: store, }).getSessionView(session.id); @@ -9098,10 +8728,10 @@ describe('SessionManager permission mode updates', () => { await stopping; assert.strictEqual((await firstEvent).done, true); assert.deepStrictEqual(backend?.sendInputs, []); - const regenerated = (await runStore.listSessionRuns(session.id)).find( + const regenerated = (await runStore.listSessionInvocations(session.id)).find( (run) => run.turnId === 'regen-stopped-preflight', ); - assert.strictEqual(regenerated?.status, 'cancelled'); + assert.strictEqual(regenerated && runtimeInvocationOutcome(regenerated), 'cancelled'); assert.strictEqual((await store.readHeader(session.id)).status === 'blocked', false); }); @@ -9322,8 +8952,9 @@ describe('SessionManager permission mode updates', () => { const secondInput = backendInstances[0]?.sendInputs[0]; if (!secondInput) throw new Error('backend input was not recorded'); assert.deepStrictEqual( + // The opening fact rides the same context as the three events it opened. secondInput.runtimeContext?.map((event) => event.turnId), - ['turn-1', 'turn-1', 'turn-1'], + ['turn-1', 'turn-1', 'turn-1', 'turn-1'], ); const turnState = secondInput.context.find( (message) => message.type === 'turn_state' && message.turnId === 'turn-1', @@ -9334,62 +8965,6 @@ describe('SessionManager permission mode updates', () => { assert.strictEqual(turnState.errorClass, 'tool_failed'); }); - test('next turn uses failed terminal RuntimeEvents when failed header commit was interrupted', async () => { - const store = new MemorySessionStore(); - const runStore = new MemoryAgentRunStore({ failUpdateRunStatusOnce: 'failed' }); - const backends = new BackendRegistry(); - let backend: TurnScriptBackend | undefined; - backends.register('ai-sdk', (ctx) => { - backend = new TurnScriptBackend(ctx, [ - [{ type: 'complete', stopReason: 'error' }], - [ - { type: 'text_delta', messageId: 'm2', text: 'second ok' }, - { type: 'complete', stopReason: 'end_turn' }, - ], - ]); - return backend; - }); - const manager = new SessionManager({ - store, - runStore, - runtimeEventStore: runStore, - backends, - newId: nextId(), - now: nextNow(6_812), - }); - const session = await manager.createSession(makeInput()); - - await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'first' })); - const [firstRun] = await runStore.listSessionRuns(session.id); - if (!firstRun) throw new Error('first run was not recorded'); - assert.strictEqual(firstRun.status, 'running'); - const firstRuntimeEvents = await runStore.readRuntimeEvents(session.id, firstRun.runId); - const firstTerminalEvents = firstRuntimeEvents.filter(isTerminalRuntimeEvent); - assert.strictEqual(firstTerminalEvents.length, 1); - assert.strictEqual(firstTerminalEvents[0]?.status, 'failed'); - assert.strictEqual(firstTerminalEvents[0]?.actions?.stateDelta?.failureClass, 'runtime_error'); - - await drain(manager.sendMessage(session.id, { turnId: 'turn-2', text: 'second' })); - - const secondInput = backend?.sendInputs[1]; - if (!secondInput) throw new Error('second backend input was not recorded'); - assert.deepStrictEqual( - secondInput.runtimeContext?.map((event) => event.turnId), - ['turn-1', 'turn-1'], - ); - const turnState = secondInput.context.find( - (message) => message.type === 'turn_state' && message.turnId === 'turn-1', - ); - if (turnState?.type !== 'turn_state') - throw new Error('prior failed turn_state was not projected'); - assert.strictEqual(turnState.status, 'failed'); - assert.strictEqual(turnState.errorClass, 'runtime_error'); - const terminalEventsAfterSecondTurn = ( - await runStore.readRuntimeEvents(session.id, firstRun.runId) - ).filter(isTerminalRuntimeEvent); - assert.strictEqual(terminalEventsAfterSecondTurn.length, 1); - }); - test('next parent turn excludes child run RuntimeEvents from model context', async () => { const store = new MemorySessionStore(); const runStore = new MemoryAgentRunStore(); @@ -9411,8 +8986,9 @@ describe('SessionManager permission mode updates', () => { const session = await manager.createSession(makeInput()); await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'first' })); - const [parentRun] = await runStore.listSessionRuns(session.id); + const [parentRun] = await runStore.listSessionInvocations(session.id); if (!parentRun) throw new Error('parent run was not recorded'); + const parentRunEndedAt = parentRun.terminalEvent?.ts ?? parentRun.openedAt; await seedRuntimeRun( runStore, makeRunHeader({ @@ -9420,9 +8996,9 @@ describe('SessionManager permission mode updates', () => { runId: 'child-run', turnId: 'child-turn', status: 'completed', - createdAt: parentRun.updatedAt + 1, - updatedAt: parentRun.updatedAt + 4, - completedAt: parentRun.updatedAt + 4, + createdAt: parentRunEndedAt + 1, + updatedAt: parentRunEndedAt + 4, + completedAt: parentRunEndedAt + 4, parentRunId: parentRun.runId, agentName: 'Researcher', }), @@ -9432,7 +9008,7 @@ describe('SessionManager permission mode updates', () => { sessionId: session.id, runId: 'child-run', turnId: 'child-turn', - ts: parentRun.updatedAt + 2, + ts: parentRunEndedAt + 2, role: 'user', author: 'user', content: { kind: 'text', text: 'child prompt' }, @@ -9442,7 +9018,7 @@ describe('SessionManager permission mode updates', () => { sessionId: session.id, runId: 'child-run', turnId: 'child-turn', - ts: parentRun.updatedAt + 3, + ts: parentRunEndedAt + 3, role: 'model', author: 'agent', content: { kind: 'text', text: 'child private answer' }, @@ -9452,7 +9028,7 @@ describe('SessionManager permission mode updates', () => { sessionId: session.id, runId: 'child-run', turnId: 'child-turn', - ts: parentRun.updatedAt + 4, + ts: parentRunEndedAt + 4, role: 'system', author: 'system', status: 'completed', @@ -9467,7 +9043,7 @@ describe('SessionManager permission mode updates', () => { if (!secondInput) throw new Error('second backend input was not recorded'); assert.deepStrictEqual( secondInput.runtimeContext?.map((event) => event.turnId), - ['turn-1', 'turn-1', 'turn-1'], + ['turn-1', 'turn-1', 'turn-1', 'turn-1'], ); assert.strictEqual( secondInput.runtimeContext?.some((event) => event.turnId === 'child-turn'), @@ -9533,14 +9109,15 @@ describe('SessionManager permission mode updates', () => { backend?.sendInputs.map((input) => input.turnId), ['active-parent-turn'], ); - const runs = await runStore.listSessionRuns(session.id); - assert.strictEqual( - runs.find((run) => run.turnId === 'active-parent-turn')?.status, - 'cancelled', - ); - assert.strictEqual( - runs.find((run) => run.turnId === 'pending-parent-turn')?.status, - 'cancelled', + const runs = await runStore.listSessionInvocations(session.id); + assert.deepStrictEqual( + runs + .filter((run) => run.turnId.endsWith('-parent-turn')) + .map((run) => [run.turnId, runtimeInvocationOutcome(run)]), + [ + ['active-parent-turn', 'cancelled'], + ['pending-parent-turn', 'cancelled'], + ], ); assert.strictEqual((await store.readHeader(session.id)).status, 'aborted'); }); @@ -9586,12 +9163,9 @@ describe('SessionManager permission mode updates', () => { releaseBuild.release(); await expectRejects(firstEvent, /backend activation timed out/); await stopping; - const run = await runStore.readRun( - session.id, - (await runStore.listSessionRuns(session.id))[0]!.runId, - ); - assert.strictEqual(run.status, 'cancelled'); - assert.strictEqual(run.failureClass, undefined); + const run = (await runStore.listSessionInvocations(session.id))[0]!; + assert.strictEqual(runtimeInvocationOutcome(run), 'cancelled'); + assert.strictEqual(runtimeInvocationFailureClass(run), undefined); assert.strictEqual((await store.readHeader(session.id)).status, 'aborted'); }); @@ -9637,8 +9211,8 @@ describe('SessionManager permission mode updates', () => { assert.strictEqual(factorySignal?.aborted, true); assert.strictEqual((await firstEvent).done, true); assert.strictEqual(dispatches, 0); - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.status, 'cancelled'); + const [run] = await runStore.listSessionInvocations(session.id); + assert.strictEqual(run && runtimeInvocationOutcome(run), 'cancelled'); }); test('node timers AbortError is cancellation only when its cause is this execution stop', async () => { @@ -9673,9 +9247,9 @@ describe('SessionManager permission mode updates', () => { await manager.stopSession(session.id, { source: 'stop_button' }); assert.strictEqual((await firstEvent).done, true); - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.status, 'cancelled'); - assert.strictEqual(run?.failureClass, undefined); + const [run] = await runStore.listSessionInvocations(session.id); + assert.strictEqual(run && runtimeInvocationOutcome(run), 'cancelled'); + assert.strictEqual(run && runtimeInvocationFailureClass(run), undefined); }); test('late ignored-signal backend is disposed once and never cached or dispatched', async () => { @@ -9732,8 +9306,8 @@ describe('SessionManager permission mode updates', () => { assert.strictEqual((await firstEvent).done, true); assert.strictEqual(firstDisposeCalls, 1); assert.strictEqual(firstDispatches, 0); - const [stoppedRun] = await runStore.listSessionRuns(session.id); - assert.strictEqual(stoppedRun?.status, 'cancelled'); + const [stoppedRun] = await runStore.listSessionInvocations(session.id); + assert.strictEqual(stoppedRun && runtimeInvocationOutcome(stoppedRun), 'cancelled'); await drain( manager.sendMessage(session.id, { @@ -9804,8 +9378,8 @@ describe('SessionManager permission mode updates', () => { await Promise.all([streamRejection, stopping]); assert.strictEqual(disposeCalls, 1); - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.status, 'cancelled'); + const [run] = await runStore.listSessionInvocations(session.id); + assert.strictEqual(run && runtimeInvocationOutcome(run), 'cancelled'); await expectRejects( drain( @@ -9925,10 +9499,10 @@ describe('SessionManager permission mode updates', () => { backend?.sendInputs.map((input) => input.turnId), ['turn-warm-cache'], ); - const registeringRun = (await runStore.listSessionRuns(session.id)).find( + const registeringRun = (await runStore.listSessionInvocations(session.id)).find( (run) => run.turnId === 'turn-registering', ); - assert.strictEqual(registeringRun?.status, 'cancelled'); + assert.strictEqual(registeringRun && runtimeInvocationOutcome(registeringRun), 'cancelled'); assert.strictEqual( (await runStore.readRuntimeEvents(session.id, registeringRun!.runId)).filter( isTerminalRuntimeEvent, @@ -9976,10 +9550,10 @@ describe('SessionManager permission mode updates', () => { releaseHook.release(); assert.strictEqual((await firstEvent).done, true); assert.deepStrictEqual(backend?.sendInputs, []); - const stoppedRun = (await runStore.listSessionRuns(session.id)).find( + const stoppedRun = (await runStore.listSessionInvocations(session.id)).find( (run) => run.turnId === 'turn-post-start-hook-stop', ); - assert.strictEqual(stoppedRun?.status, 'cancelled'); + assert.strictEqual(stoppedRun && runtimeInvocationOutcome(stoppedRun), 'cancelled'); }); test('concurrent cold turns share one backend generation without accepting an ownerless response', async () => { @@ -10341,10 +9915,10 @@ describe('SessionManager permission mode updates', () => { assert.strictEqual(list.runs[0]?.durationMs, 10); const output = await manager.readChildAgentOutput(session.id, { runId: 'child-run' }); - assert.strictEqual(output.header.runId, 'child-run'); + assert.strictEqual(output.invocation.runId, 'child-run'); assert.deepStrictEqual( output.runtimeEvents.map((event) => event.id), - ['child-user', 'child-answer', 'child-complete'], + ['child-run-invocation-opened', 'child-user', 'child-answer', 'child-complete'], ); assert.deepStrictEqual( output.artifacts.map((artifact) => artifact.id), @@ -10435,8 +10009,8 @@ describe('SessionManager permission mode updates', () => { assert.strictEqual(list.runs[0]?.durationMs, 10); const output = await manager.readChildAgentOutput(session.id, { runId: 'child-run' }); - assert.strictEqual(output.header.status, 'completed'); - assert.strictEqual(output.header.completedAt, 140); + assert.strictEqual(runtimeInvocationOutcome(output.invocation), 'completed'); + assert.strictEqual(output.invocation.terminalEvent?.ts, 140); }); test('agent output returns a bounded child inspection instead of full replay internals', async () => { @@ -10466,7 +10040,7 @@ describe('SessionManager permission mode updates', () => { agentName: 'Researcher', permissionMode: 'explore', }); - await runStore.createRun(header); + await seedInvocationFromHeader(runStore, header); for (let index = 0; index < 25; index += 1) { await runStore.appendEvent( session.id, @@ -10480,7 +10054,7 @@ describe('SessionManager permission mode updates', () => { ts: 120 + index, }), ); - await runStore.appendRuntimeEvent( + runStore.seedRuntimeEvent( session.id, 'child-run', runtimeEvent({ @@ -10502,7 +10076,7 @@ describe('SessionManager permission mode updates', () => { view: 'all', }); - assert.strictEqual(output.header.runId, 'child-run'); + assert.strictEqual(output.invocation.runId, 'child-run'); assert.deepStrictEqual( output.events.map((event) => event.id), ['op-20', 'op-21', 'op-22', 'op-23', 'op-24'], @@ -10533,7 +10107,8 @@ describe('SessionManager permission mode updates', () => { now: nextNow(6_849), }); const session = await manager.createSession(makeInput()); - await runStore.createRun( + await seedInvocationFromHeader( + runStore, makeRunHeader({ sessionId: session.id, runId: 'child-run', @@ -10548,7 +10123,7 @@ describe('SessionManager permission mode updates', () => { permissionMode: 'explore', }), ); - await runStore.appendRuntimeEvent( + runStore.seedRuntimeEvent( session.id, 'child-run', runtimeEvent({ @@ -10562,7 +10137,7 @@ describe('SessionManager permission mode updates', () => { content: { kind: 'text', text: 'x'.repeat(64 * 1024) }, }), ); - await runStore.appendRuntimeEvent( + runStore.seedRuntimeEvent( session.id, 'child-run', runtimeEvent({ @@ -10632,10 +10207,10 @@ describe('SessionManager permission mode updates', () => { (candidate) => candidate.turnId === 'turn-1', ); assert.strictEqual(turn?.status, 'failed'); - const [run] = await runStore.listSessionRuns(session.id); + const [run] = await runStore.listSessionInvocations(session.id); if (!run) throw new Error('AgentRunStore run was not created'); - assert.strictEqual(run.status, 'failed'); - assert.strictEqual(run.failureClass, 'missing_terminal_event'); + assert.strictEqual(runtimeInvocationOutcome(run), 'failed'); + assert.strictEqual(runtimeInvocationFailureClass(run), 'missing_terminal_event'); const terminalEvents = (await runStore.readRuntimeEvents(session.id, run.runId)).filter( isTerminalRuntimeEvent, ); @@ -10683,8 +10258,8 @@ describe('SessionManager permission mode updates', () => { }); assert.strictEqual((await store.readHeader(session.id)).status, 'waiting_for_user'); - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.status, 'waiting_for_user'); + const [run] = await runStore.listSessionInvocations(session.id); + assert.strictEqual(run?.terminalEvent, undefined); await expectRejects(manager.setPermissionMode(session.id, 'bypass'), /当前任务正在运行/); assert.strictEqual((await store.readHeader(session.id)).permissionMode, 'ask'); @@ -10859,7 +10434,6 @@ describe('SessionManager permission mode updates', () => { assert.strictEqual(turns.find((turn) => turn.turnId === 'turn-1')?.status, 'completed'); const view = await new RuntimeReadModel({ - runStore, runtimeEventStore: runStore, }).getSessionView(session.id); assert.strictEqual( @@ -10893,8 +10467,8 @@ describe('SessionManager permission mode updates', () => { // drain } - const [run] = await runStore.listSessionRuns(session.id); - await runStore.appendRuntimeEvent( + const [run] = await runStore.listSessionInvocations(session.id); + runStore.seedRuntimeEvent( session.id, run!.runId, runtimeEvent({ @@ -10910,7 +10484,7 @@ describe('SessionManager permission mode updates', () => { ); await assert.rejects( - new RuntimeReadModel({ runStore, runtimeEventStore: runStore }).getSessionView(session.id), + new RuntimeReadModel({ runtimeEventStore: runStore }).getSessionView(session.id), (error: unknown) => error instanceof RuntimeReadModelError && error.diagnostics.some((diagnostic) => diagnostic.code === 'unsupported_event'), @@ -10944,8 +10518,8 @@ describe('SessionManager permission mode updates', () => { const [turn] = await store.listTurns(session.id); assert.strictEqual(turn?.status, 'failed'); assert.strictEqual(turn?.errorClass, 'runtime_error'); - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.failureClass, 'runtime_error'); + const [run] = await runStore.listSessionInvocations(session.id); + assert.strictEqual(run && runtimeInvocationFailureClass(run), 'runtime_error'); }); test('marks an explicit step limit incomplete without blocking the session', async () => { @@ -10972,9 +10546,9 @@ describe('SessionManager permission mode updates', () => { const [turn] = await store.listTurns(session.id); assert.strictEqual(turn?.status, 'failed'); assert.strictEqual(turn?.errorClass, 'tool_step_cap_reached'); - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.status, 'failed'); - assert.strictEqual(run?.failureClass, 'tool_step_cap_reached'); + const [run] = await runStore.listSessionInvocations(session.id); + assert.strictEqual(run && runtimeInvocationOutcome(run), 'failed'); + assert.strictEqual(run && runtimeInvocationFailureClass(run), 'tool_step_cap_reached'); const terminal = (await runStore.readRuntimeEvents(session.id, run!.runId)).find( (event) => event.actions?.endInvocation, ); @@ -11069,7 +10643,7 @@ describe('SessionManager permission mode updates', () => { await stopPromise; while (!(await iterator.next()).done) {} - const [run] = await runStore.listSessionRuns(session.id); + const [run] = await runStore.listSessionInvocations(session.id); const runtimeEvents = await runStore.readRuntimeEvents(session.id, run!.runId); const terminalEvents = runtimeEvents.filter((event) => event.status === 'aborted'); assert.strictEqual(terminalEvents.length, 1); @@ -11121,7 +10695,7 @@ describe('SessionManager permission mode updates', () => { const emitted = await collectSessionEvents( manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' }), ); - const [run] = await runStore.listSessionRuns(session.id); + const [run] = await runStore.listSessionInvocations(session.id); const runtimeEvents = await runStore.readRuntimeEvents(session.id, run!.runId); const turnStates = (await store.readMessages(session.id)).filter( (message) => @@ -11143,8 +10717,8 @@ describe('SessionManager permission mode updates', () => { const abortedEvents = runtimeEvents.filter((event) => event.status === 'aborted'); assert.strictEqual(abortedEvents.length, 1); assert.strictEqual(abortedEvents[0]?.actions?.stateDelta?.abortSource, 'user_stop'); - assert.strictEqual(run?.status, 'cancelled'); - assert.strictEqual(run?.abortSource, 'user_stop'); + assert.strictEqual(run && runtimeInvocationOutcome(run), 'cancelled'); + assert.strictEqual(run?.terminalEvent?.actions?.stateDelta?.abortSource, 'user_stop'); assert.strictEqual(turnStates.length, 1); assert.strictEqual( turnStates[0]?.type === 'turn_state' ? turnStates[0].status : undefined, @@ -11174,7 +10748,7 @@ describe('SessionManager permission mode updates', () => { const emitted = await collectSessionEvents( manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' }), ); - const [run] = await runStore.listSessionRuns(session.id); + const [run] = await runStore.listSessionInvocations(session.id); const runtimeEvents = await runStore.readRuntimeEvents(session.id, run!.runId); const turnStates = (await store.readMessages(session.id)).filter( (message) => @@ -11187,7 +10761,7 @@ describe('SessionManager permission mode updates', () => { emitted.map((event) => event.type), ['text_delta', 'complete'], ); - assert.strictEqual(run?.status, 'completed'); + assert.strictEqual(run && runtimeInvocationOutcome(run), 'completed'); assert.deepStrictEqual( runtimeEvents .filter((event) => event.role === 'model' && event.content?.kind === 'text') @@ -11234,12 +10808,9 @@ describe('SessionManager permission mode updates', () => { const [turn] = await store.listTurns(session.id); assert.strictEqual(turn?.status, 'aborted'); assert.strictEqual(turn?.abortSource, 'renderer.stop_button'); - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.status, 'cancelled'); - assert.strictEqual(run?.failureClass, undefined); - const events = (await runStore.readEvents(session.id, run!.runId)).map((event) => event.type); - assert.ok(events.includes('run_cancelled')); - assert.strictEqual(events.includes('run_failed'), false); + const [run] = await runStore.listSessionInvocations(session.id); + assert.strictEqual(run && runtimeInvocationOutcome(run), 'cancelled'); + assert.strictEqual(run && runtimeInvocationFailureClass(run), undefined); }); test('durable run ledger records lifecycle trace events and redacts obvious secrets', async () => { @@ -11259,16 +10830,18 @@ describe('SessionManager permission mode updates', () => { await drain(manager.sendMessage(session.id, { turnId: 'turn-1', text: 'hello' })); - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.backendKind, 'ai-sdk'); - assert.strictEqual(run?.llmConnectionSlug, 'fake'); - assert.strictEqual(run?.modelId, 'fake-model'); - assert.strictEqual(run?.permissionMode, 'ask'); - assert.strictEqual(run?.status, 'completed'); + const [run] = await runStore.listSessionInvocations(session.id); + assert.partialDeepStrictEqual(run?.opening.route, { + provenance: 'unknown', + backendKind: 'ai-sdk', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }); + assert.strictEqual(run?.opening.configuration.permissionMode, 'ask'); + assert.strictEqual(run && runtimeInvocationOutcome(run), 'completed'); const events = await runStore.readEvents(session.id, run!.runId); assert.ok(events.map((event) => event.type).includes('model_stream_started')); assert.ok(events.map((event) => event.type).includes('model_stream_completed')); - assert.ok(events.map((event) => event.type).includes('run_completed')); assert.strictEqual(JSON.stringify(events).includes('sk-live-secret-token-value'), false); }); @@ -11440,7 +11013,7 @@ describe('SessionManager permission mode updates', () => { 'same-coverage-replacement:fulfilled', ]); const checkpoints: HistoryCompactCheckpoint[] = []; - for (const run of await runStore.listSessionRuns(session.id)) { + for (const run of await runStore.listSessionInvocations(session.id)) { for (const event of await runStore.readEvents(session.id, run.runId)) { if (event.type === 'history_compact_checkpoint_recorded') { checkpoints.push(event.data?.checkpoint as HistoryCompactCheckpoint); @@ -11454,12 +11027,13 @@ describe('SessionManager permission mode updates', () => { test('rejects checkpoint recording after the current AgentRun store becomes unavailable', async () => { const store = new MemorySessionStore(); - const runStoreUnavailable = makeGate(); const writeOutcomes: string[] = []; + // The store goes unavailable partway through the run, right before the + // checkpoint write asks it for anything. + let runStoreUnavailable = false; const runStore = new MemoryAgentRunStore({ - beforeAgentRunEventAppend: async (_sessionId, _runId, event) => { - if (event.type === 'run_started') throw new Error('run ledger append failed'); - if (event.type === 'trace_write_failed') runStoreUnavailable.release(); + beforeAgentRunEventAppend: async () => { + if (runStoreUnavailable) throw new Error('run ledger append failed'); }, }); const backends = new BackendRegistry(); @@ -11468,7 +11042,9 @@ describe('SessionManager permission mode updates', () => { (ctx) => new CheckpointRecorderContractProbeBackend( ctx, - async () => runStoreUnavailable.promise, + async () => { + runStoreUnavailable = true; + }, writeOutcomes, ), ); @@ -11554,7 +11130,7 @@ describe('SessionManager permission mode updates', () => { assert.deepStrictEqual(writeOutcomes, ['cold-stale-after-projection-loss:rejected']); assert.strictEqual(runStore.repairedProjection?.id, durableEvent.id); const checkpointCoverage: number[] = []; - for (const run of await runStore.listSessionRuns(session.id)) { + for (const run of await runStore.listSessionInvocations(session.id)) { for (const event of await runStore.readEvents(session.id, run.runId)) { if (event.type === 'history_compact_checkpoint_recorded') { checkpointCoverage.push( @@ -11802,7 +11378,7 @@ describe('SessionManager permission mode updates', () => { await manager.recoverInterruptedSessions(); - assert.equal((await runStore.readRun(session.id, 'run-1')).status, 'running'); + assert.equal((await readInvocation(runStore, session.id, 'run-1')).terminalEvent, undefined); assert.equal(outcomeCommitAttempts, 2); assert.equal( (await runStore.readRuntimeEvents(session.id, 'run-1')).some( @@ -11831,7 +11407,10 @@ describe('SessionManager permission mode updates', () => { }, }, ); - assert.equal((await runStore.readRun(session.id, 'run-1')).status, 'failed'); + assert.equal( + runtimeInvocationOutcome(await readInvocation(runStore, session.id, 'run-1')), + 'failed', + ); }); test('startup recovery does not leave stale permission waits stuck', async () => { @@ -11883,9 +11462,12 @@ describe('SessionManager permission mode updates', () => { // This turn owned the pending request, so its failure names the closure // rather than the bare restart. assert.strictEqual(turn?.errorClass, 'sandbox_boundary_closed_by_restart'); - const [run] = await runStore.listSessionRuns(session.id); - assert.strictEqual(run?.status, 'failed'); - assert.strictEqual(run?.failureClass, 'sandbox_boundary_closed_by_restart'); + const [run] = await runStore.listSessionInvocations(session.id); + assert.strictEqual(run && runtimeInvocationOutcome(run), 'failed'); + assert.strictEqual( + run && runtimeInvocationFailureClass(run), + 'sandbox_boundary_closed_by_restart', + ); assert.deepStrictEqual(await store.listPendingSandboxBoundaryRequests(session.id), []); }); @@ -12388,10 +11970,12 @@ class CompactingTestBackend extends TestBackend { this.compactCalls.push({ turnId: input.turnId, runtimeContextCount: input.runtimeContext.length, - sourceRoutes: (input.runtimeContextRunHeaders ?? []).map((run) => ({ + sourceRoutes: (input.runtimeContextInvocations ?? []).map((run) => ({ runId: run.runId, - ...(run.llmConnectionId ? { connectionId: run.llmConnectionId } : {}), - modelId: run.modelId, + ...(run.opening.route.provenance === 'runtime' + ? { connectionId: run.opening.route.llmConnectionId } + : {}), + modelId: run.opening.route.modelId, })), }); return compactHistoryResult(); @@ -13522,7 +13106,6 @@ class MemoryAgentRunStore readonly continuationAuthorityCapability = RUNTIME_CONTINUATION_AUTHORITY_V1; listSessionRunsCalls = 0; readEventsCalls = 0; - private headers = new Map(); private events = new Map(); private runtimeEvents = new Map(); private runtimeEventEntries: RuntimeEvent[] = []; @@ -13540,8 +13123,6 @@ class MemoryAgentRunStore failRuntimeEventAppendAfter?: number; failRuntimeEventReads?: boolean; failContinuationClaimReads?: boolean; - failUpdateRunOnce?: boolean; - failUpdateRunStatusOnce?: AgentRunHeader['status']; failContinuationCreate?: boolean; beforeListSessionRuns?: (sessionId: string) => Promise | void; beforeRuntimeEventRead?: (sessionId: string, runId: string) => Promise | void; @@ -13551,69 +13132,15 @@ class MemoryAgentRunStore event: RuntimeEvent, options?: { durable?: boolean }, ) => Promise | void; - beforeRunRead?: (sessionId: string, runId: string) => Promise | void; beforeAgentRunEventAppend?: ( sessionId: string, runId: string, event: AgentRunEvent, ) => Promise | void; beforeAgentRunEventRead?: (sessionId: string, runId: string) => Promise | void; - beforeAgentRunUpdate?: ( - sessionId: string, - runId: string, - patch: Partial, - ) => Promise | void; } = {}, ) {} - async createRun(header: AgentRunHeader): Promise { - if (this.options.failContinuationCreate && header.continuationSource) { - throw new Error('continuation claim create failed'); - } - this.headers.set(key(header.sessionId, header.runId), { ...header }); - return { ...header }; - } - - async updateRun( - sessionId: string, - runId: string, - patch: Partial, - ): Promise { - await this.options.beforeAgentRunUpdate?.(sessionId, runId, patch); - if (this.options.failUpdateRunOnce) { - this.options.failUpdateRunOnce = false; - throw new Error('update run failed'); - } - if (patch.status && patch.status === this.options.failUpdateRunStatusOnce) { - this.options.failUpdateRunStatusOnce = undefined; - throw new Error('update run failed'); - } - const current = await this.readRun(sessionId, runId); - const next = { ...current, ...patch, sessionId, runId }; - this.headers.set(key(sessionId, runId), next); - return { ...next }; - } - - async readRun(sessionId: string, runId: string): Promise { - await this.options.beforeRunRead?.(sessionId, runId); - const header = this.headers.get(key(sessionId, runId)); - if (!header) { - const error = new Error(`Unknown run ${runId}`) as NodeJS.ErrnoException; - error.code = 'ENOENT'; - throw error; - } - return { ...header }; - } - - async listSessionRuns(sessionId: string): Promise { - this.listSessionRunsCalls += 1; - await this.options.beforeListSessionRuns?.(sessionId); - return Array.from(this.headers.values()) - .filter((header) => header.sessionId === sessionId) - .sort((a, b) => a.createdAt - b.createdAt || a.runId.localeCompare(b.runId)) - .map((header) => ({ ...header })); - } - seedRootTurnAdmission( sessionId: string, turnId: string, @@ -13659,6 +13186,17 @@ class MemoryAgentRunStore this.options.failRuntimeEventAppendAfter = undefined; throw new Error('runtime event append failed'); } + assertDoubleRunNotSealed(this.runtimeEvents.get(key(sessionId, runId)) ?? [], event); + this.seedRuntimeEvent(sessionId, runId, event); + } + + /** + * Put an event into the ledger underneath the seal. + * + * A test that needs a ledger shape the store would refuse to write has to + * assemble it below the store, not through the API whose contract forbids it. + */ + seedRuntimeEvent(sessionId: string, runId: string, event: RuntimeEvent): void { const eventKey = key(sessionId, runId); this.runtimeEvents.set(eventKey, [ ...(this.runtimeEvents.get(eventKey) ?? []), @@ -13745,6 +13283,9 @@ class MemoryAgentRunStore } async claimContinuation(input: { claim: ContinuationClaimV1 }) { + if (this.options.failContinuationCreate) { + throw new Error('continuation claim create failed'); + } const claim = decodeContinuationClaim(input.claim); const existing = this.continuationClaims.get(claim.boundaryDigest); if (existing) return { kind: 'existing' as const, claim: existing }; @@ -13843,6 +13384,15 @@ class MemoryAgentRunStore ); return ordered.map((item) => item.event); } + + async listSessionInvocations(sessionId: string): Promise { + this.listSessionRunsCalls += 1; + await this.options.beforeListSessionRuns?.(sessionId); + return runtimeInvocationsFromSessionEvents( + sessionId, + await this.readSessionRuntimeEvents(sessionId), + ); + } } class ContinuationClaimBarrierRunStore extends MemoryAgentRunStore { @@ -13869,8 +13419,8 @@ class ContinuationClaimBarrierRunStore extends MemoryAgentRunStore { this.releaseContinuationClaimReadWaiter?.(); } - override async listSessionRuns(sessionId: string): Promise { - const snapshot = await super.listSessionRuns(sessionId); + override async listSessionInvocations(sessionId: string): Promise { + const snapshot = await super.listSessionInvocations(sessionId); if (!this.continuationClaimBarrierArmed) return snapshot; this.continuationClaimBarrierArmed = false; this.markContinuationClaimRead?.(); @@ -13976,24 +13526,14 @@ class ProviderRetryProgressBackend implements AgentBackend { } class ReverseOrderedAgentRunStore extends MemoryAgentRunStore { - override async listSessionRuns(sessionId: string): Promise { - return (await super.listSessionRuns(sessionId)).reverse(); + override async listSessionInvocations(sessionId: string): Promise { + return (await super.listSessionInvocations(sessionId)).reverse(); } } class OrderingAgentRunStore extends MemoryAgentRunStore { operations: string[] = []; - override async updateRun( - sessionId: string, - runId: string, - patch: Partial, - ): Promise { - const next = await super.updateRun(sessionId, runId, patch); - if (patch.status === 'completed') this.operations.push('completedRunHeader'); - return next; - } - override async appendRuntimeEvent( sessionId: string, runId: string, @@ -14038,6 +13578,17 @@ class MemoryRuntimeEventStore implements RuntimeEventStore { async appendRuntimeEvent(sessionId: string, runId: string, event: RuntimeEvent): Promise { if (this.options.failRuntimeEventAppends) throw new Error('runtime event append failed'); + assertDoubleRunNotSealed(this.runtimeEvents.get(key(sessionId, runId)) ?? [], event); + this.seedRuntimeEvent(sessionId, runId, event); + } + + /** + * Put an event into the ledger underneath the seal. + * + * A test that needs a ledger shape the store would refuse to write has to + * assemble it below the store, not through the API whose contract forbids it. + */ + seedRuntimeEvent(sessionId: string, runId: string, event: RuntimeEvent): void { const eventKey = key(sessionId, runId); this.runtimeEvents.set(eventKey, [ ...(this.runtimeEvents.get(eventKey) ?? []), @@ -14094,6 +13645,13 @@ class MemoryRuntimeEventStore implements RuntimeEventStore { ); return ordered.map((item) => item.event); } + + async listSessionInvocations(sessionId: string): Promise { + return runtimeInvocationsFromSessionEvents( + sessionId, + await this.readSessionRuntimeEvents(sessionId), + ); + } } interface Gate { @@ -14369,7 +13927,7 @@ function testTool(name: string): MakaTool { }; } -function makeRunHeader(overrides: Partial = {}): AgentRunHeader { +function makeRunHeader(overrides: Partial = {}): TestRunHeader { return { runId: 'run-1', sessionId: 'session-1', @@ -14386,10 +13944,241 @@ function makeRunHeader(overrides: Partial = {}): AgentRunHeader }; } +/** + * The facts a test states about a run it is seeding. + * + * Deliberately not a stored record: `seedInvocationFromHeader` turns it into + * the opening fact and, when the test says the run ended, the terminal event + * that ends it. Nothing keeps this shape after the seed. + */ +interface TestRunHeader { + runId: string; + sessionId: string; + turnId: string; + invocationId?: string; + status: 'created' | 'running' | 'waiting_for_user' | 'completed' | 'failed' | 'cancelled'; + backendKind: PersistedBackendKind; + llmConnectionId?: string; + llmConnectionSlug: string; + modelId: string; + providerStateIdentity?: `sha256:${string}`; + cwd: string; + workspaceIdentity?: string; + permissionMode: PermissionMode; + collaborationMode?: 'agent' | 'plan'; + orchestrationMode?: 'default' | 'graph' | 'swarm'; + orchestrationSource?: 'session' | 'turn_override'; + agentSwarmAuthorization?: 'none' | 'session_mode' | 'turn_override'; + toolMode?: ToolMode; + createdAt: number; + updatedAt: number; + completedAt?: number; + failureClass?: string; + failureMessage?: string; + abortSource?: 'stop_button' | 'graph_supervisor'; + goalId?: string; + scheduledTaskId?: string; + legacyAutomationId?: string; + agentGraphWakeId?: string; + agentGraphWakeAttemptId?: string; + parentRunId?: string; + parentTurnId?: string; + parentSessionId?: string; + resumedFromRunId?: string; + retriedFromRunId?: string; + retriedFromTurnId?: string; + regeneratedFromTurnId?: string; + branchOfTurnId?: string; + agentId?: string; + agentName?: string; + continuationSource?: { + sourceInvocationId: string; + sourceRunId: string; + sourceTurnId: string; + sourceRuntimeEventHighWater: number; + claimId?: string; + boundaryDigest?: `sha256:${string}`; + }; +} + +/** The root authority the seeded header names, defaulting to the user. */ +function testInvocationRoot(header: TestRunHeader): RuntimeInvocationRootAuthority { + if (header.goalId) return { kind: 'goal', goalId: header.goalId }; + if (header.scheduledTaskId) { + return { kind: 'scheduled_task', scheduledTaskId: header.scheduledTaskId }; + } + if (header.legacyAutomationId) { + return { kind: 'legacy_automation', legacyAutomationId: header.legacyAutomationId }; + } + if (header.agentGraphWakeId && header.agentGraphWakeAttemptId) { + return { + kind: 'agent_graph_supervisor_wake', + wakeId: header.agentGraphWakeId, + attemptId: header.agentGraphWakeAttemptId, + }; + } + return { kind: 'user' }; +} + +/** Everything the header says about lineage, with the absent edges left out. */ +function testInvocationLineage(header: TestRunHeader): RuntimeInvocationLineage { + return { + ...(header.parentRunId ? { parentRunId: header.parentRunId } : {}), + ...(header.parentTurnId ? { parentTurnId: header.parentTurnId } : {}), + ...(header.parentSessionId ? { parentSessionId: header.parentSessionId } : {}), + ...(header.resumedFromRunId ? { resumedFromRunId: header.resumedFromRunId } : {}), + ...(header.retriedFromRunId ? { retriedFromRunId: header.retriedFromRunId } : {}), + ...(header.retriedFromTurnId ? { retriedFromTurnId: header.retriedFromTurnId } : {}), + ...(header.regeneratedFromTurnId + ? { regeneratedFromTurnId: header.regeneratedFromTurnId } + : {}), + ...(header.branchOfTurnId ? { branchOfTurnId: header.branchOfTurnId } : {}), + ...(header.agentId ? { agentId: header.agentId } : {}), + ...(header.agentName ? { agentName: header.agentName } : {}), + }; +} + +function testInvocationOpening(header: TestRunHeader): RuntimeEventInvocationOpenedContent { + const lineage = testInvocationLineage(header); + return { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: + header.llmConnectionId === undefined + ? { + provenance: 'unknown', + backendKind: header.backendKind, + llmConnectionSlug: header.llmConnectionSlug, + modelId: header.modelId, + } + : { + provenance: 'runtime', + backendKind: header.backendKind, + llmConnectionId: header.llmConnectionId, + llmConnectionSlug: header.llmConnectionSlug, + modelId: header.modelId, + ...(header.providerStateIdentity + ? { providerStateIdentity: header.providerStateIdentity } + : {}), + }, + configuration: { + cwd: header.cwd, + permissionMode: header.permissionMode, + collaborationMode: header.collaborationMode ?? 'agent', + orchestrationMode: header.orchestrationMode ?? 'default', + orchestrationSource: header.orchestrationSource ?? 'session', + toolMode: header.toolMode ?? DEFAULT_TOOL_MODE, + ...(header.agentSwarmAuthorization + ? { agentSwarmAuthorization: header.agentSwarmAuthorization } + : {}), + ...(header.workspaceIdentity ? { workspaceIdentity: header.workspaceIdentity } : {}), + }, + root: testInvocationRoot(header), + source: header.continuationSource + ? { kind: 'continuation', ...header.continuationSource } + : { kind: 'fresh' }, + ...(Object.keys(lineage).length > 0 ? { lineage } : {}), + }; +} + +/** + * Open the invocation the header describes, and close it when the header says + * the run ended. Seeding writes events because events are all there is. + */ +async function seedInvocationOpening( + store: Pick, + header: TestRunHeader, +): Promise { + await store.appendRuntimeEvent( + header.sessionId, + header.runId, + buildInvocationOpenedEvent({ + id: `${header.runId}-invocation-opened`, + run: runIdentityOf(header), + openedAt: header.createdAt, + opening: testInvocationOpening(header), + }), + ); +} + +/** The one event that ends the run, when the header says the run ended. */ +async function seedInvocationTerminal( + store: Pick, + header: TestRunHeader, +): Promise { + if (header.status !== 'completed' && header.status !== 'failed' && header.status !== 'cancelled') + return; + await store.appendRuntimeEvent(header.sessionId, header.runId, { + id: `${header.runId}-terminal`, + ...runIdentityOf(header), + ts: header.completedAt ?? header.updatedAt, + partial: false, + role: 'system', + author: 'system', + status: header.status === 'cancelled' ? 'aborted' : header.status, + // The failure class and abort source live where every reader looks for + // them: on the terminal event's own state delta. + actions: { + endInvocation: true, + ...(header.failureClass || header.abortSource + ? { + stateDelta: { + ...(header.failureClass ? { failureClass: header.failureClass } : {}), + ...(header.abortSource ? { abortSource: header.abortSource } : {}), + }, + } + : {}), + }, + ...(header.failureMessage + ? { content: { kind: 'error' as const, message: header.failureMessage } } + : {}), + }); +} + +function runIdentityOf(header: TestRunHeader): { + sessionId: string; + invocationId: string; + runId: string; + turnId: string; +} { + return { + sessionId: header.sessionId, + invocationId: header.invocationId ?? header.runId, + runId: header.runId, + turnId: header.turnId, + }; +} + +async function seedInvocationFromHeader( + store: Pick, + header: TestRunHeader, +): Promise { + await seedInvocationOpening(store, header); + await seedInvocationTerminal(store, header); + return header; +} + +/** The one invocation that opened this run. */ +async function readInvocation( + store: Pick, + sessionId: string, + runId: string, +): Promise { + const found = (await store.listSessionInvocations(sessionId)).find( + (candidate) => candidate.runId === runId, + ); + if (!found) { + const error = new Error(`Unknown run ${runId}`) as NodeJS.ErrnoException; + error.code = 'ENOENT'; + throw error; + } + return found; +} + function makeRunEvent(overrides: Partial = {}): EmittedAgentRunEvent { return { - type: 'run_started', - id: `${overrides.runId ?? 'run-1'}-${overrides.type ?? 'run_started'}-${overrides.ts ?? 10}`, + type: 'turn_started', + id: `${overrides.runId ?? 'run-1'}-${overrides.type ?? 'turn_started'}-${overrides.ts ?? 10}`, runId: 'run-1', sessionId: 'session-1', turnId: 'turn-1', @@ -14532,7 +14321,7 @@ async function seedRuntimeReadTurnWithHeader(input: { userText: string; assistantText: string; legacyIdPrefix: string; - header: Partial; + header: Partial; tsBase: number; }): Promise { const header = makeRunHeader({ @@ -14618,32 +14407,53 @@ async function seedRuntimeReadTurnWithHeader(input: { } async function seedRun( - runStore: AgentRunStore, - header: AgentRunHeader, + runStore: AgentRunStore & RuntimeEventStore, + header: TestRunHeader, events: EmittedAgentRunEvent[], ): Promise { - await runStore.createRun(header); + await seedInvocationFromHeader(runStore, header); for (const event of events) { await runStore.appendEvent(header.sessionId, header.runId, event); } } +/** + * Seed one invocation whose ledger the test writes itself. + * + * The opening always comes first, and it opens the invocation the test's own + * events name, so nothing the test writes lands outside the run it seeded. The + * terminal event comes from the header only when the test did not already state + * one, so a run never ends twice. + */ async function seedRuntimeRun( - runStore: AgentRunStore & RuntimeEventStore, - header: AgentRunHeader, + runStore: RuntimeEventStore, + header: TestRunHeader, events: RuntimeEvent[], ): Promise { - await runStore.createRun(header); + const seeded: TestRunHeader = { + ...header, + invocationId: + header.invocationId ?? + events.find((event) => event.runId === header.runId)?.invocationId ?? + header.runId, + }; + await seedInvocationOpening(runStore, seeded); for (const event of events) { - await runStore.appendRuntimeEvent(header.sessionId, header.runId, event); + await runStore.appendRuntimeEvent(seeded.sessionId, seeded.runId, event); + } + if (!events.some((event) => event.status !== undefined)) { + await seedInvocationTerminal(runStore, seeded); } } function runtimeEvent(overrides: Partial): RuntimeEvent { + const runId = overrides.runId ?? 'run-1'; return { id: 'rt-event', - invocationId: 'inv-1', - runId: 'run-1', + // One invocation per run unless a test says otherwise. A shared default + // would put two runs' events on one invocation, which is two endings. + invocationId: runId, + runId, sessionId: 'session-1', turnId: 'turn-1', ts: 100, @@ -14656,7 +14466,7 @@ function runtimeEvent(overrides: Partial): RuntimeEvent { async function seedCanonicalPermissionRun( runStore: MemoryAgentRunStore, - header: AgentRunHeader, + header: TestRunHeader, includeLedgerRequest = true, ): Promise { const events = [ @@ -14721,7 +14531,7 @@ async function seedCanonicalPermissionRun( } function canonicalPermissionRecord( - header: AgentRunHeader, + header: TestRunHeader, overrides: Partial = {}, ): CanonicalPermissionOutcomeRecord { return { @@ -14830,7 +14640,7 @@ async function seedBoundaryRestartSession(input: { sessionId: session.id, runId: 'run-1', turnId: 'turn-1', - type: 'run_started', + type: 'turn_started', ts: 11, }), ], diff --git a/packages/runtime/src/__tests__/stream-graph-coordinator.test.ts b/packages/runtime/src/__tests__/stream-graph-coordinator.test.ts index e596bd0377..4201b6c807 100644 --- a/packages/runtime/src/__tests__/stream-graph-coordinator.test.ts +++ b/packages/runtime/src/__tests__/stream-graph-coordinator.test.ts @@ -33,7 +33,10 @@ import { type AgentGraphOperatorProvision, } from '@maka/core/agent-graph-topology'; import { type AgentGraphScheduleUpdate } from '@maka/core/agent-graph-schedule'; -import { type AgentRunHeader } from '@maka/core/agent-run'; +import { + runtimeInvocationOutcome, + type RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; import { type RuntimeEvent } from '@maka/core/runtime-event'; import { OPERATIONAL_STATE_DATABASE_NAME } from '@maka/storage/operational-state-store'; import { createSessionStore, isSessionNotFoundError } from '@maka/storage/session-store'; @@ -61,6 +64,7 @@ import { type UpdateAgentGraphToolInput, } from '../stream-graph-supervisor-tools.js'; import { projectAgentGraphRecords } from '../stream-graph-projection.js'; +import { testInvocationOpening } from './invocation-fixture.js'; describe('host-managed agent graph coordinator', () => { test('authorizes only selected committed results from an earlier epoch of the same root', async () => { @@ -68,20 +72,34 @@ describe('host-managed agent graph coordinator', () => { const rootSessionId = 'root-session'; const sourceGraphId = agentGraphIdForRootSession(rootSessionId); const currentGraphId = agentGraphIdForRootSessionEpoch(rootSessionId, 2); - const sourceRun: AgentRunHeader = { + const sourceRun: RuntimeInvocationRecord = { sessionId: 'source-child', runId: 'source-run', turnId: 'source-turn', invocationId: 'source-invocation', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake', - cwd: '/workspace', - permissionMode: 'explore', - status: 'completed', - createdAt: 1, - updatedAt: 2, - completedAt: 2, + openedAt: 1, + opening: testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'fake-connection', + llmConnectionSlug: 'fake', + modelId: 'fake', + }, + configuration: { cwd: '/workspace', permissionMode: 'explore' }, + }), + terminalEvent: { + id: 'source-terminal', + sessionId: 'source-child', + invocationId: 'source-invocation', + runId: 'source-run', + turnId: 'source-turn', + ts: 2, + partial: false, + role: 'system', + author: 'system', + status: 'completed', + }, }; const sourceEvent: RuntimeEvent = { id: 'source-result-event', @@ -152,11 +170,9 @@ describe('host-managed agent graph coordinator', () => { readHeader: async (sessionId: string) => ({ id: sessionId, status: 'active', isArchived: false }) as never, }, - runStore: { - listSessionRuns: async (sessionId: string) => - sessionId === sourceRun.sessionId ? [sourceRun] : [], - }, runtimeEventStore: { + listSessionInvocations: async (sessionId: string) => + sessionId === sourceRun.sessionId ? [sourceRun] : [], readImmutableRuntimeEvents: async (sessionId, runId) => sessionId === sourceRun.sessionId && runId === sourceRun.runId ? [sourceEvent] : [], }, @@ -307,7 +323,7 @@ describe('host-managed agent graph coordinator', () => { })) { // Drain the ordinary root turn so its source AgentRun is durable. } - const sourceRun = (await runStore.listSessionRuns(rootSession.id)).find( + const sourceRun = (await runtimeEventStore.listSessionInvocations(rootSession.id)).find( (run) => run.turnId === sourceTurnId, ); assert.ok(sourceRun); @@ -329,11 +345,14 @@ describe('host-managed agent graph coordinator', () => { return { kind: 'completed', turnId: input.turnId }; }, inspectAttempt: async (sessionId, attemptId, turnId) => { - const run = (await runStore.listSessionRuns(sessionId)).find( + const run = (await runtimeEventStore.listSessionInvocations(sessionId)).find( (candidate) => - candidate.agentGraphWakeAttemptId === attemptId && candidate.turnId === turnId, + candidate.opening.root.kind === 'agent_graph_supervisor_wake' && + candidate.opening.root.attemptId === attemptId && + candidate.turnId === turnId, ); - return run?.status ?? 'missing'; + if (!run) return 'missing'; + return runtimeInvocationOutcome(run) ?? 'running'; }, newId: randomUUID, onError: (_rootSessionId, error) => { @@ -424,15 +443,32 @@ describe('host-managed agent graph coordinator', () => { ), 'the original root Agent must run again and produce a deliverable response', ); - const wakeRun = (await runStore.listSessionRuns(rootSession.id)).find( + const wakeRun = (await runtimeEventStore.listSessionInvocations(rootSession.id)).find( (run) => run.turnId === graphWake.turnId, ); assert.ok(wakeRun); - assert.equal(wakeRun.agentGraphWakeId, graphWake.origin.wakeId); - assert.equal(wakeRun.agentGraphWakeAttemptId, graphWake.origin.attemptId); + assert.deepEqual(wakeRun.opening.root, { + kind: 'agent_graph_supervisor_wake', + wakeId: graphWake.origin.wakeId, + attemptId: graphWake.origin.attemptId, + }); + const wakeEvents = await runtimeEventStore.readImmutableRuntimeEvents( + rootSession.id, + wakeRun.runId, + ); + assert.deepEqual( + wakeEvents[0]?.content?.kind === 'invocation_opened' + ? wakeEvents[0].content.root + : undefined, + { + kind: 'agent_graph_supervisor_wake', + wakeId: graphWake.origin.wakeId, + attemptId: graphWake.origin.attemptId, + }, + 'the opening fact must name the host authority that woke this invocation', + ); assert.equal( - (await runtimeEventStore.readImmutableRuntimeEvents(rootSession.id, wakeRun.runId))[0] - ?.author, + wakeEvents[1]?.author, 'host', 'canonical provenance must distinguish the host-authored wake from human input', ); @@ -738,8 +774,10 @@ describe('host-managed agent graph coordinator', () => { throw new Error('wake must fail before reading the Session'); }, }, - runStore: { listSessionRuns: async () => [] }, - runtimeEventStore: { readImmutableRuntimeEvents: async () => [] }, + runtimeEventStore: { + listSessionInvocations: async () => [], + readImmutableRuntimeEvents: async () => [], + }, controlStore, epochStore: { resolveCurrentAgentGraphEpoch: async () => { @@ -797,8 +835,10 @@ describe('host-managed agent graph coordinator', () => { } as never; }, }, - runStore: { listSessionRuns: async () => [] }, - runtimeEventStore: { readImmutableRuntimeEvents: async () => [] }, + runtimeEventStore: { + listSessionInvocations: async () => [], + readImmutableRuntimeEvents: async () => [], + }, controlStore, epochStore: { resolveCurrentAgentGraphEpoch: async () => { @@ -894,8 +934,10 @@ describe('host-managed agent graph coordinator', () => { throw new Error('removed Session header must not be read during cleanup'); }, }, - runStore: { listSessionRuns: async () => [] }, - runtimeEventStore: { readImmutableRuntimeEvents: async () => [] }, + runtimeEventStore: { + listSessionInvocations: async () => [], + readImmutableRuntimeEvents: async () => [], + }, controlStore, epochStore: { resolveCurrentAgentGraphEpoch: async () => epochs[0]!, @@ -951,8 +993,10 @@ describe('host-managed agent graph coordinator', () => { orchestrationMode: 'graph', }) as never, }, - runStore: { listSessionRuns: async () => [] }, - runtimeEventStore: { readImmutableRuntimeEvents: async () => [] }, + runtimeEventStore: { + listSessionInvocations: async () => [], + readImmutableRuntimeEvents: async () => [], + }, controlStore, epochStore: controlStore, runtime: { @@ -1028,8 +1072,10 @@ describe('host-managed agent graph coordinator', () => { orchestrationMode: 'graph', }) as never, }, - runStore: { listSessionRuns: async () => [] }, - runtimeEventStore: { readImmutableRuntimeEvents: async () => [] }, + runtimeEventStore: { + listSessionInvocations: async () => [], + readImmutableRuntimeEvents: async () => [], + }, controlStore, epochStore: controlStore, runtime: { @@ -1118,7 +1164,7 @@ describe('host-managed agent graph coordinator', () => { })) { // Drain the source turn so its AgentRun is durable. } - const sourceRun = (await runStore.listSessionRuns(rootSession.id)).find( + const sourceRun = (await runtimeEventStore.listSessionInvocations(rootSession.id)).find( (run) => run.turnId === sourceTurnId, ); assert.ok(sourceRun); @@ -1132,7 +1178,6 @@ describe('host-managed agent graph coordinator', () => { const create = () => new AgentGraphCoordinator({ sessionStore, - runStore, runtimeEventStore, controlStore, runtime: failingRuntime, @@ -1217,8 +1262,10 @@ describe('host-managed agent graph coordinator', () => { isArchived: true, }) as never, }, - runStore: { listSessionRuns: async () => [] }, - runtimeEventStore: { readImmutableRuntimeEvents: async () => [] }, + runtimeEventStore: { + listSessionInvocations: async () => [], + readImmutableRuntimeEvents: async () => [], + }, epochStore: { resolveCurrentAgentGraphEpoch: async () => ({ schemaVersion: 1, @@ -1387,19 +1434,22 @@ describe('host-managed agent graph coordinator', () => { targetRunId: runId, claimedAt: 12, }; - const runningRun: AgentRunHeader = { + const runningRun: RuntimeInvocationRecord = { sessionId: childSessionId, + invocationId: 'child-invocation', runId, turnId, - invocationId: 'child-invocation', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake', - cwd: '/workspace', - permissionMode: 'explore', - status: 'running', - createdAt: 12, - updatedAt: 12, + openedAt: 12, + opening: testInvocationOpening({ + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'fake-connection', + llmConnectionSlug: 'fake', + modelId: 'fake', + }, + configuration: { cwd: '/workspace', permissionMode: 'explore' }, + }), }; const runningEvent: RuntimeEvent = { id: 'child-started', @@ -1416,7 +1466,7 @@ describe('host-managed agent graph coordinator', () => { let scheduleUpdates: AgentGraphScheduleUpdate[] = []; let provisions: AgentGraphOperatorProvision[] = []; let claims: AgentGraphIntentClaim[] = []; - let runs: AgentRunHeader[] = [runningRun]; + let runs: RuntimeInvocationRecord[] = [runningRun]; let runtimeEvents: RuntimeEvent[] = [runningEvent]; let projection: | { @@ -1438,8 +1488,10 @@ describe('host-managed agent graph coordinator', () => { isArchived: false, }) as never, }, - runStore: { listSessionRuns: async () => runs }, - runtimeEventStore: { readImmutableRuntimeEvents: async () => runtimeEvents }, + runtimeEventStore: { + listSessionInvocations: async () => runs, + readImmutableRuntimeEvents: async () => runtimeEvents, + }, controlStore: { listAgentGraphOperatorProvisions: async () => provisions, listAgentGraphScheduleUpdates: async () => scheduleUpdates, @@ -1497,7 +1549,23 @@ describe('host-managed agent graph coordinator', () => { assert.equal(await coordinator.readSessionState(rootSessionId), 'live'); assert.equal(await coordinator.hasLiveSessionState(rootSessionId), true); - runs = [{ ...runningRun, status: 'completed', completedAt: 14, updatedAt: 14 }]; + runs = [ + { + ...runningRun, + terminalEvent: { + id: 'child-terminal', + sessionId: childSessionId, + invocationId: 'child-invocation', + runId, + turnId, + ts: 14, + partial: false, + role: 'system', + author: 'system', + status: 'completed', + }, + }, + ]; runtimeEvents = [ runningEvent, { @@ -1533,8 +1601,10 @@ describe('host-managed agent graph coordinator', () => { isArchived: false, }) as never, }, - runStore: { listSessionRuns: async () => [] }, - runtimeEventStore: { readImmutableRuntimeEvents: async () => [] }, + runtimeEventStore: { + listSessionInvocations: async () => [], + readImmutableRuntimeEvents: async () => [], + }, controlStore: { listAgentGraphOperatorProvisions: async () => { throw topologyFailure; @@ -1583,8 +1653,10 @@ describe('host-managed agent graph coordinator', () => { isArchived: false, }) as never, }, - runStore: { listSessionRuns: async () => [] }, - runtimeEventStore: { readImmutableRuntimeEvents: async () => [] }, + runtimeEventStore: { + listSessionInvocations: async () => [], + readImmutableRuntimeEvents: async () => [], + }, controlStore: { listAgentGraphOperatorProvisions: async () => ['a', 'b'].map((suffix, index) => ({ @@ -1663,8 +1735,10 @@ describe('host-managed agent graph coordinator', () => { isArchived: false, }) as never, }, - runStore: { listSessionRuns: async () => [] }, - runtimeEventStore: { readImmutableRuntimeEvents: async () => [] }, + runtimeEventStore: { + listSessionInvocations: async () => [], + readImmutableRuntimeEvents: async () => [], + }, controlStore: gatedStore, runtime: { provisionAgentGraphOperator: async () => { diff --git a/packages/runtime/src/__tests__/stream-graph-handoff.test.ts b/packages/runtime/src/__tests__/stream-graph-handoff.test.ts index 950983f24f..9143cf3583 100644 --- a/packages/runtime/src/__tests__/stream-graph-handoff.test.ts +++ b/packages/runtime/src/__tests__/stream-graph-handoff.test.ts @@ -19,17 +19,18 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { hydrateAgentGraphInputHandoffs, renderAgentGraphScheduledWorkPrompt, } from '../stream-graph-handoff.js'; import { projectAgentGraphRecords } from '../stream-graph-projection.js'; +import { testInvocationRecord } from './invocation-fixture.js'; describe('agent graph operator handoffs', () => { test('hydrates a selected result or terminal record from the authoritative RuntimeEvent stream', async () => { - const run = runHeader(); + const run = runInvocation(); const events = [ runtimeEvent(run, { id: 'result-event', @@ -88,7 +89,7 @@ describe('agent graph operator handoffs', () => { }); test('bounds hydrated conclusion text across all selected inputs', async () => { - const run = runHeader(); + const run = runInvocation(); const events = [ runtimeEvent(run, { id: 'long-result', @@ -103,7 +104,7 @@ describe('agent graph operator handoffs', () => { streams: [ { operator: { operatorId: 'researcher', sessionId: run.sessionId }, - run: { ...run, status: 'running', completedAt: undefined }, + run: { ...run, terminalEvent: undefined }, events, }, ], @@ -122,7 +123,7 @@ describe('agent graph operator handoffs', () => { }); test('fails closed when a committed record cannot resolve its source event', async () => { - const run = runHeader(); + const run = runInvocation(); const event = runtimeEvent(run, { id: 'result-event', ts: 11, @@ -201,30 +202,25 @@ describe('agent graph operator handoffs', () => { }); }); -function runHeader(): AgentRunHeader { - return { - runId: 'run-child', - invocationId: 'invocation-child', +/** The child's one finished invocation, as its own events describe it. */ +function runInvocation(): RuntimeInvocationRecord { + return testInvocationRecord({ sessionId: 'child-session', + invocationId: 'invocation-child', + runId: 'run-child', turnId: 'turn-child', - status: 'completed', - backendKind: 'ai-sdk', - llmConnectionSlug: 'deepseek', - modelId: 'deepseek-chat', - cwd: '/workspace', - permissionMode: 'explore', - createdAt: 10, - updatedAt: 12, - completedAt: 12, - }; + openedAt: 10, + closedAt: 12, + outcome: 'completed', + }); } function runtimeEvent( - run: AgentRunHeader, + run: RuntimeInvocationRecord, overrides: Partial & Pick, ): RuntimeEvent { return { - invocationId: run.invocationId!, + invocationId: run.invocationId, runId: run.runId, sessionId: run.sessionId, turnId: run.turnId, diff --git a/packages/runtime/src/__tests__/stream-graph-projection.test.ts b/packages/runtime/src/__tests__/stream-graph-projection.test.ts index 81a3d10369..751e6f93dd 100644 --- a/packages/runtime/src/__tests__/stream-graph-projection.test.ts +++ b/packages/runtime/src/__tests__/stream-graph-projection.test.ts @@ -19,27 +19,28 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import { projectAgentGraphRecords, readCommittedAgentGraphProjection, replayAgentGraphRecords, } from '../stream-graph-projection.js'; +import { testInvocationRecord } from './invocation-fixture.js'; const baseTs = 1_800_000_000_000; describe('committed stream graph projection', () => { test('projects immutable child-session events into a stable reference-only graph trace', async () => { - const runA = runHeader({ + const runA = runInvocation({ sessionId: 'child-a', runId: 'run-a', turnId: 'turn-a', status: 'completed', createdAt: baseTs, }); - const runB = runHeader({ + const runB = runInvocation({ sessionId: 'child-b', runId: 'run-b', turnId: 'turn-b', @@ -139,12 +140,10 @@ describe('committed stream graph projection', () => { { operatorId: 'research', sessionId: runA.sessionId }, { operatorId: 'verify', sessionId: runB.sessionId }, ], - runStore: { - async listSessionRuns(sessionId) { + runtimeEventStore: { + async listSessionInvocations(sessionId) { return sessionId === runA.sessionId ? [runA] : [runB]; }, - }, - runtimeEventStore: { async readImmutableRuntimeEvents(_sessionId, runId) { return eventsByRun.get(runId) ?? []; }, @@ -212,7 +211,7 @@ describe('committed stream graph projection', () => { }); test('replay is deterministic for reordered delivery and idempotent duplicates', () => { - const run = runHeader({ + const run = runInvocation({ sessionId: 'child-a', runId: 'run-a', turnId: 'turn-a', @@ -254,7 +253,7 @@ describe('committed stream graph projection', () => { }); test('replays reserved JavaScript property names as ordinary graph identities', () => { - const run = runHeader({ + const run = runInvocation({ sessionId: 'reserved-session', runId: 'constructor', turnId: 'reserved-turn', @@ -286,7 +285,7 @@ describe('committed stream graph projection', () => { }); test('rejects one Session projected under different operators across observations', () => { - const run = runHeader({ + const run = runInvocation({ sessionId: 'child-a', runId: 'run-a', turnId: 'turn-a', @@ -329,14 +328,14 @@ describe('committed stream graph projection', () => { }); test('keeps existing records byte-stable when a late operator contributes earlier event time', () => { - const runA = runHeader({ + const runA = runInvocation({ sessionId: 'child-a', runId: 'run-a', turnId: 'turn-a', status: 'running', createdAt: baseTs, }); - const runB = runHeader({ + const runB = runInvocation({ sessionId: 'child-b', runId: 'run-b', turnId: 'turn-b', @@ -391,14 +390,14 @@ describe('committed stream graph projection', () => { }); test('allows equal event times and resolves them with the stable source order key', () => { - const first = runHeader({ + const first = runInvocation({ sessionId: 'child-z', runId: 'run-z', turnId: 'turn-z', status: 'running', createdAt: baseTs, }); - const second = runHeader({ + const second = runInvocation({ sessionId: 'child-a', runId: 'run-a', turnId: 'turn-a', @@ -434,7 +433,7 @@ describe('committed stream graph projection', () => { }); test('projects concurrent tool commits whose immutable event times are not commit-monotonic', () => { - const run = runHeader({ + const run = runInvocation({ sessionId: 'child-concurrent', runId: 'run-concurrent', turnId: 'turn-concurrent', @@ -478,14 +477,14 @@ describe('committed stream graph projection', () => { const precomposedId = '\u00e9'; const decomposedId = 'e\u0301'; assert.equal(precomposedId.localeCompare(decomposedId), 0); - const precomposed = runHeader({ + const precomposed = runInvocation({ sessionId: 'child-precomposed', runId: precomposedId, turnId: 'turn-precomposed', status: 'running', createdAt: baseTs, }); - const decomposed = runHeader({ + const decomposed = runInvocation({ sessionId: 'child-decomposed', runId: decomposedId, turnId: 'turn-decomposed', @@ -518,7 +517,7 @@ describe('committed stream graph projection', () => { }); test('routes human-interaction facts to the always-on supervisor without blocking lifecycle', () => { - const run = runHeader({ + const run = runInvocation({ sessionId: 'child-a', runId: 'run-a', turnId: 'turn-a', @@ -599,14 +598,14 @@ describe('committed stream graph projection', () => { }); test('keeps later session-inline runs as distinct activations of one operator', () => { - const first = runHeader({ + const first = runInvocation({ sessionId: 'child-a', runId: 'run-1', turnId: 'turn-1', status: 'completed', createdAt: baseTs, }); - const followup = runHeader({ + const followup = runInvocation({ sessionId: 'child-a', runId: 'run-2', turnId: 'turn-2', @@ -647,24 +646,26 @@ describe('committed stream graph projection', () => { }); test('fails closed on ambiguous authority or impossible replay order', async () => { - const run = runHeader({ + const run = runInvocation({ sessionId: 'child-a', runId: 'run-a', turnId: 'turn-a', status: 'completed', createdAt: baseTs, }); - const runtimeEventStore: Pick = {}; + const runtimeEventStore = { + async listSessionInvocations() { + return [run]; + }, + } as unknown as Pick< + RuntimeEventStore, + 'listSessionInvocations' | 'readImmutableRuntimeEvents' + >; await assert.rejects( readCommittedAgentGraphProjection({ graphId: 'graph-no-immutable-reader', operators: [{ operatorId: 'research', sessionId: run.sessionId }], - runStore: { - async listSessionRuns() { - return [run]; - }, - }, runtimeEventStore, }), /requires immutable RuntimeEvent reads/, @@ -696,12 +697,10 @@ describe('committed stream graph projection', () => { const projection = await readCommittedAgentGraphProjection({ graphId: 'graph-empty', operators: [], - runStore: { - async listSessionRuns() { + runtimeEventStore: { + async listSessionInvocations() { return []; }, - }, - runtimeEventStore: { async readImmutableRuntimeEvents() { return []; }, @@ -716,34 +715,34 @@ describe('committed stream graph projection', () => { }); }); -function runHeader(input: { +/** One invocation, as its opening fact and its terminal event describe it. */ +function runInvocation(input: { sessionId: string; runId: string; turnId: string; - status: AgentRunHeader['status']; + status: 'created' | 'running' | 'completed' | 'failed' | 'aborted'; createdAt: number; -}): AgentRunHeader { - return { - ...input, +}): RuntimeInvocationRecord { + const ended = + input.status === 'completed' || input.status === 'failed' || input.status === 'aborted' + ? input.status + : undefined; + return testInvocationRecord({ + sessionId: input.sessionId, invocationId: `invocation-${input.runId}`, - backendKind: 'ai-sdk', - llmConnectionSlug: 'deepseek', - modelId: 'deepseek-chat', - cwd: '/workspace', - permissionMode: 'explore', - updatedAt: input.createdAt + 1, - ...(input.status === 'completed' || input.status === 'failed' || input.status === 'cancelled' - ? { completedAt: input.createdAt + 1 } - : {}), - }; + runId: input.runId, + turnId: input.turnId, + openedAt: input.createdAt, + ...(ended ? { outcome: ended } : {}), + }); } function runtimeEvent( - run: AgentRunHeader, + run: RuntimeInvocationRecord, overrides: Partial & Pick, ): RuntimeEvent { return { - invocationId: run.invocationId ?? `invocation-${run.runId}`, + invocationId: run.invocationId, runId: run.runId, sessionId: run.sessionId, turnId: run.turnId, diff --git a/packages/runtime/src/__tests__/stream-graph-readiness.test.ts b/packages/runtime/src/__tests__/stream-graph-readiness.test.ts index 59a425136e..533311efd2 100644 --- a/packages/runtime/src/__tests__/stream-graph-readiness.test.ts +++ b/packages/runtime/src/__tests__/stream-graph-readiness.test.ts @@ -19,7 +19,7 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { AGENT_GRAPH_READINESS_SCHEMA_VERSION, @@ -28,13 +28,14 @@ import { } from '../stream-graph-readiness.js'; import { projectAgentGraphRecords } from '../stream-graph-projection.js'; import type { AgentGraphTraceTopology } from '../stream-graph-trace.js'; +import { testInvocationRecord } from './invocation-fixture.js'; const baseTs = 1_800_000_000_000; describe('operator-local stream graph readiness', () => { test('derives one stable map intent per direct input route without supervisor gating', () => { - const source = runHeader('source', baseTs); - const worker = runHeader('worker', baseTs + 1); + const source = runInvocation('source', baseTs); + const worker = runInvocation('worker', baseTs + 1); const projection = projectAgentGraphRecords({ graphId: 'graph-map', streams: [ @@ -116,8 +117,8 @@ describe('operator-local stream graph readiness', () => { }); test('keeps a map operator waiting while exposing that state to the supervisor', () => { - const source = runHeader('empty-source', baseTs); - const worker = runHeader('empty-worker', baseTs + 1); + const source = runInvocation('empty-source', baseTs); + const worker = runInvocation('empty-worker', baseTs + 1); const snapshot = buildAgentGraphReadinessSnapshot({ topology: { graphId: 'graph-map-waiting', @@ -144,15 +145,15 @@ describe('operator-local stream graph readiness', () => { }); test('waits for an exact all-settled activation frontier and accepts every terminal outcome', () => { - const branchA = runHeader('branch-a', baseTs, 'completed'); - const branchBRunning = runHeader('branch-b', baseTs + 1); + const branchA = runInvocation('branch-a', baseTs, 'completed'); + const branchBRunning = runInvocation('branch-b', baseTs + 1); const branchBFailed = { ...branchBRunning, status: 'failed' as const, completedAt: baseTs + 4, }; - const branchC = runHeader('branch-c', baseTs + 2, 'completed'); - const join = runHeader('join', baseTs + 3); + const branchC = runInvocation('branch-c', baseTs + 2, 'completed'); + const join = runInvocation('join', baseTs + 3); const topology: AgentGraphTraceTopology = { graphId: 'graph-all-settled', operators: [ @@ -245,15 +246,15 @@ describe('operator-local stream graph readiness', () => { }); test('does not let a later follow-up activation rewrite a sealed all-settled intent', () => { - const branchA = runHeader('sealed-a', baseTs, 'completed'); - const branchAFollowup = runHeader( + const branchA = runInvocation('sealed-a', baseTs, 'completed'); + const branchAFollowup = runInvocation( 'sealed-a-followup', baseTs + 20, 'running', branchA.sessionId, ); - const branchB = runHeader('sealed-b', baseTs + 1, 'completed'); - const join = runHeader('sealed-join', baseTs + 2); + const branchB = runInvocation('sealed-b', baseTs + 1, 'completed'); + const join = runInvocation('sealed-join', baseTs + 2); const topology: AgentGraphTraceTopology = { graphId: 'graph-sealed-frontier', operators: [binding(branchA, 'a'), binding(branchB, 'b'), binding(join, 'join')], @@ -308,9 +309,9 @@ describe('operator-local stream graph readiness', () => { }); test('keeps local intent identity stable across unrelated and downstream-only topology changes', () => { - const source = runHeader('fingerprint-source', baseTs); - const worker = runHeader('fingerprint-worker', baseTs + 1); - const observer = runHeader('fingerprint-observer', baseTs + 2); + const source = runInvocation('fingerprint-source', baseTs); + const worker = runInvocation('fingerprint-worker', baseTs + 1); + const observer = runInvocation('fingerprint-observer', baseTs + 2); const records = projectAgentGraphRecords({ graphId: 'graph-fingerprint', streams: [stream(source, 'source', [runtimeEvent(source, 'record', baseTs + 1, 'record')])], @@ -392,9 +393,9 @@ describe('operator-local stream graph readiness', () => { }); test('orders distinct Unicode identities canonically across topology and sealed inputs', () => { - const precomposed = runHeader('unicode-precomposed', baseTs); - const decomposed = runHeader('unicode-decomposed', baseTs + 1); - const join = runHeader('unicode-join', baseTs + 2); + const precomposed = runInvocation('unicode-precomposed', baseTs); + const decomposed = runInvocation('unicode-decomposed', baseTs + 1); + const join = runInvocation('unicode-join', baseTs + 2); const precomposedId = '\u00e9'; const decomposedId = 'e\u0301'; assert.equal(precomposedId.localeCompare(decomposedId), 0); @@ -453,8 +454,8 @@ describe('operator-local stream graph readiness', () => { }); test('keeps reserved JavaScript property names safe in readiness identities', () => { - const source = runHeader('reserved-source', baseTs); - const worker = runHeader('reserved-worker', baseTs + 1); + const source = runInvocation('reserved-source', baseTs); + const worker = runInvocation('reserved-worker', baseTs + 1); const records = projectAgentGraphRecords({ graphId: 'graph-reserved-readiness', streams: [stream(source, 'source', [runtimeEvent(source, 'record', baseTs + 1, 'record')])], @@ -485,9 +486,9 @@ describe('operator-local stream graph readiness', () => { }); test('fails closed on ambiguous or incomplete local readiness policies', () => { - const sourceA = runHeader('invalid-a', baseTs); - const sourceB = runHeader('invalid-b', baseTs + 1); - const target = runHeader('invalid-target', baseTs + 2); + const sourceA = runInvocation('invalid-a', baseTs); + const sourceB = runInvocation('invalid-b', baseTs + 1); + const target = runInvocation('invalid-target', baseTs + 2); const topology: AgentGraphTraceTopology = { graphId: 'graph-invalid-readiness', operators: [binding(sourceA, 'a'), binding(sourceB, 'b'), binding(target, 'target')], @@ -572,36 +573,28 @@ describe('operator-local stream graph readiness', () => { }); }); -function runHeader( +/** One invocation, open or ended, as its opening fact and terminal event say. */ +function runInvocation( name: string, - createdAt: number, - status: AgentRunHeader['status'] = 'running', + openedAt: number, + status: 'running' | 'completed' | 'failed' | 'aborted' = 'running', sessionId = `session-${name}`, -): AgentRunHeader { - return { +): RuntimeInvocationRecord { + return testInvocationRecord({ sessionId, + invocationId: `invocation-${name}`, runId: `run-${name}`, turnId: `turn-${name}`, - invocationId: `invocation-${name}`, - backendKind: 'ai-sdk', - llmConnectionSlug: 'deepseek', - modelId: 'deepseek-chat', - cwd: '/workspace', - permissionMode: 'explore', - status, - createdAt, - updatedAt: createdAt + 1, - ...(status === 'completed' || status === 'failed' || status === 'cancelled' - ? { completedAt: createdAt + 1 } - : {}), - }; + openedAt, + ...(status === 'running' ? {} : { outcome: status }), + }); } -function binding(run: AgentRunHeader, operatorId: string) { +function binding(run: RuntimeInvocationRecord, operatorId: string) { return { operatorId, sessionId: run.sessionId }; } -function stream(run: AgentRunHeader, operatorId: string, events: readonly RuntimeEvent[]) { +function stream(run: RuntimeInvocationRecord, operatorId: string, events: readonly RuntimeEvent[]) { return { operator: binding(run, operatorId), run, @@ -609,10 +602,15 @@ function stream(run: AgentRunHeader, operatorId: string, events: readonly Runtim }; } -function runtimeEvent(run: AgentRunHeader, id: string, ts: number, text: string): RuntimeEvent { +function runtimeEvent( + run: RuntimeInvocationRecord, + id: string, + ts: number, + text: string, +): RuntimeEvent { return { id, - invocationId: run.invocationId ?? `invocation-${run.runId}`, + invocationId: run.invocationId, runId: run.runId, sessionId: run.sessionId, turnId: run.turnId, @@ -625,14 +623,14 @@ function runtimeEvent(run: AgentRunHeader, id: string, ts: number, text: string) } function terminalEvent( - run: AgentRunHeader, + run: RuntimeInvocationRecord, id: string, ts: number, - status: Extract, + status: 'completed' | 'failed' | 'aborted', ): RuntimeEvent { return { id, - invocationId: run.invocationId ?? `invocation-${run.runId}`, + invocationId: run.invocationId, runId: run.runId, sessionId: run.sessionId, turnId: run.turnId, diff --git a/packages/runtime/src/__tests__/stream-graph-trace.test.ts b/packages/runtime/src/__tests__/stream-graph-trace.test.ts index 58549fd744..2366a5ef08 100644 --- a/packages/runtime/src/__tests__/stream-graph-trace.test.ts +++ b/packages/runtime/src/__tests__/stream-graph-trace.test.ts @@ -19,7 +19,7 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { projectAgentGraphRecords } from '../stream-graph-projection.js'; import { @@ -27,15 +27,16 @@ import { buildAgentGraphTraceSnapshot, type AgentGraphTraceTopology, } from '../stream-graph-trace.js'; +import { testInvocationRecord } from './invocation-fixture.js'; const baseTs = 1_800_000_000_000; describe('stream graph trace topology', () => { test('materializes deterministic direct-edge routes without putting the supervisor in the path', () => { - const research = runHeader('research', baseTs); - const verify = runHeader('verify', baseTs + 1); - const synthesize = runHeader('synthesize', baseTs + 2); - const audit = runHeader('audit', baseTs + 3); + const research = runInvocation('research', baseTs); + const verify = runInvocation('verify', baseTs + 1); + const synthesize = runInvocation('synthesize', baseTs + 2); + const audit = runInvocation('audit', baseTs + 3); const projection = projectAgentGraphRecords({ graphId: 'graph-trace', streams: [ @@ -123,8 +124,8 @@ describe('stream graph trace topology', () => { }); test('is deterministic and idempotent for reordered duplicate observations', () => { - const source = runHeader('source', baseTs); - const target = runHeader('target', baseTs + 1); + const source = runInvocation('source', baseTs); + const target = runInvocation('target', baseTs + 1); const projection = projectAgentGraphRecords({ graphId: 'graph-replay', streams: [ @@ -169,9 +170,9 @@ describe('stream graph trace topology', () => { }); test('fingerprints only declared topology fields in raw identity order', () => { - const precomposed = runHeader('unicode-precomposed', baseTs); - const decomposed = runHeader('unicode-decomposed', baseTs + 1); - const target = runHeader('unicode-target', baseTs + 2); + const precomposed = runInvocation('unicode-precomposed', baseTs); + const decomposed = runInvocation('unicode-decomposed', baseTs + 1); + const target = runInvocation('unicode-target', baseTs + 2); const precomposedId = '\u00e9'; const decomposedId = 'e\u0301'; assert.equal(precomposedId.localeCompare(decomposedId), 0); @@ -238,8 +239,8 @@ describe('stream graph trace topology', () => { }); test('keeps existing route identities stable as later observations arrive', () => { - const source = runHeader('source', baseTs); - const target = runHeader('target', baseTs + 1); + const source = runInvocation('source', baseTs); + const target = runInvocation('target', baseTs + 1); const initialProjection = projectAgentGraphRecords({ graphId: 'graph-incremental', streams: [stream(source, 'source', [runtimeEvent(source, 'first', baseTs + 10, 'first')])], @@ -279,8 +280,8 @@ describe('stream graph trace topology', () => { }); test('retains an observable topology before any runtime facts arrive', () => { - const source = runHeader('source', baseTs); - const target = runHeader('target', baseTs + 1); + const source = runInvocation('source', baseTs); + const target = runInvocation('target', baseTs + 1); const snapshot = buildAgentGraphTraceSnapshot({ topology: { @@ -308,11 +309,11 @@ describe('stream graph trace topology', () => { test('materializes reserved JavaScript property names as own snapshot keys', () => { const source = { - ...runHeader('reserved-source', baseTs), + ...runInvocation('reserved-source', baseTs), runId: 'constructor', invocationId: 'reserved-invocation', }; - const target = runHeader('reserved-target', baseTs + 1); + const target = runInvocation('reserved-target', baseTs + 1); const projection = projectAgentGraphRecords({ graphId: 'graph-reserved-keys', streams: [ @@ -351,9 +352,9 @@ describe('stream graph trace topology', () => { }); test('binds route identity to immutable edge endpoints', () => { - const source = runHeader('route-source', baseTs); - const targetA = runHeader('route-target-a', baseTs + 1); - const targetB = runHeader('route-target-b', baseTs + 2); + const source = runInvocation('route-source', baseTs); + const targetA = runInvocation('route-target-a', baseTs + 1); + const targetB = runInvocation('route-target-b', baseTs + 2); const projection = projectAgentGraphRecords({ graphId: 'graph-edge-rebinding', streams: [ @@ -401,9 +402,9 @@ describe('stream graph trace topology', () => { }); test('fails closed on invalid topology and record ownership', () => { - const one = runHeader('one', baseTs); - const two = runHeader('two', baseTs + 1); - const three = runHeader('three', baseTs + 2); + const one = runInvocation('one', baseTs); + const two = runInvocation('two', baseTs + 1); + const three = runInvocation('three', baseTs + 2); const projection = projectAgentGraphRecords({ graphId: 'graph-invalid', streams: [stream(one, 'one', [runtimeEvent(one, 'one-message', baseTs + 1, 'one')])], @@ -490,28 +491,22 @@ describe('stream graph trace topology', () => { }); }); -function runHeader(name: string, createdAt: number): AgentRunHeader { - return { +/** One still-open invocation, as its opening fact describes it. */ +function runInvocation(name: string, openedAt: number): RuntimeInvocationRecord { + return testInvocationRecord({ sessionId: `session-${name}`, + invocationId: `invocation-${name}`, runId: `run-${name}`, turnId: `turn-${name}`, - invocationId: `invocation-${name}`, - backendKind: 'ai-sdk', - llmConnectionSlug: 'deepseek', - modelId: 'deepseek-chat', - cwd: '/workspace', - permissionMode: 'explore', - status: 'running', - createdAt, - updatedAt: createdAt + 1, - }; + openedAt, + }); } -function binding(run: AgentRunHeader, operatorId: string) { +function binding(run: RuntimeInvocationRecord, operatorId: string) { return { operatorId, sessionId: run.sessionId }; } -function stream(run: AgentRunHeader, operatorId: string, events: readonly RuntimeEvent[]) { +function stream(run: RuntimeInvocationRecord, operatorId: string, events: readonly RuntimeEvent[]) { return { operator: binding(run, operatorId), run, @@ -519,10 +514,15 @@ function stream(run: AgentRunHeader, operatorId: string, events: readonly Runtim }; } -function runtimeEvent(run: AgentRunHeader, id: string, ts: number, text: string): RuntimeEvent { +function runtimeEvent( + run: RuntimeInvocationRecord, + id: string, + ts: number, + text: string, +): RuntimeEvent { return { id, - invocationId: run.invocationId ?? `invocation-${run.runId}`, + invocationId: run.invocationId, runId: run.runId, sessionId: run.sessionId, turnId: run.turnId, diff --git a/packages/runtime/src/agent-graph-supervisor-wake.ts b/packages/runtime/src/agent-graph-supervisor-wake.ts index 30e96a28ae..30d088a552 100644 --- a/packages/runtime/src/agent-graph-supervisor-wake.ts +++ b/packages/runtime/src/agent-graph-supervisor-wake.ts @@ -23,9 +23,9 @@ import { type AgentGraphSupervisorWakeStore, } from '@maka/core/agent-graph-supervisor-wake'; import type { ContextCompactionOutcome } from '@maka/core/events'; -import { type AgentRunHeader } from '@maka/core/agent-run'; import { type SessionEvent } from '@maka/core/events'; import { type UserMessageInput } from '@maka/core/runtime-inputs'; +import type { RuntimeInvocationOutcome } from '@maka/core/runtime-invocation'; import type { GoalTurnOutcome, SessionActivityLease, @@ -174,6 +174,14 @@ export type AgentGraphSupervisorWakeDiagnostic = }; }; +/** + * What the delivering invocation has to say for itself when the wake is settled. + * + * An invocation the events never closed is `running`, whether it is still on a + * provider or was parked on an interaction the host restart threw away. + */ +export type AgentGraphWakeAttemptStatus = RuntimeInvocationOutcome | 'running' | 'missing'; + export interface AgentGraphSupervisorWakeInput { activityRegistry: SessionActivityRegistry; wakeStore: AgentGraphSupervisorWakeStore; @@ -189,7 +197,7 @@ export interface AgentGraphSupervisorWakeInput { rootSessionId: string, attemptId: string, turnId: string, - ): Promise; + ): Promise; shouldWake?( rootSessionId: string, result: AgentGraphScheduleReconciliationResult | undefined, diff --git a/packages/runtime/src/agent-graph-timeline.ts b/packages/runtime/src/agent-graph-timeline.ts index 08d31fb4ae..d282d08100 100644 --- a/packages/runtime/src/agent-graph-timeline.ts +++ b/packages/runtime/src/agent-graph-timeline.ts @@ -27,7 +27,8 @@ import type { AgentGraphTimelineMetadataSnapshot, AgentGraphTimelineMetadataStore, } from '@maka/core/agent-graph-timeline'; -import type { AgentRunHeader, AgentRunStore } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; +import { runtimeInvocationOutcome } from '@maka/core/runtime-invocation'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; import { stableHash } from './request-shape.js'; import { @@ -72,7 +73,7 @@ export type AgentGraphTimelineEvent = | (AgentGraphTimelineEventBase & { kind: 'supervisor_turn_terminal'; run: AgentGraphTimelineRunRef; - status: Extract; + status: 'completed' | 'failed' | 'cancelled'; wake?: { wakeId: string; attemptId: string }; }) | (AgentGraphTimelineEventBase & { @@ -200,8 +201,10 @@ export interface ReadAgentGraphTimelinePageInput { rootSessionId: string; graphId: string; controlStore: AgentGraphTimelineMetadataStore; - runStore: Pick; - runtimeEventStore: Pick; + runtimeEventStore: Pick< + RuntimeEventStore, + 'readImmutableRuntimeEvents' | 'listSessionInvocations' + >; options?: AgentGraphTimelinePageOptions; } @@ -209,8 +212,8 @@ export interface BuildAgentGraphTimelineInput { rootSessionId: string; graphId: string; metadata: AgentGraphTimelineMetadataSnapshot; - rootRuns: readonly AgentRunHeader[]; - childRuns: readonly AgentRunHeader[]; + rootRuns: readonly RuntimeInvocationRecord[]; + childRuns: readonly RuntimeInvocationRecord[]; projection: AgentGraphProjection; } @@ -259,11 +262,10 @@ export async function readAgentGraphTimelinePage( sessionId: provision.targetSessionId, })); const [rootRuns, projected] = await Promise.all([ - input.runStore.listSessionRuns(input.rootSessionId), + input.runtimeEventStore.listSessionInvocations(input.rootSessionId), readCommittedAgentGraphProjectionWithRuns({ graphId: input.graphId, operators, - runStore: input.runStore, runtimeEventStore: input.runtimeEventStore, }), ]); @@ -303,9 +305,10 @@ export function buildAgentGraphTimeline( ), ); for (const run of input.rootRuns) { + const root = run.opening.root; const wake = - run.agentGraphWakeId && run.agentGraphWakeAttemptId - ? { wakeId: run.agentGraphWakeId, attemptId: run.agentGraphWakeAttemptId } + root.kind === 'agent_graph_supervisor_wake' + ? { wakeId: root.wakeId, attemptId: root.attemptId } : undefined; if (!relevantRootRunIds.has(run.runId) && !wakeAttemptIds.has(wake?.attemptId ?? '')) { continue; @@ -320,16 +323,14 @@ export function buildAgentGraphTimeline( sessionId: run.sessionId, runId: run.runId, }, - run.createdAt, + run.openedAt, ), kind: 'supervisor_turn_started', run: runRef, ...(wake ? { wake } : {}), }); - if ( - run.completedAt !== undefined && - (run.status === 'completed' || run.status === 'failed' || run.status === 'cancelled') - ) { + const outcome = runtimeInvocationOutcome(run); + if (outcome && run.terminalEvent) { push({ ...eventBase( input.graphId, @@ -337,13 +338,13 @@ export function buildAgentGraphTimeline( { sessionId: run.sessionId, runId: run.runId, - status: run.status, + status: outcome, }, - run.completedAt, + run.terminalEvent.ts, ), kind: 'supervisor_turn_terminal', run: runRef, - status: run.status, + status: outcome, ...(wake ? { wake } : {}), }); } @@ -408,7 +409,7 @@ export function buildAgentGraphTimeline( provision.targetSessionId, ]), ); - const childRunByIdentity = new Map(); + const childRunByIdentity = new Map(); for (const run of input.childRuns) { const key = `${run.sessionId}\0${run.runId}`; if (childRunByIdentity.has(key)) { @@ -447,7 +448,7 @@ export function buildAgentGraphTimeline( input.graphId, 'activation_started', { operatorId: claim.targetOperatorId, runId: run.runId }, - run.createdAt, + run.openedAt, ), kind: 'activation_started', operatorId: claim.targetOperatorId, @@ -751,7 +752,7 @@ function eventBase( }; } -function timelineRunRef(run: AgentRunHeader): AgentGraphTimelineRunRef { +function timelineRunRef(run: RuntimeInvocationRecord): AgentGraphTimelineRunRef { return { sessionId: run.sessionId, runId: run.runId, @@ -759,7 +760,7 @@ function timelineRunRef(run: AgentRunHeader): AgentGraphTimelineRunRef { }; } -function assertRunRef(run: AgentRunHeader, sessionId: string): void { +function assertRunRef(run: RuntimeInvocationRecord, sessionId: string): void { if (run.sessionId !== sessionId) { throw new Error(`AgentRun ${run.runId} belongs to ${run.sessionId}, expected ${sessionId}`); } diff --git a/packages/runtime/src/agent-run-inspect.ts b/packages/runtime/src/agent-run-inspect.ts index 977f97cd42..b53cef5443 100644 --- a/packages/runtime/src/agent-run-inspect.ts +++ b/packages/runtime/src/agent-run-inspect.ts @@ -17,9 +17,10 @@ * under the License. */ -import type { AgentRunEvent, AgentRunHeader, AgentRunStore } from '@maka/core/agent-run'; +import type { AgentRunEvent, AgentRunStore } from '@maka/core/agent-run'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { StoredMessage } from '@maka/core/session'; import { classifyRuntimeEventTerminalFact, @@ -35,11 +36,9 @@ import { export type AgentRunInspectDiagnosticCode = | 'operational_ledger_read_failed' | 'operational_event_corrupt' - | 'operational_terminal_missing' | 'missing_runtime_ledger' | 'runtime_ledger_read_failed' | 'runtime_terminal_missing' - | 'status_consistency_mismatch' | RuntimeEventReadModelDiagnostic['code']; export interface AgentRunInspectDiagnostic { @@ -54,8 +53,6 @@ export interface AgentRunInspectDiagnostic { export interface AgentRunInspectSourceHealth { runtimeLedger: 'present' | 'missing' | 'read_failed'; runtimeTerminalPresent: boolean; - operationalTerminalPresent: boolean; - statusConsistency: 'consistent' | 'inconsistent' | 'incomplete'; } export interface AgentRunInspectProjectionSummary { @@ -64,11 +61,10 @@ export interface AgentRunInspectProjectionSummary { } export interface AgentRunInspectModel { - header: AgentRunHeader; + invocation: RuntimeInvocationRecord; events: AgentRunEvent[]; runtimeEvents: RuntimeEvent[]; terminalRuntimeFact?: RuntimeEventTerminalFact; - operationalTerminalEvent?: AgentRunEvent; modelReplay?: RuntimeEventModelReplayPlan; projection?: AgentRunInspectProjectionSummary; sourceHealth: AgentRunInspectSourceHealth; @@ -78,63 +74,55 @@ export interface AgentRunInspectModel { export interface InspectAgentRunOptions { sessionId: string; runId: string; - header?: AgentRunHeader; + invocation?: RuntimeInvocationRecord; isFatalReadError?: (error: unknown) => boolean; includeModelReplay?: boolean; } -export type AgentRunInspectReader = Pick; +export type AgentRunInspectReader = Pick; -export type SessionAgentRunInspectReader = AgentRunInspectReader & - Pick; - -export type RuntimeEventInspectReader = Pick; +export type RuntimeEventInspectReader = Pick & + Required>; +/** + * One run, read from both ledgers it actually has: the RuntimeEvent spine that + * owns its facts, and the AgentRunEvent ledger that records what the runtime did + * operationally. There is no third record to reconcile them against any more. + */ export async function inspectAgentRunReadModel( runStore: AgentRunInspectReader, runtimeEventStore: RuntimeEventInspectReader, options: InspectAgentRunOptions, ): Promise { - const header = options.header ?? (await runStore.readRun(options.sessionId, options.runId)); + const invocation = options.invocation ?? (await readInvocation(runtimeEventStore, options)); const diagnostics: AgentRunInspectDiagnostic[] = []; const events = await readOperationalEvents( runStore, - header, + invocation, diagnostics, options.isFatalReadError, ); const runtimeRead = await readRuntimeEvents( runtimeEventStore, - header, + invocation, diagnostics, options.isFatalReadError, ); const runtimeEvents = runtimeRead.events; - const operationalTerminalEvent = latestOperationalTerminalEvent(events); - if (!operationalTerminalEvent) { - diagnostics.push( - inspectDiagnostic( - header, - 'operational_terminal_missing', - 'operational AgentRunEvent ledger has no terminal run event', - ), - ); - } - let terminalRuntimeFact: RuntimeEventTerminalFact | undefined; - if (runtimeRead.state === 'present') { - const terminalFactResult = classifyRuntimeEventTerminalFact(header, runtimeEvents); + if (runtimeRead.state === 'present' && invocation.terminalEvent) { + const terminalFactResult = classifyRuntimeEventTerminalFact(invocation, runtimeEvents); terminalRuntimeFact = terminalFactResult.fact; diagnostics.push( ...terminalFactResult.diagnostics.map((diagnostic) => - fromRuntimeReadModelDiagnostic(header, diagnostic), + fromRuntimeReadModelDiagnostic(invocation, diagnostic), ), ); if (!terminalRuntimeFact) { diagnostics.push( inspectDiagnostic( - header, + invocation, 'runtime_terminal_missing', 'runtime ledger has no complete terminal RuntimeEvent fact', ), @@ -144,12 +132,12 @@ export async function inspectAgentRunReadModel( const projection = runtimeEvents.length > 0 - ? projectRuntimeEventsToStoredMessages(runtimeEvents, { runHeaders: [header] }) + ? projectRuntimeEventsToStoredMessages(runtimeEvents, { invocations: [invocation] }) : undefined; if (projection) { diagnostics.push( ...projection.diagnostics.map((diagnostic) => - fromRuntimeReadModelDiagnostic(header, diagnostic), + fromRuntimeReadModelDiagnostic(invocation, diagnostic), ), ); } @@ -159,60 +147,35 @@ export async function inspectAgentRunReadModel( ? buildRuntimeEventModelReplayPlan(runtimeEvents) : undefined; - const statusConsistency = computeStatusConsistency( - header, - operationalTerminalEvent, - terminalRuntimeFact, - ); - if (statusConsistency === 'inconsistent') { - diagnostics.push( - inspectDiagnostic( - header, - 'status_consistency_mismatch', - 'AgentRunHeader, operational terminal event, and RuntimeEvent terminal fact disagree', - { - headerStatus: header.status, - operationalStatus: operationalTerminalEvent - ? operationalStatusFor(operationalTerminalEvent) - : undefined, - runtimeStatus: terminalRuntimeFact?.runStatus, - }, - ), - ); - } - return { - header, + invocation, events, runtimeEvents, ...(terminalRuntimeFact ? { terminalRuntimeFact } : {}), - ...(operationalTerminalEvent ? { operationalTerminalEvent } : {}), ...(modelReplay ? { modelReplay } : {}), ...(projection ? { projection } : {}), sourceHealth: { runtimeLedger: runtimeRead.state, runtimeTerminalPresent: terminalRuntimeFact !== undefined, - operationalTerminalPresent: operationalTerminalEvent !== undefined, - statusConsistency, }, diagnostics, }; } export async function inspectSessionRunReadModels( - runStore: SessionAgentRunInspectReader, + runStore: AgentRunInspectReader, runtimeEventStore: RuntimeEventInspectReader, sessionId: string, options: Pick = {}, ): Promise { - const headers = await runStore.listSessionRuns(sessionId); + const invocations = await runtimeEventStore.listSessionInvocations(sessionId); const models: AgentRunInspectModel[] = []; - for (const header of headers) { + for (const invocation of invocations) { models.push( await inspectAgentRunReadModel(runStore, runtimeEventStore, { sessionId, - runId: header.runId, - header, + runId: invocation.runId, + invocation, ...(options.isFatalReadError ? { isFatalReadError: options.isFatalReadError } : {}), }), ); @@ -220,19 +183,33 @@ export async function inspectSessionRunReadModels( return models; } +// A run is not its invocation: a continuation is a new run on the invocation it +// resumes. This reader is addressed by run, so it looks the invocation up by the +// id it was actually given. +async function readInvocation( + runtimeEventStore: RuntimeEventInspectReader, + options: InspectAgentRunOptions, +): Promise { + const found = (await runtimeEventStore.listSessionInvocations(options.sessionId)).find( + (invocation) => invocation.runId === options.runId, + ); + if (!found) throw new Error(`Runtime invocation not found: ${options.runId}`); + return found; +} + async function readOperationalEvents( runStore: AgentRunInspectReader, - header: AgentRunHeader, + invocation: RuntimeInvocationRecord, diagnostics: AgentRunInspectDiagnostic[], isFatalReadError: InspectAgentRunOptions['isFatalReadError'], ): Promise { try { - const events = await runStore.readEvents(header.sessionId, header.runId); + const events = await runStore.readEvents(invocation.sessionId, invocation.runId); for (const event of events) { if (event.type !== 'event_corrupt') continue; diagnostics.push( inspectDiagnostic( - header, + invocation, 'operational_event_corrupt', 'operational AgentRunEvent ledger contains a corrupt row', event.data, @@ -245,7 +222,7 @@ async function readOperationalEvents( if (isFatalReadError?.(error)) throw error; diagnostics.push( inspectDiagnostic( - header, + invocation, 'operational_ledger_read_failed', 'AgentRunStore.readEvents failed', errorMessage(error), @@ -257,16 +234,19 @@ async function readOperationalEvents( async function readRuntimeEvents( runtimeEventStore: RuntimeEventInspectReader, - header: AgentRunHeader, + invocation: RuntimeInvocationRecord, diagnostics: AgentRunInspectDiagnostic[], isFatalReadError: InspectAgentRunOptions['isFatalReadError'], ): Promise<{ state: AgentRunInspectSourceHealth['runtimeLedger']; events: RuntimeEvent[] }> { try { - const events = await runtimeEventStore.readRuntimeEvents(header.sessionId, header.runId); + const events = await runtimeEventStore.readRuntimeEvents( + invocation.sessionId, + invocation.runId, + ); if (events.length === 0) { diagnostics.push( inspectDiagnostic( - header, + invocation, 'missing_runtime_ledger', 'runtime-events ledger is missing or empty for this run', ), @@ -278,7 +258,7 @@ async function readRuntimeEvents( if (isFatalReadError?.(error)) throw error; diagnostics.push( inspectDiagnostic( - header, + invocation, 'runtime_ledger_read_failed', 'RuntimeEventStore.readRuntimeEvents failed', errorMessage(error), @@ -288,55 +268,14 @@ async function readRuntimeEvents( } } -function computeStatusConsistency( - header: AgentRunHeader, - operationalTerminalEvent: AgentRunEvent | undefined, - terminalRuntimeFact: RuntimeEventTerminalFact | undefined, -): AgentRunInspectSourceHealth['statusConsistency'] { - const statuses = [ - isTerminalRunStatus(header.status) ? header.status : undefined, - operationalTerminalEvent ? operationalStatusFor(operationalTerminalEvent) : undefined, - terminalRuntimeFact?.runStatus, - ].filter((status): status is 'completed' | 'failed' | 'cancelled' => status !== undefined); - - if (statuses.length < 2) return 'incomplete'; - return statuses.every((status) => status === statuses[0]) ? 'consistent' : 'inconsistent'; -} - -function latestOperationalTerminalEvent( - events: readonly AgentRunEvent[], -): AgentRunEvent | undefined { - for (let index = events.length - 1; index >= 0; index -= 1) { - const event = events[index]; - if (!event) continue; - if (operationalStatusFor(event)) return event; - } - return undefined; -} - -function operationalStatusFor( - event: AgentRunEvent, -): 'completed' | 'failed' | 'cancelled' | undefined { - if (event.type === 'run_completed') return 'completed'; - if (event.type === 'run_failed') return 'failed'; - if (event.type === 'run_cancelled') return 'cancelled'; - return undefined; -} - -function isTerminalRunStatus( - status: AgentRunHeader['status'], -): status is 'completed' | 'failed' | 'cancelled' { - return status === 'completed' || status === 'failed' || status === 'cancelled'; -} - function fromRuntimeReadModelDiagnostic( - header: AgentRunHeader, + invocation: RuntimeInvocationRecord, diagnostic: RuntimeEventReadModelDiagnostic, ): AgentRunInspectDiagnostic { return { code: diagnostic.code, - runId: diagnostic.runId ?? header.runId, - turnId: diagnostic.turnId ?? header.turnId, + runId: diagnostic.runId ?? invocation.runId, + turnId: diagnostic.turnId ?? invocation.turnId, message: diagnostic.message, ...(diagnostic.eventId ? { eventId: diagnostic.eventId } : {}), ...(diagnostic.detail !== undefined ? { detail: diagnostic.detail } : {}), @@ -344,7 +283,7 @@ function fromRuntimeReadModelDiagnostic( } function inspectDiagnostic( - header: AgentRunHeader, + invocation: RuntimeInvocationRecord, code: AgentRunInspectDiagnosticCode, message: string, detail?: unknown, @@ -352,8 +291,8 @@ function inspectDiagnostic( ): AgentRunInspectDiagnostic { return { code, - runId: header.runId, - turnId: header.turnId, + runId: invocation.runId, + turnId: invocation.turnId, message, ...(eventId ? { eventId } : {}), ...(detail !== undefined ? { detail } : {}), diff --git a/packages/runtime/src/agent-run-recovery.ts b/packages/runtime/src/agent-run-recovery.ts index 7a04ef94b0..fe5cb68a32 100644 --- a/packages/runtime/src/agent-run-recovery.ts +++ b/packages/runtime/src/agent-run-recovery.ts @@ -21,7 +21,9 @@ import { SANDBOX_BOUNDARY_RESTART_CLOSURE_CLASS, isSandboxBoundaryRestartClosure, } from '@maka/core/sandbox-boundary'; -import type { AgentRunEvent, AgentRunHeader } from '@maka/core/agent-run'; +import type { AgentRunEvent } from '@maka/core/agent-run'; +import type { RuntimeInvocationLineage } from '@maka/core/runtime-event'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { SandboxBoundaryRequest } from '@maka/core/sandbox-boundary'; export interface AgentRunRecoveryDecision { @@ -34,78 +36,49 @@ export interface AgentRunRecoveryDecision { lineage: AgentRunRecoveryLineage; } -type AgentRunRecoveryLineage = Partial< - Pick< - AgentRunHeader, - | 'parentRunId' - | 'parentTurnId' - | 'retriedFromTurnId' - | 'regeneratedFromTurnId' - | 'branchOfTurnId' - | 'parentSessionId' - > +type AgentRunRecoveryLineage = Pick< + RuntimeInvocationLineage, + | 'parentRunId' + | 'parentTurnId' + | 'retriedFromTurnId' + | 'regeneratedFromTurnId' + | 'branchOfTurnId' + | 'parentSessionId' >; +/** + * Why a run the events never closed has to be failed closed. + * + * The caller has already established that there is no terminal event, so the + * outcome is settled before this runs. All that is left is to say what the run + * was doing when the host went away, and only its own ledger can say that. + */ export function classifyAgentRunRecovery( - header: AgentRunHeader, + invocation: RuntimeInvocationRecord, events: readonly AgentRunEvent[], -): AgentRunRecoveryDecision | undefined { - if (isTerminalRunStatus(header.status)) return undefined; - +): AgentRunRecoveryDecision { const lastEvent = lastNonCorruptEvent(events); const hasCorruptEvent = events.some((event) => event.type === 'event_corrupt'); const lastEventType = lastEvent?.type; - if (lastEventType === 'model_stream_completed' && !hasTerminalRunEvent(events)) { - return failedDecision( - header, - 'app_restarted', - diagnostic('model_stream_completed_without_runtime_terminal', lastEventType, hasCorruptEvent), - ); - } - - if ( - header.status === 'waiting_for_user' || - lastEventType === 'permission_requested' || - lastEventType === 'permission_failed' - ) { - return failedDecision( - header, - 'app_restarted', - diagnostic('stale_user_wait', lastEventType, hasCorruptEvent), - ); - } - - if (lastEventType === 'tool_started') { - return failedDecision( - header, - 'app_restarted', - diagnostic('tool_interrupted', lastEventType, hasCorruptEvent), - ); - } - - if ( - header.status === 'created' || - header.status === 'running' || - lastEventType === undefined || - lastEventType === 'run_created' || - lastEventType === 'run_started' || - lastEventType === 'turn_started' || - lastEventType === 'model_resolved' || - lastEventType === 'model_stream_started' || - lastEventType === 'run_status_changed' - ) { - return failedDecision( - header, - 'app_restarted', - diagnostic('run_interrupted', lastEventType, hasCorruptEvent), - ); - } + const reason = + lastEventType === 'model_stream_completed' + ? 'model_stream_completed_without_runtime_terminal' + : lastEventType === 'permission_requested' || lastEventType === 'permission_failed' + ? 'stale_user_wait' + : lastEventType === 'tool_started' + ? 'tool_interrupted' + : lastEventType === undefined || + lastEventType === 'turn_started' || + lastEventType === 'model_resolved' || + lastEventType === 'model_stream_started' + ? 'run_interrupted' + : 'non_terminal_run_recovered'; return failedDecision( - header, + invocation, 'app_restarted', - diagnostic('non_terminal_run_recovered', lastEventType, hasCorruptEvent), + diagnostic(reason, lastEventType, hasCorruptEvent), ); } @@ -148,33 +121,20 @@ export function attributeSandboxBoundaryRestartClosure( } function failedDecision( - header: AgentRunHeader, + invocation: RuntimeInvocationRecord, failureClass: string, diagnostic?: Record, ): AgentRunRecoveryDecision { return { - runId: header.runId, - turnId: header.turnId, + runId: invocation.runId, + turnId: invocation.turnId, status: 'failed', failureClass, diagnostic, - lineage: headerLineage(header), + lineage: openingLineage(invocation), }; } -function isTerminalRunStatus(status: AgentRunHeader['status']): boolean { - return status === 'completed' || status === 'failed' || status === 'cancelled'; -} - -function hasTerminalRunEvent(events: readonly AgentRunEvent[]): boolean { - return events.some( - (event) => - event.type === 'run_completed' || - event.type === 'run_failed' || - event.type === 'run_cancelled', - ); -} - function lastNonCorruptEvent(events: readonly AgentRunEvent[]): AgentRunEvent | undefined { for (let index = events.length - 1; index >= 0; index -= 1) { const event = events[index]; @@ -197,15 +157,17 @@ function diagnostic( }; } -function headerLineage(header: AgentRunHeader): AgentRunRecoveryLineage { +function openingLineage(invocation: RuntimeInvocationRecord): AgentRunRecoveryLineage { + const lineage = invocation.opening.lineage; + if (!lineage) return {}; return { - ...(header.parentRunId ? { parentRunId: header.parentRunId } : {}), - ...(header.parentTurnId ? { parentTurnId: header.parentTurnId } : {}), - ...(header.retriedFromTurnId ? { retriedFromTurnId: header.retriedFromTurnId } : {}), - ...(header.regeneratedFromTurnId - ? { regeneratedFromTurnId: header.regeneratedFromTurnId } + ...(lineage.parentRunId ? { parentRunId: lineage.parentRunId } : {}), + ...(lineage.parentTurnId ? { parentTurnId: lineage.parentTurnId } : {}), + ...(lineage.retriedFromTurnId ? { retriedFromTurnId: lineage.retriedFromTurnId } : {}), + ...(lineage.regeneratedFromTurnId + ? { regeneratedFromTurnId: lineage.regeneratedFromTurnId } : {}), - ...(header.branchOfTurnId ? { branchOfTurnId: header.branchOfTurnId } : {}), - ...(header.parentSessionId ? { parentSessionId: header.parentSessionId } : {}), + ...(lineage.branchOfTurnId ? { branchOfTurnId: lineage.branchOfTurnId } : {}), + ...(lineage.parentSessionId ? { parentSessionId: lineage.parentSessionId } : {}), }; } diff --git a/packages/runtime/src/agent-run.ts b/packages/runtime/src/agent-run.ts index 03213161d3..7f119bfda3 100644 --- a/packages/runtime/src/agent-run.ts +++ b/packages/runtime/src/agent-run.ts @@ -17,18 +17,25 @@ * under the License. */ +import type { AgentRunEvent, AgentRunStore, EmittedAgentRunEvent } from '@maka/core/agent-run'; +import { RUN_COMPOSITION_RECORDED_EVENT_TYPE } from '@maka/core/agent-run'; import type { - AgentRunEvent, - AgentRunHeader, - AgentRunStore, - EmittedAgentRunEvent, -} from '@maka/core/agent-run'; -import type { RuntimeEvent, ToolBoundaryProtocol } from '@maka/core/runtime-event'; + RuntimeEvent, + RuntimeEventInvocationOpenedContent, + RuntimeInvocationRootAuthority, + ToolBoundaryProtocol, +} from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; import type { RunCompositionSnapshot } from '@maka/core/run-composition'; import { decodeRunCompositionSnapshot } from '@maka/core/run-composition'; import { DurableStoreWriteError, RunSealedError } from '@maka/core/runtime-event-store'; -import { isSessionInlineRun } from '@maka/core/agent-run'; +import { + buildInvocationOpenedEvent, + buildSyntheticTerminalRuntimeEvent, + isSessionInlineInvocation, +} from '@maka/core/runtime-invocation'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; +import type { RuntimeInvocationLineage } from '@maka/core/runtime-event'; import { MODEL_PROJECTION_TRANSITION_EVENT_TYPE, type ModelProjectionTransition, @@ -79,10 +86,7 @@ import { statusFromEvent, turnStatusFromEvent, } from './session-projection-helpers.js'; -import { - buildSyntheticTerminalRuntimeEvent, - commitOrCreateTerminalRunFact, -} from './terminal-run-commit.js'; +import { commitOrCreateTerminalRunFact } from './terminal-run-commit.js'; import type { RuntimeContinuation } from './runtime-resume.js'; import { createRuntimeContinuationStartAdmissionProof, @@ -124,7 +128,7 @@ export interface AgentRunHooks { } export type AgentRunLineage = Partial< - Pick & + Pick & Pick< UserMessageInput, | 'parentTurnId' @@ -143,20 +147,21 @@ export interface AgentRunInput { userInput: UserMessageInput; /** Internal lineage for runtime-owned continuations; never accepted by live turn input. */ runLineage?: Pick; - rootExecutionKind?: AgentRunHeader['rootExecutionKind']; + rootExecutionKind?: 'context_compact'; runId?: string; userMessageId?: string | null; durability?: AgentRunDurability; store: AgentRunSessionStore; runStore?: AgentRunStore; runtimeEventStore?: RuntimeEventStore; - repairRunRuntimeLedger?: (sessionId: string, runId: string) => Promise; newId: () => string; now: () => number; workspaceIdentity?: string; continuationFailpoint?: (point: RuntimeContinuationFailpoint) => Promise; - /** Exact target header already committed inside the durable continuation claim. */ - claimedRunHeader?: AgentRunHeader; + /** Exact target opening fact already committed inside the durable continuation claim. */ + claimedOpening?: RuntimeEventInvocationOpenedContent; + /** The moment that claim was taken; the target invocation opens at it. */ + claimedOpenedAt?: number; /** Commits the claimed continuation provider-call T1 after Run creation. */ commitContinuationStart?: (startedAt: number) => Promise<{ startEventId: string; created: true }>; hooks: AgentRunHooks; @@ -176,10 +181,8 @@ export interface AgentRunSessionStore { export type RuntimeContinuationFailpoint = | 'after_continuation_claim_committed' - | 'after_run_created' | 'after_continuation_start_committed' - | 'after_terminal_event_committed' - | 'after_terminal_header_committed'; + | 'after_terminal_event_committed'; export class ContinuationStartCommitError extends Error { readonly name = 'ContinuationStartCommitError'; @@ -202,7 +205,7 @@ export interface AgentRunBeginResult { export interface AgentRunOperationBeginResult { backend: AgentBackend; runtimeContext: RuntimeEvent[]; - runtimeContextRunHeaders: AgentRunHeader[]; + runtimeContextInvocations: RuntimeInvocationRecord[]; startedAt: number; } @@ -249,9 +252,11 @@ export class AgentRun { private finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined; private turnFailed = false; private finalized = false; - private terminalRunHeaderCommitted = false; + private terminalRunFactCommitted = false; private continuationActive = false; private providerStateIdentity: `sha256:${string}` | undefined; + private invocationOpening: RuntimeEventInvocationOpenedContent | undefined; + private invocationOpeningCommitted = false; private terminalClaim: | { owner: 'event' | 'stop'; @@ -338,7 +343,7 @@ export class AgentRun { } bindProviderStateIdentity(identity: `sha256:${string}` | undefined): void { - const claimed = this.input.claimedRunHeader?.providerStateIdentity; + const claimed = claimedProviderStateIdentity(this.input.claimedOpening); const expected = claimed ?? this.providerStateIdentity; if (expected !== undefined && expected !== identity) { throw new Error('Prepared backend provider state does not match the AgentRun admission'); @@ -347,10 +352,11 @@ export class AgentRun { } isSessionInline(): boolean { - return isSessionInlineRun({ - ...(this.lineage.parentRunId ? { parentRunId: this.lineage.parentRunId } : {}), - ...(this.continuationActive ? { continuationSource: true } : {}), - }); + const opening = this.invocationOpening; + if (opening) return isSessionInlineInvocation(opening); + // Before the opening fact exists there is only the lineage the turn was + // admitted with, which decides the same question the same way. + return this.lineage.parentRunId === undefined; } hasPendingStop(): boolean { @@ -373,7 +379,7 @@ export class AgentRun { * produces its own terminal event finds the claim taken and writes nothing. */ async settleStopTerminal(): Promise { - if (this.terminalClaim?.owner !== 'stop' || this.terminalRunHeaderCommitted) return; + if (this.terminalClaim?.owner !== 'stop' || this.terminalRunFactCommitted) return; // Nothing durable is configured, so there is no fact to land. Every other // failure below is real and must reach the stop's caller: a stop that // reports success while the run stays non-terminal is the silent loss this @@ -399,7 +405,7 @@ export class AgentRun { // the latch, one that cannot fails the settlement loudly so the stop // stays retryable. try { - await runStore.readRun(this.sessionId, this.runId); + await runStore.readEvents(this.sessionId, this.runId); this.runStoreAvailable = true; } catch (error) { throw new Error('AgentRun store is unavailable for stop settlement', { cause: error }); @@ -446,10 +452,18 @@ export class AgentRun { this.runComposition ??= normalized; if (this.runCompositionWrite) return this.runCompositionWrite; const write = this.enqueueRequiredRunStoreWrite('commit Run Composition', async () => { - await this.input.runStore?.updateRun( + await this.input.runStore?.appendEvent( this.sessionId, this.runId, - { runComposition: normalized }, + { + type: RUN_COMPOSITION_RECORDED_EVENT_TYPE, + id: this.input.newId(), + runId: this.runId, + sessionId: this.sessionId, + turnId: this.turnId, + ts: this.input.now(), + data: { runComposition: normalized }, + }, { durable: this.requiresDurablePersistence() }, ); }); @@ -608,8 +622,8 @@ export class AgentRun { } if (this.requiresDurablePersistence() && isInteractionResumeAck(sessionEvent)) { // A hosted continuation may resume execution only after its identity-only - // settlement fact is durable. Run status advances next, then Session - // status; the queue consumer acknowledges the event only after all three. + // settlement fact is durable. Session status advances next, and the queue + // consumer acknowledges the event only after both. await this.recordRuntimeEvents([runtimeEvent], { requireDurableWrite: true }); await this.recordSessionEvent(sessionEvent, options); return; @@ -642,7 +656,7 @@ export class AgentRun { } async begin(): Promise { - await this.createRunRecord(); + await this.openInvocation(); let initialRuntimeEventId: string; @@ -686,14 +700,13 @@ export class AgentRun { }); this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); - await this.markRunStarted(this.lastTs); await this.input.hooks.updateStatus(this.sessionId, 'running', undefined, this.lastTs); const priorRuntimeContext = await this.buildPriorRuntimeContext(); const projectionContext = priorRuntimeContext ? projectRuntimeEventsToStoredMessages(priorRuntimeContext.events, { - runHeaders: priorRuntimeContext.runs, + invocations: priorRuntimeContext.invocations, }).messages : []; @@ -718,7 +731,7 @@ export class AgentRun { ...(priorRuntimeContext ? { runtimeContext: priorRuntimeContext.events, - runtimeContextRunHeaders: priorRuntimeContext.runs, + runtimeContextInvocations: priorRuntimeContext.invocations, } : {}), }), @@ -727,7 +740,7 @@ export class AgentRun { } async beginOperation(): Promise { - await this.createRunRecord(); + await this.openInvocation(); const startedAt = this.input.now(); this.lastTs = startedAt; @@ -737,7 +750,6 @@ export class AgentRun { }); this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); - await this.markRunStarted(startedAt); await this.input.hooks.updateStatus(this.sessionId, 'running', undefined, startedAt); @@ -745,7 +757,7 @@ export class AgentRun { return { backend: this.active.backend, runtimeContext: priorRuntimeContext?.events ?? [], - runtimeContextRunHeaders: priorRuntimeContext?.runs ?? [], + runtimeContextInvocations: priorRuntimeContext?.invocations ?? [], startedAt, }; } @@ -762,8 +774,7 @@ export class AgentRun { } this.continuationActive = true; - await this.createRunRecord(continuation); - await this.input.continuationFailpoint?.('after_run_created'); + await this.openInvocation(continuation); const startedAt = this.input.now(); this.lastTs = startedAt; if (!this.input.commitContinuationStart) { @@ -782,7 +793,6 @@ export class AgentRun { }); this.active = await this.input.hooks.reserveRun(this.sessionId, this.header, this); - await this.markRunStarted(startedAt); await this.input.hooks.updateStatus(this.sessionId, 'running', undefined, startedAt); return { @@ -831,7 +841,10 @@ export class AgentRun { ? { inlineReferences: input.inlineReferences } : {}), }, - ...(this.toolBoundaryProtocol + // The marker belongs to the invocation's first event. Once an opening + // fact exists it holds the marker, and a second copy here would read as + // a stray marker to RecoveryResolver. + ...(this.toolBoundaryProtocol && !this.invocationOpeningCommitted ? { actions: { runtimeProtocol: { toolBoundary: this.toolBoundaryProtocol } } } : {}), }; @@ -871,7 +884,6 @@ export class AgentRun { this.markRunFailed( turnStatus.errorClass, `turn ended with stopReason=${ev.type === 'complete' ? ev.stopReason : 'unknown'}`, - ev.ts, ); } } @@ -890,15 +902,7 @@ export class AgentRun { ev.ts, ); }; - // On resume, advance the Run before the Session so an interrupted pair - // remains conservatively waiting rather than advertising false readiness. - if (this.requiresDurablePersistence() && isInteractionResumeAck(ev)) { - await this.recordStatusFromTransition(ev, transition, ev.ts); - await updateSessionStatus(); - } else { - await updateSessionStatus(); - await this.recordStatusFromTransition(ev, transition, ev.ts); - } + await updateSessionStatus(); } if (turnStatus && !this.stopped) { const appendTurnState = this.input.hooks.appendTurnState( @@ -936,7 +940,7 @@ export class AgentRun { }) .catch((error) => this.enqueueTraceWriteFailure(error, 'terminal session projection')); - this.markRunFailed(ev.reason ?? ev.code ?? 'unknown', ev.message, ev.ts); + this.markRunFailed(ev.reason ?? ev.code ?? 'unknown', ev.message); } } } @@ -1051,26 +1055,24 @@ export class AgentRun { }) .catch(() => {}); - this.markRunFailed( - error instanceof Error ? error.name : 'unknown', - errorMessage(error), - this.input.now(), - ); + this.markRunFailed(error instanceof Error ? error.name : 'unknown', errorMessage(error)); } async finalize(): Promise { if (this.finalized) return; this.finalized = true; + // A run cannot end without having begun. Finalizing one that never reached + // its start would otherwise leave a terminal event on an invocation the + // inventory cannot see, because nothing opened it. A continuation is the + // exception at both ends: its opening rides the continuation-start event, + // and a continuation that never committed one has no invocation to end. + if (!this.input.commitContinuationStart) await this.openInvocation().catch(() => {}); await this.flushRuntimePartialBuffer(true); const lastTs = this.lastTs || this.input.now(); if (this.stopped) this.finalStatus = { status: 'aborted' }; if (!this.finalStatus) { this.finalStatus = { status: 'blocked', blockedReason: 'unknown' }; - this.markRunFailed( - 'missing_terminal_event', - 'run finalized without a terminal SessionEvent', - lastTs, - ); + this.markRunFailed('missing_terminal_event', 'run finalized without a terminal SessionEvent'); } this.reserveFinalizationTerminal(this.finalStatus, lastTs); if (this.active) { @@ -1103,120 +1105,153 @@ export class AgentRun { await this.finishRun(this.finalStatus, lastTs); } - private async createRunRecord(continuation?: RuntimeContinuation): Promise { - if (!this.input.runStore) { - if (continuation) throw new Error('Runtime continuation requires a durable run store'); - return; + private async openInvocation(continuation?: RuntimeContinuation): Promise { + if (!this.input.runStore && continuation) { + throw new Error('Runtime continuation requires a durable run store'); } + // The opening fact is a RuntimeEvent, so it opens whenever this run has a + // spine to open on. The operational ledger is a separate store with its own + // availability, and a run without one still exists. + if (!this.input.runtimeEventStore) return; const createdAt = - continuation && this.input.claimedRunHeader - ? this.input.claimedRunHeader.createdAt + continuation && this.input.claimedOpenedAt !== undefined + ? this.input.claimedOpenedAt : this.input.now(); const providerStateIdentity = - this.input.claimedRunHeader?.providerStateIdentity ?? this.providerStateIdentity; + claimedProviderStateIdentity(this.input.claimedOpening) ?? this.providerStateIdentity; this.providerStateIdentity = providerStateIdentity; - const computedHeader: AgentRunHeader = { - runId: this.runId, - invocationId: this.invocationId, - sessionId: this.sessionId, - turnId: this.turnId, - status: 'created', - backendKind: this.header.backend, - ...(this.header.llmConnectionId === undefined - ? {} - : { llmConnectionId: this.header.llmConnectionId }), - ...(providerStateIdentity ? { providerStateIdentity } : {}), - llmConnectionSlug: this.header.llmConnectionSlug, - modelId: this.header.model, - cwd: this.header.cwd, - ...(this.input.workspaceIdentity ? { workspaceIdentity: this.input.workspaceIdentity } : {}), - permissionMode: this.header.permissionMode, - collaborationMode: this.header.collaborationMode ?? 'agent', - orchestrationMode: this.effectiveOrchestration.mode, - orchestrationSource: this.effectiveOrchestration.source, - agentSwarmAuthorization: this.effectiveOrchestration.agentSwarmAuthorization, - toolMode: this.toolMode, - createdAt, - updatedAt: createdAt, + const computedOpening = this.buildInvocationOpening(continuation, providerStateIdentity); + if ( + continuation && + this.input.claimedOpening && + !isDeepStrictEqual(this.input.claimedOpening, computedOpening) + ) { + throw new Error('Claimed continuation target opening no longer matches execution'); + } + this.invocationOpening = this.input.claimedOpening ?? computedOpening; + // A continuation's opening fact rides its continuation-start event, which + // the store requires to be event 1 of the target invocation. Every other + // invocation opens with its own event, committed before any provider or + // tool dispatch. + if (!continuation) await this.commitInvocationOpening(createdAt); + } + + /** + * The one immutable statement of how this invocation was opened. + * + * Everything a later reader needs to know about the run's route, + * configuration, root authority and lineage is decided here, once, and never + * restated anywhere else. + */ + private buildInvocationOpening( + continuation: RuntimeContinuation | undefined, + providerStateIdentity: `sha256:${string}` | undefined, + ): RuntimeEventInvocationOpenedContent { + const lineage = { ...this.lineage, - ...(continuation - ? { - continuationSource: - continuation.claimId && continuation.boundary - ? { - protocol: 'continuation_source_v2' as const, - claimId: continuation.claimId, - boundaryDigest: continuation.boundary.manifestDigest, - sourceInvocationId: continuation.sourceInvocationId, - sourceRunId: continuation.sourceRunId, - sourceTurnId: continuation.sourceTurnId, - sourceRuntimeEventHighWater: continuation.sourceRuntimeEventHighWater, - sourcePrefixDigest: continuation.boundary.segments.at(-1)!.prefixDigest, - replayManifestDigest: continuation.boundary.manifestDigest, - } - : { - sourceInvocationId: continuation.sourceInvocationId, - sourceRunId: continuation.sourceRunId, - sourceTurnId: continuation.sourceTurnId, - sourceRuntimeEventHighWater: continuation.sourceRuntimeEventHighWater, - }, - } - : {}), ...(this.input.userInput.agentId ? { agentId: this.input.userInput.agentId } : {}), ...(this.input.userInput.agentName ? { agentName: this.input.userInput.agentName } : {}), - ...(this.input.userInput.origin?.kind === 'scheduled_task' - ? { scheduledTaskId: this.input.userInput.origin.scheduledTaskId } - : {}), - ...(this.input.userInput.origin?.kind === 'goal' - ? { goalId: this.input.userInput.origin.goalId } - : {}), - ...(this.input.userInput.origin?.kind === 'agent_graph' + ...(continuation ? { parentRunId: continuation.sourceRunId } : {}), + }; + return { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: + this.header.llmConnectionId === undefined + ? { + provenance: 'unknown', + backendKind: this.header.backend, + llmConnectionSlug: this.header.llmConnectionSlug, + modelId: this.header.model, + } + : { + provenance: 'runtime', + backendKind: this.header.backend, + llmConnectionId: this.header.llmConnectionId, + llmConnectionSlug: this.header.llmConnectionSlug, + modelId: this.header.model, + ...(providerStateIdentity ? { providerStateIdentity } : {}), + }, + configuration: { + cwd: this.header.cwd, + permissionMode: this.header.permissionMode, + collaborationMode: this.header.collaborationMode ?? 'agent', + orchestrationMode: this.effectiveOrchestration.mode, + orchestrationSource: this.effectiveOrchestration.source, + toolMode: this.toolMode, + ...(this.effectiveOrchestration.agentSwarmAuthorization !== undefined + ? { agentSwarmAuthorization: this.effectiveOrchestration.agentSwarmAuthorization } + : {}), + ...(this.input.workspaceIdentity + ? { workspaceIdentity: this.input.workspaceIdentity } + : {}), + }, + root: this.invocationRootAuthority(), + source: continuation ? { - agentGraphWakeId: this.input.userInput.origin.wakeId, - agentGraphWakeAttemptId: this.input.userInput.origin.attemptId, + kind: 'continuation', + sourceInvocationId: continuation.sourceInvocationId, + sourceRunId: continuation.sourceRunId, + sourceTurnId: continuation.sourceTurnId, + sourceRuntimeEventHighWater: continuation.sourceRuntimeEventHighWater, + ...(continuation.claimId ? { claimId: continuation.claimId } : {}), + ...(continuation.boundary + ? { boundaryDigest: continuation.boundary.manifestDigest } + : {}), } - : {}), - ...(this.input.rootExecutionKind ? { rootExecutionKind: this.input.rootExecutionKind } : {}), + : { kind: 'fresh' }, + ...(Object.keys(lineage).length > 0 ? { lineage } : {}), }; - const header = - continuation && this.input.claimedRunHeader ? this.input.claimedRunHeader : computedHeader; - if ( - continuation && - this.input.claimedRunHeader && - !isDeepStrictEqual(this.input.claimedRunHeader, computedHeader) - ) { - throw new Error('Claimed continuation target Run header no longer matches execution'); + } + + private invocationRootAuthority(): RuntimeInvocationRootAuthority { + const origin = this.input.userInput.origin; + if (origin?.kind === 'scheduled_task') { + return { kind: 'scheduled_task', scheduledTaskId: origin.scheduledTaskId }; } - try { - const durable = this.requiresDurablePersistence(); - await this.input.runStore.createRun(header, { durable }); - await this.input.runStore.appendEvent( - this.sessionId, - this.runId, + if (origin?.kind === 'goal') return { kind: 'goal', goalId: origin.goalId }; + if (origin?.kind === 'agent_graph') { + return { + kind: 'agent_graph_supervisor_wake', + wakeId: origin.wakeId, + attemptId: origin.attemptId, + }; + } + if (this.input.rootExecutionKind === 'context_compact') return { kind: 'context_compact' }; + return { kind: 'user' }; + } + + /** + * Make the invocation's opening fact durable before anything can dispatch. + * + * It is the invocation's first event, so it also carries the protocol marker + * RecoveryResolver reads off event one. + */ + private async commitInvocationOpening(ts: number): Promise { + const opening = this.invocationOpening; + if (!opening || this.invocationOpeningCommitted) return; + await this.recordRuntimeEvents( + [ { - type: 'run_created', - id: this.input.newId(), - runId: this.runId, - sessionId: this.sessionId, - turnId: this.turnId, - ts: createdAt, - data: { - textLength: this.input.userInput.text.length, - attachmentCount: this.input.userInput.attachments?.length ?? 0, - orchestrationMode: this.effectiveOrchestration.mode, - orchestrationSource: this.effectiveOrchestration.source, - agentSwarmAuthorization: this.effectiveOrchestration.agentSwarmAuthorization, - toolMode: this.toolMode, - }, + ...buildInvocationOpenedEvent({ + id: this.input.newId(), + run: { + sessionId: this.sessionId, + invocationId: this.invocationId, + runId: this.runId, + turnId: this.turnId, + }, + openedAt: ts, + opening, + }), + ...(this.toolBoundaryProtocol + ? { actions: { runtimeProtocol: { toolBoundary: this.toolBoundaryProtocol } } } + : {}), }, - { durable }, - ); - } catch (error) { - this.runStoreAvailable = false; - if (this.requiresDurablePersistence()) throw error; - this.enqueueTraceWriteFailure(error); - if (continuation) throw error; - } + ], + { requireDurableWrite: this.requiresDurablePersistence() }, + ); + this.invocationOpeningCommitted = true; } private requiresDurablePersistence(): boolean { @@ -1228,215 +1263,41 @@ export class AgentRun { sessionId: this.sessionId, currentRunId: this.runId, currentTurnId: this.turnId, - runStore: this.input.runStore, runtimeEventStore: this.input.runtimeEventStore, - runStoreAvailable: this.runStoreAvailable, runtimeEventStoreAvailable: this.runtimeEventStoreAvailable, - repairRunRuntimeLedger: this.input.repairRunRuntimeLedger, - readMessages: () => this.input.store.readMessages(this.sessionId), }); } - private async markRunStarted(ts: number): Promise { - if (!this.input.runStore || !this.runStoreAvailable) return; - const durable = this.requiresDurablePersistence(); - const write = this.enqueueRunStore( - 'mark run started', - async () => { - await this.input.runStore?.appendEvent( - this.sessionId, - this.runId, - { - type: 'run_started', - id: this.input.newId(), - runId: this.runId, - sessionId: this.sessionId, - turnId: this.turnId, - ts, - }, - { durable }, - ); - await this.input.runStore?.updateRun( - this.sessionId, - this.runId, - { status: 'running', updatedAt: ts }, - { durable }, - ); - }, - { rethrow: durable }, - ); - if (durable) await write; - } - - private async recordStatusFromTransition( - ev: SessionEvent, - transition: { status: SessionStatus; blockedReason?: SessionBlockedReason }, - ts: number, - ): Promise { - const durable = this.requiresDurablePersistence(); - const runStore = this.input.runStore; - if (!runStore) { - if (durable) { - throw new Error('AgentRun store is unavailable for a required status transition'); - } - return; - } - const status = - transition.status === 'waiting_for_user' - ? 'waiting_for_user' - : transition.status === 'aborted' - ? 'cancelled' - : transition.status === 'blocked' - ? 'failed' - : transition.status === 'active' - ? 'completed' - : 'running'; - if (isTerminalRunStatus(status)) return; - const appendAudit = async (): Promise => { - await runStore.appendEvent( - this.sessionId, - this.runId, - { - type: 'run_status_changed', - id: this.input.newId(), - runId: this.runId, - sessionId: this.sessionId, - turnId: this.turnId, - ts, - data: { - sessionStatus: transition.status, - ...(transition.blockedReason ? { blockedReason: transition.blockedReason } : {}), - }, - }, - { durable }, - ); - }; - if (durable) { - await this.enqueueRequiredRunStoreWrite('record required run status', async () => { - await runStore.updateRun( - this.sessionId, - this.runId, - { status, updatedAt: ts }, - { durable: true }, - ); - }); - // The audit remains best-effort, but its physical write belongs to this - // required transition and must settle before the resume acknowledgement. - await this.enqueueRunStore('append run status audit', appendAudit); - } else { - this.enqueueRunStore('record run status', async () => { - await runStore.updateRun(this.sessionId, this.runId, { status, updatedAt: ts }); - await appendAudit(); - }); - } - if (ev.type === 'abort') { - this.markRunCancelled(ev.reason, ts); - } - } - - private markRunFailed(failureClass: string, message: string, ts: number): void { - if (!this.input.runStore || !this.runStoreAvailable) return; + /** + * Remember why this run is going to fail. + * + * Nothing is written here: the terminal RuntimeEvent carries the failure, and + * it is committed once, at the end, by `commitTerminalRun`. + */ + private markRunFailed(failureClass: string, message: string): void { this.failureClass = failureClass; this.failureMessage = redactTraceString(message); - if (this.input.runtimeEventStore) return; - this.enqueueRunStore('mark run failed', async () => { - await this.input.runStore?.updateRun(this.sessionId, this.runId, { - status: 'failed', - updatedAt: ts, - completedAt: ts, - failureClass, - failureMessage: this.failureMessage, - }); - await this.input.runStore?.appendEvent(this.sessionId, this.runId, { - type: 'run_failed', - id: this.input.newId(), - runId: this.runId, - sessionId: this.sessionId, - turnId: this.turnId, - ts, - message: redactTraceString(message), - data: { failureClass }, - }); - }); - } - - private markRunCancelled(reason: string | undefined, ts: number): void { - if (!this.input.runStore || !this.runStoreAvailable) return; - if (this.input.runtimeEventStore) return; - this.enqueueRunStore('mark run cancelled', async () => { - await this.input.runStore?.updateRun(this.sessionId, this.runId, { - status: 'cancelled', - updatedAt: ts, - completedAt: ts, - }); - await this.input.runStore?.appendEvent(this.sessionId, this.runId, { - type: 'run_cancelled', - id: this.input.newId(), - runId: this.runId, - sessionId: this.sessionId, - turnId: this.turnId, - ts, - ...(reason ? { message: redactTraceString(reason) } : {}), - }); - }); } + /** + * End the run by committing its terminal RuntimeEvent, and nothing else. + * + * A turn that parks on an interaction has not ended, so it commits nothing: + * the absence of a terminal event is exactly what "still open" means. + */ private async finishRun( finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined, ts: number, ): Promise { await this.traceQueue.catch(() => {}); - if (!this.input.runStore || !this.runStoreAvailable) return; - const status = this.runStatusForFinalStatus(finalStatus); - const isTerminal = status === 'completed' || status === 'failed' || status === 'cancelled'; - if (isTerminal && this.input.runtimeEventStore) { - await this.commitTerminalRun(finalStatus, ts); - return; - } - await this.enqueueRunStore('finish run', async () => { - await this.input.runStore?.updateRun(this.sessionId, this.runId, { - status, - updatedAt: ts, - ...(isTerminal ? { completedAt: ts } : {}), - ...(status === 'failed' - ? { - failureClass: this.failureClass ?? finalStatus?.blockedReason ?? 'unknown', - ...(this.failureMessage ? { failureMessage: this.failureMessage } : {}), - } - : {}), - }); - await this.input.runStore?.appendEvent(this.sessionId, this.runId, { - type: - status === 'cancelled' - ? 'run_cancelled' - : status === 'failed' - ? 'run_failed' - : status === 'completed' - ? 'run_completed' - : 'run_status_changed', - id: this.input.newId(), - runId: this.runId, - sessionId: this.sessionId, - turnId: this.turnId, - ts, - ...(status === 'failed' - ? { data: { failureClass: this.failureClass ?? finalStatus?.blockedReason ?? 'unknown' } } - : status === 'waiting_for_user' - ? { - data: { - sessionStatus: 'waiting_for_user', - blockedReason: finalStatus?.blockedReason ?? 'permission_required', - }, - } - : {}), - }); - }); - await this.traceQueue.catch(() => {}); + if (!this.input.runtimeEventStore) return; + if (this.runStatusForFinalStatus(finalStatus) === 'waiting_for_user') return; + await this.commitTerminalRun(finalStatus, ts); } private runStatusForFinalStatus( finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined, - ): AgentRunHeader['status'] { + ): 'completed' | 'failed' | 'cancelled' | 'waiting_for_user' { if (this.stopped || finalStatus?.status === 'aborted') return 'cancelled'; if (this.failureClass || finalStatus?.status === 'blocked') return 'failed'; if (finalStatus?.status === 'waiting_for_user') return 'waiting_for_user'; @@ -1447,10 +1308,9 @@ export class AgentRun { finalStatus: { status: SessionStatus; blockedReason?: SessionBlockedReason } | undefined, ts: number, ): Promise { - if (this.terminalRunHeaderCommitted) return; - const runStore = this.input.runStore; + if (this.terminalRunFactCommitted) return; const runtimeEventStore = this.input.runtimeEventStore; - if (!runStore || !runtimeEventStore) return; + if (!runtimeEventStore) return; // A latched RuntimeEvent store normally keeps the skip below: the latch // marks a write failure, and a transient one leaves the run non-terminal // on purpose so startup recovery repairs it with its own bookkeeping. @@ -1466,7 +1326,6 @@ export class AgentRun { if (!(this.runtimeEventStoreFailure instanceof ToolLedgerCorruptionError)) return; corruptionRecovery = true; } - if (!this.runStoreAvailable) return; const fallbackStatus = this.stopped || finalStatus?.status === 'aborted' ? 'cancelled' : 'failed'; const fallbackFailureClass = 'missing_terminal_event'; @@ -1490,9 +1349,8 @@ export class AgentRun { // Re-check after the await, not only at entry. Two callers — a stop // settling the claim and the stream's own finalize — can both pass the // entry guard and then queue behind the same write. The claim slot - // dedupes the RuntimeEvent, but the run-store projection would append a - // second terminal AgentRunEvent for the one run. - if (this.terminalRunHeaderCommitted) return; + // dedupes the RuntimeEvent, so a second pass has nothing left to do. + if (this.terminalRunFactCommitted) return; // On the recovery path the claimed event's write never committed, so // the boundary named after that commit must wait for the durability // barrier inside commitOrCreateTerminalRunFact; firing it here would @@ -1503,7 +1361,6 @@ export class AgentRun { await this.input.continuationFailpoint?.('after_terminal_event_committed'); } const commit = commitOrCreateTerminalRunFact({ - runStore, runtimeEventStore, ...(this.continuationActive && deferContinuationBoundary ? { @@ -1522,38 +1379,30 @@ export class AgentRun { ? { failureClass: this.failureClass ?? finalStatus?.blockedReason } : {}), ...(this.failureMessage ? { failureMessage: this.failureMessage } : {}), - ...(this.traceWriteError ? { traceWriteError: this.traceWriteError } : {}), ...(this.abortSource || fallbackStatus === 'cancelled' ? { abortSource: this.abortSource ?? 'user_stop' } : {}), fallbackStatus, fallbackInvocationId: this.runId, ...(fallbackStatus === 'failed' ? { fallbackFailureClass, fallbackFailureMessage } : {}), - allowHeaderCommitFailure: true, }); if (!terminalClaim.write) { terminalClaim.write = commit.then(() => undefined); void terminalClaim.write.catch(() => {}); } - const result = await commit; - this.terminalRunHeaderCommitted = result.headerCommitted; - if (result.headerCommitted && this.continuationActive) { - await this.input.continuationFailpoint?.('after_terminal_header_committed'); - } - if (result.headerCommitError !== undefined) { - await this.enqueueTraceWriteFailure(result.headerCommitError, 'commit terminal run header'); - } + await commit; + this.terminalRunFactCommitted = true; } catch (error) { if (corruptionRecovery) { // The scoped barrier lost its bet: the ledger refused even the // terminal fact. The latch never lifted, so there is nothing to // restore; record the failure and keep the finalize path's // historical silence for a store that stays broken. - await this.enqueueTraceWriteFailure(error, 'commit terminal run header'); + await this.enqueueTraceWriteFailure(error, 'commit terminal run fact'); return; } this.runStoreAvailable = false; - await this.enqueueTraceWriteFailure(error, 'commit terminal run header'); + await this.enqueueTraceWriteFailure(error, 'commit terminal run fact'); throw error; } await this.traceQueue.catch(() => {}); @@ -1781,14 +1630,6 @@ export class AgentRun { ): Promise { const message = errorMessage(error); this.traceWriteError ??= `${label}: ${message}`; - try { - await this.input.runStore?.updateRun(this.sessionId, this.runId, { - traceWriteError: this.traceWriteError, - updatedAt: this.input.now(), - }); - } catch { - // The terminal header commit retries the in-memory latch. - } try { await this.input.runStore?.appendEvent(this.sessionId, this.runId, { type: 'trace_write_failed', @@ -1926,3 +1767,11 @@ function isAtomicToolBoundaryProjection( if (!protocol || event.refs?.operationId === undefined) return false; return event.content?.kind === 'function_call' || event.content?.kind === 'function_response'; } + +/** The provider endpoint identity a continuation claim froze, if it named one. */ +function claimedProviderStateIdentity( + opening: RuntimeEventInvocationOpenedContent | undefined, +): `sha256:${string}` | undefined { + const route = opening?.route; + return route?.provenance === 'runtime' ? route.providerStateIdentity : undefined; +} diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index 39ae42a93c..8f641c56dd 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -1943,7 +1943,7 @@ export class AiSdkBackend implements AgentBackend { projectionCheckpoint, compatibleProviderReasoningReplayEventIds( replayEvents, - input.runtimeContextRunHeaders, + input.runtimeContextInvocations, this.input.providerStateIdentity, this.input.modelId, scope.runId, @@ -3670,7 +3670,7 @@ export class AiSdkBackend implements AgentBackend { const priorRuntimeContext = preparedContextBudget.events; const providerReasoningReplayEventIds = compatibleProviderReasoningReplayEventIds( priorRuntimeContext, - input.runtimeContextRunHeaders, + input.runtimeContextInvocations, this.input.providerStateIdentity, this.input.modelId, ); diff --git a/packages/runtime/src/ai-sdk-compaction-contract.ts b/packages/runtime/src/ai-sdk-compaction-contract.ts index 698d7de64e..62831805fa 100644 --- a/packages/runtime/src/ai-sdk-compaction-contract.ts +++ b/packages/runtime/src/ai-sdk-compaction-contract.ts @@ -22,7 +22,7 @@ import type { HistoryCompactRoute } from '@maka/core/model-call-attempt'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { ModelProjectionTransition } from '@maka/core/model-projection-transition'; import type { LoadedModelProjectionTransitions } from './model-projection-transition-ledger.js'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { ProviderRequestTracker } from './provider-request-telemetry.js'; import type { ContextBudgetPolicy } from './context-budget.js'; @@ -47,7 +47,7 @@ export interface HistoryCompactSummaryInput { runId?: string; source: { foldedRuntimeEvents: RuntimeEvent[]; - runHeaders?: readonly AgentRunHeader[]; + invocations?: readonly RuntimeInvocationRecord[]; }; previousCheckpoint?: HistoryCompactCheckpoint; newlyFoldedRuntimeEvents?: RuntimeEvent[]; diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts index fec8bcbfbc..4199e4541a 100644 --- a/packages/runtime/src/ai-sdk-compaction.ts +++ b/packages/runtime/src/ai-sdk-compaction.ts @@ -29,7 +29,7 @@ */ import type { RuntimeEvent } from '@maka/core/runtime-event'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { BackendCompactHistoryInput, BackendCompactHistoryResult, @@ -366,7 +366,9 @@ export class AiSdkCompaction { sessionId: this.sessionId, phase: 'standalone', orderedEvents: runtimeContext, - ...(input.runtimeContextRunHeaders ? { runHeaders: input.runtimeContextRunHeaders } : {}), + ...(input.runtimeContextInvocations + ? { invocations: input.runtimeContextInvocations } + : {}), acceptedRoute: { modelId: this.input.modelId, ...(this.targetConnectionId !== undefined @@ -388,8 +390,8 @@ export class AiSdkCompaction { runId: input.runId, source: { foldedRuntimeEvents: [...coveredRuntimeEvents], - ...(input.runtimeContextRunHeaders - ? { runHeaders: input.runtimeContextRunHeaders } + ...(input.runtimeContextInvocations + ? { invocations: input.runtimeContextInvocations } : {}), }, newlyFoldedRuntimeEvents: [...newlyFoldedRuntimeEvents], @@ -479,13 +481,16 @@ export class AiSdkCompaction { input: HistoryCompactSummaryInput, ): Promise { const foldedRunIds = new Set(input.source.foldedRuntimeEvents.map((event) => event.runId)); - const sourceRunRoutes = input.source.runHeaders - ?.filter((run) => foldedRunIds.has(run.runId)) - .map((run) => ({ - runId: run.runId, - connectionId: run.llmConnectionId, - modelId: run.modelId, - })) + const sourceRunRoutes = input.source.invocations + ?.filter((invocation) => foldedRunIds.has(invocation.runId)) + .map((invocation) => { + const route = invocation.opening.route; + return { + runId: invocation.runId, + ...(route.provenance === 'runtime' ? { connectionId: route.llmConnectionId } : {}), + modelId: route.modelId, + }; + }) .sort((left, right) => left.runId.localeCompare(right.runId)); const fingerprint = sha256( stableStringifyForSignature({ @@ -805,7 +810,7 @@ export class AiSdkCompaction { const state = new MidTurnCapacityCompactState( headAnchor, priorContentEvents, - input.runtimeContextRunHeaders ?? [], + input.runtimeContextInvocations ?? [], resolveDeclaredContextWindow(this.input.connection, this.input.modelId), ); // Seed the turn's FIRST request with the last request the provider @@ -814,7 +819,7 @@ export class AiSdkCompaction { // decides, and its rejection is recovered from. const persisted = persistedRequestAnchor( input.runtimeContext ?? [], - state.priorRunHeaders, + state.priorInvocations, this.input.modelId, this.targetConnectionId, ); @@ -1077,7 +1082,7 @@ export class AiSdkCompaction { phase: input.phase ?? 'mid_turn', orderedEvents, headAnchor: { runtimeEventId: state.headAnchor.id, turnId }, - runHeaders: state.priorRunHeaders, + invocations: state.priorInvocations, acceptedRoute: { modelId: this.input.modelId, ...(this.targetConnectionId !== undefined ? { connectionId: this.targetConnectionId } : {}), @@ -1106,7 +1111,7 @@ export class AiSdkCompaction { ...(input.origin.runId ? { runId: input.origin.runId } : {}), source: { foldedRuntimeEvents: [...coveredRuntimeEvents], - runHeaders: state.priorRunHeaders, + invocations: state.priorInvocations, }, ...(previousCheckpoint ? { previousCheckpoint } : {}), newlyFoldedRuntimeEvents: [...newlyFoldedRuntimeEvents], @@ -1154,7 +1159,7 @@ export class AiSdkCompaction { plan.checkpoint, compatibleProviderReasoningReplayEventIds( plan.replacementEvents, - state.priorRunHeaders, + state.priorInvocations, this.targetProviderStateIdentity, this.input.modelId, input.origin.runId, @@ -1484,7 +1489,7 @@ export class MidTurnCapacityCompactState { constructor( readonly headAnchor: RuntimeEvent, readonly priorContentEvents: readonly RuntimeEvent[], - readonly priorRunHeaders: readonly AgentRunHeader[], + readonly priorInvocations: readonly RuntimeInvocationRecord[], /** * The Maka window: the context window the USER declared for this model, * a compaction target and nothing else. Absent when none is declared, @@ -1538,7 +1543,7 @@ function usageBaselineTokens(usage: NormalizedUsage | undefined): number | undef */ function persistedRequestAnchor( events: readonly RuntimeEvent[], - runHeaders: readonly AgentRunHeader[], + invocations: readonly RuntimeInvocationRecord[], modelId: string, connectionId: string | undefined, ): LastRequestAnchor | undefined { @@ -1546,8 +1551,12 @@ function persistedRequestAnchor( const event = events[index]; const anchor = event?.actions?.tokenUsage?.lastRequestAnchor; if (!anchor) continue; - const header = runHeaders.find((candidate) => candidate.runId === event?.runId); - if (!header || header.modelId !== modelId || header.llmConnectionId !== connectionId) { + const route = invocations.find((candidate) => candidate.runId === event?.runId)?.opening.route; + if ( + route?.provenance !== 'runtime' || + route.modelId !== modelId || + route.llmConnectionId !== connectionId + ) { return undefined; } return anchor; diff --git a/packages/runtime/src/context-diagnostics.ts b/packages/runtime/src/context-diagnostics.ts index bc45466ab0..81416800d7 100644 --- a/packages/runtime/src/context-diagnostics.ts +++ b/packages/runtime/src/context-diagnostics.ts @@ -18,7 +18,6 @@ */ import { - isSessionInlineRun, supersedesLatestContext, type AgentRunEvent, type AgentRunStore, @@ -112,11 +111,7 @@ export interface ContextDiagnosticsComposition { type ContextRunStore = Pick< AgentRunStore, - | 'listSessionRuns' - | 'readEvents' - | 'readEventProjection' - | 'readEventLedgerRevision' - | 'repairEventProjection' + 'readEvents' | 'readEventProjection' | 'readEventLedgerRevision' | 'repairEventProjection' >; /** @@ -141,6 +136,8 @@ type ContextRunStore = Pick< export async function readLatestContextDiagnostics( runStore: ContextRunStore, sessionId: string, + /** The session-inline runs to scan on the cold path, from the event spine. */ + runIds: readonly string[], ): Promise { try { let replaceProjectionId: string | undefined; @@ -165,7 +162,13 @@ export async function readLatestContextDiagnostics( runStore.readEventLedgerRevision && runStore.repairEventProjection ? await runStore.readEventLedgerRevision(sessionId) : undefined; - return await rebuildContextFromLedger(runStore, sessionId, replaceProjectionId, ledgerRevision); + return await rebuildContextFromLedger( + runStore, + sessionId, + runIds, + replaceProjectionId, + ledgerRevision, + ); } catch { return { status: 'unavailable', reason: 'trace_unavailable' }; } @@ -184,10 +187,10 @@ export async function readLatestContextDiagnostics( async function rebuildContextFromLedger( runStore: ContextRunStore, sessionId: string, + runIds: readonly string[], replaceProjectionId?: string, ledgerRevision?: string, ): Promise { - const runs = (await runStore.listSessionRuns(sessionId)).filter(isSessionInlineRun); let anchor: MeteringAnchor | undefined; // Only consulted when the scan finds no canonical attempt at all: a session // written before canonical metering existed has provider attempts and @@ -207,8 +210,8 @@ async function rebuildContextFromLedger( const historicalAttempts: LegacyProviderAnchor[] = []; const checkpoints: CheckpointCandidate[] = []; - for (const run of runs) { - for (const event of await runStore.readEvents(sessionId, run.runId)) { + for (const runId of runIds) { + for (const event of await runStore.readEvents(sessionId, runId)) { if (event.type === METERING_EVENT_TYPE) { sawCanonicalRecord = true; const candidate = meteringAnchor(event); diff --git a/packages/runtime/src/continuation-replay.ts b/packages/runtime/src/continuation-replay.ts index f9d6fc6e28..c26b7c6ad2 100644 --- a/packages/runtime/src/continuation-replay.ts +++ b/packages/runtime/src/continuation-replay.ts @@ -18,7 +18,7 @@ */ import { createHash } from 'node:crypto'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { stableJsonStringify } from '@maka/core/tool-args-identity'; import { @@ -83,7 +83,7 @@ export type ContinuationReplayPlanResult = }; export interface ContinuationReplayAdmissionRoute { - runHeaders: readonly AgentRunHeader[]; + invocations: readonly RuntimeInvocationRecord[]; targetProviderStateIdentity: `sha256:${string}` | undefined; targetModelId: string; } @@ -112,7 +112,7 @@ export function buildContinuationReplayPlan(input: { const runtimeContext = segments.flatMap((segment) => segment.replayRuntimeEvents); const providerReasoningReplayEventIds = compatibleProviderReasoningReplayEventIds( runtimeContext, - input.admissionRoute.runHeaders, + input.admissionRoute.invocations, input.admissionRoute.targetProviderStateIdentity, input.admissionRoute.targetModelId, ); diff --git a/packages/runtime/src/conversation-copy.ts b/packages/runtime/src/conversation-copy.ts index 1b5d78bd02..fc158e67a8 100644 --- a/packages/runtime/src/conversation-copy.ts +++ b/packages/runtime/src/conversation-copy.ts @@ -17,20 +17,20 @@ * under the License. */ -import type { - AgentRunEvent, - AgentRunHeader, - AgentRunStore, - EmittedAgentRunEvent, -} from '@maka/core/agent-run'; -import type { RuntimeEvent } from '@maka/core/runtime-event'; +import type { AgentRunEvent, AgentRunStore, EmittedAgentRunEvent } from '@maka/core/agent-run'; +import type { RuntimeEvent, RuntimeEventInvocationOpenedContent } from '@maka/core/runtime-event'; +import { + buildInvocationOpenedEvent, + isSessionInlineInvocation, +} from '@maka/core/runtime-invocation'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; import { type StorageRef, type ToolResultContent } from '@maka/core/events'; import { parseAttachmentResourceRef } from '@maka/core/attachments'; import { markPersisted } from '@maka/core/persisted-value'; import type { StoredMessage } from '@maka/core/session'; import { decodePersistedToolResultContent } from '@maka/core/tool-result-record-schema'; -import { isEmittedAgentRunEventType, isSessionInlineRun } from '@maka/core/agent-run'; +import { isEmittedAgentRunEventType } from '@maka/core/agent-run'; import { decodeModelCallAttempt, MODEL_CALL_ATTEMPT_EVENT_TYPE, @@ -154,7 +154,7 @@ export interface ConversationRuntimeLedgerCopyPlan { readonly copyTurnIds: readonly string[]; readonly inlineRuntimeEvents: readonly RuntimeEvent[]; readonly runs: readonly { - readonly run: AgentRunHeader; + readonly run: RuntimeInvocationRecord; readonly runtimeEvents: readonly RuntimeEvent[]; readonly operationalEvents: readonly AgentRunEvent[]; }[]; @@ -319,10 +319,13 @@ export async function prepareConversationRuntimeLedgerCopy(input: { readonly sourceSessionId: string; readonly sourceEvents: readonly RuntimeEvent[]; readonly copiedMessages: readonly StoredMessage[]; - readonly runStore: Pick; - readonly runtimeEventStore: Pick; + readonly runStore: Pick; + readonly runtimeEventStore: Pick< + RuntimeEventStore, + 'readRuntimeEvents' | 'listSessionInvocations' + >; }): Promise { - const sourceRuns = await input.runStore.listSessionRuns(input.sourceSessionId); + const sourceRuns = await input.runtimeEventStore.listSessionInvocations(input.sourceSessionId); const transcriptTurnIds = [ ...new Set( input.copiedMessages.map(messageTurnId).filter((turnId): turnId is string => !!turnId), @@ -338,21 +341,31 @@ export async function prepareConversationRuntimeLedgerCopy(input: { const runs = await Promise.all( selectedRunEvents.map(async ({ run, events }) => { const operationalEvents = await input.runStore.readEvents(run.sessionId, run.runId); - if (events.length === 0) { - throw new Error(`Cannot copy AgentRun ${run.runId} without RuntimeEvent facts`); - } const terminal = classifyTerminalRuntimeLedger(run, events); - if (isTerminalRunStatus(run.status) && terminal.kind !== 'fact') { + if (run.terminalEvent && terminal.kind !== 'fact') { throw new Error(`Cannot copy terminal AgentRun ${run.runId} without one terminal fact`); } return { run, runtimeEvents: events, operationalEvents }; }), ); await rebuildCopiedProjectionTransitions(input.sourceSessionId, sourceRuns, runs, input.runStore); + // A restored opening takes the place the migration could not give it: right + // before the first event of its run in the Session's order. + const restoredOpenings = new Map( + selectedRunEvents.flatMap(({ run, restoredOpening }) => + restoredOpening ? [[run.runId, restoredOpening] as const] : [], + ), + ); + const inlineRuntimeEvents = input.sourceEvents.flatMap((event) => { + const opening = restoredOpenings.get(event.runId); + if (!opening) return [event]; + restoredOpenings.delete(event.runId); + return [opening, event]; + }); const plan = { sourceSessionId: input.sourceSessionId, copyTurnIds, - inlineRuntimeEvents: [...input.sourceEvents], + inlineRuntimeEvents, runs, }; assertConversationRuntimeLedgerCopySupported(plan); @@ -376,15 +389,18 @@ export async function prepareConversationRuntimeLedgerCopy(input: { */ async function rebuildCopiedProjectionTransitions( sessionId: string, - sourceRuns: readonly AgentRunHeader[], + sourceRuns: readonly RuntimeInvocationRecord[], runs: readonly { - readonly run: AgentRunHeader; + readonly run: RuntimeInvocationRecord; readonly runtimeEvents: readonly RuntimeEvent[]; readonly operationalEvents: AgentRunEvent[]; }[], runStore: Pick, ): Promise { - const owningRun = new Map(); + const owningRun = new Map< + string, + { run: RuntimeInvocationRecord; operationalEvents: AgentRunEvent[] } + >(); const copiedRuntimeEvents: RuntimeEvent[] = []; for (const { run, runtimeEvents, operationalEvents } of runs) { for (const event of runtimeEvents) { @@ -453,7 +469,8 @@ function assertConversationRuntimeLedgerCopySupported( ): void { const unsupported = plan.runs.some( ({ run, runtimeEvents }) => - run.continuationSource !== undefined || runtimeEvents.some(isContinuationStartRuntimeEvent), + run.opening.source.kind === 'continuation' || + runtimeEvents.some(isContinuationStartRuntimeEvent), ); if (!unsupported) return; @@ -480,8 +497,12 @@ export async function cloneConversationRuntimeLedger( flattenedPlans, input.plan.inlineRuntimeEvents, ); + // One physical execution attempt, one identity: a copied run and its copied + // invocation get the same fresh value rather than two independent ones. const runIds = new Map(flattenedPlans.map(({ run }) => [run.runId, input.newId()])); - const targetInvocationIds = new Map(flattenedPlans.map(({ run }) => [run.runId, input.newId()])); + const targetInvocationIds = new Map( + flattenedPlans.map(({ run }) => [run.runId, runIds.get(run.runId)!]), + ); const invocationIds = new Map( flattenedPlans.flatMap(({ run }) => run.invocationId ? [[run.invocationId, targetInvocationIds.get(run.runId)!] as const] : [], @@ -548,7 +569,6 @@ export async function cloneConversationRuntimeLedger( >(); const preparedPlans = flattenedPlans.map((plan) => { const runId = runIds.get(plan.run.runId)!; - const invocationId = targetInvocationIds.get(plan.run.runId)!; const clonedOperationalEvents = plan.operationalEvents.flatMap((event) => { const clonedEvent = cloneAgentRunEvent( event, @@ -570,22 +590,15 @@ export async function cloneConversationRuntimeLedger( return clonedEvent ? [clonedEvent] : []; }); const terminalEvent = - plan.terminal.kind === 'fact' && isTerminalRunStatus(plan.run.status) + plan.terminal.kind === 'fact' ? clonedEventBySourceId.get(plan.terminal.fact.terminalEvent.id) : undefined; - if (plan.terminal.kind === 'fact' && isTerminalRunStatus(plan.run.status) && !terminalEvent) { + if (plan.terminal.kind === 'fact' && !terminalEvent) { throw new Error(`Copied AgentRun ${plan.run.runId} lost its terminal RuntimeEvent`); } return { plan, runId, - clonedRun: cloneRunHeader( - plan.run, - input.referenceMap.targetSessionId, - runId, - invocationId, - references, - ), clonedOperationalEvents, terminalEvent, }; @@ -594,10 +607,6 @@ export async function cloneConversationRuntimeLedger( rewriteConversationCopyMessage(message, references), ); - for (const { clonedRun } of preparedPlans) { - await input.runStore.createRun(clonedRun); - } - const importedSourceEventIds = new Set(); const orderedBatches = input.plan.inlineRuntimeEvents.flatMap((event) => { const cloned = clonedEventBySourceId.get(event.id); @@ -624,9 +633,8 @@ export async function cloneConversationRuntimeLedger( await input.runStore.appendEvent(input.referenceMap.targetSessionId, runId, clonedEvent); } - if (plan.terminal.kind === 'fact' && isTerminalRunStatus(plan.run.status) && terminalEvent) { + if (plan.terminal.kind === 'fact' && terminalEvent) { await commitTerminalRunWithRuntimeFact({ - runStore: input.runStore, runtimeEventStore: input.runtimeEventStore, newId: input.newId, sessionId: input.referenceMap.targetSessionId, @@ -638,14 +646,7 @@ export async function cloneConversationRuntimeLedger( ...(plan.terminal.fact.failureClass ? { failureClass: plan.terminal.fact.failureClass } : {}), - ...(plan.run.failureMessage ? { failureMessage: plan.run.failureMessage } : {}), ...(plan.terminal.fact.abortSource ? { abortSource: plan.terminal.fact.abortSource } : {}), - runEventData: { - recovered: true, - recoveryReason: 'conversation_runtime_ledger_clone', - sourceSessionId: plan.run.sessionId, - sourceRunId: plan.run.runId, - }, }); } } @@ -660,12 +661,20 @@ export async function cloneConversationRuntimeLedger( } interface ConversationCopyRunEvents { - readonly run: AgentRunHeader; + readonly run: RuntimeInvocationRecord; + /** The run's events, beginning with its opening. */ readonly events: readonly RuntimeEvent[]; + /** + * The opening as an event, when the run's own events did not carry one: + * the migration shelved openings of runs that already owned an immutable + * sequence, and a copy is where such a run gets its opening back as event + * one, because the copy is a fresh sequence. + */ + readonly restoredOpening?: RuntimeEvent; } async function loadConversationCopyRunEvents( - sourceRuns: readonly AgentRunHeader[], + sourceRuns: readonly RuntimeInvocationRecord[], sourceEvents: readonly RuntimeEvent[], copyTurnIds: readonly string[], runtimeEventStore: Pick, @@ -682,7 +691,18 @@ async function loadConversationCopyRunEvents( projectedEvents.length > 0 ? projectedEvents : runtimeEventStore.readRuntimeEvents(run.sessionId, run.runId), - ).then((events) => ({ run, events })), + ).then((events) => { + if (events.some((event) => event.content?.kind === 'invocation_opened')) { + return { run, events }; + } + const restoredOpening = buildInvocationOpenedEvent({ + id: `invocation_opened:${run.runId}`, + run, + openedAt: run.openedAt, + opening: run.opening, + }); + return { run, events: [restoredOpening, ...events], restoredOpening }; + }), ]; }), ); @@ -867,17 +887,7 @@ function cloneAgentRunEvent( } let data = event.data; - if (event.type === 'provider_request_captured') { - data = rewriteProviderRequestCapture(event, ids.eventId, references, providerTraceIds); - } else if (event.type === 'provider_request_attempt_recorded') { - data = rewriteProviderRequestAttempt( - event, - ids.eventId, - references, - operationalEventIds, - providerTraceIds, - ); - } else if (event.type === MODEL_CALL_ATTEMPT_EVENT_TYPE) { + if (event.type === MODEL_CALL_ATTEMPT_EVENT_TYPE) { data = rewriteModelCallAttempt( event, { sessionId: ids.sessionId, runId: ids.runId, attemptId: ids.eventId }, @@ -1045,46 +1055,6 @@ function cloneModelProjectionTransition( return transition; } -function rewriteProviderRequestCapture( - event: AgentRunEvent, - eventId: string, - references: ConversationCopyReferenceMap, - providerTraceIds: ReadonlyMap, -): Record { - const data = providerRequestCapture(event); - return { - ...data, - traceId: requiredMappedId(providerTraceIds, data.traceId, 'provider trace'), - captureId: eventId, - artifactId: rewriteOwnedArtifactId(data.artifactId, references), - }; -} - -function rewriteProviderRequestAttempt( - event: AgentRunEvent, - eventId: string, - references: ConversationCopyReferenceMap, - operationalEventIds: ReadonlyMap, - providerTraceIds: ReadonlyMap, -): Record { - const data = providerRequestAttempt(event); - return { - ...data, - traceId: requiredMappedId(providerTraceIds, data.traceId, 'provider trace'), - attemptId: eventId, - ...(data.captureId !== undefined && data.captureArtifactId !== undefined - ? { - captureId: requiredMappedId( - operationalEventIds, - data.captureId, - 'provider request capture', - ), - captureArtifactId: rewriteOwnedArtifactId(data.captureArtifactId, references), - } - : {}), - }; -} - function rewriteModelCallAttempt( event: AgentRunEvent, ids: { @@ -1199,6 +1169,16 @@ function rewriteOwnedId(sourceId: string, ids: ReadonlyMap, kind return requiredMappedId(ids, sourceId, kind); } +const PROVIDER_TRACE_BEARING_EVENT_TYPES: ReadonlySet = new Set([ + MODEL_CALL_ATTEMPT_EVENT_TYPE, + 'provider_request_captured', + 'provider_request_attempt_recorded', +]); + +function isProviderTraceBearingEventType(type: string): boolean { + return PROVIDER_TRACE_BEARING_EVENT_TYPES.has(type); +} + function providerTraceIdMap( plans: readonly { readonly operationalEvents: readonly AgentRunEvent[] }[], newId: () => string, @@ -1206,13 +1186,11 @@ function providerTraceIdMap( const result = new Map(); for (const { operationalEvents } of plans) { for (const event of operationalEvents) { - if ( - event.type !== 'provider_request_captured' && - event.type !== 'provider_request_attempt_recorded' && - event.type !== MODEL_CALL_ATTEMPT_EVENT_TYPE - ) { - continue; - } + // Harvest from retired writers too. Their rows are not copied, but a + // copied RuntimeEvent may still point at a trace only they recorded, and + // carrying the source's trace id into the target would be worse than + // pointing at a fresh one nothing describes. + if (!isProviderTraceBearingEventType(event.type)) continue; const traceId = event.data?.traceId; if (typeof traceId === 'string' && !result.has(traceId)) result.set(traceId, newId()); } @@ -1227,7 +1205,11 @@ function logicalModelCallIdMap( const result = new Map(); for (const { operationalEvents } of plans) { for (const event of operationalEvents) { - if (event.type !== MODEL_CALL_ATTEMPT_EVENT_TYPE) continue; + // Harvest from retired writers too. Their rows are not copied, but a + // copied RuntimeEvent may still point at a trace only they recorded, and + // carrying the source's trace id into the target would be worse than + // pointing at a fresh one nothing describes. + if (!isProviderTraceBearingEventType(event.type)) continue; const logicalCallId = event.data?.logicalCallId; if (typeof logicalCallId === 'string' && !result.has(logicalCallId)) { result.set(logicalCallId, newId()); @@ -1239,7 +1221,7 @@ function logicalModelCallIdMap( function toolOperationIdMap( plans: readonly { - readonly run: AgentRunHeader; + readonly run: RuntimeInvocationRecord; readonly events: readonly RuntimeEvent[]; }[], targetInvocationIds: ReadonlyMap, @@ -1270,12 +1252,7 @@ function isCopiedAgentRunEvent(event: AgentRunEvent): event is EmittedAgentRunEv // into the target with source identities intact. The ledger's `type` is open, so such an event // may predate a retired writer or postdate this build entirely (#1942). if (!isEmittedAgentRunEventType(event.type)) return false; - return ( - event.type !== 'run_completed' && - event.type !== 'run_failed' && - event.type !== 'run_cancelled' && - event.type !== 'event_corrupt' - ); + return event.type !== 'event_corrupt'; } function cloneRuntimeEvent( @@ -1322,60 +1299,52 @@ function cloneRuntimeEvent( return cloned; } -function cloneRunHeader( - source: AgentRunHeader, - targetSessionId: string, - runId: string, - invocationId: string, +/** + * Rewrite the lineage a copied invocation's opening fact carries. + * + * The opening is an ordinary RuntimeEvent, so the copy rewrites its owned ids + * the way it rewrites every other reference. Its `source` needs no rewriting: + * a copy that contains a continuation is refused before it gets this far. + */ +function rewriteInvocationOpening( + opening: RuntimeEventInvocationOpenedContent, references: ConversationCopyReferenceMap, -): AgentRunHeader { - const cloned: AgentRunHeader = { - ...source, - invocationId, - sessionId: targetSessionId, - runId, - ...(source.parentRunId - ? { parentRunId: rewriteOwnedId(source.parentRunId, references.runIds, 'AgentRun') } - : {}), - ...(source.resumedFromRunId - ? { - resumedFromRunId: rewriteOwnedId(source.resumedFromRunId, references.runIds, 'AgentRun'), - } - : {}), - ...(source.retriedFromRunId - ? { - retriedFromRunId: rewriteOwnedId(source.retriedFromRunId, references.runIds, 'AgentRun'), - } - : {}), - ...(source.parentSessionId === references.sourceSessionId - ? { parentSessionId: targetSessionId } - : {}), - ...(source.continuationSource +): RuntimeEventInvocationOpenedContent { + const lineage = opening.lineage; + return { + ...opening, + ...(lineage ? { - continuationSource: { - ...source.continuationSource, - sourceInvocationId: rewriteOwnedId( - source.continuationSource.sourceInvocationId, - references.invocationIds, - 'invocation', - ), - sourceRunId: rewriteOwnedId( - source.continuationSource.sourceRunId, - references.runIds, - 'AgentRun', - ), + lineage: { + ...lineage, + ...(lineage.parentRunId + ? { parentRunId: rewriteOwnedId(lineage.parentRunId, references.runIds, 'AgentRun') } + : {}), + ...(lineage.resumedFromRunId + ? { + resumedFromRunId: rewriteOwnedId( + lineage.resumedFromRunId, + references.runIds, + 'AgentRun', + ), + } + : {}), + ...(lineage.retriedFromRunId + ? { + retriedFromRunId: rewriteOwnedId( + lineage.retriedFromRunId, + references.runIds, + 'AgentRun', + ), + } + : {}), + ...(lineage.parentSessionId === references.sourceSessionId + ? { parentSessionId: references.targetSessionId } + : {}), }, } : {}), }; - if (isTerminalRunStatus(source.status)) { - cloned.status = 'running'; - delete cloned.completedAt; - delete cloned.failureClass; - delete cloned.failureMessage; - delete cloned.abortSource; - } - return cloned; } function rewriteRuntimeEventReferences( @@ -1411,7 +1380,9 @@ function rewriteRuntimeEventReferences( } : {}), } - : event.content; + : event.content?.kind === 'invocation_opened' + ? rewriteInvocationOpening(event.content, references) + : event.content; const refs = event.refs ? (() => { const { @@ -1890,7 +1861,7 @@ function messageTurnId(message: StoredMessage): string | undefined { } function conversationCopyTurnClosure( - runs: readonly AgentRunHeader[], + runs: readonly RuntimeInvocationRecord[], retainedTurnIds: readonly string[], ): string[] { const result = [...new Set(retainedTurnIds)]; @@ -1902,9 +1873,9 @@ function conversationCopyTurnClosure( changed = false; for (const run of runs) { if ( - isSessionInlineRun(run) || - !run.parentRunId || - !includedRunIds.has(run.parentRunId) || + isSessionInlineInvocation(run.opening) || + !run.opening.lineage?.parentRunId || + !includedRunIds.has(run.opening.lineage.parentRunId) || includedRunIds.has(run.runId) ) { continue; @@ -1922,7 +1893,7 @@ function conversationCopyTurnClosure( function sourceCompactableEventsByRunId( plans: readonly { - readonly run: AgentRunHeader; + readonly run: RuntimeInvocationRecord; readonly events: readonly RuntimeEvent[]; }[], sessionEvents: readonly RuntimeEvent[], @@ -1932,7 +1903,7 @@ function sourceCompactableEventsByRunId( const result = new Map(); for (const plan of plans) { - if (isSessionInlineRun(plan.run)) { + if (isSessionInlineInvocation(plan.run.opening)) { result.set(plan.run.runId, inlineEvents); continue; } @@ -1948,7 +1919,7 @@ function sourceCompactableEventsByRunId( } visited.add(cursor.run.runId); reverseChain.push(cursor); - const sourceRunId = cursor.run.resumedFromRunId; + const sourceRunId = cursor.run.opening.lineage?.resumedFromRunId; if (!sourceRunId) break; cursor = plansByRunId.get(sourceRunId); if (!cursor) { @@ -1968,7 +1939,3 @@ function sourceCompactableEventsByRunId( return result; } - -function isTerminalRunStatus(status: AgentRunHeader['status']): boolean { - return status === 'completed' || status === 'failed' || status === 'cancelled'; -} diff --git a/packages/runtime/src/execution-inspect.ts b/packages/runtime/src/execution-inspect.ts index 5647c313e5..22b8eeb72d 100644 --- a/packages/runtime/src/execution-inspect.ts +++ b/packages/runtime/src/execution-inspect.ts @@ -17,8 +17,9 @@ * under the License. */ -import type { AgentRunHeader } from '@maka/core/agent-run'; import type { RuntimeEvent } from '@maka/core/runtime-event'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; +import { runtimeInvocationOutcome } from '@maka/core/runtime-invocation'; import type { SessionHeader } from '@maka/core/session'; import { AGENT_RUN_INSPECT_DOCUMENT_VERSION, @@ -39,7 +40,6 @@ import { type AgentRunInspectDiagnostic as SourceDiagnostic, type InspectAgentRunOptions, type RuntimeEventInspectReader, - type SessionAgentRunInspectReader, } from './agent-run-inspect.js'; import { isSupersededHistoryCompactCheckpoint, @@ -53,7 +53,7 @@ export interface SessionHeaderReader { export interface InspectSessionDocumentOptions { header?: SessionHeader; - runHeaders?: readonly AgentRunHeader[]; + invocations?: readonly RuntimeInvocationRecord[]; isFatalReadError?: InspectAgentRunOptions['isFatalReadError']; } @@ -63,30 +63,30 @@ export async function inspectAgentRunDocument( input: { sessionId: string; agentRunId: string; - header?: AgentRunHeader; + invocation?: RuntimeInvocationRecord; isFatalReadError?: InspectAgentRunOptions['isFatalReadError']; }, ): Promise { const model = await inspectAgentRunReadModel(runStore, runtimeEventStore, { sessionId: input.sessionId, runId: input.agentRunId, - ...(input.header ? { header: input.header } : {}), + ...(input.invocation ? { invocation: input.invocation } : {}), ...(input.isFatalReadError ? { isFatalReadError: input.isFatalReadError } : {}), includeModelReplay: false, }); - const diagnostics = model.diagnostics.map((item) => sourceDiagnostic(model.header, item)); - const tools = inspectTools(model.header, model.runtimeEvents, diagnostics); + const diagnostics = model.diagnostics.map((item) => sourceDiagnostic(model.invocation, item)); + const tools = inspectTools(model.invocation, model.runtimeEvents, diagnostics); const compactionCheckpoints = inspectCompactionCheckpoints( - model.header, + model.invocation, model.events, diagnostics, ); - const runtimeCoverage = coverageFor(model.header.runId, model.runtimeEvents); + const runtimeCoverage = coverageFor(model.invocation.runId, model.runtimeEvents); return { schemaVersion: AGENT_RUN_INSPECT_DOCUMENT_VERSION, kind: 'agent_run', - agentRun: inspectIdentity(model.header), + agentRun: inspectIdentity(model.invocation), sources: { operationalEventCount: model.events.length, runtimeEventCount: model.runtimeEvents.length, @@ -101,20 +101,21 @@ export async function inspectAgentRunDocument( export async function inspectSessionDocument( sessionStore: SessionHeaderReader, - runStore: SessionAgentRunInspectReader, + runStore: AgentRunInspectReader, runtimeEventStore: RuntimeEventInspectReader, sessionId: string, options: InspectSessionDocumentOptions = {}, ): Promise { const resolvedHeader = options.header ?? (await sessionStore.readHeader(sessionId)); - const runHeaders = options.runHeaders ?? (await runStore.listSessionRuns(sessionId)); + const invocations = + options.invocations ?? (await runtimeEventStore.listSessionInvocations(sessionId)); const agentRuns: AgentRunInspectDocument[] = []; - for (const runHeader of runHeaders) { + for (const invocation of invocations) { agentRuns.push( await inspectAgentRunDocument(runStore, runtimeEventStore, { sessionId, - agentRunId: runHeader.runId, - header: runHeader, + agentRunId: invocation.runId, + invocation, ...(options.isFatalReadError ? { isFatalReadError: options.isFatalReadError } : {}), }), ); @@ -162,7 +163,7 @@ export function renderAgentRunInspectTree(document: AgentRunInspectDocument): st `├─ Turn ${run.turnId}`, `├─ Runtime Events ${formatCoverage(document.sources.runtimeCoverage)} (${document.sources.runtimeEventCount})`, `├─ Operational Events ${document.sources.operationalEventCount}`, - `├─ Source Health [${document.sources.health.statusConsistency}]`, + `├─ Source Health [runtime ledger ${document.sources.health.runtimeLedger}]`, `├─ Tools ${document.tools.callCount} calls / ${document.tools.responseCount} responses`, ]; for (const checkpoint of document.compactionCheckpoints) { @@ -198,28 +199,34 @@ export function renderSessionInspectTree(document: SessionInspectDocument): stri return `${lines.join('\n')}\n`; } -function inspectIdentity(header: AgentRunHeader): AgentRunInspectIdentity { +function inspectIdentity(invocation: RuntimeInvocationRecord): AgentRunInspectIdentity { + const lineage = invocation.opening.lineage; + const terminal = invocation.terminalEvent; + const stateDelta = terminal?.actions?.stateDelta; + const failureClass = + typeof stateDelta?.failureClass === 'string' ? stateDelta.failureClass : undefined; + const abortSource = + typeof stateDelta?.abortSource === 'string' ? stateDelta.abortSource : undefined; return { - sessionId: header.sessionId, - agentRunId: header.runId, - ...(header.invocationId ? { invocationId: header.invocationId } : {}), - turnId: header.turnId, - ...(header.parentRunId ? { parentRunId: header.parentRunId } : {}), - ...(header.resumedFromRunId ? { resumedFromRunId: header.resumedFromRunId } : {}), - ...(header.retriedFromRunId ? { retriedFromRunId: header.retriedFromRunId } : {}), - ...(header.parentTurnId ? { parentTurnId: header.parentTurnId } : {}), - ...(header.agentId ? { agentId: header.agentId } : {}), - status: header.status, - createdAt: header.createdAt, - updatedAt: header.updatedAt, - ...(header.completedAt !== undefined ? { completedAt: header.completedAt } : {}), - ...(header.failureClass ? { failureClass: header.failureClass } : {}), - ...(header.abortSource ? { abortSource: header.abortSource } : {}), + sessionId: invocation.sessionId, + agentRunId: invocation.runId, + invocationId: invocation.invocationId, + turnId: invocation.turnId, + ...(lineage?.parentRunId ? { parentRunId: lineage.parentRunId } : {}), + ...(lineage?.resumedFromRunId ? { resumedFromRunId: lineage.resumedFromRunId } : {}), + ...(lineage?.retriedFromRunId ? { retriedFromRunId: lineage.retriedFromRunId } : {}), + ...(lineage?.parentTurnId ? { parentTurnId: lineage.parentTurnId } : {}), + ...(lineage?.agentId ? { agentId: lineage.agentId } : {}), + status: runtimeInvocationOutcome(invocation) ?? 'running', + openedAt: invocation.openedAt, + ...(terminal ? { endedAt: terminal.ts } : {}), + ...(failureClass ? { failureClass } : {}), + ...(abortSource ? { abortSource } : {}), }; } function inspectTools( - header: AgentRunHeader, + invocation: RuntimeInvocationRecord, events: readonly RuntimeEvent[], diagnostics: ExecutionInspectDiagnostic[], ): AgentRunInspectToolSummary { @@ -250,7 +257,7 @@ function inspectTools( for (const call of callsWithoutResponse) { diagnostics.push( diagnostic( - header, + invocation, 'tool_response_missing', 'warning', `Tool Call ${call.toolCallId} has no committed Runtime response; its outcome and external side effects are unknown.`, @@ -261,7 +268,7 @@ function inspectTools( for (const response of responsesWithoutCall) { diagnostics.push( diagnostic( - header, + invocation, 'tool_call_missing', 'warning', `Tool response ${response.toolCallId} has no matching Runtime call fact.`, @@ -279,7 +286,7 @@ function inspectTools( } function inspectCompactionCheckpoints( - header: AgentRunHeader, + invocation: RuntimeInvocationRecord, events: readonly { type: string; id: string; data?: Record }[], diagnostics: ExecutionInspectDiagnostic[], ): AgentRunInspectCompactionCheckpoint[] { @@ -287,7 +294,7 @@ function inspectCompactionCheckpoints( for (const event of events) { if (event.type !== 'history_compact_checkpoint_recorded') continue; const checkpoint = event.data?.checkpoint; - if (!validateHistoryCompactCheckpointShape(checkpoint, header.sessionId)) { + if (!validateHistoryCompactCheckpointShape(checkpoint, invocation.sessionId)) { // A checkpoint recorded under an older source policy is expected history, // not corruption: the ledger keeps every checkpoint it ever wrote, and // every consumer fails open on it. Reporting it as an error would drown @@ -295,7 +302,7 @@ function inspectCompactionCheckpoints( const superseded = isSupersededHistoryCompactCheckpoint(checkpoint); diagnostics.push( diagnostic( - header, + invocation, superseded ? 'compaction_checkpoint_superseded' : 'compaction_checkpoint_invalid', superseded ? 'info' : 'error', superseded @@ -323,7 +330,7 @@ function inspectCompactionCheckpoints( } function sourceDiagnostic( - header: AgentRunHeader, + invocation: RuntimeInvocationRecord, source: SourceDiagnostic, ): ExecutionInspectDiagnostic { const severity: ExecutionInspectSeverity = /read_failed|corrupt|mismatch/.test(source.code) @@ -331,11 +338,11 @@ function sourceDiagnostic( : source.code.includes('missing') ? 'warning' : 'info'; - return diagnostic(header, source.code, severity, source.message, source.eventId); + return diagnostic(invocation, source.code, severity, source.message, source.eventId); } function diagnostic( - header: AgentRunHeader, + invocation: RuntimeInvocationRecord, code: string, severity: ExecutionInspectSeverity, message: string, @@ -345,9 +352,9 @@ function diagnostic( severity, code, message, - sessionId: header.sessionId, - agentRunId: header.runId, - turnId: header.turnId, + sessionId: invocation.sessionId, + agentRunId: invocation.runId, + turnId: invocation.turnId, ...(eventId ? { eventId } : {}), }; } diff --git a/packages/runtime/src/history-compact-checkpoint-coordinator.ts b/packages/runtime/src/history-compact-checkpoint-coordinator.ts index 684e71c7b5..05533e500f 100644 --- a/packages/runtime/src/history-compact-checkpoint-coordinator.ts +++ b/packages/runtime/src/history-compact-checkpoint-coordinator.ts @@ -20,7 +20,7 @@ import type { AgentRunStore } from '@maka/core/agent-run'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; -import { isSessionInlineRun } from '@maka/core/agent-run'; +import { isSessionInlineInvocation } from '@maka/core/runtime-invocation'; import { loadLatestHistoryCompactCheckpointFromRunLedger } from './history-compact-ledger.js'; import { canReplaceHistoryCompactCheckpoint, @@ -54,10 +54,14 @@ export class HistoryCompactCheckpointCoordinator { } const existing = this.loads.get(sessionId); if (existing) return existing; - if (!this.deps.runStore) return Promise.resolve(undefined); + const runStore = this.deps.runStore; + if (!runStore) return Promise.resolve(undefined); let guardedLoad: Promise; - guardedLoad = loadLatestHistoryCompactCheckpointFromRunLedger(this.deps.runStore, sessionId) + guardedLoad = this.inlineRunIds(sessionId) + .then((runIds) => + loadLatestHistoryCompactCheckpointFromRunLedger(runStore, sessionId, runIds), + ) .then((checkpoint) => { if (checkpoint) this.scheduleCleanup(sessionId, checkpoint); if (this.loads.get(sessionId) === guardedLoad && !this.checkpoints.has(sessionId)) { @@ -107,6 +111,15 @@ export class HistoryCompactCheckpointCoordinator { this.loads.delete(sessionId); } + /** The session's own runs, enumerated from the event spine that defines them. */ + private async inlineRunIds(sessionId: string): Promise { + const store = this.deps.runtimeEventStore; + if (!store) return []; + return (await store.listSessionInvocations(sessionId)) + .filter((invocation) => isSessionInlineInvocation(invocation.opening)) + .map((invocation) => invocation.runId); + } + private scheduleCleanup(sessionId: string, checkpoint: HistoryCompactCheckpoint): void { if ( !this.deps.cleanupHistoryCompactArtifacts || @@ -119,13 +132,10 @@ export class HistoryCompactCheckpointCoordinator { tracked = previous .catch(() => {}) .then(async () => { - const runs = (await this.deps.runStore!.listSessionRuns(sessionId)).filter( - isSessionInlineRun, - ); const runtimeEvents: RuntimeEvent[] = []; - for (const run of runs) { + for (const runId of await this.inlineRunIds(sessionId)) { runtimeEvents.push( - ...(await this.deps.runtimeEventStore!.readRuntimeEvents(sessionId, run.runId)), + ...(await this.deps.runtimeEventStore!.readRuntimeEvents(sessionId, runId)), ); } await this.deps.cleanupHistoryCompactArtifacts!({ diff --git a/packages/runtime/src/history-compact-ledger.ts b/packages/runtime/src/history-compact-ledger.ts index 9698b92a44..6eb9c0fb65 100644 --- a/packages/runtime/src/history-compact-ledger.ts +++ b/packages/runtime/src/history-compact-ledger.ts @@ -54,12 +54,13 @@ function hasLoadableHistoryCompactSummary(checkpoint: HistoryCompactCheckpoint): } export async function loadHistoryCompactCheckpointsFromRunLedger( - runStore: Pick, + runStore: Pick, sessionId: string, + runIds: readonly string[], ): Promise { const checkpoints = new Map(); - for (const run of await runStore.listSessionRuns(sessionId)) { - for (const event of await runStore.readEvents(sessionId, run.runId)) { + for (const runId of runIds) { + for (const event of await runStore.readEvents(sessionId, runId)) { if (event.type !== 'history_compact_checkpoint_recorded') continue; const checkpoint = event.data?.checkpoint; if ( @@ -76,13 +77,10 @@ export async function loadHistoryCompactCheckpointsFromRunLedger( export async function loadLatestHistoryCompactCheckpointFromRunLedger( runStore: Pick< AgentRunStore, - | 'listSessionRuns' - | 'readEvents' - | 'readEventProjection' - | 'readEventLedgerRevision' - | 'repairEventProjection' + 'readEvents' | 'readEventProjection' | 'readEventLedgerRevision' | 'repairEventProjection' >, sessionId: string, + runIds: readonly string[], ): Promise { let replaceEventId: string | undefined; if (runStore.readEventProjection) { @@ -108,11 +106,9 @@ export async function loadLatestHistoryCompactCheckpointFromRunLedger( runStore.readEventLedgerRevision && runStore.repairEventProjection ? await runStore.readEventLedgerRevision(sessionId) : undefined; - const runs = await runStore.listSessionRuns(sessionId); const candidates: LedgerCheckpointCandidate[] = []; - for (let runIndex = runs.length - 1; runIndex >= 0; runIndex -= 1) { - const run = runs[runIndex]!; - const events = await runStore.readEvents(sessionId, run.runId); + for (let runIndex = runIds.length - 1; runIndex >= 0; runIndex -= 1) { + const events = await runStore.readEvents(sessionId, runIds[runIndex]!); for (let eventIndex = events.length - 1; eventIndex >= 0; eventIndex -= 1) { const event = events[eventIndex]!; if (event.type !== 'history_compact_checkpoint_recorded') continue; diff --git a/packages/runtime/src/history-compaction.ts b/packages/runtime/src/history-compaction.ts index 268aa59f50..de33cc1280 100644 --- a/packages/runtime/src/history-compaction.ts +++ b/packages/runtime/src/history-compaction.ts @@ -17,7 +17,7 @@ * under the License. */ -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { ContextBudgetDiagnostic } from '@maka/core/usage-stats/types'; import { finitePositive } from './context-budget-helpers.js'; @@ -185,12 +185,12 @@ export interface PlanHistoryCompactionInput { highWaterSeq?: number; previousCheckpoint?: HistoryCompactCheckpoint; /** - * Run headers for the ordered events, and the route this fold is dispatched - * on. Together they name the newest reply this route produced, which is the - * only span a retreat may target: a rejection of a larger one says nothing - * about a span another model accepted. + * The invocations behind the ordered events, and the route this fold is + * dispatched on. Together they name the newest reply this route produced, + * which is the only span a retreat may target: a rejection of a larger one + * says nothing about a span another model accepted. */ - runHeaders?: readonly AgentRunHeader[]; + invocations?: readonly RuntimeInvocationRecord[]; acceptedRoute?: { modelId: string; connectionId?: string }; /** Present only when this automatic Compaction should create a Memory task. */ memoryExtractionBoundary?: HistoryCompactMemoryExtractionBoundary; @@ -229,22 +229,24 @@ export type HistoryCompactionFailReason = 'no_safe_completed_span' | 'summarizer * A span is only proven for the model and connection that accepted it: a token * count is a number in one tokenizer, and a session's history can span runs on * several routes. So the newest reply produced on the summarizer's own route - * ends the span, found through the run headers rather than by role alone — + * ends the span, found through each run's opening rather than by role alone — * everything before its first event was in a request that route accepted. * A ledger with no reply from this route has nothing proven, and the caller - * must not invent a boundary. + * must not invent a boundary. Nor does a run whose opening could not prove its + * route — a migrated header with no Connection — even when the current run has + * no Connection of its own: two unknowns are not a match. */ function acceptedInputBoundary( events: readonly RuntimeEvent[], - runHeaders: readonly AgentRunHeader[], + invocations: readonly RuntimeInvocationRecord[], route: { modelId: string; connectionId?: string } | undefined, ): number | undefined { if (!route) return undefined; const onRoute = (event: RuntimeEvent | undefined): boolean => { if (event?.role !== 'model') return false; - const header = runHeaders.find((candidate) => candidate.runId === event.runId); - if (!header || header.modelId !== route.modelId) return false; - return header.llmConnectionId === route.connectionId; + const opened = invocations.find((candidate) => candidate.runId === event.runId)?.opening.route; + if (opened?.provenance !== 'runtime' || opened.modelId !== route.modelId) return false; + return opened.llmConnectionId === route.connectionId; }; let index = -1; for (let cursor = events.length - 1; cursor >= 0; cursor -= 1) { @@ -337,7 +339,7 @@ export async function planHistoryCompaction( // (#4559). const proven = acceptedInputBoundary( input.orderedEvents, - input.runHeaders ?? [], + input.invocations ?? [], input.acceptedRoute, ); if (proven === undefined || proven >= boundary.coveredCount) { diff --git a/packages/runtime/src/interaction-authority.ts b/packages/runtime/src/interaction-authority.ts index b4463624f2..e1fe23ca35 100644 --- a/packages/runtime/src/interaction-authority.ts +++ b/packages/runtime/src/interaction-authority.ts @@ -199,6 +199,27 @@ export class RuntimeInteractionFailStopError extends Error { } } +/** + * Whether shutdown, rather than the work itself, is what refused this admission. + * + * A draining authority turns anything it rejects into a cancellation: the run + * did not fail, it was never allowed to proceed. Callers use this to settle the + * run as cancelled and to keep the rejection from reading as a Host fault. + */ +export function isShutdownCancelledInteractionAdmission(error: unknown): boolean { + if ( + error instanceof RuntimeInteractionAdmissionRejectedError && + error.reason === 'authority_draining' + ) { + return true; + } + return ( + error instanceof RuntimeInteractionFailStopError && + error.authorityFailure instanceof RuntimeInteractionAdmissionRejectedError && + error.authorityFailure.reason === 'authority_draining' + ); +} + type LocalClosureFinalizer = () => void; type HostedInteractionRequestEvent = diff --git a/packages/runtime/src/message-authority.ts b/packages/runtime/src/message-authority.ts index 55498687f8..ad9df40ad3 100644 --- a/packages/runtime/src/message-authority.ts +++ b/packages/runtime/src/message-authority.ts @@ -17,8 +17,8 @@ * under the License. */ -import type { SteeringLease } from '@maka/core/backend-types'; -import type { RootExecutionDescriptor } from '@maka/core/agent-run'; +import type { BackendStopMode, SteeringLease } from '@maka/core/backend-types'; +import type { RootExecutionDescriptor } from '@maka/core/runtime-invocation'; import type { MessageContent, SessionEvent } from '@maka/core/events'; import type { StopSessionInput } from './session-manager.js'; diff --git a/packages/runtime/src/model-history.ts b/packages/runtime/src/model-history.ts index 840e7392f0..64feb3605e 100644 --- a/packages/runtime/src/model-history.ts +++ b/packages/runtime/src/model-history.ts @@ -63,7 +63,7 @@ import { } from '@maka/core/runtime-event'; import { formatAttachmentResourceRef } from '@maka/core/attachments'; import type { AttachmentRef, DirectoryReference, QuoteRef } from '@maka/core/events'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { ModelMessage, ToolResultOutput, @@ -83,24 +83,29 @@ export const PROVIDER_REPLAY_PROJECTION_VERSION = 2; /** * Resolve the RuntimeEvents whose provider-owned reasoning may cross the - * current provider boundary. Route provenance remains on AgentRunHeader; - * current-run events are same-route by construction during mid-turn replay. + * current provider boundary. + * + * Route provenance is stated once, by the opening fact of the invocation that + * produced the events, and joined here by `runId`. Current-run events are + * same-route by construction during mid-turn replay. */ export function compatibleProviderReasoningReplayEventIds( events: readonly RuntimeEvent[], - runHeaders: readonly AgentRunHeader[] | undefined, + invocations: readonly RuntimeInvocationRecord[] | undefined, targetProviderStateIdentity: `sha256:${string}` | undefined, targetModelId: string, currentRunId?: string, ): ReadonlySet { const compatibleRunIds = new Set(currentRunId ? [currentRunId] : []); - if (targetProviderStateIdentity && runHeaders) { - for (const run of runHeaders) { + if (targetProviderStateIdentity && invocations) { + for (const invocation of invocations) { + const route = invocation.opening.route; if ( - run.providerStateIdentity === targetProviderStateIdentity && - run.modelId === targetModelId + route.provenance === 'runtime' && + route.providerStateIdentity === targetProviderStateIdentity && + route.modelId === targetModelId ) { - compatibleRunIds.add(run.runId); + compatibleRunIds.add(invocation.runId); } } } diff --git a/packages/runtime/src/model-projection-transition-ledger.ts b/packages/runtime/src/model-projection-transition-ledger.ts index 93cd9aec2e..c4285f5d0e 100644 --- a/packages/runtime/src/model-projection-transition-ledger.ts +++ b/packages/runtime/src/model-projection-transition-ledger.ts @@ -86,14 +86,15 @@ export interface EffectiveModelProjectionReduction { * here: the whole set is the state. */ export async function loadModelProjectionTransitionsFromRunLedger( - runStore: Pick, + runStore: Pick, sessionId: string, + runIds: readonly string[], ): Promise { const byId = new Map(); const unreadableTargets = new Set(); let unscopedUnreadable = 0; - for (const run of await runStore.listSessionRuns(sessionId)) { - for (const event of await runStore.readEvents(sessionId, run.runId)) { + for (const runId of runIds) { + for (const event of await runStore.readEvents(sessionId, runId)) { if (event.type !== MODEL_PROJECTION_TRANSITION_EVENT_TYPE) continue; const transition = decodeLedgerTransition(event, sessionId); if (!transition) { diff --git a/packages/runtime/src/openai-codex-history-compactor.ts b/packages/runtime/src/openai-codex-history-compactor.ts index 8cb442d4a8..c9018cbdce 100644 --- a/packages/runtime/src/openai-codex-history-compactor.ts +++ b/packages/runtime/src/openai-codex-history-compactor.ts @@ -78,7 +78,7 @@ export function buildOpenAiCodexHistoryCompactor(options: BuildOpenAiCodexHistor : input.source.foldedRuntimeEvents; const providerReasoningReplayEventIds = compatibleProviderReasoningReplayEventIds( events, - input.source.runHeaders, + input.source.invocations, options.providerStateIdentity, options.modelId, input.runId, diff --git a/packages/runtime/src/prior-run-context.ts b/packages/runtime/src/prior-run-context.ts index 01533dfafe..baf31ce6d8 100644 --- a/packages/runtime/src/prior-run-context.ts +++ b/packages/runtime/src/prior-run-context.ts @@ -17,162 +17,59 @@ * under the License. */ -import type { AgentRunHeader, AgentRunStore } from '@maka/core/agent-run'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; -import { isSessionInlineRun } from '@maka/core/agent-run'; -import { isTerminalRuntimeEvent } from '@maka/core/runtime-event'; -import type { StoredMessage } from '@maka/core/session'; +import { isSessionInlineInvocation } from '@maka/core/runtime-invocation'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import { buildRuntimeEventModelReplayPlan } from './model-history.js'; -import { backfillRuntimeEventsFromStoredMessages } from './runtime-event-backfill.js'; -import { classifyRuntimeEventTerminalFact } from './runtime-event-read-model.js'; -import { isTerminalRunStatus } from './session-projection-helpers.js'; -import { effectiveRunHeaderFromTerminalFact } from './terminal-run-commit.js'; export interface PriorRuntimeContext { events: RuntimeEvent[]; - runs: AgentRunHeader[]; + invocations: RuntimeInvocationRecord[]; } export interface BuildPriorRuntimeContextInput { sessionId: string; currentRunId: string; currentTurnId: string; - runStore?: AgentRunStore; runtimeEventStore?: RuntimeEventStore; - runStoreAvailable: boolean; runtimeEventStoreAvailable: boolean; - repairRunRuntimeLedger?: (sessionId: string, runId: string) => Promise; - readMessages: () => Promise; -} - -interface PriorRunTerminalFactContext { - events: RuntimeEvent[]; - run: AgentRunHeader; } +/** + * The conversation the model must see before this turn: every earlier + * session-inline invocation's events, in the order the Session committed them. + * + * A prior invocation that never reached a terminal event is still replayed. It + * was stopped while parked on an interaction, or the process died mid-turn, and + * its turn — the user's message included — is conversation either way. There is + * nothing left to reconcile here: the events are the run, so an invocation + * cannot claim an outcome its ledger does not show. + */ export async function buildPriorRuntimeContext( input: BuildPriorRuntimeContextInput, ): Promise { - if ( - !input.runStore || - !input.runtimeEventStore || - !input.runStoreAvailable || - !input.runtimeEventStoreAvailable - ) - return undefined; + const store = input.runtimeEventStore; + if (!store || !input.runtimeEventStoreAvailable) return undefined; - const runs = await input.runStore.listSessionRuns(input.sessionId); - const priorRuns = runs.filter( - (run) => - run.runId !== input.currentRunId && - run.turnId !== input.currentTurnId && - isSessionInlineRun(run), + const invocations = (await store.listSessionInvocations(input.sessionId)).filter( + (invocation) => + invocation.runId !== input.currentRunId && + invocation.turnId !== input.currentTurnId && + isSessionInlineInvocation(invocation.opening), ); - if (priorRuns.length === 0) return undefined; + if (invocations.length === 0) return undefined; - const ordered: Array<{ event: RuntimeEvent; runIndex: number; eventIndex: number }> = []; - for (let runIndex = 0; runIndex < priorRuns.length; runIndex += 1) { - const run = priorRuns[runIndex]!; - if (!isTerminalRunStatus(run.status)) { - const nonTerminal = await readNonTerminalPriorRun(input, run); - if (nonTerminal.run) priorRuns[runIndex] = nonTerminal.run; - appendEvents(ordered, nonTerminal.events, runIndex, input); - continue; - } - let events = await input.runtimeEventStore.readRuntimeEvents(input.sessionId, run.runId); - if (events.length === 0 && (await input.repairRunRuntimeLedger?.(input.sessionId, run.runId))) { - events = await input.runtimeEventStore.readRuntimeEvents(input.sessionId, run.runId); - } - if (events.length === 0) { - const recovered = await backfillMissingPriorRuntimeEvents(input, run); - if (recovered.length === 0 || !recovered.some(isTerminalRuntimeEvent)) { - throw new Error( - `Cannot build model context: RuntimeEvent ledger is missing for prior run ${run.runId}`, - ); + const events: RuntimeEvent[] = []; + for (const invocation of invocations) { + const committed = await store.readRuntimeEvents(input.sessionId, invocation.runId); + for (const event of committed) { + if (event.runId !== input.currentRunId && event.turnId !== input.currentTurnId) { + events.push(event); } - events = recovered; - } - if ( - !events.some(isTerminalRuntimeEvent) && - (await input.repairRunRuntimeLedger?.(input.sessionId, run.runId)) - ) { - events = await input.runtimeEventStore.readRuntimeEvents(input.sessionId, run.runId); - } - if (!events.some(isTerminalRuntimeEvent)) { - throw new Error( - `Cannot build model context: RuntimeEvent ledger has no terminal fact for prior run ${run.runId}`, - ); } - let terminalFact = classifyRuntimeEventTerminalFact(run, events).fact; - if (!terminalFact && (await input.repairRunRuntimeLedger?.(input.sessionId, run.runId))) { - events = await input.runtimeEventStore.readRuntimeEvents(input.sessionId, run.runId); - terminalFact = classifyRuntimeEventTerminalFact(run, events).fact; - } - if (!terminalFact) { - throw new Error( - `Cannot build model context: RuntimeEvent ledger has no valid terminal fact for prior run ${run.runId}`, - ); - } - priorRuns[runIndex] = effectiveRunHeaderFromTerminalFact(run, terminalFact); - appendEvents(ordered, events, runIndex, input); } - - ordered.sort((a, b) => a.runIndex - b.runIndex || a.eventIndex - b.eventIndex); - const events = ordered.map((item) => item.event); if (events.length === 0 || buildRuntimeEventModelReplayPlan(events).items.length === 0) return undefined; - return { events, runs: priorRuns }; -} - -/** - * A prior run whose header never reached a terminal status: it was stopped - * while parked on an interaction, or the process died mid-turn. Its turn is - * still conversation the model must see, so the ledger it does have is - * replayed either way. Dropping the run instead would delete a whole turn — - * the user message included — from every later turn's context, silently and - * for good, because the header never becomes terminal on its own. - */ -async function readNonTerminalPriorRun( - input: BuildPriorRuntimeContextInput, - run: AgentRunHeader, -): Promise<{ events: RuntimeEvent[]; run?: AgentRunHeader }> { - if (!input.runtimeEventStore) return { events: [] }; - // No repair attempt here, unlike the terminal branch: `repairRunTerminalFact` - // returns false for a non-terminal header before reading anything, so calling - // it would be a promise that only ever answers "no". - const events = await input.runtimeEventStore - .readRuntimeEvents(input.sessionId, run.runId) - .catch(() => []); - const terminalFact = classifyRuntimeEventTerminalFact(run, events).fact; - return terminalFact - ? { events, run: effectiveRunHeaderFromTerminalFact(run, terminalFact) } - : { events }; -} - -async function backfillMissingPriorRuntimeEvents( - input: BuildPriorRuntimeContextInput, - run: AgentRunHeader, -): Promise { - let messages: StoredMessage[]; - try { - messages = await input.readMessages(); - } catch { - return []; - } - return backfillRuntimeEventsFromStoredMessages({ run, messages }).events; -} - -function appendEvents( - ordered: Array<{ event: RuntimeEvent; runIndex: number; eventIndex: number }>, - events: readonly RuntimeEvent[], - runIndex: number, - input: BuildPriorRuntimeContextInput, -): void { - for (let eventIndex = 0; eventIndex < events.length; eventIndex += 1) { - const event = events[eventIndex]!; - if (event.runId !== input.currentRunId && event.turnId !== input.currentTurnId) { - ordered.push({ event, runIndex, eventIndex }); - } - } + return { events, invocations }; } diff --git a/packages/runtime/src/runtime-event-backfill.ts b/packages/runtime/src/runtime-event-backfill.ts index ac02be04bb..2b5ab6137c 100644 --- a/packages/runtime/src/runtime-event-backfill.ts +++ b/packages/runtime/src/runtime-event-backfill.ts @@ -17,7 +17,8 @@ * under the License. */ -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationOutcome } from '@maka/core/runtime-invocation'; +import type { RunIdentity } from './terminal-run-commit.js'; import type { PermissionDecisionMessage, StoredMessage, @@ -44,8 +45,22 @@ export interface RuntimeEventBackfillDiagnostic { detail?: unknown; } +/** + * How the imported turn ended, as the importer read it off the transcript. + * + * Without one there is no terminal RuntimeEvent to write: nothing else in a + * StoredMessage transcript states an outcome the ledger can be held to. + */ +export interface RuntimeEventBackfillOutcome { + status: RuntimeInvocationOutcome; + ts: number; + failureClass?: string; + abortSource?: string; +} + export interface RuntimeEventBackfillInput { - run: AgentRunHeader; + run: RunIdentity & { invocationId?: string }; + outcome?: RuntimeEventBackfillOutcome; messages: readonly StoredMessage[]; invocationId?: string; modelHistory?: 'full' | 'conversation_text'; @@ -358,7 +373,14 @@ export function backfillRuntimeEventsFromStoredMessages( } } - const terminal = terminalRuntimeEvent({ run: input.run, turnMessages, invocationId, newId, now }); + const terminal = terminalRuntimeEvent({ + run: input.run, + outcome: input.outcome, + turnMessages, + invocationId, + newId, + now, + }); if (terminal.event) { events.push(terminal.event); } else if (terminal.diagnostic) { @@ -418,14 +440,15 @@ function terminalRecoveryState( } function terminalRuntimeEvent(input: { - run: AgentRunHeader; + run: RunIdentity; + outcome: RuntimeEventBackfillOutcome | undefined; turnMessages: readonly StoredMessage[]; invocationId: string; newId: () => string; now: () => number; }): { event?: RuntimeEvent; diagnostic?: RuntimeEventBackfillDiagnostic } { const turnState = latestTurnState(input.turnMessages); - const status = terminalStatus(input.run, turnState); + const status = terminalStatus(input.outcome, turnState); if (!status) { return { diagnostic: { @@ -435,19 +458,19 @@ function terminalRuntimeEvent(input: { detail: { runId: input.run.runId, turnId: input.run.turnId, - runStatus: input.run.status, + declaredStatus: input.outcome?.status, turnStatus: turnState?.status, }, }, }; } - const ts = turnState?.ts ?? input.run.completedAt ?? input.run.updatedAt; + const ts = turnState?.ts ?? input.outcome?.ts ?? input.now(); const failureClass = - status === 'failed' ? (turnState?.errorClass ?? input.run.failureClass) : undefined; + status === 'failed' ? (turnState?.errorClass ?? input.outcome?.failureClass) : undefined; const abortSource = status === 'aborted' ? (turnState?.abortSource ?? - input.run.abortSource ?? + input.outcome?.abortSource ?? (turnState?.status === 'aborted' ? 'unknown' : undefined)) : undefined; return { @@ -476,19 +499,20 @@ function terminalRuntimeEvent(input: { } function terminalStatus( - run: AgentRunHeader, + outcome: RuntimeEventBackfillOutcome | undefined, turnState: TurnStateMessage | undefined, ): RuntimeEventStatus | undefined { const legacyStatus = turnState?.status; - if (legacyStatus === 'completed' || run.status === 'completed') return 'completed'; - if (legacyStatus === 'failed' && run.status === 'failed') return 'failed'; + const declared = outcome?.status; + if (legacyStatus === 'completed' || declared === 'completed') return 'completed'; + if (legacyStatus === 'failed' && declared === 'failed') return 'failed'; if ( - (legacyStatus === 'failed' || run.status === 'failed') && - (run.failureClass || turnState?.errorClass) + (legacyStatus === 'failed' || declared === 'failed') && + (outcome?.failureClass || turnState?.errorClass) ) return 'failed'; - if (legacyStatus === 'aborted' && run.status === 'cancelled') return 'aborted'; - if ((legacyStatus === 'aborted' || run.status === 'cancelled') && turnState?.abortSource) + if (legacyStatus === 'aborted' && declared === 'cancelled') return 'aborted'; + if ((legacyStatus === 'aborted' || declared === 'cancelled') && turnState?.abortSource) return 'aborted'; return undefined; } diff --git a/packages/runtime/src/runtime-event-read-model.ts b/packages/runtime/src/runtime-event-read-model.ts index 4de08bacb0..d3cbba2a6e 100644 --- a/packages/runtime/src/runtime-event-read-model.ts +++ b/packages/runtime/src/runtime-event-read-model.ts @@ -17,7 +17,7 @@ * under the License. */ -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { AssistantStepContentKind, StoredMessage, TurnStatus } from '@maka/core/session'; import type { RuntimeEvent, RuntimeEventStatus } from '@maka/core/runtime-event'; import type { ToolActivityKind, ToolResultContent } from '@maka/core/events'; @@ -148,7 +148,9 @@ export interface RuntimeEventReadModelProjection { } export interface ProjectRuntimeEventsToStoredMessagesOptions { - runHeaders: readonly AgentRunHeader[] | Readonly>; + invocations: + | readonly RuntimeInvocationRecord[] + | Readonly>; canonicalPermissionOutcomes?: ReadonlyMap; } @@ -179,7 +181,7 @@ export interface RuntimeEventTerminalFactResult { } interface ProjectionState { - headers: Map; + invocations: Map; diagnostics: RuntimeEventReadModelDiagnostic[]; toolNameByUseId: Map; permissionRequestById: Map< @@ -217,7 +219,7 @@ export function projectRuntimeEventsToStoredMessages( options: ProjectRuntimeEventsToStoredMessagesOptions, ): RuntimeEventReadModelProjection { const state: ProjectionState = { - headers: normalizeHeaders(options.runHeaders), + invocations: normalizeInvocations(options.invocations), diagnostics: [], toolNameByUseId: new Map(), permissionRequestById: new Map(), @@ -249,6 +251,11 @@ export function projectRuntimeEventsToStoredMessages( case 'thinking': projected = projectThinking(event, state, messages) || projected; break; + case 'invocation_opened': + // The opening fact records route, configuration and lineage once per + // invocation. Every reader joins it by invocationId; it has no chat row. + projected = true; + break; case 'error': if (!isTerminalRuntimeEvent(event)) { diagnostic( @@ -513,15 +520,15 @@ export function compareRuntimeReadModelMessages( } export function classifyRuntimeEventTerminalFact( - header: AgentRunHeader, + invocation: Pick, events: readonly RuntimeEvent[], ): RuntimeEventTerminalFactResult { const diagnostics: RuntimeEventReadModelDiagnostic[] = []; if (events.length === 0) { diagnostics.push( readModelDiagnostic('incomplete_event', 'runtime ledger has no readable RuntimeEvents', { - runId: header.runId, - turnId: header.turnId, + runId: invocation.runId, + turnId: invocation.turnId, }), ); return { diagnostics }; @@ -530,9 +537,9 @@ export function classifyRuntimeEventTerminalFact( const terminalSignals = events.filter( (event) => !isPartialRuntimeEvent(event) && - event.sessionId === header.sessionId && - event.runId === header.runId && - event.turnId === header.turnId && + event.sessionId === invocation.sessionId && + event.runId === invocation.runId && + event.turnId === invocation.turnId && isTerminalRuntimeEvent(event), ); @@ -541,7 +548,7 @@ export function classifyRuntimeEventTerminalFact( readModelDiagnostic( 'incomplete_event', 'runtime ledger has no matching terminal RuntimeEvent', - { runId: header.runId, turnId: header.turnId }, + { runId: invocation.runId, turnId: invocation.turnId }, ), ); return { diagnostics }; @@ -552,8 +559,8 @@ export function classifyRuntimeEventTerminalFact( 'incomplete_event', 'runtime ledger has multiple matching terminal RuntimeEvents', { - runId: header.runId, - turnId: header.turnId, + runId: invocation.runId, + turnId: invocation.turnId, eventIds: terminalSignals.map((event) => event.id), }, ), @@ -575,8 +582,8 @@ export function classifyRuntimeEventTerminalFact( if (terminalEvent.status === 'completed') { const fact: RuntimeEventTerminalFact = { - runId: header.runId, - turnId: header.turnId, + runId: invocation.runId, + turnId: invocation.turnId, runStatus: 'completed', turnStatus: 'completed', terminalEvent, @@ -585,48 +592,51 @@ export function classifyRuntimeEventTerminalFact( return { fact, diagnostics }; } + // A terminal event is the run's ending, and it is immutable once written, so + // an omitted failure class or abort source is a detail nobody can ever supply + // afterwards. Withholding the fact over it would only leave the reader with a + // run that ended and no way to say so; the omission is worth a diagnostic, not + // a refusal. if (terminalEvent.status === 'failed') { - const failureClass = failureClassFromRuntimeEvent(terminalEvent, header); + const failureClass = failureClassFromRuntimeEvent(terminalEvent); if (!failureClass) { diagnostics.push( readModelDiagnostic( 'incomplete_event', - 'failed terminal RuntimeEvent requires a stable failure class', + 'failed terminal RuntimeEvent states no failure class', terminalEvent, ), ); - return { diagnostics }; } const fact: RuntimeEventTerminalFact = { - runId: header.runId, - turnId: header.turnId, + runId: invocation.runId, + turnId: invocation.turnId, runStatus: 'failed', turnStatus: 'failed', terminalEvent, - failureClass, + failureClass: failureClass ?? 'unknown', diagnostics, }; return { fact, diagnostics }; } - const abortSource = abortSourceFromRuntime(terminalEvent, header); + const abortSource = abortSourceFromRuntime(terminalEvent); if (!abortSource) { diagnostics.push( readModelDiagnostic( 'incomplete_event', - 'aborted terminal RuntimeEvent requires an abort source', + 'aborted terminal RuntimeEvent states no abort source', terminalEvent, ), ); - return { diagnostics }; } const fact: RuntimeEventTerminalFact = { - runId: header.runId, - turnId: header.turnId, + runId: invocation.runId, + turnId: invocation.turnId, runStatus: 'cancelled', turnStatus: 'aborted', terminalEvent, - abortSource, + abortSource: abortSource ?? 'unknown', diagnostics, }; return { fact, diagnostics }; @@ -646,13 +656,13 @@ function projectText( } if (event.role === 'model') { - const header = state.headers.get(event.runId); - if (!header?.modelId) { + const invocation = state.invocations.get(event.runId); + if (!invocation?.opening.route.modelId) { diagnostic( state, event, 'incomplete_event', - 'model text RuntimeEvent requires AgentRunHeader.modelId', + 'model text RuntimeEvent requires the opening fact of its invocation', ); return false; } @@ -668,7 +678,7 @@ function projectText( ? { providerOptions: structuredClone(event.content.providerOptions) } : {}), ...(contentOrder ? { contentOrder } : {}), - modelId: header.modelId, + modelId: invocation.opening.route.modelId, }); attachPendingThinking(event, state, messages, assistantId); return true; @@ -1139,17 +1149,18 @@ function projectTerminalTurnState( state: ProjectionState, messages: StoredMessage[], ): boolean { - const header = state.headers.get(event.runId); - if (!header) { + const invocation = state.invocations.get(event.runId); + if (!invocation) { diagnostic( state, event, 'incomplete_event', - 'terminal RuntimeEvent requires an AgentRunHeader', + 'terminal RuntimeEvent requires the opening fact of its invocation', ); return false; } - const status = turnStatusFor(event.status, header.status); + const lineage = invocation.opening.lineage; + const status = turnStatusFor(event.status); if (!status) { diagnostic( state, @@ -1159,9 +1170,8 @@ function projectTerminalTurnState( ); return false; } - const abortSource = status === 'aborted' ? abortSourceFromRuntime(event, header) : undefined; - const failureClass = - status === 'failed' ? failureClassFromRuntimeEvent(event, header) : undefined; + const abortSource = status === 'aborted' ? abortSourceFromRuntime(event) : undefined; + const failureClass = status === 'failed' ? failureClassFromRuntimeEvent(event) : undefined; const partialOutputRetained = messages.some( (message) => message.turnId === event.turnId && @@ -1174,13 +1184,13 @@ function projectTerminalTurnState( turnId: event.turnId, ts: event.ts, status, - ...(header.parentTurnId ? { parentTurnId: header.parentTurnId } : {}), - ...(header.retriedFromTurnId ? { retriedFromTurnId: header.retriedFromTurnId } : {}), - ...(header.regeneratedFromTurnId - ? { regeneratedFromTurnId: header.regeneratedFromTurnId } + ...(lineage?.parentTurnId ? { parentTurnId: lineage.parentTurnId } : {}), + ...(lineage?.retriedFromTurnId ? { retriedFromTurnId: lineage.retriedFromTurnId } : {}), + ...(lineage?.regeneratedFromTurnId + ? { regeneratedFromTurnId: lineage.regeneratedFromTurnId } : {}), - ...(header.branchOfTurnId ? { branchOfTurnId: header.branchOfTurnId } : {}), - ...(header.parentSessionId ? { parentSessionId: header.parentSessionId } : {}), + ...(lineage?.branchOfTurnId ? { branchOfTurnId: lineage.branchOfTurnId } : {}), + ...(lineage?.parentSessionId ? { parentSessionId: lineage.parentSessionId } : {}), ...(status === 'aborted' ? { abortedAt: event.ts } : {}), ...(abortSource ? { abortSource } : {}), ...(status === 'failed' ? { errorClass: failureClass ?? 'unknown' } : {}), @@ -1195,22 +1205,9 @@ function projectTerminalTurnState( kind: 'step_limit', }); } - if (status === 'failed' && !failureClass) { - diagnostic( - state, - event, - 'incomplete_event', - 'failed terminal event did not carry an exact AgentRunHeader.failureClass', - ); - } - if (status === 'aborted' && !abortSource) { - diagnostic( - state, - event, - 'incomplete_event', - 'abortSource is not present in RuntimeEvent or AgentRunHeader metadata', - ); - } + // An omitted failure class or abort source is `classifyRuntimeEventTerminalFact`'s + // observation to make. Repeating it here would only turn a transcript row that + // already reads `unknown` into an unreadable Session. return true; } @@ -1273,28 +1270,37 @@ function thinkingMessageId(event: RuntimeEvent): string { return event.refs?.providerEventId ?? event.refs?.storedMessageId ?? event.id; } -function abortSourceFromRuntime(event: RuntimeEvent, header: AgentRunHeader): string | undefined { +/** + * Why this invocation failed, according to its own terminal event. + * + * `undefined` for an invocation that is still running or did not fail. There is + * no second place to look: the event that ends the run also states the class. + */ +export function runtimeInvocationFailureClass(invocation: { + terminalEvent?: RuntimeEvent; +}): string | undefined { + const terminalEvent = invocation.terminalEvent; + if (terminalEvent?.status !== 'failed') return undefined; + return failureClassFromRuntimeEvent(terminalEvent); +} + +function abortSourceFromRuntime(event: RuntimeEvent): string | undefined { return ( stringStateDelta(event, 'abortSource') ?? stringStateDelta(event, 'source') ?? stringRecordValue(event.refs, 'abortSource') ?? - stringRecordValue(event.refs, 'source') ?? - stringRecordValue(header as unknown as Record, 'abortSource') + stringRecordValue(event.refs, 'source') ); } -function failureClassFromRuntimeEvent( - event: RuntimeEvent, - header: AgentRunHeader, -): string | undefined { +function failureClassFromRuntimeEvent(event: RuntimeEvent): string | undefined { const failureClass = stringStateDelta(event, 'failureClass') ?? stringStateDelta(event, 'errorClass') ?? stringStateDelta(event, 'reason') ?? stringStateDelta(event, 'code') ?? (event.content?.kind === 'error' ? nonEmptyString(event.content.reason) : undefined) ?? - (event.content?.kind === 'error' ? nonEmptyString(event.content.code) : undefined) ?? - header.failureClass; + (event.content?.kind === 'error' ? nonEmptyString(event.content.code) : undefined); // Retired outcome. The runtime no longer decides locally that a request // cannot be shaped to fit — the provider rejects it and recovery compacts and // retries — so a turn that ends over the window is a context overflow like any @@ -1337,25 +1343,22 @@ function toolUseIdFor(event: RuntimeEvent): string | undefined { return event.content.id || event.refs?.toolCallId; } -function normalizeHeaders( - headers: readonly AgentRunHeader[] | Readonly>, -): Map { - if (Array.isArray(headers)) { - return new Map(headers.map((header) => [header.runId, header])); - } - return new Map(Object.values(headers).map((header) => [header.runId, header])); +function normalizeInvocations( + invocations: + | readonly RuntimeInvocationRecord[] + | Readonly>, +): Map { + const values = Array.isArray(invocations) + ? (invocations as readonly RuntimeInvocationRecord[]) + : Object.values(invocations as Readonly>); + return new Map(values.map((invocation) => [invocation.runId, invocation])); } -function turnStatusFor( - eventStatus: RuntimeEventStatus | undefined, - runStatus: AgentRunHeader['status'], -): TurnStatus | undefined { +/** The terminal event states the outcome; nothing else is allowed to disagree. */ +function turnStatusFor(eventStatus: RuntimeEventStatus | undefined): TurnStatus | undefined { if (eventStatus === 'completed') return 'completed'; if (eventStatus === 'failed') return 'failed'; if (eventStatus === 'aborted' || eventStatus === 'cancelled') return 'aborted'; - if (runStatus === 'completed') return 'completed'; - if (runStatus === 'failed') return 'failed'; - if (runStatus === 'cancelled') return 'aborted'; return undefined; } diff --git a/packages/runtime/src/runtime-kernel.ts b/packages/runtime/src/runtime-kernel.ts index 2ec0173e0a..80fa08b312 100644 --- a/packages/runtime/src/runtime-kernel.ts +++ b/packages/runtime/src/runtime-kernel.ts @@ -17,7 +17,8 @@ * under the License. */ -import type { AgentRunHeader, AgentRunStore } from '@maka/core/agent-run'; +import type { AgentRunStore } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import { decodeRuntimeBoundaryCursor, type ContinuationClaimV1, @@ -26,6 +27,7 @@ import { import { isTerminalRuntimeEvent, type RuntimeEvent, + type RuntimeEventInvocationOpenedContent, type ToolBoundaryProtocol, } from '@maka/core/runtime-event'; import type { @@ -148,6 +150,7 @@ import { bindRuntimeInteractionRun, isHostedInteractionRequestEvent, isHostedInteractionSettlementAckEvent, + isShutdownCancelledInteractionAdmission, type RuntimeInteractionAuthority, type RuntimeInteractionRunBinding, type RuntimeInteractionRunClosureReason, @@ -267,7 +270,6 @@ export interface RuntimeKernelDeps { now: () => number; childTools?: readonly MakaTool[]; resolveChildTools?: (sessionId: string) => Promise; - repairRunRuntimeLedger?: (sessionId: string, runId: string) => Promise; shellRuns?: ShellRunProcessManager; cleanupHistoryCompactArtifacts?: (input: HistoryCompactCleanupRequest) => Promise; inspectContinuationSafety?: (sessionId: string) => Promise; @@ -664,7 +666,6 @@ export class RuntimeKernel implements RuntimeKernelLike { ...(this.deps.toolBoundaryProtocol ? { toolBoundaryProtocol: this.deps.toolBoundaryProtocol } : {}), - repairRunRuntimeLedger: this.deps.repairRunRuntimeLedger, newId: this.deps.newId, now: this.deps.now, ...(workspaceIdentity ? { workspaceIdentity } : {}), @@ -747,10 +748,16 @@ export class RuntimeKernel implements RuntimeKernelLike { } const header = await this.deps.store.readHeader(continuation.sessionId); - const [sourceRun, sessionRuns] = await Promise.all([ - this.deps.runStore.readRun(continuation.sessionId, continuation.sourceRunId), - this.deps.runStore.listSessionRuns(continuation.sessionId), - ]); + const sessionRuns = await this.deps.runtimeEventStore.listSessionInvocations( + continuation.sessionId, + ); + const sourceRun = sessionRuns.find((run) => run.runId === continuation.sourceRunId); + if (!sourceRun) { + throw new RuntimeContinuationRevalidationError( + 'source_identity_changed', + 'Runtime continuation source run no longer exists', + ); + } const targetProviderStateIdentity = ( await this.deps.backends.prepare(header.backend, { sessionId: continuation.sessionId, @@ -760,7 +767,7 @@ export class RuntimeKernel implements RuntimeKernelLike { }) ).providerStateIdentity; const admissionRoute: ContinuationReplayAdmissionRoute = { - runHeaders: sessionRuns, + invocations: sessionRuns, targetProviderStateIdentity, targetModelId: header.model, }; @@ -780,7 +787,7 @@ export class RuntimeKernel implements RuntimeKernelLike { const effectiveOrchestration = effectiveOrchestrationForRun(sourceRun, header); const effectiveToolMode = effectiveToolModeForRun(sourceRun); const claimedAt = this.deps.now(); - const targetRunHeader = continuationTargetRunHeaderForExecution({ + const targetOpening = continuationTargetOpeningForExecution({ continuation, sessionHeader: header, userInput, @@ -788,9 +795,8 @@ export class RuntimeKernel implements RuntimeKernelLike { effectiveOrchestration, effectiveToolMode, targetProviderStateIdentity, - claimedAt, }); - const claim = continuationClaimForExecution(continuation, claimedAt, targetRunHeader); + const claim = continuationClaimForExecution(continuation, claimedAt, targetOpening); const claimResult = await continuationAuthority.claimContinuation({ claim }); if (claimResult.kind !== 'acquired') { throw new RuntimeContinuationRevalidationError( @@ -800,19 +806,21 @@ export class RuntimeKernel implements RuntimeKernelLike { } await this.deps.continuationFailpoint?.('after_continuation_claim_committed'); - const existingClaim = sessionRuns.find( - (runHeader) => - runHeader.continuationSource?.sourceRunId === continuation.sourceRunId && - runHeader.continuationSource.sourceRuntimeEventHighWater === - continuation.sourceRuntimeEventHighWater, - ); + const existingClaim = sessionRuns.find((candidate) => { + const source = candidate.opening.source; + return ( + source.kind === 'continuation' && + source.sourceRunId === continuation.sourceRunId && + source.sourceRuntimeEventHighWater === continuation.sourceRuntimeEventHighWater + ); + }); if (existingClaim) { throw new RuntimeContinuationRevalidationError( 'continuation_claim_conflict', `Runtime continuation source already has a continuation child: ${existingClaim.runId}`, ); } - const existingTarget = sessionRuns.find((runHeader) => runHeader.runId === continuation.runId); + const existingTarget = sessionRuns.find((candidate) => candidate.runId === continuation.runId); if (existingTarget) { throw new RuntimeContinuationRevalidationError( 'target_run_conflict', @@ -834,12 +842,15 @@ export class RuntimeKernel implements RuntimeKernelLike { ...(continuationToolBoundaryProtocol ? { toolBoundaryProtocol: continuationToolBoundaryProtocol } : {}), - repairRunRuntimeLedger: this.deps.repairRunRuntimeLedger, newId: this.deps.newId, now: this.deps.now, workspaceIdentity: continuation.safetySnapshot.workspaceIdentity, effectiveOrchestration, - claimedRunHeader: claim.targetRunHeader, + // Round-tripped through the claim on purpose: openInvocation compares it + // against the opening it computes, so every continuation proves the claim + // still authorises the run about to execute. + claimedOpening: claim.targetOpening, + claimedOpenedAt: claimedAt, effectiveToolMode, continuationFailpoint: this.deps.continuationFailpoint, commitContinuationStart: async (startedAt) => { @@ -854,6 +865,10 @@ export class RuntimeKernel implements RuntimeKernelLike { partial: false, role: 'system', author: 'system', + modelVisibility: 'hidden', + // The start event is event 1 of the target invocation, so it is + // also where that invocation's opening fact lives. + content: claim.targetOpening, actions: { ...(continuationToolBoundaryProtocol ? { @@ -987,7 +1002,6 @@ export class RuntimeKernel implements RuntimeKernelLike { ...(this.deps.toolBoundaryProtocol ? { toolBoundaryProtocol: this.deps.toolBoundaryProtocol } : {}), - repairRunRuntimeLedger: this.deps.repairRunRuntimeLedger, newId: this.deps.newId, now: this.deps.now, effectiveOrchestration: resolveEffectiveOrchestration('default', undefined), @@ -1045,7 +1059,7 @@ export class RuntimeKernel implements RuntimeKernelLike { turnId: run.turnId, runId: run.runId, runtimeContext: begin.runtimeContext, - runtimeContextRunHeaders: begin.runtimeContextRunHeaders, + runtimeContextInvocations: begin.runtimeContextInvocations, }); if (run.isStopped()) return; const tokenUsageEvent: TokenUsageEvent = { @@ -1380,7 +1394,7 @@ export class RuntimeKernel implements RuntimeKernelLike { text: '', context: [], runtimeContext: continuation.runtimeContext, - runtimeContextRunHeaders: admissionRoute.runHeaders, + runtimeContextInvocations: admissionRoute.invocations, continuation: continuationMetadata, }, onSessionEvent: async (sessionEvent, runtimeEvent) => { @@ -1492,6 +1506,9 @@ export class RuntimeKernel implements RuntimeKernelLike { execution: PendingExecutionClaim, error: unknown, ): Promise { + // A draining authority refused the start because everything is stopping, not + // because this run went wrong, so the run ends cancelled rather than failed. + if (isShutdownCancelledInteractionAdmission(error)) run.stop(undefined); try { await owners.failStart(error); } catch (failure) { @@ -2270,6 +2287,13 @@ export class RuntimeKernel implements RuntimeKernelLike { ); } + /** Every run this Session has opened, enumerated from the event spine. */ + private async sessionRunIds(sessionId: string): Promise { + const store = this.deps.runtimeEventStore; + if (!store) return []; + return (await store.listSessionInvocations(sessionId)).map((invocation) => invocation.runId); + } + private buildBackendRecorderHooks(input: { sessionId: string; }): Pick< @@ -2313,8 +2337,12 @@ export class RuntimeKernel implements RuntimeKernelLike { checkpoint: HistoryCompactCheckpoint, turnId: string, ) => this.historyCompactCoordinator.record(sessionId, checkpoint, runFor(turnId)), - loadModelProjectionTransitions: () => - loadModelProjectionTransitionsFromRunLedger(this.deps.runStore!, sessionId), + loadModelProjectionTransitions: async () => + loadModelProjectionTransitionsFromRunLedger( + this.deps.runStore!, + sessionId, + await this.sessionRunIds(sessionId), + ), recordModelProjectionTransition: ( transition: ModelProjectionTransition, turnId: string, @@ -2838,7 +2866,7 @@ async function revalidateContinuationBoundary( function continuationClaimForExecution( continuation: RuntimeContinuation, claimedAt: number, - targetRunHeader: AgentRunHeader, + targetOpening: RuntimeEventInvocationOpenedContent, ): ContinuationClaimV1 { if ( !continuation.claimId || @@ -2864,12 +2892,19 @@ function continuationClaimForExecution( runId: continuation.runId, turnId: continuation.turnId, }, - targetRunHeader, + targetOpening, claimedAt, }; } -function continuationTargetRunHeaderForExecution(input: { +/** + * The opening fact the claim freezes for its target invocation. + * + * It has to be byte-identical to the one the target's own AgentRun computes: + * the run compares them before it starts, so a claim can only admit the + * execution it actually authorised. + */ +function continuationTargetOpeningForExecution(input: { continuation: RuntimeContinuation; sessionHeader: SessionHeader; userInput: UserMessageInput; @@ -2877,48 +2912,16 @@ function continuationTargetRunHeaderForExecution(input: { effectiveOrchestration: EffectiveOrchestration; effectiveToolMode: ToolMode; targetProviderStateIdentity: `sha256:${string}` | undefined; - claimedAt: number; -}): AgentRunHeader { - const { - continuation, - sessionHeader, - userInput, - effectiveOrchestration, - effectiveToolMode, - claimedAt, - } = input; +}): RuntimeEventInvocationOpenedContent { + const { continuation, sessionHeader, userInput, effectiveOrchestration, effectiveToolMode } = + input; if (!continuation.claimId || !continuation.boundary) { throw new RuntimeContinuationRevalidationError( 'source_identity_changed', 'Runtime continuation is missing its durable target-header identity', ); } - const source = continuation.boundary.segments.at(-1)!; - return { - runId: continuation.runId, - invocationId: continuation.invocationId, - sessionId: continuation.sessionId, - turnId: continuation.turnId, - status: 'created', - backendKind: sessionHeader.backend, - ...(sessionHeader.llmConnectionId === undefined - ? {} - : { llmConnectionId: sessionHeader.llmConnectionId }), - ...(input.targetProviderStateIdentity - ? { providerStateIdentity: input.targetProviderStateIdentity } - : {}), - llmConnectionSlug: sessionHeader.llmConnectionSlug, - modelId: sessionHeader.model, - cwd: sessionHeader.cwd, - workspaceIdentity: input.workspaceIdentity, - permissionMode: sessionHeader.permissionMode, - collaborationMode: sessionHeader.collaborationMode ?? 'agent', - orchestrationMode: effectiveOrchestration.mode, - orchestrationSource: effectiveOrchestration.source, - agentSwarmAuthorization: effectiveOrchestration.agentSwarmAuthorization, - toolMode: effectiveToolMode, - createdAt: claimedAt, - updatedAt: claimedAt, + const lineage = { parentRunId: continuation.sourceRunId, ...(userInput.parentTurnId ? { parentTurnId: userInput.parentTurnId } : {}), ...(userInput.retriedFromTurnId ? { retriedFromTurnId: userInput.retriedFromTurnId } : {}), @@ -2929,17 +2932,51 @@ function continuationTargetRunHeaderForExecution(input: { ...(userInput.parentSessionId ? { parentSessionId: userInput.parentSessionId } : {}), ...(userInput.agentId ? { agentId: userInput.agentId } : {}), ...(userInput.agentName ? { agentName: userInput.agentName } : {}), - continuationSource: { - protocol: 'continuation_source_v2', + }; + return { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: + sessionHeader.llmConnectionId === undefined + ? { + provenance: 'unknown', + backendKind: sessionHeader.backend, + llmConnectionSlug: sessionHeader.llmConnectionSlug, + modelId: sessionHeader.model, + } + : { + provenance: 'runtime', + backendKind: sessionHeader.backend, + llmConnectionId: sessionHeader.llmConnectionId, + llmConnectionSlug: sessionHeader.llmConnectionSlug, + modelId: sessionHeader.model, + ...(input.targetProviderStateIdentity + ? { providerStateIdentity: input.targetProviderStateIdentity } + : {}), + }, + configuration: { + cwd: sessionHeader.cwd, + permissionMode: sessionHeader.permissionMode, + collaborationMode: sessionHeader.collaborationMode ?? 'agent', + orchestrationMode: effectiveOrchestration.mode, + orchestrationSource: effectiveOrchestration.source, + toolMode: effectiveToolMode, + ...(effectiveOrchestration.agentSwarmAuthorization !== undefined + ? { agentSwarmAuthorization: effectiveOrchestration.agentSwarmAuthorization } + : {}), + workspaceIdentity: input.workspaceIdentity, + }, + root: { kind: 'user' }, + source: { + kind: 'continuation', + sourceInvocationId: continuation.sourceInvocationId, + sourceRunId: continuation.sourceRunId, + sourceTurnId: continuation.sourceTurnId, + sourceRuntimeEventHighWater: continuation.sourceRuntimeEventHighWater, claimId: continuation.claimId, boundaryDigest: continuation.boundary.manifestDigest, - sourceInvocationId: source.identity.invocationId, - sourceRunId: source.identity.runId, - sourceTurnId: source.identity.turnId, - sourceRuntimeEventHighWater: source.position.lastEventSeq, - sourcePrefixDigest: source.prefixDigest, - replayManifestDigest: continuation.boundary.manifestDigest, }, + lineage, }; } @@ -2994,7 +3031,7 @@ function consumeAdmittedRuntimeContinuation(input: { const replay = buildRuntimeEventModelReplayPlan(continuation.runtimeContext); const providerReasoningReplayEventIds = compatibleProviderReasoningReplayEventIds( continuation.runtimeContext, - input.admissionRoute.runHeaders, + input.admissionRoute.invocations, input.admissionRoute.targetProviderStateIdentity, input.admissionRoute.targetModelId, ); @@ -3057,7 +3094,7 @@ function assertRuntimeContinuationEnvelope(continuation: RuntimeContinuation): v function assertContinuationSourceUnchanged( continuation: RuntimeContinuation, - sourceRun: AgentRunHeader, + sourceRun: RuntimeInvocationRecord, sourceEvents: readonly RuntimeEvent[], ): void { if ( @@ -3071,9 +3108,10 @@ function assertContinuationSourceUnchanged( ); } const terminalEvents = matchingTerminalRuntimeEvents(sourceRun, sourceEvents); - const terminalStatus = - terminalEvents.length === 1 ? terminalRunStatusFromRuntimeEvent(terminalEvents[0]!) : undefined; - if (terminalStatus === undefined || terminalStatus !== sourceRun.status) { + if ( + terminalEvents.length !== 1 || + terminalRunStatusFromRuntimeEvent(terminalEvents[0]!) === undefined + ) { throw new RuntimeContinuationRevalidationError( 'source_terminal_changed', 'Runtime continuation source is no longer terminal', @@ -3296,25 +3334,22 @@ class RuntimeRunOwnerScope { } function effectiveOrchestrationForRun( - run: AgentRunHeader, + run: RuntimeInvocationRecord, session: SessionHeader, ): EffectiveOrchestration { - if ( - run.orchestrationMode !== undefined && - run.orchestrationSource !== undefined && - run.agentSwarmAuthorization !== undefined - ) { + const configuration = run.opening.configuration; + if (configuration.agentSwarmAuthorization !== undefined) { return { - mode: run.orchestrationMode, - source: run.orchestrationSource, - agentSwarmAuthorization: run.agentSwarmAuthorization, + mode: configuration.orchestrationMode, + source: configuration.orchestrationSource, + agentSwarmAuthorization: configuration.agentSwarmAuthorization, }; } return resolveEffectiveOrchestration(session.orchestrationMode, undefined); } -function effectiveToolModeForRun(run: AgentRunHeader): ToolMode { - return run.toolMode ?? DEFAULT_TOOL_MODE; +function effectiveToolModeForRun(run: RuntimeInvocationRecord): ToolMode { + return run.opening.configuration.toolMode; } function assertNoRemovedChildAgentRunLineage(input: UserMessageInput): void { diff --git a/packages/runtime/src/runtime-ledger-repair.ts b/packages/runtime/src/runtime-ledger-repair.ts index 0c00e01f24..8ab8949c99 100644 --- a/packages/runtime/src/runtime-ledger-repair.ts +++ b/packages/runtime/src/runtime-ledger-repair.ts @@ -19,33 +19,28 @@ import { createHash } from 'node:crypto'; import { deriveTurnRecords } from '@maka/core/session'; -import { isSessionInlineRun } from '@maka/core/agent-run'; +import { DEFAULT_TOOL_MODE } from '@maka/core/tool-mode'; import { isTerminalRuntimeEvent } from '@maka/core/runtime-event'; -import type { AgentRunHeader, AgentRunStore } from '@maka/core/agent-run'; -import type { RuntimeEvent } from '@maka/core/runtime-event'; +import type { RuntimeEvent, RuntimeEventInvocationOpenedContent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; +import { + buildInvocationOpenedEvent, + isSessionInlineInvocation, +} from '@maka/core/runtime-invocation'; +import type { + RuntimeInvocationOutcome, + RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; import type { SessionHeader } from '@maka/core/session'; import type { StoredMessage, TurnRecord } from '@maka/core/session'; -import type { AgentRunLineage } from './agent-run.js'; import { backfillRuntimeEventsFromStoredMessages } from './runtime-event-backfill.js'; +import type { RuntimeEventBackfillOutcome } from './runtime-event-backfill.js'; import { projectRuntimeEventUserMessage } from './runtime-event-read-model.js'; -import { - buildRecoveredTerminalRuntimeEvent, - commitTerminalRunWithRuntimeFact, -} from './terminal-run-commit.js'; export interface RuntimeLedgerRepairDeps { - runStore: AgentRunStore; runtimeEventStore: RuntimeEventStore; readMessages(sessionId: string): Promise; appendMessage(sessionId: string, message: StoredMessage): Promise; - appendTurnState( - sessionId: string, - turnId: string, - status: TurnRecord['status'], - lineage?: AgentRunLineage, - options?: { ts?: number; errorClass?: string; abortSource?: string }, - ): Promise; newId: () => string; now: () => number; } @@ -76,24 +71,23 @@ export class RuntimeLedgerRepair { constructor(private readonly deps: RuntimeLedgerRepairDeps) {} - async repairMissingTerminalFactOnce(sessionId: string, runId: string): Promise { - const run = await this.deps.runStore.readRun(sessionId, runId).catch(() => undefined); - if (!run) return false; - return this.repairRunTerminalFact(sessionId, run); - } - + /** + * Give an imported transcript a runtime spine: one invocation per turn, opened + * by its own opening fact and closed by its own terminal event. + * + * The transcript is the only evidence there is, so a turn it cannot close is + * refused rather than imported half-formed. Re-running is a no-op: a turn + * whose invocation already exists is left exactly as it is. + */ async materializeTranscriptLedger(header: SessionHeader): Promise { const sessionId = header.id; return this.withRepairQueue(sessionId, 'transcript-runs', async () => { - const [messages, runs] = await Promise.all([ - this.deps.readMessages(sessionId), - this.deps.runStore.listSessionRuns(sessionId), - ]); + const messages = await this.deps.readMessages(sessionId); const ledgerMessages = messages.filter( (message) => message.type !== 'user' || message.steeringEventId === undefined, ); - const inlineRunsByTurn = new Map( - runs.filter(isSessionInlineRun).map((run) => [run.turnId, run] as const), + const openedTurnIds = new Set( + (await this.listInlineInvocations(sessionId)).map((invocation) => invocation.turnId), ); const messagesByTurn = groupMessagesByTurn(ledgerMessages); const turns = deriveTurnRecords(ledgerMessages).filter((turn) => @@ -101,33 +95,30 @@ export class RuntimeLedgerRepair { ); if (turns.length === 0) return; - const firstCreatedAt = Math.max(0, header.createdAt - turns.length); + const firstOpenedAt = Math.max(0, header.createdAt - turns.length); for (const [index, turn] of turns.entries()) { + if (openedTurnIds.has(turn.turnId)) continue; const turnMessages = messagesByTurn.get(turn.turnId) ?? []; const runId = transcriptRunId(sessionId, turn.turnId); - const existing = inlineRunsByTurn.get(turn.turnId); - if (existing && existing.runId !== runId) continue; - const run = - existing ?? - (await this.deps.runStore.createRun( - transcriptRunHeader({ - header, - turn, - turnMessages, - runId, - createdAt: firstCreatedAt + index, - }), - )); - if (!(await this.materializeTranscriptRun(sessionId, run))) { - throw new Error(`Imported transcript Run ${run.runId} could not be materialized`); + const openedAt = firstOpenedAt + index; + const run = { sessionId, runId, turnId: turn.turnId, invocationId: runId }; + const events = [ + transcriptOpeningEvent({ header, run, openedAt, newId: this.deps.newId }), + ...backfillRuntimeEventsFromStoredMessages({ + run, + outcome: transcriptOutcome(turn, turnMessages, openedAt), + messages: turnMessages, + modelHistory: 'conversation_text', + newId: this.deps.newId, + now: this.deps.now, + }).events, + ]; + if (!events.some(isTerminalRuntimeEvent)) { + throw new Error(`Imported transcript Run ${runId} has no terminal RuntimeEvent`); } - const runtimeEvents = await this.deps.runtimeEventStore.readRuntimeEvents( - sessionId, - run.runId, - ); - if (!runtimeEvents.some((event) => isMatchingTerminalRuntimeEvent(run, event))) { - throw new Error(`Imported transcript Run ${run.runId} has no terminal RuntimeEvent`); + for (const event of events) { + await this.deps.runtimeEventStore.appendRuntimeEvent(sessionId, runId, event); } } }); @@ -138,9 +129,7 @@ export class RuntimeLedgerRepair { const messages = await this.deps.readMessages(sessionId); const messageIds = new Set(messages.map((message) => message.id)); const inlineRunIds = new Set( - (await this.deps.runStore.listSessionRuns(sessionId)) - .filter(isSessionInlineRun) - .map((run) => run.runId), + (await this.listInlineInvocations(sessionId)).map((invocation) => invocation.runId), ); let repaired = 0; for (const event of await this.deps.runtimeEventStore.readSessionRuntimeEvents(sessionId)) { @@ -155,169 +144,10 @@ export class RuntimeLedgerRepair { }); } - private async repairRunTerminalFact( - sessionId: string, - staleRun: AgentRunHeader, - ): Promise { - return this.withRepairQueue(sessionId, staleRun.runId, async () => { - const run = await this.deps.runStore.readRun(sessionId, staleRun.runId).catch(() => staleRun); - if (!isTerminalRunStatus(run.status)) return false; - const runtimeEvents = await this.deps.runtimeEventStore - .readRuntimeEvents(sessionId, run.runId) - .catch(() => undefined); - if (!runtimeEvents) return false; - const messages = await this.deps.readMessages(sessionId).catch(() => undefined); - if (!messages) return false; - return this.repairRunTerminalFactFromSnapshot( - sessionId, - run, - runtimeEvents, - messages, - 'full', - ); - }); - } - - private async materializeTranscriptRun( - sessionId: string, - createdRun: AgentRunHeader, - ): Promise { - return this.withRepairQueue(sessionId, createdRun.runId, async () => { - const run = await this.deps.runStore.readRun(sessionId, createdRun.runId); - if (!isTerminalRunStatus(run.status)) return false; - const [runtimeEvents, messages] = await Promise.all([ - this.deps.runtimeEventStore.readRuntimeEvents(sessionId, run.runId), - this.deps.readMessages(sessionId), - ]); - return this.repairRunTerminalFactFromSnapshot( - sessionId, - run, - runtimeEvents, - messages, - 'conversation_text', - ); - }); - } - - private async repairRunTerminalFactFromSnapshot( - sessionId: string, - run: AgentRunHeader, - runtimeEvents: readonly RuntimeEvent[], - messages: readonly StoredMessage[], - modelHistory: 'full' | 'conversation_text', - ): Promise { - const recovered = backfillRuntimeEventsFromStoredMessages({ - run, - messages, - modelHistory, - invocationId: runtimeEvents[0]?.invocationId, - newId: this.deps.newId, - now: this.deps.now, - }).events; - const recoveredTerminal = recovered.find((event) => isMatchingTerminalRuntimeEvent(run, event)); - const legacyTerminal = latestTurnState(messages, run.turnId); - const canTrustRecoveredTerminal = recoveredTerminal - ? isTrustworthyRecoveredTerminal(run, legacyTerminal, recoveredTerminal) - : false; - const recoveredEventsToPersist = canTrustRecoveredTerminal - ? recovered - : recovered.filter((event) => !isMatchingTerminalRuntimeEvent(run, event)); - const eventsToAppend = missingRecoveredRuntimeEvents( - run, - runtimeEvents, - recoveredEventsToPersist, - ); - for (const event of eventsToAppend) { - await this.deps.runtimeEventStore.appendRuntimeEvent(sessionId, run.runId, event); - } - - const existingTerminal = [...runtimeEvents, ...eventsToAppend].find((event) => - isMatchingTerminalRuntimeEvent(run, event), + private async listInlineInvocations(sessionId: string): Promise { + return (await this.deps.runtimeEventStore.listSessionInvocations(sessionId)).filter( + (invocation) => isSessionInlineInvocation(invocation.opening), ); - if (existingTerminal) { - return ( - (await this.repairRunHeaderFromExistingTerminal( - sessionId, - run, - messages, - legacyTerminal, - existingTerminal, - )) || eventsToAppend.length > 0 - ); - } - - await this.repairMissingTerminalAsFailed(sessionId, run, messages, [ - ...runtimeEvents, - ...eventsToAppend, - ]); - return true; - } - - private async repairRunHeaderFromExistingTerminal( - sessionId: string, - run: AgentRunHeader, - messages: readonly StoredMessage[], - turnState: Extract | undefined, - terminal: RuntimeEvent, - ): Promise { - const status = terminalRunStatusFromEvent(run, terminal); - if (!status) return false; - const ts = run.completedAt ?? terminal.ts ?? run.updatedAt ?? this.deps.now(); - const failureClass = - status === 'failed' - ? (failureClassFromExistingTerminal(terminal) ?? - (turnState?.status === 'failed' ? turnState.errorClass : undefined) ?? - 'missing_terminal_event') - : undefined; - const abortSource = - status === 'cancelled' - ? (abortSourceFromExistingTerminal(terminal) ?? - (turnState?.status === 'aborted' ? turnState.abortSource : undefined) ?? - 'unknown') - : undefined; - const existingEvents = await this.deps.runStore - .readEvents(sessionId, run.runId) - .catch(() => []); - await commitTerminalRunWithRuntimeFact({ - runStore: this.deps.runStore, - runtimeEventStore: this.deps.runtimeEventStore, - newId: this.deps.newId, - sessionId, - runId: run.runId, - turnId: run.turnId, - status, - ts, - terminalEvent: terminal, - ...(failureClass ? { failureClass } : {}), - ...(abortSource ? { abortSource } : {}), - runEventData: { - recovered: true, - recoveryReason: 'runtime_event_terminal_fact', - runtimeEventId: terminal.id, - runtimeEventStatus: terminal.status, - }, - existingEvents, - }); - await this.appendTerminalTurnStateIfNeeded( - sessionId, - messages, - run, - { - runId: run.runId, - turnId: run.turnId, - status, - ...(failureClass ? { failureClass } : {}), - diagnostic: { recoveryReason: 'runtime_event_terminal_fact', runtimeEventId: terminal.id }, - lineage: headerLineage(run), - }, - terminalTurnStatus(status), - { - ts, - ...(failureClass ? { errorClass: failureClass } : {}), - ...(abortSource ? { abortSource } : {}), - }, - ).catch(() => {}); - return true; } private async withRepairQueue( @@ -341,72 +171,6 @@ export class RuntimeLedgerRepair { } } } - - private async repairMissingTerminalAsFailed( - sessionId: string, - run: AgentRunHeader, - messages: readonly StoredMessage[], - runtimeEvents: readonly RuntimeEvent[], - ): Promise { - const ts = run.completedAt ?? run.updatedAt ?? this.deps.now(); - const failureClass = 'missing_terminal_event'; - const terminalEvent = buildRecoveredTerminalRuntimeEvent({ - id: this.deps.newId(), - run, - status: 'failed', - ts, - invocationId: runtimeEvents[0]?.invocationId ?? `recovery-${run.runId}`, - failureClass, - recoveryReason: failureClass, - message: 'terminal run header had no terminal RuntimeEvent', - }); - const existingEvents = await this.deps.runStore - .readEvents(sessionId, run.runId) - .catch(() => []); - await commitTerminalRunWithRuntimeFact({ - runStore: this.deps.runStore, - runtimeEventStore: this.deps.runtimeEventStore, - newId: this.deps.newId, - sessionId, - runId: run.runId, - turnId: run.turnId, - status: 'failed', - ts, - terminalEvent, - failureClass, - runEventData: { recovered: true, recoveryReason: failureClass }, - existingEvents, - }); - await this.appendTerminalTurnStateIfNeeded( - sessionId, - messages, - run, - { - runId: run.runId, - turnId: run.turnId, - status: 'failed', - failureClass, - diagnostic: { recoveryReason: failureClass }, - lineage: headerLineage(run), - }, - 'failed', - { ts, errorClass: failureClass }, - ).catch(() => {}); - } - - private async appendTerminalTurnStateIfNeeded( - sessionId: string, - messages: readonly StoredMessage[], - run: AgentRunHeader, - decision: RuntimeLedgerRepairDecision, - status: TurnRecord['status'], - options: { ts: number; errorClass?: string; abortSource?: string }, - ): Promise { - if (!isSessionInlineRun(run)) return; - const latest = latestTurnState(messages, decision.turnId); - if (latest && isTerminalTurnStatus(latest.status) && latest.status === status) return; - await this.deps.appendTurnState(sessionId, decision.turnId, status, decision.lineage, options); - } } function transcriptRunId(sessionId: string, turnId: string): string { @@ -414,46 +178,76 @@ function transcriptRunId(sessionId: string, turnId: string): string { return `transcript-${digest.slice(0, 48)}`; } -function transcriptRunHeader(input: { +/** + * The opening fact of an imported turn. + * + * Its route is `unknown` on purpose: an external transcript records which model + * produced the text, never which credential the host would have used, so the + * import must not let anything treat the route as authenticated. + */ +function transcriptOpeningEvent(input: { header: SessionHeader; - turn: TurnRecord; - turnMessages: readonly StoredMessage[]; - runId: string; - createdAt: number; -}): AgentRunHeader { - const updatedAt = Math.max(input.createdAt, ...input.turnMessages.map((message) => message.ts)); - const status = transcriptRunStatus(input.turn.status); + run: { sessionId: string; runId: string; turnId: string; invocationId: string }; + openedAt: number; + newId: () => string; +}): RuntimeEvent { + const opening: RuntimeEventInvocationOpenedContent = { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: { + provenance: 'unknown', + backendKind: input.header.backend, + llmConnectionSlug: input.header.llmConnectionSlug, + modelId: input.header.model, + }, + configuration: { + cwd: input.header.cwd, + permissionMode: input.header.permissionMode, + collaborationMode: input.header.collaborationMode ?? 'agent', + orchestrationMode: input.header.orchestrationMode ?? 'default', + orchestrationSource: 'session', + toolMode: DEFAULT_TOOL_MODE, + }, + root: { kind: 'user' }, + source: { kind: 'fresh' }, + }; + return buildInvocationOpenedEvent({ + id: input.newId(), + run: input.run, + openedAt: input.openedAt, + opening, + }); +} + +/** How the imported turn ended, read off the transcript's own turn record. */ +function transcriptOutcome( + turn: TurnRecord, + turnMessages: readonly StoredMessage[], + openedAt: number, +): RuntimeEventBackfillOutcome { + const ts = Math.max(openedAt, ...turnMessages.map((message) => message.ts)); + // A transcript that never stated how a turn ended does not get to claim it + // completed. The terminal event is written once and cannot be corrected later, + // so an inferred status is recorded as the failure it actually is — which is + // also the reason an adapter emits a cutoff of its own. + if (turn.statusSource !== 'recorded') { + return { status: 'failed', ts, failureClass: 'missing_terminal_event' }; + } + const status = transcriptOutcomeStatus(turn.status); return { - runId: input.runId, - invocationId: `invocation-${input.runId}`, - sessionId: input.header.id, - turnId: input.turn.turnId, status, - backendKind: input.header.backend, - ...(input.header.llmConnectionId === undefined - ? {} - : { llmConnectionId: input.header.llmConnectionId }), - llmConnectionSlug: input.header.llmConnectionSlug, - modelId: input.header.model, - cwd: input.header.cwd, - permissionMode: input.header.permissionMode, - collaborationMode: input.header.collaborationMode, - orchestrationMode: input.header.orchestrationMode, - createdAt: input.createdAt, - updatedAt, - completedAt: updatedAt, + ts, ...(status === 'failed' - ? { failureClass: input.turn.errorClass ?? 'external_transcript_failed' } + ? { failureClass: turn.errorClass ?? 'external_transcript_failed' } : {}), ...(status === 'cancelled' - ? { abortSource: input.turn.abortSource ?? 'external_session_snapshot' } + ? { abortSource: turn.abortSource ?? 'external_session_snapshot' } : {}), }; } -function transcriptRunStatus(status: TurnRecord['status']): AgentRunHeader['status'] { +function transcriptOutcomeStatus(status: TurnRecord['status']): RuntimeInvocationOutcome { if (status === 'failed') return 'failed'; - if (status === 'aborted') return 'cancelled'; if (status === 'completed') return 'completed'; return 'cancelled'; } @@ -470,50 +264,6 @@ function groupMessagesByTurn(messages: readonly StoredMessage[]): Map; - lineage: AgentRunLineage; -} - -export function firstRuntimeRepairRunId( - diagnostics: readonly { code: string; message: string; runId?: string; detail?: unknown }[], - alreadyRepaired: ReadonlySet = new Set(), -): string | undefined { - for (const diagnostic of diagnostics) { - const runId = diagnostic.runId ?? diagnosticDetailRunId(diagnostic.detail); - if (!runId || alreadyRepaired.has(runId)) continue; - if (diagnostic.code !== 'incomplete_event') continue; - if ( - diagnostic.message === 'terminal run recovered from legacy projection cache' || - diagnostic.message === 'terminal run has no readable RuntimeEvent ledger' || - diagnostic.message === 'terminal run has no terminal RuntimeEvent' || - diagnostic.message === 'terminal run header does not match RuntimeEvent terminal fact' || - diagnostic.message === 'failed terminal RuntimeEvent requires a stable failure class' || - diagnostic.message === - 'failed terminal event did not carry an exact AgentRunHeader.failureClass' || - diagnostic.message === 'aborted terminal RuntimeEvent requires an abort source' || - diagnostic.message === 'abortSource is not present in RuntimeEvent or AgentRunHeader metadata' - ) { - return runId; - } - } - return undefined; -} - -function diagnosticDetailRunId(detail: unknown): string | undefined { - if (!detail || typeof detail !== 'object') return undefined; - const runId = (detail as { runId?: unknown }).runId; - return typeof runId === 'string' && runId.length > 0 ? runId : undefined; -} - -function isTerminalRunStatus(status: AgentRunHeader['status']): boolean { - return status === 'completed' || status === 'failed' || status === 'cancelled'; -} - function steeringMessageFromRuntimeEvent(event: RuntimeEvent): StoredMessage | undefined { const messageId = event.refs?.providerEventId; if ( @@ -527,183 +277,3 @@ function steeringMessageFromRuntimeEvent(event: RuntimeEvent): StoredMessage | u } return projectRuntimeEventUserMessage(event, messageId); } - -function isTerminalTurnStatus(status: TurnRecord['status']): boolean { - return status === 'completed' || status === 'failed' || status === 'aborted'; -} - -function terminalRunStatusFromEvent( - run: AgentRunHeader, - event: RuntimeEvent, -): 'completed' | 'failed' | 'cancelled' | undefined { - if (event.status === 'completed') return 'completed'; - if (event.status === 'failed') return 'failed'; - if (event.status === 'aborted' || event.status === 'cancelled') return 'cancelled'; - if (run.status === 'completed' || run.status === 'failed' || run.status === 'cancelled') - return run.status; - return undefined; -} - -function terminalTurnStatus(status: 'completed' | 'failed' | 'cancelled'): TurnRecord['status'] { - if (status === 'cancelled') return 'aborted'; - return status; -} - -function isMatchingTerminalRuntimeEvent(run: AgentRunHeader, event: RuntimeEvent): boolean { - return ( - !event.partial && - event.sessionId === run.sessionId && - event.runId === run.runId && - event.turnId === run.turnId && - (run.invocationId === undefined || event.invocationId === run.invocationId) && - isTerminalRuntimeEvent(event) - ); -} - -function missingRecoveredRuntimeEvents( - run: AgentRunHeader, - existing: readonly RuntimeEvent[], - recovered: readonly RuntimeEvent[], -): RuntimeEvent[] { - const recoveredEventKeys = new Set( - existing.map(recoveredEventKey).filter((key): key is string => key !== undefined), - ); - const hasTerminal = existing.some((event) => isMatchingTerminalRuntimeEvent(run, event)); - const matchedExistingEventIndexes = new Set(); - const missing: RuntimeEvent[] = []; - for (const event of recovered) { - if (isMatchingTerminalRuntimeEvent(run, event)) { - if (!hasTerminal) missing.push(event); - continue; - } - const eventKey = recoveredEventKey(event); - if (!eventKey) continue; - if (recoveredEventKeys.has(eventKey)) continue; - recoveredEventKeys.add(eventKey); - const existingIndex = existing.findIndex( - (candidate, index) => - !matchedExistingEventIndexes.has(index) && isSameRecoveredRuntimeEvent(candidate, event), - ); - if (existingIndex >= 0) { - matchedExistingEventIndexes.add(existingIndex); - } else { - missing.push(event); - } - } - return missing; -} - -function recoveredEventKey(event: RuntimeEvent): string | undefined { - const storedMessageId = event.refs?.storedMessageId; - if (typeof storedMessageId !== 'string' || storedMessageId.length === 0) return undefined; - return JSON.stringify({ - storedMessageId, - role: event.role, - author: event.author, - status: event.status, - content: event.content, - toolCallId: event.refs?.toolCallId, - tokenUsage: event.actions?.tokenUsage, - permissionDecision: event.actions?.permissionDecision, - }); -} - -function failureClassFromExistingTerminal(event: RuntimeEvent): string | undefined { - return ( - stringStateDelta(event, 'failureClass') ?? - stringStateDelta(event, 'errorClass') ?? - stringStateDelta(event, 'reason') ?? - stringStateDelta(event, 'code') ?? - (event.content?.kind === 'error' ? nonEmptyString(event.content.reason) : undefined) ?? - (event.content?.kind === 'error' ? nonEmptyString(event.content.code) : undefined) - ); -} - -function abortSourceFromExistingTerminal(event: RuntimeEvent): string | undefined { - return ( - stringStateDelta(event, 'abortSource') ?? - stringStateDelta(event, 'source') ?? - stringRecordValue(event.refs, 'abortSource') ?? - stringRecordValue(event.refs, 'source') - ); -} - -function stringStateDelta(event: RuntimeEvent, key: string): string | undefined { - const value = event.actions?.stateDelta?.[key]; - return typeof value === 'string' && value.length > 0 ? value : undefined; -} - -function stringRecordValue(value: unknown, key: string): string | undefined { - if (!value || typeof value !== 'object') return undefined; - const result = (value as Record)[key]; - return typeof result === 'string' && result.length > 0 ? result : undefined; -} - -function nonEmptyString(value: unknown): string | undefined { - return typeof value === 'string' && value.length > 0 ? value : undefined; -} - -function isSameRecoveredRuntimeEvent(existing: RuntimeEvent, recovered: RuntimeEvent): boolean { - return ( - !existing.partial && - existing.sessionId === recovered.sessionId && - existing.runId === recovered.runId && - existing.turnId === recovered.turnId && - existing.role === recovered.role && - existing.author === recovered.author && - existing.status === recovered.status && - JSON.stringify(existing.content) === JSON.stringify(recovered.content) && - JSON.stringify(existing.actions?.tokenUsage) === - JSON.stringify(recovered.actions?.tokenUsage) && - JSON.stringify(existing.actions?.permissionDecision) === - JSON.stringify(recovered.actions?.permissionDecision) - ); -} - -function isTrustworthyRecoveredTerminal( - run: AgentRunHeader, - turnState: Extract | undefined, - terminal: RuntimeEvent, -): boolean { - if (!turnState || !isTerminalTurnStatus(turnState.status)) return false; - if (terminal.status === 'completed') { - return run.status === 'completed' && turnState.status === 'completed'; - } - if (terminal.status === 'failed') { - return ( - run.status === 'failed' && - turnState.status === 'failed' && - (!run.failureClass || !turnState.errorClass || turnState.errorClass === run.failureClass) - ); - } - if (terminal.status === 'aborted' || terminal.status === 'cancelled') { - return run.status === 'cancelled' && turnState.status === 'aborted'; - } - return false; -} - -function latestTurnState( - messages: readonly StoredMessage[], - turnId: string, -): Extract | undefined { - for (let index = messages.length - 1; index >= 0; index -= 1) { - const message = messages[index]; - if (message?.type === 'turn_state' && message.turnId === turnId) return message; - } - return undefined; -} - -function headerLineage(header: AgentRunHeader): AgentRunLineage { - return { - ...(header.parentRunId ? { parentRunId: header.parentRunId } : {}), - ...(header.resumedFromRunId ? { resumedFromRunId: header.resumedFromRunId } : {}), - ...(header.retriedFromRunId ? { retriedFromRunId: header.retriedFromRunId } : {}), - ...(header.parentTurnId ? { parentTurnId: header.parentTurnId } : {}), - ...(header.retriedFromTurnId ? { retriedFromTurnId: header.retriedFromTurnId } : {}), - ...(header.regeneratedFromTurnId - ? { regeneratedFromTurnId: header.regeneratedFromTurnId } - : {}), - ...(header.branchOfTurnId ? { branchOfTurnId: header.branchOfTurnId } : {}), - ...(header.parentSessionId ? { parentSessionId: header.parentSessionId } : {}), - }; -} diff --git a/packages/runtime/src/runtime-read-model.ts b/packages/runtime/src/runtime-read-model.ts index eca372de48..e103ae0dd5 100644 --- a/packages/runtime/src/runtime-read-model.ts +++ b/packages/runtime/src/runtime-read-model.ts @@ -17,13 +17,12 @@ * under the License. */ -import type { AgentRunHeader, AgentRunStore } from '@maka/core/agent-run'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { StoredMessage, TurnRecord } from '@maka/core/session'; import { deriveTurnRecords } from '@maka/core/session'; -import { isSessionInlineRun } from '@maka/core/agent-run'; -import { isTerminalRuntimeEvent } from '@maka/core/runtime-event'; +import { isSessionInlineInvocation } from '@maka/core/runtime-invocation'; import type { CanonicalPermissionOutcomeReader, CanonicalPermissionOutcomeRecord, @@ -40,11 +39,6 @@ import { buildRuntimeEventModelReplayPlan, type RuntimeEventModelReplayPlan, } from './model-history.js'; -import { backfillRuntimeEventsFromStoredMessages } from './runtime-event-backfill.js'; -import { - effectiveRunHeaderFromTerminalFact, - terminalRunHeaderMatchesFact, -} from './terminal-run-commit.js'; const CANONICAL_PERMISSION_READ_CONCURRENCY = 8; @@ -53,7 +47,6 @@ export interface RuntimeReadModelProjectionCache { } export interface RuntimeReadModelDeps { - runStore: AgentRunStore; runtimeEventStore: RuntimeEventStore; projectionCache?: RuntimeReadModelProjectionCache; canonicalPermissionOutcomes?: CanonicalPermissionOutcomeReader; @@ -64,7 +57,7 @@ export interface RuntimeReadModelSessionView { messages: StoredMessage[]; turns: TurnRecord[]; events: RuntimeEvent[]; - runs: AgentRunHeader[]; + invocations: RuntimeInvocationRecord[]; diagnostics: RuntimeEventReadModelDiagnostic[]; terminalFacts: RuntimeEventTerminalFact[]; replayPlan: RuntimeEventModelReplayPlan; @@ -94,21 +87,25 @@ export class RuntimeReadModel { async getSessionView(sessionId: string): Promise { const diagnostics: RuntimeEventReadModelDiagnostic[] = []; const inFlightTurnIds = new Set(); - let runs: AgentRunHeader[]; + let invocations: RuntimeInvocationRecord[]; try { - runs = await this.deps.runStore.listSessionRuns(sessionId); + invocations = (await this.deps.runtimeEventStore.listSessionInvocations(sessionId)).filter( + (invocation) => isSessionInlineInvocation(invocation.opening), + ); } catch (error) { - throw new RuntimeReadModelError('RuntimeReadModel could not list AgentRun headers', [ - readModelDiagnostic('unsupported_event', 'AgentRunStore.listSessionRuns failed', { - error: errorMessage(error), - }), + throw new RuntimeReadModelError('RuntimeReadModel could not list Session invocations', [ + readModelDiagnostic( + 'unsupported_event', + 'RuntimeEventStore.listSessionInvocations failed', + { + error: errorMessage(error), + }, + ), ]); } - const inlineRuns = runs.filter(isSessionInlineRun); - - if (inlineRuns.length === 0) { - return this.buildView({ runs: inlineRuns, events: [], diagnostics }); + if (invocations.length === 0) { + return this.buildView({ invocations, events: [], diagnostics }); } const durableEventOrdinals = await this.readSessionRuntimeEventOrdinals(sessionId); @@ -117,98 +114,51 @@ export class RuntimeReadModel { ); const ordered: OrderedRuntimeEvent[] = []; const terminalFacts: RuntimeEventTerminalFact[] = []; - for (let runIndex = 0; runIndex < inlineRuns.length; runIndex += 1) { - const run = inlineRuns[runIndex]!; - if (!isTerminalRunStatus(run.status)) { - const activeRunContext = await this.readNonTerminalRunContext(sessionId, run); - if (activeRunContext?.fact) { - inlineRuns[runIndex] = effectiveRunHeaderFromTerminalFact(run, activeRunContext.fact); - terminalFacts.push(activeRunContext.fact); - diagnostics.push(...activeRunContext.fact.diagnostics); - appendOrderedEvents(ordered, activeRunContext.events, runIndex, durableEventOrdinalById); - continue; - } - - const diagnostic = readModelDiagnostic( - 'incomplete_event', - 'active run is using the in-flight projection cache', - { - runId: run.runId, - turnId: run.turnId, - status: run.status, - }, - ); - diagnostics.push(diagnostic); - inFlightTurnIds.add(run.turnId); - if (!this.deps.projectionCache) { - throw new RuntimeReadModelError('RuntimeEvent ledger is incomplete for an active run', [ - readModelDiagnostic( - 'incomplete_event', - 'active run has no stable RuntimeEvent read projection', - { - runId: run.runId, - turnId: run.turnId, - status: run.status, - }, - ), - ]); - } - const overlayEvents = activeRunContext?.events.flatMap(activeInteractionOverlayEvent) ?? []; - appendOrderedEvents(ordered, overlayEvents, runIndex); - continue; - } - + for (let runIndex = 0; runIndex < invocations.length; runIndex += 1) { + const invocation = invocations[runIndex]!; let runEvents: RuntimeEvent[]; try { - runEvents = await this.deps.runtimeEventStore.readRuntimeEvents(sessionId, run.runId); + runEvents = await this.deps.runtimeEventStore.readRuntimeEvents( + sessionId, + invocation.runId, + ); } catch (error) { throw new RuntimeReadModelError('RuntimeEvent ledger read failed', [ readModelDiagnostic('unsupported_event', 'RuntimeEventStore.readRuntimeEvents failed', { - runId: run.runId, + runId: invocation.runId, error: errorMessage(error), }), ]); } - if (runEvents.length === 0) { - const recovered = await this.backfillMissingRuntimeEvents(sessionId, run); - if (recovered.length === 0 || !recovered.some(isTerminalRuntimeEvent)) { - throw new RuntimeReadModelError('RuntimeEvent ledger is missing for a terminal run', [ - readModelDiagnostic( - 'incomplete_event', - 'terminal run has no readable RuntimeEvent ledger', - { - runId: run.runId, - turnId: run.turnId, - }, - ), - ]); - } + // No terminal event yet: the invocation is still open, or the process died + // holding it. Either way the ledger is the whole truth about it, so the + // in-flight projection cache supplies the rows a live turn has not + // committed instead of a status field claiming otherwise. + if (!invocation.terminalEvent) { diagnostics.push( readModelDiagnostic( 'incomplete_event', - 'terminal run recovered from legacy projection cache', - { - runId: run.runId, - turnId: run.turnId, - }, + 'active run is using the in-flight projection cache', + { runId: invocation.runId, turnId: invocation.turnId }, ), ); - runEvents = recovered; - } - if (!runEvents.some(isTerminalRuntimeEvent)) { - throw new RuntimeReadModelError( - 'RuntimeEvent ledger has no terminal fact for a terminal run', - [ - readModelDiagnostic('incomplete_event', 'terminal run has no terminal RuntimeEvent', { - runId: run.runId, - turnId: run.turnId, - }), - ], - ); + inFlightTurnIds.add(invocation.turnId); + if (!this.deps.projectionCache) { + throw new RuntimeReadModelError('RuntimeEvent ledger is incomplete for an active run', [ + readModelDiagnostic( + 'incomplete_event', + 'active run has no stable RuntimeEvent read projection', + { runId: invocation.runId, turnId: invocation.turnId }, + ), + ]); + } + const overlayEvents = runEvents.flatMap(activeInteractionOverlayEvent); + appendOrderedEvents(ordered, overlayEvents, runIndex); + continue; } - const terminalFact = classifyRuntimeEventTerminalFact(run, runEvents); + const terminalFact = classifyRuntimeEventTerminalFact(invocation, runEvents); diagnostics.push(...terminalFact.diagnostics); if (!terminalFact.fact) { throw new RuntimeReadModelError( @@ -216,25 +166,6 @@ export class RuntimeReadModel { diagnostics, ); } - if (!terminalRunHeaderMatchesFact(run, terminalFact.fact)) { - diagnostics.push( - readModelDiagnostic( - 'incomplete_event', - 'terminal run header does not match RuntimeEvent terminal fact', - { - runId: run.runId, - turnId: run.turnId, - headerStatus: run.status, - factStatus: terminalFact.fact.runStatus, - headerFailureClass: run.failureClass, - factFailureClass: terminalFact.fact.failureClass, - headerAbortSource: run.abortSource, - factAbortSource: terminalFact.fact.abortSource, - }, - ), - ); - } - inlineRuns[runIndex] = effectiveRunHeaderFromTerminalFact(run, terminalFact.fact); terminalFacts.push(terminalFact.fact); appendOrderedEvents(ordered, runEvents, runIndex, durableEventOrdinalById); @@ -243,7 +174,7 @@ export class RuntimeReadModel { ordered.sort(compareOrderedRuntimeEvents); return this.buildView({ - runs: inlineRuns, + invocations, events: ordered.map((item) => item.event), diagnostics, terminalFacts, @@ -251,23 +182,6 @@ export class RuntimeReadModel { }); } - private async readNonTerminalRunContext( - sessionId: string, - run: AgentRunHeader, - ): Promise<{ events: RuntimeEvent[]; fact?: RuntimeEventTerminalFact } | undefined> { - let runEvents: RuntimeEvent[]; - try { - runEvents = await this.deps.runtimeEventStore.readRuntimeEvents(sessionId, run.runId); - } catch { - return undefined; - } - const fact = classifyRuntimeEventTerminalFact(run, runEvents).fact; - return { - events: runEvents, - ...(fact ? { fact } : {}), - }; - } - private async readSessionRuntimeEventOrdinals( sessionId: string, ): Promise> { @@ -284,22 +198,8 @@ export class RuntimeReadModel { } } - private async backfillMissingRuntimeEvents( - sessionId: string, - run: AgentRunHeader, - ): Promise { - if (!this.deps.projectionCache) return []; - let messages: StoredMessage[]; - try { - messages = await this.deps.projectionCache.readMessages(sessionId); - } catch { - return []; - } - return backfillRuntimeEventsFromStoredMessages({ run, messages }).events; - } - private async buildView(input: { - runs: AgentRunHeader[]; + invocations: RuntimeInvocationRecord[]; events: RuntimeEvent[]; diagnostics: RuntimeEventReadModelDiagnostic[]; terminalFacts?: RuntimeEventTerminalFact[]; @@ -307,7 +207,7 @@ export class RuntimeReadModel { }): Promise { const canonicalPermissionRead = await this.readCanonicalPermissionOutcomes(input.events); const projected = projectRuntimeEventsToStoredMessages(input.events, { - runHeaders: input.runs, + invocations: input.invocations, canonicalPermissionOutcomes: canonicalPermissionRead.outcomes, }); const diagnostics = [ @@ -322,7 +222,7 @@ export class RuntimeReadModel { throw new RuntimeReadModelError('RuntimeEvent read projection is incomplete', diagnostics); } - const sessionId = input.runs[0]?.sessionId; + const sessionId = input.invocations[0]?.sessionId; let cachedMessages: StoredMessage[] | undefined; if (sessionId && this.deps.projectionCache) { try { @@ -363,7 +263,7 @@ export class RuntimeReadModel { messages, turns: deriveTurnRecords(messages), events: input.events, - runs: input.runs, + invocations: input.invocations, diagnostics, terminalFacts: input.terminalFacts ?? [], replayPlan: buildRuntimeEventModelReplayPlan(input.events), @@ -505,10 +405,6 @@ function readModelDiagnostic( }; } -function isTerminalRunStatus(status: AgentRunHeader['status']): boolean { - return status === 'completed' || status === 'failed' || status === 'cancelled'; -} - function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } diff --git a/packages/runtime/src/runtime-resume.ts b/packages/runtime/src/runtime-resume.ts index 849972c74a..d368aaad2e 100644 --- a/packages/runtime/src/runtime-resume.ts +++ b/packages/runtime/src/runtime-resume.ts @@ -26,13 +26,18 @@ import { type RuntimeEventFunctionCallContent, type RuntimeEventFunctionResponseContent, } from '@maka/core/runtime-event'; +import { + continuationStartEventMatchesClaim, + invocationMatchesClaimTarget, +} from '@maka/core/runtime-boundary'; import type { ContinuationClaimV1, ImmutableRuntimePrefixV1, RuntimeBoundaryCursorV1, RuntimeBoundaryDigest, } from '@maka/core/runtime-boundary'; -import type { AgentRunHeader } from '@maka/core/agent-run'; +import type { RuntimeEventInvocationOpenedContent } from '@maka/core/runtime-event'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; import type { ContinuationClaimStateV1 } from '@maka/core/runtime-event-store'; import { isDeepStrictEqual } from 'node:util'; import { @@ -46,7 +51,6 @@ import { } from './model-history.js'; import { resolveRuntimeRecovery, type RuntimeRecoveryResolution } from './recovery-resolver.js'; import { classifyRuntimeEventTerminalFact } from './runtime-event-read-model.js'; -import { terminalRunHeaderMatchesFact } from './terminal-run-commit.js'; export type ToolOperationStatus = | 'succeeded' @@ -370,7 +374,7 @@ export interface RuntimeContinuationPlannerInput { } export interface RuntimeContinuationPlannerDeps { - readSourceRun(sessionId: string, runId: string): Promise; + readSourceInvocation(sessionId: string, runId: string): Promise; readImmutableRuntimePrefix(input: { sessionId: string; runId: string; @@ -391,9 +395,9 @@ export class RuntimeContinuationPlanner { constructor(private readonly deps: RuntimeContinuationPlannerDeps) {} async plan(input: RuntimeContinuationPlannerInput): Promise { - let sourceRun: Awaited>; + let sourceInvocation: RuntimeInvocationRecord; try { - sourceRun = await this.deps.readSourceRun(input.sessionId, input.sourceRunId); + sourceInvocation = await this.deps.readSourceInvocation(input.sessionId, input.sourceRunId); } catch { return parkedPlan('source_run_unreadable', 'source AgentRun could not be read'); } @@ -403,8 +407,8 @@ export class RuntimeContinuationPlanner { prefixes = await this.readLineagePrefixes( input.sessionId, input.sourceRunId, - sourceRun, - input.admissionRoute.runHeaders, + sourceInvocation.opening, + input.admissionRoute.invocations, ); } catch (error) { if (error instanceof RuntimeLineageError) { @@ -488,18 +492,20 @@ export class RuntimeContinuationPlanner { return buildSafeBoundaryContinuationPlan(events, { ledgerReadable: true, - terminalRepairSucceeded: hasConsistentTerminalBoundary(sourceRun, events), - sourceCwd: sourceRun.cwd, + terminalRepairSucceeded: hasConsistentTerminalBoundary(events), + sourceCwd: sourceInvocation.opening.configuration.cwd, currentCwd: input.currentCwd, sourceWorkspaceIdentity: input.sourceWorkspaceIdentity, currentWorkspaceIdentity: input.currentWorkspaceIdentity, backgroundOperationsSettled: input.backgroundOperationsSettled, availableToolNames: input.availableToolNames, - continuationIdentity: { - invocationId: this.deps.newId(), - runId: this.deps.newId(), - turnId: this.deps.newId(), - }, + // One physical execution attempt, one identity. Run and invocation are + // the same value at every mint site so the opening fact can be joined + // either way while the two names are still being retired. + continuationIdentity: (() => { + const invocationId = this.deps.newId(); + return { invocationId, runId: invocationId, turnId: this.deps.newId() }; + })(), continuationClaimId: this.deps.newId(), continuationReplayPlan: replay.plan, ...(input.expectedRuntimeEventHighWater !== undefined @@ -520,9 +526,9 @@ export class RuntimeContinuationPlanner { continuationClaimId: claim.claimId, continuationRunId: claim.target.runId, }; - let run: Awaited>; + let targetInvocation: RuntimeInvocationRecord; try { - run = await this.deps.readSourceRun(sessionId, claim.target.runId); + targetInvocation = await this.deps.readSourceInvocation(sessionId, claim.target.runId); } catch { return parkedPlan( 'continuation_claim_repair_required', @@ -530,8 +536,7 @@ export class RuntimeContinuationPlanner { detail, ); } - const targetRun = run; - if (!claimTargetRunHeaderMatches(targetRun, claim)) { + if (!invocationMatchesClaimTarget(targetInvocation, claim)) { return parkedPlan( 'continuation_claim_repair_required', 'durable continuation claim target Run identity does not match its claim', @@ -555,7 +560,7 @@ export class RuntimeContinuationPlanner { if ( !state.startEventId || prefix.events[0]?.id !== state.startEventId || - !continuationStartMatchesClaim(prefix.events[0], claim, state.startKind) + !continuationStartEventMatchesClaim(prefix.events[0], claim, state.startKind) ) { return parkedPlan( 'continuation_claim_repair_required', @@ -563,7 +568,10 @@ export class RuntimeContinuationPlanner { detail, ); } - const terminalClassification = classifyRuntimeEventTerminalFact(targetRun, prefix.events); + const terminalClassification = classifyRuntimeEventTerminalFact( + targetInvocation, + prefix.events, + ); const terminal = prefix.events.find(isTerminalRuntimeEvent); if (terminal && prefix.events.at(-1)?.id !== terminal.id) { return parkedPlan( @@ -579,31 +587,13 @@ export class RuntimeContinuationPlanner { detail, ); } - if (terminalClassification.fact && !isTerminalRunStatus(targetRun.status)) { - return parkedPlan( - 'continuation_claim_repair_required', - 'continuation target has a terminal fact whose Run header requires repair', - detail, - ); - } - if ( - terminalClassification.fact && - isTerminalRunStatus(targetRun.status) && - terminalRunHeaderMatchesFact(targetRun, terminalClassification.fact) - ) { + if (terminalClassification.fact) { return parkedPlan( 'continuation_already_exists', 'source boundary already has a terminal continuation', detail, ); } - if (terminalClassification.fact || isTerminalRunStatus(targetRun.status)) { - return parkedPlan( - 'continuation_claim_repair_required', - 'continuation target terminal Run header does not match its RuntimeEvent fact', - detail, - ); - } if (start) { return parkedPlan( 'continuation_started_indeterminate', @@ -621,8 +611,8 @@ export class RuntimeContinuationPlanner { private async readLineagePrefixes( sessionId: string, sourceRunId: string, - sourceRun: Awaited>, - runHeaders: readonly AgentRunHeader[], + sourceOpening: RuntimeEventInvocationOpenedContent, + invocations: readonly RuntimeInvocationRecord[], ): Promise<[ImmutableRuntimePrefixV1, ...ImmutableRuntimePrefixV1[]]> { const immediate = await this.deps.readImmutableRuntimePrefix({ sessionId, @@ -630,9 +620,9 @@ export class RuntimeContinuationPlanner { }); const segments: ImmutableRuntimePrefixV1[] = [immediate]; const seen = new Set([sourceRunId]); - const v2Edges: Array<{ + const claimedEdges: Array<{ childRunId: string; - childRunHeader: AgentRunHeader; + childInvocation: RuntimeInvocationRecord; startEvent: RuntimeEvent; startKind: 'runtime_admission' | 'claim_repair'; claimId: string; @@ -640,48 +630,56 @@ export class RuntimeContinuationPlanner { providerProjectionVersion: 1 | typeof PROVIDER_REPLAY_PROJECTION_VERSION; providerReplayDigest: RuntimeBoundaryDigest; }> = []; - let childRun = sourceRun; + let childInvocation: RuntimeInvocationRecord = { + sessionId, + invocationId: immediate.identity.invocationId, + runId: sourceRunId, + turnId: immediate.identity.turnId, + openedAt: 0, + opening: sourceOpening, + }; let childRunId = sourceRunId; let childPrefix = immediate; let depth = 1; while (true) { - const current = childRun.continuationSource; + const opened = childInvocation.opening.source; + const current = opened.kind === 'continuation' ? opened : undefined; const start = childPrefix.events[0]?.actions?.continuationStart; - const currentV2 = - current && 'protocol' in current && current.protocol === 'continuation_source_v2' - ? current + // A migrated opening keeps the lineage edge but names no claim, so only + // an edge that names one can be authenticated against a durable claim. + const claimed = + current?.claimId !== undefined && current.boundaryDigest !== undefined + ? { ...current, claimId: current.claimId, boundaryDigest: current.boundaryDigest } : undefined; - if (start && !currentV2) { + if (start && !claimed) { throw new RuntimeLineageError( 'runtime_lineage_start_mismatch', `canonical continuation-start cannot be downgraded to legacy lineage for ${childRunId}`, ); } - if (currentV2) { + if (claimed) { if ( !start || - start.claimId !== currentV2.claimId || - start.boundaryDigest !== currentV2.boundaryDigest || - start.replayManifestDigest !== currentV2.replayManifestDigest || + start.claimId !== claimed.claimId || + start.boundaryDigest !== claimed.boundaryDigest || start.immediateSource.sessionId !== sessionId || - start.immediateSource.invocationId !== currentV2.sourceInvocationId || - start.immediateSource.runId !== currentV2.sourceRunId || - start.immediateSource.turnId !== currentV2.sourceTurnId || - start.immediateSource.highWater !== currentV2.sourceRuntimeEventHighWater || - start.immediateSource.prefixDigest !== currentV2.sourcePrefixDigest + start.immediateSource.invocationId !== claimed.sourceInvocationId || + start.immediateSource.runId !== claimed.sourceRunId || + start.immediateSource.turnId !== claimed.sourceTurnId || + start.immediateSource.highWater !== claimed.sourceRuntimeEventHighWater ) { throw new RuntimeLineageError( 'runtime_lineage_start_mismatch', `continuation-start does not authenticate lineage edge for ${childRunId}`, ); } - v2Edges.push({ + claimedEdges.push({ childRunId, - childRunHeader: childRun, + childInvocation, startEvent: childPrefix.events[0]!, startKind: start.provenance, claimId: start.claimId, - boundaryDigest: currentV2.boundaryDigest, + boundaryDigest: claimed.boundaryDigest, providerProjectionVersion: start.providerProjectionVersion, providerReplayDigest: start.providerReplayDigest, }); @@ -700,11 +698,11 @@ export class RuntimeContinuationPlanner { ); } seen.add(current.sourceRunId); - let run: Awaited>; + let invocation: RuntimeInvocationRecord; let prefix: ImmutableRuntimePrefixV1; try { - [run, prefix] = await Promise.all([ - this.deps.readSourceRun(sessionId, current.sourceRunId), + [invocation, prefix] = await Promise.all([ + this.deps.readSourceInvocation(sessionId, current.sourceRunId), this.deps.readImmutableRuntimePrefix({ sessionId, runId: current.sourceRunId, @@ -729,11 +727,9 @@ export class RuntimeContinuationPlanner { `continuation ancestor ${current.sourceRunId} identity does not match its lineage edge`, ); } - if ( - 'protocol' in current && - current.protocol === 'continuation_source_v2' && - current.sourcePrefixDigest !== prefix.prefixDigest - ) { + // The child's continuation-start is what froze the ancestor's prefix, so + // it is also the only record that can say the prefix has since changed. + if (start && start.immediateSource.prefixDigest !== prefix.prefixDigest) { throw new RuntimeLineageError( 'source_prefix_digest_mismatch', `continuation ancestor ${current.sourceRunId} prefix digest changed`, @@ -741,11 +737,11 @@ export class RuntimeContinuationPlanner { } segments.unshift(prefix); childPrefix = prefix; - childRun = run; + childInvocation = invocation; childRunId = current.sourceRunId; depth += 1; } - for (const edge of v2Edges) { + for (const edge of claimedEdges) { const childIndex = segments.findIndex((prefix) => prefix.identity.runId === edge.childRunId); if (childIndex <= 0) { throw new RuntimeLineageError( @@ -774,8 +770,8 @@ export class RuntimeContinuationPlanner { state.claim.boundaryDigest !== edge.boundaryDigest || state.startEventId !== edge.startEvent.id || state.startKind !== edge.startKind || - !claimTargetRunHeaderMatches(edge.childRunHeader, state.claim) || - !continuationStartMatchesClaim(edge.startEvent, state.claim, state.startKind) + !invocationMatchesClaimTarget(edge.childInvocation, state.claim) || + !continuationStartEventMatchesClaim(edge.startEvent, state.claim, state.startKind) ) { throw new RuntimeLineageError( 'runtime_lineage_claim_mismatch', @@ -795,9 +791,12 @@ export class RuntimeContinuationPlanner { ], providerProjectionVersion: edge.providerProjectionVersion, admissionRoute: { - runHeaders, - targetProviderStateIdentity: state.claim.targetRunHeader.providerStateIdentity, - targetModelId: state.claim.targetRunHeader.modelId, + invocations, + targetProviderStateIdentity: + state.claim.targetOpening.route.provenance === 'runtime' + ? state.claim.targetOpening.route.providerStateIdentity + : undefined, + targetModelId: state.claim.targetOpening.route.modelId, }, }); if ( @@ -834,21 +833,13 @@ class RuntimeLineageError extends Error { } } -function isTerminalRunStatus(status: string): boolean { - return status === 'completed' || status === 'failed' || status === 'cancelled'; -} - -function hasConsistentTerminalBoundary( - run: AgentRunHeader, - events: readonly RuntimeEvent[], -): boolean { - if (!isTerminalRunStatus(run.status)) return false; - const classification = classifyRuntimeEventTerminalFact(run, events); - return ( - classification.fact !== undefined && - events.at(-1)?.id === classification.fact.terminalEvent.id && - terminalRunHeaderMatchesFact(run, classification.fact) - ); +/** + * Has this run ended, with the terminal event last where a sealed run must + * leave it? There is nothing else to agree with: the events are the run. + */ +function hasConsistentTerminalBoundary(events: readonly RuntimeEvent[]): boolean { + const last = events.at(-1); + return last !== undefined && isTerminalRuntimeEvent(last); } export const INDETERMINATE_TOOL_RESULT_DIRECTIVE = [ @@ -1485,71 +1476,3 @@ function hasMatchingCall( ): boolean { return call !== undefined && call.name === response.name; } - -function claimTargetRunHeaderMatches(actual: AgentRunHeader, claim: ContinuationClaimV1): boolean { - const candidate = actual as unknown as Record; - const expected = claim.targetRunHeader as unknown as Record; - const immutable = (header: Record) => { - const { - status: _status, - updatedAt: _updatedAt, - completedAt: _completedAt, - failureClass: _failureClass, - failureMessage: _failureMessage, - abortSource: _abortSource, - traceWriteError: _traceWriteError, - ...rest - } = header; - return rest; - }; - return isDeepStrictEqual(immutable(candidate), immutable(expected)); -} - -function continuationStartMatchesClaim( - event: RuntimeEvent | undefined, - claim: ContinuationClaimV1, - startKind: ContinuationClaimStateV1['startKind'], -): boolean { - const start = event?.actions?.continuationStart; - const runtimeProtocol = event?.actions?.runtimeProtocol; - const actionKeys = event?.actions ? Object.keys(event.actions) : []; - const actionShapeMatches = - actionKeys.includes('continuationStart') && - actionKeys.every((key) => key === 'continuationStart' || key === 'runtimeProtocol') && - actionKeys.length === (runtimeProtocol === undefined ? 1 : 2); - const runtimeProtocolMatches = - runtimeProtocol === undefined || - (startKind === 'runtime_admission' && - runtimeProtocol.toolBoundary === TOOL_BOUNDARY_PROTOCOL_V1); - const source = claim.boundary.segments.at(-1)!; - return Boolean( - event && - event.sessionId === claim.target.sessionId && - event.invocationId === claim.target.invocationId && - event.runId === claim.target.runId && - event.turnId === claim.target.turnId && - event.partial !== true && - event.role === 'system' && - event.author === 'system' && - event.status === undefined && - event.content === undefined && - event.actions && - actionShapeMatches && - runtimeProtocolMatches && - start?.protocol === 'continuation_start_v2' && - start.provenance === startKind && - start.claimId === claim.claimId && - start.boundaryDigest === claim.boundaryDigest && - start.replayManifestDigest === claim.boundary.manifestDigest && - start.providerProjectionVersion === claim.providerProjectionVersion && - start.providerReplayDigest === claim.providerReplayDigest && - isDeepStrictEqual(start.immediateSource, { - sessionId: source.identity.sessionId, - invocationId: source.identity.invocationId, - runId: source.identity.runId, - turnId: source.identity.turnId, - highWater: source.position.lastEventSeq, - prefixDigest: source.prefixDigest, - }), - ); -} diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index e602dba5c4..91bd2d1021 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -100,8 +100,14 @@ import { decodeAgentGraphIntentClaim } from '@maka/core/agent-graph-control'; import { executionBoundaryContains } from '@maka/core/sandbox-boundary'; import { failureClassFromCompleteStopReason } from '@maka/core/events'; import { isActiveShellRunStatus } from '@maka/core/shell-run'; -import { isSessionInlineRun } from '@maka/core/agent-run'; import { isTerminalRuntimeEvent } from '@maka/core/runtime-event'; +import { + buildInvocationOpenedEvent, + isSessionInlineInvocation, + runtimeInvocationOutcome, + type RootExecutionDescriptor, + type RuntimeInvocationRecord, +} from '@maka/core/runtime-invocation'; import type { AgentGraphIntentClaim, AgentGraphIntentClaimStore, @@ -112,19 +118,23 @@ import type { AgentGraphProvisionedEdge, } from '@maka/core/agent-graph-topology'; import type { AgentGraphScheduleUpdateSource } from '@maka/core/agent-graph-schedule'; -import type { - AgentRunEvent, - AgentRunHeader, - AgentRunStore, - RootExecutionDescriptor, -} from '@maka/core/agent-run'; +import type { AgentRunEvent, AgentRunStore } from '@maka/core/agent-run'; import type { ArtifactRecord } from '@maka/core/artifacts'; +import { invocationMatchesClaimTarget } from '@maka/core/runtime-boundary'; import type { ContinuationClaimV1 } from '@maka/core/runtime-boundary'; -import type { - RuntimeEventStore, - RuntimeContinuationAuthorityStore, +import { + readRunInvocation, + type RuntimeEventStore, + type RuntimeContinuationAuthorityStore, } from '@maka/core/runtime-event-store'; -import type { RuntimeEvent, ToolBoundaryProtocol } from '@maka/core/runtime-event'; +import type { + RuntimeEvent, + RuntimeEventInvocationOpenedContent, + RuntimeInvocationConfiguration, + RuntimeInvocationLineage, + RuntimeInvocationRootAuthority, + ToolBoundaryProtocol, +} from '@maka/core/runtime-event'; import type { RunCompositionSnapshot } from '@maka/core/run-composition'; import type { SubagentWorkspaceBinding, @@ -133,7 +143,10 @@ import type { import type { SubagentPreset } from '@maka/core/subagent-settings'; import type { ResolvedSubagentPreset } from './configured-subagent-catalog.js'; import { AGENT_GRAPH_OPERATOR_PROVISION_SCHEMA_VERSION } from '@maka/core/agent-graph-topology'; -import type { RuntimeEventTerminalFact } from './runtime-event-read-model.js'; +import { + runtimeInvocationFailureClass, + type RuntimeEventTerminalFact, +} from './runtime-event-read-model.js'; import { RuntimeReadModel, RuntimeReadModelError, @@ -141,12 +154,11 @@ import { type RuntimeReadModelSessionView, } from './runtime-read-model.js'; import { inspectAgentRunReadModel, type AgentRunInspectModel } from './agent-run-inspect.js'; -import { firstRuntimeRepairRunId, RuntimeLedgerRepair } from './runtime-ledger-repair.js'; +import { RuntimeLedgerRepair } from './runtime-ledger-repair.js'; import { buildRecoveredTerminalRuntimeEvent, classifyTerminalRuntimeLedger, commitTerminalRunWithRuntimeFact, - effectiveRunHeaderFromTerminalFact, terminalRunStatusFromRuntimeEvent, } from './terminal-run-commit.js'; @@ -198,6 +210,7 @@ import { buildStatusPatch, buildTurnStateMessage, turnHasRetainedOutput as messagesHaveRetainedOutput, + type RunLifecycleStatus, } from './session-projection-helpers.js'; import { assertAgentDefinitionRunnable, @@ -398,7 +411,6 @@ type ResolvedClaimedAgentGraphIntentInput = Omit< }; const CHILD_AGENT_SUMMARY_MAX_CHARS = 4_000; -const MAX_RUNTIME_LEDGER_REPAIR_ATTEMPTS = 8; export interface AgentListItem { runId: string; @@ -406,8 +418,8 @@ export interface AgentListItem { parentRunId: string; agentId?: string; agentName?: string; - status: AgentRunHeader['status']; - permissionMode: AgentRunHeader['permissionMode']; + status: RunLifecycleStatus; + permissionMode: PermissionMode; createdAt: number; updatedAt: number; completedAt?: number; @@ -421,7 +433,7 @@ export interface SubagentExecutionListItem { agentName?: string; profile?: string; turnId?: string; - status: AgentRunHeader['status']; + status: RunLifecycleStatus; permissionMode: PermissionMode; createdAt: number; updatedAt: number; @@ -453,7 +465,7 @@ export type AgentOutputView = 'result' | 'events' | 'runtime_events' | 'all'; export interface AgentOutputCommittedResult { schemaVersion: 1; - status: AgentRunHeader['status']; + status: RunLifecycleStatus; graph?: { graphId: string; workId: string; @@ -473,7 +485,7 @@ export interface AgentOutputCommittedResult { export interface AgentOutputResult { execution: SubagentExecutionRef; - header: AgentRunHeader; + invocation: RuntimeInvocationRecord; result?: AgentOutputCommittedResult; events: AgentRunEvent[]; runtimeEvents: RuntimeEvent[]; @@ -634,7 +646,6 @@ export interface StrictRecoverySessionStore extends SessionStore { } export interface StrictRecoveryAgentRunStore extends AgentRunStore { - listSessionRunsForRecovery(sessionId: string): Promise; readEventsForRecovery(sessionId: string, runId: string): Promise; } @@ -891,23 +902,14 @@ export class SessionManager { deps.runtimeCommitSink ?? runtimeCommitSinkFromEventStore(deps.runtimeEventStore); if (deps.runStore && deps.runtimeEventStore) { this.runtimeLedgerRepair = new RuntimeLedgerRepair({ - runStore: deps.runStore, runtimeEventStore: deps.runtimeEventStore, readMessages: (sessionId) => deps.store.readMessages(sessionId), appendMessage: (sessionId, message) => deps.store.appendMessage(sessionId, message), - appendTurnState: (sessionId, turnId, status, lineage, options) => - this.appendTurnState(sessionId, turnId, status, lineage, options), newId: deps.newId, now: deps.now, }); } - this.runtimeKernel = - deps.runtimeKernel ?? - new RuntimeKernel({ - ...deps, - repairRunRuntimeLedger: (sessionId, runId) => - this.repairMissingTerminalFactOnce(sessionId, runId), - }); + this.runtimeKernel = deps.runtimeKernel ?? new RuntimeKernel({ ...deps }); } // -------------------------------------------------------------------------- @@ -1021,7 +1023,7 @@ export class SessionManager { private async finalizeAndListChildTurnArtifacts( sessionId: string, turnId: string, - status: AgentRunHeader['status'], + status: RunLifecycleStatus, ): Promise { const list = this.deps.listArtifactsForTurn; if (!list) return []; @@ -1042,6 +1044,30 @@ export class SessionManager { return finalized; } + /** + * The Session's invocations, enumerated from the events that define them. + * + * There is no run table to consult: an invocation exists because its opening + * fact does, and it has ended because its terminal event does. + */ + private async listInvocations(sessionId: string): Promise { + const store = this.deps.runtimeEventStore; + if (!store) return []; + return store.listSessionInvocations(sessionId); + } + + /** One invocation by run id. Absent means no opening fact ever named it. */ + private async readInvocation(sessionId: string, runId: string): Promise { + const store = this.deps.runtimeEventStore; + const invocation = store ? await readRunInvocation(store, sessionId, runId) : undefined; + if (!invocation) { + const error = new Error(`AgentRun ${runId} not found`) as Error & { code?: string }; + error.code = 'ENOENT'; + throw error; + } + return invocation; + } + /** Publish the recoverable write-back owed by the latest terminal worktree child Run. */ async finalizeChildWorkspacePatches(sessionId: string): Promise { if (!this.hasWorktreePatchWriteBack() || !this.deps.runStore) return; @@ -1049,13 +1075,13 @@ export class SessionManager { const binding = header.subagentWorkspace; if (!binding) return; - const latest = (await this.deps.runStore.listSessionRuns(sessionId)) - .filter(isSessionInlineRun) - .sort( - (left, right) => right.createdAt - left.createdAt || right.runId.localeCompare(left.runId), - )[0]; + const latest = latestInvocation( + (await this.listInvocations(sessionId)).filter((run) => + isSessionInlineInvocation(run.opening), + ), + ); if (!latest) return; - if (!isTerminalRunStatus(latest.status)) { + if (!latest.terminalEvent) { throw new Error( `Child Session ${sessionId} cannot finalize its workspace while Run ${latest.runId} is nonterminal`, ); @@ -1246,7 +1272,13 @@ export class SessionManager { async getContextDiagnostics(sessionId: string): Promise { const runStore = this.deps.runStore; return runStore - ? readLatestContextDiagnostics(runStore, sessionId) + ? readLatestContextDiagnostics( + runStore, + sessionId, + (await this.listInvocations(sessionId)) + .filter((run) => isSessionInlineInvocation(run.opening)) + .map((run) => run.runId), + ) : { status: 'unavailable', reason: 'trace_unavailable' }; } @@ -2030,9 +2062,9 @@ export class SessionManager { let admissionRoute: RuntimeContinuationPlannerInput['admissionRoute']; try { if (!this.deps.runStore) throw new Error('AgentRunStore is not configured'); - const [header, runHeaders] = await Promise.all([ + const [header, invocations] = await Promise.all([ this.deps.store.readHeader(sessionId), - this.deps.runStore.listSessionRuns(sessionId), + this.listInvocations(sessionId), ]); const targetProviderStateIdentity = ( await this.deps.backends.prepare(header.backend, { @@ -2042,7 +2074,7 @@ export class SessionManager { }) ).providerStateIdentity; admissionRoute = { - runHeaders, + invocations, targetProviderStateIdentity, targetModelId: header.model, }; @@ -2061,9 +2093,9 @@ export class SessionManager { return plan; } const planner = new RuntimeContinuationPlanner({ - readSourceRun: async (targetSessionId, runId) => { + readSourceInvocation: async (targetSessionId, runId) => { if (!this.deps.runStore) throw new Error('AgentRunStore is not configured'); - return this.deps.runStore.readRun(targetSessionId, runId); + return this.readInvocation(targetSessionId, runId); }, readImmutableRuntimePrefix: async (prefixInput) => { const authority = runtimeContinuationAuthority(this.deps.runtimeEventStore); @@ -2083,11 +2115,14 @@ export class SessionManager { sourceRuntimeEventHighWater, ) => { if (!this.deps.runStore) throw new Error('AgentRunStore is not configured'); - return (await this.deps.runStore.listSessionRuns(targetSessionId)).find( - (run) => - run.continuationSource?.sourceRunId === sourceRunId && - run.continuationSource.sourceRuntimeEventHighWater === sourceRuntimeEventHighWater, - ); + return (await this.listInvocations(targetSessionId)).find((run) => { + const source = run.opening.source; + return ( + source.kind === 'continuation' && + source.sourceRunId === sourceRunId && + source.sourceRuntimeEventHighWater === sourceRuntimeEventHighWater + ); + }); }, newId: this.deps.newId, }); @@ -2119,9 +2154,9 @@ export class SessionManager { this.recordContinuationPlan(sessionId, input.sourceRunId, plan); return plan; } - const sourceRun = await this.deps.runStore - .readRun(sessionId, input.sourceRunId) - .catch(() => undefined); + const sourceRun = await this.readInvocation(sessionId, input.sourceRunId).catch( + () => undefined, + ); if (!sourceRun) { const plan: SafeBoundaryContinuationPlan = { disposition: 'park', @@ -2133,7 +2168,7 @@ export class SessionManager { this.recordContinuationPlan(sessionId, input.sourceRunId, plan); return plan; } - if (!sourceRun.workspaceIdentity) { + if (!sourceRun.opening.configuration.workspaceIdentity) { const plan: SafeBoundaryContinuationPlan = { disposition: 'park', rejectionReasons: ['workspace_identity_missing'], @@ -2168,7 +2203,7 @@ export class SessionManager { return this.planSafeBoundaryContinuation(sessionId, { sourceRunId: input.sourceRunId, currentCwd: header.cwd, - sourceWorkspaceIdentity: sourceRun.workspaceIdentity, + sourceWorkspaceIdentity: sourceRun.opening.configuration.workspaceIdentity, currentWorkspaceIdentity: observation.workspaceIdentity, backgroundOperationsSettled: observation.backgroundOperationsSettled, availableToolNames: observation.availableToolNames, @@ -2203,13 +2238,15 @@ export class SessionManager { this.recordContinuationPlan(sessionId, '', plan); return plan; } - const candidate = (await this.deps.runStore.listSessionRuns(sessionId)) - .filter( - (run) => (run.status === 'failed' || run.status === 'cancelled') && isSessionInlineRun(run), - ) - .sort( - (left, right) => right.createdAt - left.createdAt || right.runId.localeCompare(left.runId), - )[0]; + const candidate = latestInvocation( + (await this.listInvocations(sessionId)).filter((run) => { + const outcome = runtimeInvocationOutcome(run); + return ( + (outcome === 'failed' || outcome === 'cancelled') && + isSessionInlineInvocation(run.opening) + ); + }), + ); if (!candidate) { const plan: SafeBoundaryContinuationPlan = { disposition: 'park', @@ -2389,7 +2426,7 @@ export class SessionManager { } const [parentHeader, sourceRun, parentBoundary] = await Promise.all([ this.deps.store.readHeader(input.source.sessionId), - this.deps.runStore.readRun(input.source.sessionId, input.source.runId), + this.readInvocation(input.source.sessionId, input.source.runId), this.deps.store.readExecutionBoundary(input.source.sessionId), ]); if ( @@ -2716,7 +2753,7 @@ export class SessionManager { return readyNotification; }; - let run = await this.deps.runStore.readRun(child.id, claim.targetRunId).catch((error) => { + let run = await this.readInvocation(child.id, claim.targetRunId).catch((error) => { if (isNotFoundError(error)) return undefined; throw error; }); @@ -2740,7 +2777,7 @@ export class SessionManager { ); }, }); - run = await this.deps.runStore.readRun(child.id, claim.targetRunId); + run = await this.readInvocation(child.id, claim.targetRunId); this.assertClaimedAgentGraphRun(child, snapshot, claim, run); await this.assertClaimedAgentGraphPrompt( child.id, @@ -2757,15 +2794,15 @@ export class SessionManager { await this.assertClaimedAgentGraphPrompt(child.id, claim.targetTurnId, input.prompt); await notifyReady(); while ( - !isTerminalRunStatus(run.status) && + !run.terminalEvent && this.runtimeKernel.hasActiveRun?.(child.id, run.runId, run.turnId) ) { await delay(25, undefined, input.abortSignal ? { signal: input.abortSignal } : undefined); - run = await this.deps.runStore.readRun(child.id, claim.targetRunId); + run = await this.readInvocation(child.id, claim.targetRunId); } - if (!isTerminalRunStatus(run.status)) { + if (!run.terminalEvent) { await this.recoverAgentRunsFromLedger(child.id); - run = await this.deps.runStore.readRun(child.id, claim.targetRunId); + run = await this.readInvocation(child.id, claim.targetRunId); } this.assertClaimedAgentGraphRun(child, snapshot, claim, run); await this.assertClaimedAgentGraphPrompt(child.id, claim.targetTurnId, input.prompt); @@ -2775,7 +2812,7 @@ export class SessionManager { await this.finalizeChildWorkspacePatches(child.id); const [runs, messages] = await Promise.all([ - this.deps.runStore.listSessionRuns(child.id), + this.listInvocations(child.id), this.deps.store.readMessages(child.id), ]); const turnOwner = runs.find((candidate) => candidate.turnId === claim.targetTurnId); @@ -2860,13 +2897,14 @@ export class SessionManager { } const completedAt = this.deps.now(); - const completedRun = await this.deps.runStore.readRun(child.id, claim.targetRunId); + const completedRun = await this.readInvocation(child.id, claim.targetRunId); this.assertClaimedAgentGraphRun(child, snapshot, claim, completedRun); - const failureClass = completedRun.failureClass ?? summary.failureClass; + const completedFacts = invocationListingFacts(completedRun); + const failureClass = completedFacts.failureClass ?? summary.failureClass; const artifacts = await this.finalizeAndListChildTurnArtifacts( child.id, claim.targetTurnId, - completedRun.status, + completedFacts.status, ); return { claimId: claim.claimId, @@ -2879,7 +2917,7 @@ export class SessionManager { profile: snapshot.profile, turnId: claim.targetTurnId, runId: claim.targetRunId, - status: agentRunStatusForSpawnResult(completedRun.status), + status: agentRunStatusForSpawnResult(completedFacts.status), permissionMode: child.permissionMode, summary: summary.text(), artifactIds: artifacts.map((artifact) => artifact.id), @@ -2944,15 +2982,16 @@ export class SessionManager { child: SessionHeader, snapshot: NonNullable, claim: AgentGraphIntentClaim, - run: AgentRunHeader, + run: RuntimeInvocationRecord, ): void { + const lineage = run.opening.lineage; if ( run.sessionId !== child.id || run.runId !== claim.targetRunId || run.turnId !== claim.targetTurnId || - !isSessionInlineRun(run) || - run.agentId !== snapshot.agentId || - (run.agentName !== undefined && run.agentName !== snapshot.agentName) + !isSessionInlineInvocation(run.opening) || + lineage?.agentId !== snapshot.agentId || + (lineage?.agentName !== undefined && lineage.agentName !== snapshot.agentName) ) { throw new Error('Existing AgentRun does not match the claimed graph activation identity'); } @@ -2999,7 +3038,7 @@ export class SessionManager { } const [parentHeader, parentRun, parentBoundary] = await Promise.all([ this.deps.store.readHeader(parentSessionId), - this.deps.runStore.readRun(parentSessionId, input.spawnedBy.parentRunId), + this.readInvocation(parentSessionId, input.spawnedBy.parentRunId), this.deps.store.readExecutionBoundary(parentSessionId), ]); this.assertActiveParentRun(parentSessionId, parentRun, input.spawnedBy.parentTurnId); @@ -3129,7 +3168,7 @@ export class SessionManager { // crash boundary. Revalidate admission after the lookup: the parent or // caller may have settled while durable state was being inspected. try { - const latestParentRun = await this.deps.runStore.readRun( + const latestParentRun = await this.readInvocation( parentSessionId, input.spawnedBy.parentRunId, ); @@ -3203,9 +3242,10 @@ export class SessionManager { const completedAt = this.deps.now(); const run = await this.findRunByTurnId(child.id, turnId); - const failureClass = run?.failureClass ?? summary.failureClass; - const artifacts = run - ? await this.finalizeAndListChildTurnArtifacts(child.id, turnId, run.status) + const facts = run ? invocationListingFacts(run) : undefined; + const failureClass = facts?.failureClass ?? summary.failureClass; + const artifacts = facts + ? await this.finalizeAndListChildTurnArtifacts(child.id, turnId, facts.status) : []; return { childSessionId: child.id, @@ -3214,7 +3254,7 @@ export class SessionManager { profile: snapshot.profile, turnId, runId, - status: run ? agentRunStatusForSpawnResult(run.status) : summary.status(aborted), + status: facts ? agentRunStatusForSpawnResult(facts.status) : summary.status(aborted), permissionMode: child.permissionMode, summary: summary.text(), artifactIds: artifacts.map((artifact) => artifact.id), @@ -3240,7 +3280,7 @@ export class SessionManager { if (!snapshot || !spawn) { throw new Error('Stored child session is missing its durable runtime or spawn identity'); } - let run = await this.deps.runStore.readRun(child.id, spawn.initialRunId).catch((error) => { + let run = await this.readInvocation(child.id, spawn.initialRunId).catch((error) => { if (isNotFoundError(error)) return undefined; throw error; }); @@ -3248,32 +3288,33 @@ export class SessionManager { await notifyReady(); while ( - !isTerminalRunStatus(run.status) && + !run.terminalEvent && this.runtimeKernel.hasActiveRun?.(child.id, run.runId, run.turnId) ) { await delay(25, undefined, input.abortSignal ? { signal: input.abortSignal } : undefined); - run = await this.deps.runStore.readRun(child.id, spawn.initialRunId); + run = await this.readInvocation(child.id, spawn.initialRunId); } - if (!isTerminalRunStatus(run.status)) { + if (!run.terminalEvent) { await this.recoverAgentRunsFromLedger(child.id); - run = await this.deps.runStore.readRun(child.id, spawn.initialRunId); + run = await this.readInvocation(child.id, spawn.initialRunId); } return await this.projectExistingChildSpawn(child, run); } private async projectExistingChildSpawn( child: SessionHeader, - run: AgentRunHeader, + run: RuntimeInvocationRecord, ): Promise { if (!this.deps.runtimeEventStore) { throw new Error('Child session projection requires RuntimeEventStore'); } const snapshot = child.subagentRuntime; if (!snapshot) throw new Error('Stored child session is missing its durable runtime snapshot'); + const facts = invocationListingFacts(run); const [messages, runtimeEvents, artifacts] = await Promise.all([ this.deps.store.readMessages(child.id), this.deps.runtimeEventStore.readRuntimeEvents(child.id, run.runId), - this.finalizeAndListChildTurnArtifacts(child.id, run.turnId, run.status), + this.finalizeAndListChildTurnArtifacts(child.id, run.turnId, facts.status), ]); const storedSummary = messages @@ -3295,7 +3336,6 @@ export class SessionManager { .filter((event) => event.partial) .map((event) => event.content.text) .join(''); - const completedAt = run.completedAt ?? run.updatedAt; return { childSessionId: child.id, agentId: snapshot.agentId, @@ -3303,15 +3343,15 @@ export class SessionManager { profile: snapshot.profile, turnId: run.turnId, runId: run.runId, - status: agentRunStatusForSpawnResult(run.status), + status: agentRunStatusForSpawnResult(facts.status), permissionMode: child.permissionMode, summary: trimSummary(durableRuntimeSummary ?? (storedSummary || partialRuntimeSummary)), artifactIds: artifacts.map((artifact) => artifact.id), - startedAt: run.createdAt, - completedAt, - durationMs: Math.max(0, completedAt - run.createdAt), + startedAt: facts.createdAt, + completedAt: facts.updatedAt, + durationMs: facts.durationMs ?? 0, eventCount: runtimeEvents.length, - ...(run.failureClass ? { failureClass: run.failureClass } : {}), + ...(facts.failureClass ? { failureClass: facts.failureClass } : {}), }; } @@ -3373,36 +3413,20 @@ export class SessionManager { }); const presets = this.deps.subagentCatalog ? await this.deps.subagentCatalog.list() : []; if (!this.deps.runStore) return { definitions, presets, executions: [], runs: [] }; - const runs = await this.deps.runStore.listSessionRuns(sessionId); - const childRuns = await Promise.all( - runs - .filter( - (run): run is AgentRunHeader & { parentRunId: string } => - !!run.parentRunId && !isSessionInlineRun(run), - ) - .map( - async (run): Promise => ({ - ...(await this.effectiveRunHeaderFromRuntimeLedger(run)), - parentRunId: run.parentRunId, - }), - ), + const childRuns = (await this.listInvocations(sessionId)).filter( + (run) => !!run.opening.lineage?.parentRunId && !isSessionInlineInvocation(run.opening), ); - const legacyRuns = childRuns.map((run) => ({ - runId: run.runId, - turnId: run.turnId, - parentRunId: run.parentRunId, - ...(run.agentId ? { agentId: run.agentId } : {}), - ...(run.agentName ? { agentName: run.agentName } : {}), - status: run.status, - permissionMode: run.permissionMode, - createdAt: run.createdAt, - updatedAt: run.updatedAt, - ...(run.completedAt !== undefined ? { completedAt: run.completedAt } : {}), - ...(run.completedAt !== undefined - ? { durationMs: Math.max(0, run.completedAt - run.createdAt) } - : {}), - ...(run.failureClass ? { failureClass: run.failureClass } : {}), - })); + const legacyRuns = childRuns.map((run) => { + const facts = invocationListingFacts(run); + return { + runId: run.runId, + turnId: run.turnId, + parentRunId: run.opening.lineage!.parentRunId!, + ...(run.opening.lineage?.agentId ? { agentId: run.opening.lineage.agentId } : {}), + ...(run.opening.lineage?.agentName ? { agentName: run.opening.lineage.agentName } : {}), + ...facts, + }; + }); const childSessionHeaders = await Promise.all( (await this.listChildSessions(sessionId)).map((child) => this.deps.store.readHeader(child.id), @@ -3410,16 +3434,8 @@ export class SessionManager { ); const childSessionExecutions = await Promise.all( childSessionHeaders.map(async (child): Promise => { - const childRuns = await this.deps.runStore!.listSessionRuns(child.id); - const latest = childRuns - .slice() - .sort( - (left, right) => - right.createdAt - left.createdAt || - right.updatedAt - left.updatedAt || - right.runId.localeCompare(left.runId), - )[0]; - const run = latest ? await this.effectiveRunHeaderFromRuntimeLedger(latest) : undefined; + const run = latestInvocation(await this.listInvocations(child.id)); + const facts = run ? invocationListingFacts(run) : undefined; const currentRunId = run?.runId ?? child.subagentSpawn?.initialRunId; return { execution: { @@ -3433,15 +3449,13 @@ export class SessionManager { : {}), ...(child.subagentRuntime?.profile ? { profile: child.subagentRuntime.profile } : {}), ...(run?.turnId ? { turnId: run.turnId } : {}), - status: run?.status ?? (child.status === 'aborted' ? 'cancelled' : 'created'), - permissionMode: run?.permissionMode ?? child.permissionMode, - createdAt: run?.createdAt ?? child.createdAt, - updatedAt: run?.updatedAt ?? child.lastMessageAt ?? child.createdAt, - ...(run?.completedAt !== undefined ? { completedAt: run.completedAt } : {}), - ...(run?.completedAt !== undefined - ? { durationMs: Math.max(0, run.completedAt - run.createdAt) } - : {}), - ...(run?.failureClass ? { failureClass: run.failureClass } : {}), + status: facts?.status ?? (child.status === 'aborted' ? 'cancelled' : 'running'), + permissionMode: facts?.permissionMode ?? child.permissionMode, + createdAt: facts?.createdAt ?? child.createdAt, + updatedAt: facts?.updatedAt ?? child.lastMessageAt ?? child.createdAt, + ...(facts?.completedAt !== undefined ? { completedAt: facts.completedAt } : {}), + ...(facts?.durationMs !== undefined ? { durationMs: facts.durationMs } : {}), + ...(facts?.failureClass ? { failureClass: facts.failureClass } : {}), }; }), ); @@ -3450,7 +3464,7 @@ export class SessionManager { presets, executions: [ ...childSessionExecutions, - ...childRuns.map( + ...legacyRuns.map( (run): SubagentExecutionListItem => ({ execution: { kind: 'legacy_child_run', @@ -3465,9 +3479,7 @@ export class SessionManager { createdAt: run.createdAt, updatedAt: run.updatedAt, ...(run.completedAt !== undefined ? { completedAt: run.completedAt } : {}), - ...(run.completedAt !== undefined - ? { durationMs: Math.max(0, run.completedAt - run.createdAt) } - : {}), + ...(run.durationMs !== undefined ? { durationMs: run.durationMs } : {}), ...(run.failureClass ? { failureClass: run.failureClass } : {}), }), ), @@ -3489,27 +3501,23 @@ export class SessionManager { throw new Error('agent_output requires AgentRunStore and RuntimeEventStore'); } const located = await this.findChildRunForOutput(sessionId, input); - const { header } = located; + const { invocation } = located; const inspected = await inspectAgentRunReadModel( this.deps.runStore, this.deps.runtimeEventStore, - { - sessionId: header.sessionId, - runId: header.runId, - header, - }, + { sessionId: invocation.sessionId, runId: invocation.runId, invocation }, ); const artifacts = await this.finalizeAndListChildTurnArtifacts( - header.sessionId, - header.turnId, - inspected.header.status, + invocation.sessionId, + invocation.turnId, + runtimeInvocationOutcome(inspected.invocation) ?? 'running', ); const maxEvents = normalizeAgentOutputMaxEvents(input.maxEvents); const maxBytes = normalizeAgentOutputMaxBytes(input.maxBytes); const view = input.view ?? 'runtime_events'; if (view === 'result') { const boundedResult = buildAgentOutputCommittedResult({ - header: inspected.header, + invocation: inspected.invocation, runtimeEvents: inspected.runtimeEvents, artifacts, maxArtifacts: maxEvents, @@ -3518,7 +3526,7 @@ export class SessionManager { }); return { execution: located.execution, - header: inspected.header, + invocation: inspected.invocation, result: boundedResult.result, events: [], runtimeEvents: [], @@ -3550,7 +3558,7 @@ export class SessionManager { ); return { execution: located.execution, - header: inspected.header, + invocation: inspected.invocation, events: bounded.events, runtimeEvents: bounded.runtimeEvents, sourceHealth: inspected.sourceHealth, @@ -3658,30 +3666,44 @@ export class SessionManager { throw new Error('Hosted admission recovery requires execution stores'); } const session = await this.deps.store.readHeader(input.sessionId); - const headerExtras: Partial = {}; + let root: RuntimeInvocationRootAuthority = { kind: 'user' }; + let orchestration: Pick< + RuntimeInvocationConfiguration, + 'orchestrationMode' | 'orchestrationSource' | 'agentSwarmAuthorization' + > = { + orchestrationMode: session.orchestrationMode ?? 'default', + orchestrationSource: 'session', + agentSwarmAuthorization: 'none', + }; + const lineage: RuntimeInvocationLineage = {}; let recoveryReason: string; let diagnostic: Record; let workspaceIdentity: string | undefined; if (input.execution.kind === 'goal') { - headerExtras.goalId = input.execution.goalId; + root = { kind: 'goal', goalId: input.execution.goalId }; recoveryReason = 'goal_internal_admission_without_run'; diagnostic = { executionKind: input.execution.kind, goalId: input.execution.goalId, }; } else if (input.execution.kind === 'legacy_automation') { - headerExtras.legacyAutomationId = input.execution.automationId; + root = { kind: 'legacy_automation', legacyAutomationId: input.execution.automationId }; recoveryReason = 'legacy_automation_authority_removed'; diagnostic = { executionKind: input.execution.kind, automationId: input.execution.automationId, }; } else if (input.execution.kind === 'agent_graph_supervisor_wake') { - headerExtras.agentGraphWakeId = input.execution.wakeId; - headerExtras.agentGraphWakeAttemptId = input.execution.attemptId; - headerExtras.orchestrationMode = 'graph'; - headerExtras.orchestrationSource = 'turn_override'; - headerExtras.agentSwarmAuthorization = 'none'; + root = { + kind: 'agent_graph_supervisor_wake', + wakeId: input.execution.wakeId, + attemptId: input.execution.attemptId, + }; + orchestration = { + orchestrationMode: 'graph', + orchestrationSource: 'turn_override', + agentSwarmAuthorization: 'none', + }; recoveryReason = 'agent_graph_supervisor_internal_admission_without_run'; diagnostic = { executionKind: input.execution.kind, @@ -3733,27 +3755,24 @@ export class SessionManager { input.execution.kind === 'linked_child_resume' || input.execution.kind === 'linked_child_provider_retry' ) { - const sourceRun = await this.deps.runStore.readRun( - input.sessionId, - input.execution.sourceRunId, - ); + const sourceRun = await this.readInvocation(input.sessionId, input.execution.sourceRunId); if ( - sourceRun.agentId !== input.execution.agentId || - sourceRun.agentName !== input.execution.agentName + sourceRun.opening.lineage?.agentId !== input.execution.agentId || + sourceRun.opening.lineage.agentName !== input.execution.agentName ) { throw new Error( `Admitted Turn ${input.turnId} source changed its trusted agent identity`, ); } - workspaceIdentity = sourceRun.workspaceIdentity; + workspaceIdentity = sourceRun.opening.configuration.workspaceIdentity; if (input.execution.kind === 'linked_child_resume') { - headerExtras.resumedFromRunId = input.execution.sourceRunId; + lineage.resumedFromRunId = input.execution.sourceRunId; } else { - headerExtras.retriedFromRunId = input.execution.sourceRunId; + lineage.retriedFromRunId = input.execution.sourceRunId; } } - headerExtras.agentId = input.execution.agentId; - headerExtras.agentName = input.execution.agentName; + lineage.agentId = input.execution.agentId; + lineage.agentName = input.execution.agentName; recoveryReason = 'child_internal_admission_without_run'; diagnostic = { executionKind: input.execution.kind, @@ -3766,27 +3785,55 @@ export class SessionManager { throw new Error('External message recovery closure is not supported'); } - const run: AgentRunHeader = { - runId: input.runId, - invocationId: input.runId, + const run = { sessionId: input.sessionId, + invocationId: input.runId, + runId: input.runId, turnId: input.turnId, - status: 'created', - backendKind: session.backend, - ...(session.llmConnectionId === undefined - ? {} - : { llmConnectionId: session.llmConnectionId }), - llmConnectionSlug: session.llmConnectionSlug, - modelId: session.model, - cwd: session.cwd, - ...(workspaceIdentity !== undefined ? { workspaceIdentity } : {}), - permissionMode: session.permissionMode, - collaborationMode: session.collaborationMode ?? 'agent', - createdAt: input.admittedAt, - updatedAt: input.admittedAt, - ...headerExtras, }; - await this.deps.runStore.createRun(run, { durable: true }); + const opening: RuntimeEventInvocationOpenedContent = { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: + session.llmConnectionId === undefined + ? { + provenance: 'unknown', + backendKind: session.backend, + llmConnectionSlug: session.llmConnectionSlug, + modelId: session.model, + } + : { + provenance: 'runtime', + backendKind: session.backend, + llmConnectionId: session.llmConnectionId, + llmConnectionSlug: session.llmConnectionSlug, + modelId: session.model, + }, + configuration: { + cwd: session.cwd, + permissionMode: session.permissionMode, + collaborationMode: session.collaborationMode ?? 'agent', + toolMode: DEFAULT_TOOL_MODE, + ...orchestration, + ...(workspaceIdentity !== undefined ? { workspaceIdentity } : {}), + }, + root, + source: { kind: 'fresh' }, + ...(Object.keys(lineage).length > 0 ? { lineage } : {}), + }; + // The admission never reached an AgentRun, so nothing else will ever open + // this invocation. Recovery opens and closes it in one pass so the Turn + // ends up on the spine like any other, with its own reason for ending. + await this.deps.runtimeEventStore.appendRuntimeEvent( + input.sessionId, + input.runId, + buildInvocationOpenedEvent({ + id: this.deps.newId(), + run, + openedAt: input.admittedAt, + opening, + }), + ); const ts = this.deps.now(); const terminalEvent = buildRecoveredTerminalRuntimeEvent({ @@ -3800,7 +3847,6 @@ export class SessionManager { message: 'app_restarted', }); await commitTerminalRunWithRuntimeFact({ - runStore: this.deps.runStore, runtimeEventStore: this.deps.runtimeEventStore, newId: this.deps.newId, sessionId: input.sessionId, @@ -3810,12 +3856,6 @@ export class SessionManager { ts, terminalEvent, failureClass: 'app_restarted', - runEventData: { - recovered: true, - recoveryReason, - ...diagnostic, - }, - existingEvents: [], }); } @@ -3948,7 +3988,7 @@ export class SessionManager { } const readMessages = readMessagesSnapshot.bind(this.deps.store); const view = await this.getSessionView(sessionId, { readMessages }); - if (view.runs.length > 0 || view.messages.length > 0) return view; + if (view.invocations.length > 0 || view.messages.length > 0) return view; const messages = await readMessages(sessionId); if (messages.length === 0) return view; return { @@ -3986,16 +4026,15 @@ export class SessionManager { private async findRunByTurnId( sessionId: string, turnId: string, - ): Promise { + ): Promise { if (!this.deps.runStore) return undefined; - const runs = await this.deps.runStore.listSessionRuns(sessionId).catch(() => []); - const run = runs.find((candidate) => candidate.turnId === turnId); - return run ? this.effectiveRunHeaderFromRuntimeLedger(run) : undefined; + const runs = await this.listInvocations(sessionId).catch(() => []); + return runs.find((candidate) => candidate.turnId === turnId); } private assertActiveParentRun( parentSessionId: string, - parentRun: AgentRunHeader, + parentRun: RuntimeInvocationRecord, parentTurnId: string, ): void { if ( @@ -4058,7 +4097,7 @@ export class SessionManager { sessionId: string, input: AgentOutputInput, ): Promise<{ - header: AgentRunHeader; + invocation: RuntimeInvocationRecord; execution: SubagentExecutionRef; graph?: NonNullable; }> { @@ -4078,27 +4117,19 @@ export class SessionManager { ) { throw new Error('agent_output could not find the requested child session'); } - const runs = await this.deps.runStore?.listSessionRuns(child.id); + const runs = await this.listInvocations(child.id); const selected = execution.currentRunId - ? runs?.find((run) => run.runId === execution.currentRunId) - : runs - ?.slice() - .sort( - (left, right) => - right.createdAt - left.createdAt || - right.updatedAt - left.updatedAt || - right.runId.localeCompare(left.runId), - )[0]; - if (!selected || !isSessionInlineRun(selected)) { + ? runs.find((run) => run.runId === execution.currentRunId) + : latestInvocation(runs); + if (!selected || !isSessionInlineInvocation(selected.opening)) { throw new Error('agent_output could not find the requested child session run'); } - const header = await this.effectiveRunHeaderFromRuntimeLedger(selected); return { - header, + invocation: selected, execution: { kind: 'child_session', sessionId: child.id, - currentRunId: header.runId, + currentRunId: selected.runId, }, ...(child.subagentParent.graph ? { graph: child.subagentParent.graph } : {}), }; @@ -4109,8 +4140,7 @@ export class SessionManager { if (legacyExecution && legacyExecution.sessionId !== sessionId) { throw new Error('agent_output could not find the requested legacy child run'); } - const runs = await this.deps.runStore?.listSessionRuns(sessionId); - const header = runs?.find((run) => + const invocation = (await this.listInvocations(sessionId)).find((run) => legacyExecution ? run.runId === legacyExecution.runId : input.runId @@ -4119,30 +4149,20 @@ export class SessionManager { ? run.turnId === input.turnId : false, ); - if (!header) throw new Error('agent_output could not find the requested child agent run'); - if (!header.parentRunId || isSessionInlineRun(header)) { + if (!invocation) throw new Error('agent_output could not find the requested child agent run'); + if (!invocation.opening.lineage?.parentRunId || isSessionInlineInvocation(invocation.opening)) { throw new Error('agent_output only reads child agent runs'); } return { - header: await this.effectiveRunHeaderFromRuntimeLedger(header), + invocation, execution: { kind: 'legacy_child_run', sessionId, - runId: header.runId, + runId: invocation.runId, }, }; } - private async effectiveRunHeaderFromRuntimeLedger(run: AgentRunHeader): Promise { - if (!this.deps.runtimeEventStore) return run; - const runtimeEvents = await this.deps.runtimeEventStore - .readRuntimeEvents(run.sessionId, run.runId) - .catch(() => undefined); - if (!runtimeEvents) return run; - const ledger = classifyTerminalRuntimeLedger(run, runtimeEvents); - return ledger.kind === 'fact' ? effectiveRunHeaderFromTerminalFact(run, ledger.fact) : run; - } - private async updateStatus( sessionId: string, status: SessionStatus, @@ -4323,22 +4343,6 @@ export class SessionManager { sessionId: string, projectionCache: RuntimeReadModelProjectionCache = this.deps.store, ): Promise { - const repaired = new Set(); - for (let attempt = 0; attempt < MAX_RUNTIME_LEDGER_REPAIR_ATTEMPTS; attempt += 1) { - try { - const view = await this.readModel(projectionCache).getSessionView(sessionId); - const runId = firstRuntimeRepairRunId(view.diagnostics, repaired); - if (!runId) return view; - if (!(await this.repairMissingTerminalFactOnce(sessionId, runId))) return view; - repaired.add(runId); - } catch (error) { - if (!(error instanceof RuntimeReadModelError)) throw error; - const runId = firstRuntimeRepairRunId(error.diagnostics, repaired); - if (!runId) throw error; - if (!(await this.repairMissingTerminalFactOnce(sessionId, runId))) throw error; - repaired.add(runId); - } - } return this.readModel(projectionCache).getSessionView(sessionId); } @@ -4349,7 +4353,6 @@ export class SessionManager { throw new Error('RuntimeReadModel requires AgentRunStore and RuntimeEventStore'); } return new RuntimeReadModel({ - runStore: this.deps.runStore, runtimeEventStore: this.deps.runtimeEventStore, projectionCache, ...(this.deps.canonicalPermissionOutcomes @@ -4358,12 +4361,6 @@ export class SessionManager { }); } - private async repairMissingTerminalFactOnce(sessionId: string, runId: string): Promise { - return ( - (await this.runtimeLedgerRepair?.repairMissingTerminalFactOnce(sessionId, runId)) ?? false - ); - } - async prepareImportedSessionHistory(sessionId: string): Promise { const repair = this.runtimeLedgerRepair; if (!repair) throw new Error('Imported Session history requires canonical Runtime stores'); @@ -4409,33 +4406,19 @@ export class SessionManager { let recovered = false; for (const initialState of states) { const { claim } = initialState; - let run: AgentRunHeader; - try { - run = await this.deps.runStore.readRun(sessionId, claim.target.runId); - } catch (error) { - if (!isMissingRunError(error)) throw error; - try { - await this.deps.runStore.createRun(claim.targetRunHeader, { durable: true }); - run = await this.deps.runStore.readRun(sessionId, claim.target.runId); - } catch (createError) { - try { - run = await this.deps.runStore.readRun(sessionId, claim.target.runId); - } catch { - throw createError; - } - } - recovered = true; - } + // The target's opening fact rides its continuation-start event, so an + // invocation that does not exist yet is exactly the case the repair + // start below commits. There is no separate run record to create. + const invocation = await this.readInvocation(sessionId, claim.target.runId).catch((error) => { + if (isMissingRunError(error)) return undefined; + throw error; + }); let state = (await authority.readContinuationClaimStateByBoundary(claim.boundaryDigest)) ?? initialState; - if ( - state.startEventId - ? !claimTargetRunHeaderIsCompatible(run, claim.targetRunHeader) - : !isDeepStrictEqual(run, claim.targetRunHeader) - ) { + if (invocation && !invocationMatchesClaimTarget(invocation, claim)) { throw new Error( - `Continuation claim target Run header conflicts with claim ${claim.claimId}`, + `Continuation claim target invocation conflicts with claim ${claim.claimId}`, ); } @@ -4474,7 +4457,7 @@ export class SessionManager { const failureClass = 'continuation_abandoned_before_provider_dispatch'; const expectedTerminal = buildRecoveredTerminalRuntimeEvent({ id: continuationRepairEventId('terminal', claim.claimId), - run, + run: claim.target, status: 'failed', ts: Math.max(start.ts + 1, claim.claimedAt + 1), recoveryReason: failureClass, @@ -4486,18 +4469,8 @@ export class SessionManager { if (terminal && !isDeepStrictEqual(terminal, expectedTerminal)) { throw new Error(`Continuation claim ${claim.claimId} has a conflicting repair terminal`); } - const existingRunEvents = await this.deps.runStore.readEvents( - claim.target.sessionId, - claim.target.runId, - ); - const projectionComplete = - terminal !== undefined && - run.status === 'failed' && - run.failureClass === failureClass && - existingRunEvents.some((event) => event.type === 'run_failed'); - if (projectionComplete) continue; + if (terminal) continue; await commitTerminalRunWithRuntimeFact({ - runStore: this.deps.runStore, runtimeEventStore: authority, newId: () => continuationRepairEventId('run-terminal', claim.claimId), sessionId: claim.target.sessionId, @@ -4507,12 +4480,6 @@ export class SessionManager { ts: expectedTerminal.ts, terminalEvent: expectedTerminal, failureClass, - runEventData: { - recovered: true, - recoveryReason: failureClass, - continuationClaimId: claim.claimId, - }, - existingEvents: existingRunEvents, }); recovered = true; } @@ -4525,10 +4492,7 @@ export class SessionManager { ): Promise<{ hasLedger: boolean; recovered: boolean }> { if (!this.deps.runStore || !this.deps.runtimeEventStore) return { hasLedger: false, recovered: false }; - const runs = - policy.kind === 'strict' - ? await policy.stores.agentRunStore.listSessionRunsForRecovery(sessionId) - : await this.deps.runStore.listSessionRuns(sessionId); + const runs = await this.listInvocations(sessionId); if (runs.length === 0) return { hasLedger: false, recovered: false }; const continuationAuthority = runtimeContinuationAuthority(this.deps.runtimeEventStore); const claimOwnedUnsettledRunIds = new Set(); @@ -4570,7 +4534,7 @@ export class SessionManager { let inspected = await inspectAgentRunReadModel( this.deps.runStore, this.deps.runtimeEventStore, - { sessionId, runId: run.runId, header: run }, + { sessionId, runId: run.runId, invocation: run }, ); if (inspected.sourceHealth.runtimeLedger === 'read_failed') { if (policy.kind === 'strict') { @@ -4601,7 +4565,7 @@ export class SessionManager { const interruptedOutcomes = buildInterruptedCodeModeOutcomeCommits( inspected.runtimeEvents, this.deps.now(), - run.toolMode ?? DEFAULT_TOOL_MODE, + run.opening.configuration.toolMode, ); let outcomeCommitFailed = false; for (const outcome of interruptedOutcomes) { @@ -4623,23 +4587,16 @@ export class SessionManager { inspected = await inspectAgentRunReadModel( this.deps.runStore, this.deps.runtimeEventStore, - { sessionId, runId: run.runId, header: run }, + { sessionId, runId: run.runId, invocation: run }, ); } } const terminalLedger = classifyTerminalRuntimeLedger(run, inspected.runtimeEvents); - if (terminalLedger.kind === 'ambiguous') { + if (terminalLedger.kind === 'corrupt') { if (policy.kind === 'strict') { - throw new Error(`RuntimeEvent ledger has ambiguous terminal facts for run ${run.runId}`); - } - continue; - } - if (isTerminalRunStatus(run.status) && !inspected.terminalRuntimeFact) { - const repaired = await this.repairMissingTerminalFactOnce(sessionId, run.runId); - if (repaired) { - recovered = true; - } else if (policy.kind === 'strict') { - throw new Error(`Unable to repair the terminal RuntimeEvent fact for run ${run.runId}`); + throw new Error( + `RuntimeEvent ledger has more than one terminal event for run ${run.runId}`, + ); } continue; } @@ -4660,9 +4617,11 @@ export class SessionManager { private classifyRuntimeEventRecovery( inspected: AgentRunInspectModel, ): AgentRunRecoveryDecision | undefined { - if (isTerminalRunStatus(inspected.header.status) || !inspected.terminalRuntimeFact) - return undefined; - return runtimeTerminalFactToRecoveryDecision(inspected.header, inspected.terminalRuntimeFact); + if (!inspected.terminalRuntimeFact) return undefined; + return runtimeTerminalFactToRecoveryDecision( + inspected.invocation, + inspected.terminalRuntimeFact, + ); } private async applyAgentRunRecovery( @@ -4673,12 +4632,7 @@ export class SessionManager { ): Promise { if (!this.deps.runStore || !this.deps.runtimeEventStore) return false; const ts = this.deps.now(); - const terminalLedger = classifyTerminalRuntimeLedger(inspected.header, inspected.runtimeEvents); - const existingTerminal = - inspected.terminalRuntimeFact?.terminalEvent ?? - (terminalLedger.kind === 'incomplete_single_terminal' - ? terminalLedger.terminalEvent - : undefined); + const existingTerminal = inspected.terminalRuntimeFact?.terminalEvent; const status = existingTerminal ? (terminalRunStatusFromRuntimeEvent(existingTerminal) ?? decision.status) : decision.status; @@ -4689,7 +4643,7 @@ export class SessionManager { existingTerminal ?? buildRecoveredTerminalRuntimeEvent({ id: this.deps.newId(), - run: inspected.header, + run: inspected.invocation, status, ts, recoveryReason: diagnosticRecoveryReason(decision.diagnostic), @@ -4702,7 +4656,6 @@ export class SessionManager { }); try { await commitTerminalRunWithRuntimeFact({ - runStore: this.deps.runStore, runtimeEventStore: this.deps.runtimeEventStore, newId: this.deps.newId, sessionId, @@ -4713,20 +4666,18 @@ export class SessionManager { terminalEvent, ...(failureClass ? { failureClass } : {}), ...(abortSource ? { abortSource } : {}), - runEventData: { recovered: true, ...decision.diagnostic }, - existingEvents: inspected.events, }); } catch (error) { if (policy.kind === 'strict') throw error; return false; } - await recoverOr( + const appendedTurnState = await recoverOr( policy, () => this.appendTerminalTurnStateIfNeeded( sessionId, - inspected.header, + inspected.invocation, decision, terminalTurnStatus(status), { @@ -4736,28 +4687,32 @@ export class SessionManager { }, policy, ), - undefined, + false, ); - return true; + // A run that already carried a complete terminal fact and a terminal Turn + // state had nothing to recover. Saying otherwise makes recovery rewrite the + // Session status of every healthy run it walks past. + return inspected.terminalRuntimeFact === undefined || appendedTurnState; } private async appendTerminalTurnStateIfNeeded( sessionId: string, - run: AgentRunHeader, + run: RuntimeInvocationRecord, decision: AgentRunRecoveryDecision, status: TurnRecord['status'], options: { ts: number; errorClass?: string; abortSource?: string }, policy: RecoveryPolicy = { kind: 'best_effort' }, - ): Promise { - if (!isSessionInlineRun(run)) return; + ): Promise { + if (!isSessionInlineInvocation(run.opening)) return false; const messages = await recoverOr( policy, () => this.deps.store.readMessages(sessionId), [] as StoredMessage[], ); const latest = latestTurnState(messages, decision.turnId); - if (latest && isTerminalTurnStatus(latest.status) && latest.status === status) return; + if (latest && isTerminalTurnStatus(latest.status) && latest.status === status) return false; await this.appendTurnState(sessionId, decision.turnId, status, decision.lineage, options); + return true; } } @@ -4847,6 +4802,8 @@ function buildContinuationRepairStartEvent(claim: ContinuationClaimV1): RuntimeE partial: false, role: 'system', author: 'system', + modelVisibility: 'hidden', + content: claim.targetOpening, actions: { continuationStart: { protocol: 'continuation_start_v2', @@ -4891,38 +4848,26 @@ function assertClaimOwnsHostedLinkedChildAdmission( ) { throw new Error('Linked child admission conflicts with its continuation claim target'); } - const header = claim.targetRunHeader; + const { lineage, source: openSource } = claim.targetOpening; const source = claim.boundary.segments.at(-1)!; - const continuationSource = header.continuationSource; - const continuationSourceV2 = - continuationSource !== undefined && - 'protocol' in continuationSource && - continuationSource.protocol === 'continuation_source_v2' - ? continuationSource - : undefined; if ( - header.sessionId !== claim.target.sessionId || - header.invocationId !== claim.target.invocationId || - header.runId !== claim.target.runId || - header.turnId !== claim.target.turnId || - header.status !== 'created' || - header.agentId !== input.execution.agentId || - header.agentName !== input.execution.agentName || + lineage?.agentId !== input.execution.agentId || + lineage.agentName !== input.execution.agentName || source.identity.sessionId !== input.sessionId || source.identity.runId !== input.execution.sourceRunId || - !continuationSourceV2 || - continuationSourceV2.claimId !== claim.claimId || - continuationSourceV2.boundaryDigest !== claim.boundaryDigest || - continuationSourceV2.sourceRunId !== input.execution.sourceRunId + openSource.kind !== 'continuation' || + openSource.claimId !== claim.claimId || + openSource.boundaryDigest !== claim.boundaryDigest || + openSource.sourceRunId !== input.execution.sourceRunId ) { throw new Error('Linked child admission continuation claim identity is inconsistent'); } if ( input.execution.kind === 'linked_child_resume' - ? header.resumedFromRunId !== input.execution.sourceRunId || - header.retriedFromRunId !== undefined - : header.retriedFromRunId !== input.execution.sourceRunId || - header.resumedFromRunId !== undefined + ? lineage.resumedFromRunId !== input.execution.sourceRunId || + lineage.retriedFromRunId !== undefined + : lineage.retriedFromRunId !== input.execution.sourceRunId || + lineage.resumedFromRunId !== undefined ) { throw new Error('Linked child admission continuation lineage is inconsistent'); } @@ -4936,26 +4881,6 @@ async function readImmutableRuntimeEventsOrEmpty( return authority.readImmutableRuntimeEvents(sessionId, runId); } -function claimTargetRunHeaderIsCompatible( - actual: AgentRunHeader, - expected: AgentRunHeader, -): boolean { - const immutable = (header: AgentRunHeader) => { - const { - status: _status, - updatedAt: _updatedAt, - completedAt: _completedAt, - failureClass: _failureClass, - failureMessage: _failureMessage, - abortSource: _abortSource, - traceWriteError: _traceWriteError, - ...rest - } = header; - return rest; - }; - return isDeepStrictEqual(immutable(actual), immutable(expected)); -} - function isMissingRunError(error: unknown): boolean { return ( isNotFoundError(error) || @@ -5008,6 +4933,48 @@ export function headerToSummary(h: SessionHeader): SessionSummary { return summary; } +/** + * What a listing shows about one invocation, read entirely off its own facts. + * + * Every field here used to be a mutable column on the Run header that a writer + * had to keep in step with the events. Deriving them means a listing cannot + * disagree with the ledger it is listing. + */ +function invocationListingFacts(invocation: RuntimeInvocationRecord): { + status: RunLifecycleStatus; + permissionMode: PermissionMode; + createdAt: number; + updatedAt: number; + completedAt?: number; + durationMs?: number; + failureClass?: string; +} { + const completedAt = invocation.terminalEvent?.ts; + const failureClass = runtimeInvocationFailureClass(invocation); + return { + status: runtimeInvocationOutcome(invocation) ?? 'running', + permissionMode: invocation.opening.configuration.permissionMode, + createdAt: invocation.openedAt, + updatedAt: completedAt ?? invocation.openedAt, + ...(completedAt !== undefined ? { completedAt } : {}), + ...(completedAt !== undefined + ? { durationMs: Math.max(0, completedAt - invocation.openedAt) } + : {}), + ...(failureClass ? { failureClass } : {}), + }; +} + +/** The most recently opened invocation, breaking ties on run id. */ +function latestInvocation( + invocations: readonly RuntimeInvocationRecord[], +): RuntimeInvocationRecord | undefined { + return invocations + .slice() + .sort( + (left, right) => right.openedAt - left.openedAt || right.runId.localeCompare(left.runId), + )[0]; +} + function isNotFoundError(error: unknown): error is NodeJS.ErrnoException { return error instanceof Error && 'code' in error && error.code === 'ENOENT'; } @@ -5138,12 +5105,12 @@ function narrowsExecutionAuthority( } function agentRunStatusForSpawnResult( - status: AgentRunHeader['status'], + status: RunLifecycleStatus, ): SpawnChildSessionResult['status'] { if (status === 'waiting_for_user') return 'waiting_for_user'; if (status === 'cancelled') return 'cancelled'; if (status === 'failed') return 'failed'; - if (status === 'running' || status === 'created') return 'running'; + if (status === 'running') return 'running'; return 'completed'; } @@ -5305,7 +5272,7 @@ function turnStateLineage( }; } -function isTerminalRunStatus(status: AgentRunHeader['status']): boolean { +function isTerminalRunStatus(status: RunLifecycleStatus): boolean { return status === 'completed' || status === 'failed' || status === 'cancelled'; } @@ -5337,7 +5304,7 @@ function latestTurnState( } function runtimeTerminalFactToRecoveryDecision( - header: AgentRunHeader, + invocation: RuntimeInvocationRecord, fact: RuntimeEventTerminalFact, ): AgentRunRecoveryDecision { return { @@ -5351,20 +5318,22 @@ function runtimeTerminalFactToRecoveryDecision( runtimeEventId: fact.terminalEvent.id, runtimeEventStatus: fact.terminalEvent.status, }, - lineage: headerLineage(header), + lineage: openingLineage(invocation), }; } -function headerLineage(header: AgentRunHeader): AgentRunRecoveryDecision['lineage'] { +function openingLineage(invocation: RuntimeInvocationRecord): AgentRunRecoveryDecision['lineage'] { + const lineage = invocation.opening.lineage; + if (!lineage) return {}; return { - ...(header.parentRunId ? { parentRunId: header.parentRunId } : {}), - ...(header.parentTurnId ? { parentTurnId: header.parentTurnId } : {}), - ...(header.retriedFromTurnId ? { retriedFromTurnId: header.retriedFromTurnId } : {}), - ...(header.regeneratedFromTurnId - ? { regeneratedFromTurnId: header.regeneratedFromTurnId } + ...(lineage.parentRunId ? { parentRunId: lineage.parentRunId } : {}), + ...(lineage.parentTurnId ? { parentTurnId: lineage.parentTurnId } : {}), + ...(lineage.retriedFromTurnId ? { retriedFromTurnId: lineage.retriedFromTurnId } : {}), + ...(lineage.regeneratedFromTurnId + ? { regeneratedFromTurnId: lineage.regeneratedFromTurnId } : {}), - ...(header.branchOfTurnId ? { branchOfTurnId: header.branchOfTurnId } : {}), - ...(header.parentSessionId ? { parentSessionId: header.parentSessionId } : {}), + ...(lineage.branchOfTurnId ? { branchOfTurnId: lineage.branchOfTurnId } : {}), + ...(lineage.parentSessionId ? { parentSessionId: lineage.parentSessionId } : {}), }; } @@ -5384,7 +5353,7 @@ function normalizeAgentOutputMaxBytes(value: number | undefined): number { } function buildAgentOutputCommittedResult(input: { - header: AgentRunHeader; + invocation: RuntimeInvocationRecord; runtimeEvents: readonly RuntimeEvent[]; artifacts: readonly ArtifactRecord[]; maxArtifacts: number; @@ -5411,9 +5380,9 @@ function buildAgentOutputCommittedResult(input: { { operator: { operatorId: input.graph.operatorId, - sessionId: input.header.sessionId, + sessionId: input.invocation.sessionId, }, - run: input.header, + run: input.invocation, events: input.runtimeEvents, }, ], @@ -5434,7 +5403,7 @@ function buildAgentOutputCommittedResult(input: { ); const base = (): AgentOutputCommittedResult => ({ schemaVersion: 1, - status: input.header.status, + status: runtimeInvocationOutcome(input.invocation) ?? 'running', ...(input.graph ? { graph: { ...input.graph } } : {}), ...(outputRecord || terminalRecord ? { resultRecordId: (outputRecord ?? terminalRecord)!.recordId } @@ -5445,7 +5414,9 @@ function buildAgentOutputCommittedResult(input: { textTruncated: false, artifactIds, omittedArtifactIds: Math.max(0, input.artifacts.length - artifactIds.length), - ...(input.header.failureClass ? { failureClass: input.header.failureClass } : {}), + ...(runtimeInvocationFailureClass(input.invocation) + ? { failureClass: runtimeInvocationFailureClass(input.invocation) } + : {}), }); while (artifactIds.length > 0 && serializedBytes(base()) > input.maxBytes) { diff --git a/packages/runtime/src/session-projection-helpers.ts b/packages/runtime/src/session-projection-helpers.ts index f470699290..456f0ddd7a 100644 --- a/packages/runtime/src/session-projection-helpers.ts +++ b/packages/runtime/src/session-projection-helpers.ts @@ -18,8 +18,8 @@ */ import { createHash } from 'node:crypto'; -import type { AgentRunHeader } from '@maka/core/agent-run'; import { failureClassFromCompleteStopReason, type SessionEvent } from '@maka/core/events'; +import type { RuntimeInvocationOutcome } from '@maka/core/runtime-invocation'; import type { SessionBlockedReason, SessionHeader, @@ -122,7 +122,13 @@ export function workHubDirectStopAbortSource(actionId: string | undefined): stri return `workhub.direct_stop.${suffix}`; } -export function isTerminalRunStatus(status: AgentRunHeader['status']): boolean { +/** + * What a live run says about itself before its events close it. Only the + * outcomes are durable; the other two describe a run still in flight. + */ +export type RunLifecycleStatus = RuntimeInvocationOutcome | 'running' | 'waiting_for_user'; + +export function isTerminalRunStatus(status: RunLifecycleStatus): boolean { return status === 'completed' || status === 'failed' || status === 'cancelled'; } diff --git a/packages/runtime/src/stream-graph-coordinator.ts b/packages/runtime/src/stream-graph-coordinator.ts index fe838a7a8c..8ebb0885ca 100644 --- a/packages/runtime/src/stream-graph-coordinator.ts +++ b/packages/runtime/src/stream-graph-coordinator.ts @@ -106,8 +106,10 @@ export interface AgentGraphCoordinatorRuntime { export interface AgentGraphCoordinatorInput { sessionStore: AgentGraphCoordinatorSessionStore; - runStore: Pick; - runtimeEventStore: Pick; + runtimeEventStore: Pick< + RuntimeEventStore, + 'readImmutableRuntimeEvents' | 'listSessionInvocations' + >; controlStore: AgentGraphScheduleControlStore & AgentGraphClientProjectionStore & AgentGraphTimelineMetadataStore; @@ -405,7 +407,6 @@ export class AgentGraphCoordinator { rootSessionId, graphId, controlStore: this.#input.controlStore, - runStore: this.#input.runStore, runtimeEventStore: this.#input.runtimeEventStore, options, }); @@ -547,7 +548,6 @@ export class AgentGraphCoordinator { readCommittedAgentGraphProjection({ graphId, operators: topology.operators, - runStore: this.#input.runStore, runtimeEventStore: this.#input.runtimeEventStore, }), this.#input.controlStore.listAgentGraphIntentClaims(graphId), @@ -1242,7 +1242,6 @@ export class AgentGraphCoordinator { const projection = await readCommittedAgentGraphProjection({ graphId: sourceGraphId, operators: topology.operators, - runStore: this.#input.runStore, runtimeEventStore: this.#input.runtimeEventStore, }); recordsBySource.set( diff --git a/packages/runtime/src/stream-graph-projection.ts b/packages/runtime/src/stream-graph-projection.ts index 3a6068af95..f0f42f6acf 100644 --- a/packages/runtime/src/stream-graph-projection.ts +++ b/packages/runtime/src/stream-graph-projection.ts @@ -17,10 +17,11 @@ * under the License. */ -import type { AgentRunHeader, AgentRunStore } from '@maka/core/agent-run'; +import type { RuntimeInvocationRecord } from '@maka/core/runtime-invocation'; +import { runtimeInvocationOutcome } from '@maka/core/runtime-invocation'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; -import { isSessionInlineRun } from '@maka/core/agent-run'; +import { isSessionInlineInvocation } from '@maka/core/runtime-invocation'; import { stableHash, stableStringify } from './request-shape.js'; import { compareAgentGraphIdentity } from './stream-graph-identity.js'; @@ -185,7 +186,7 @@ export interface AgentGraphReplayState { export interface AgentGraphRunStream { operator: AgentGraphOperatorBinding; - run: AgentRunHeader; + run: RuntimeInvocationRecord; events: readonly RuntimeEvent[]; } @@ -206,18 +207,20 @@ export interface AgentGraphProjection { export interface ReadCommittedAgentGraphProjectionInput { graphId: string; operators: readonly AgentGraphOperatorBinding[]; - runStore: Pick; - runtimeEventStore: Pick; + runtimeEventStore: Pick< + RuntimeEventStore, + 'readImmutableRuntimeEvents' | 'listSessionInvocations' + >; } export interface AgentGraphProjectionWithRuns { projection: AgentGraphProjection; - runs: AgentRunHeader[]; + runs: RuntimeInvocationRecord[]; } interface OrderedRuntimeEvent { operator: AgentGraphOperatorBinding; - run: AgentRunHeader; + run: RuntimeInvocationRecord; event: RuntimeEvent; committedEventOrdinal: number; } @@ -244,10 +247,10 @@ export async function readCommittedAgentGraphProjectionWithRuns( const streams = ( await Promise.all( input.operators.map(async (operator) => { - const runs = await input.runStore.listSessionRuns(operator.sessionId); + const runs = await input.runtimeEventStore.listSessionInvocations(operator.sessionId); const orderedRuns = runs - .filter(isSessionInlineRun) - .sort((a, b) => a.createdAt - b.createdAt || compareAgentGraphIdentity(a.runId, b.runId)); + .filter((run) => isSessionInlineInvocation(run.opening)) + .sort((a, b) => a.openedAt - b.openedAt || compareAgentGraphIdentity(a.runId, b.runId)); return await Promise.all( orderedRuns.map(async (run): Promise => { if (run.sessionId !== operator.sessionId) { @@ -350,7 +353,7 @@ export function projectAgentGraphRecords(input: ProjectAgentGraphRecordsInput): agentRunId: item.run.runId, eventTime: item.event.ts, orderKey: { - runCreatedAt: item.run.createdAt, + runCreatedAt: item.run.openedAt, operatorId: item.operator.operatorId, runId: item.run.runId, committedEventOrdinal: item.committedEventOrdinal, @@ -499,7 +502,10 @@ export function replayAgentGraphRecords( }; } -function runtimeEventFacets(event: RuntimeEvent, run: AgentRunHeader): AgentGraphRecordFacet[] { +function runtimeEventFacets( + event: RuntimeEvent, + run: RuntimeInvocationRecord, +): AgentGraphRecordFacet[] { const facets: AgentGraphRecordFacet[] = []; switch (event.content?.kind) { case 'text': @@ -537,7 +543,7 @@ function runtimeEventFacets(event: RuntimeEvent, run: AgentRunHeader): AgentGrap function runtimeEventSupervisorSignals( event: RuntimeEvent, - run: AgentRunHeader, + run: RuntimeInvocationRecord, ): AgentGraphSupervisorSignal[] { const signals: AgentGraphSupervisorSignal[] = []; if (event.actions?.permissionRequest) { @@ -558,7 +564,7 @@ function runtimeEventSupervisorSignals( function runtimeEventTerminalStatus( event: RuntimeEvent, - run: AgentRunHeader, + run: RuntimeInvocationRecord, ): | Extract | undefined { @@ -574,18 +580,13 @@ function runtimeEventTerminalStatus( } function terminalStatusFromRun( - run: AgentRunHeader, + run: RuntimeInvocationRecord, ): Extract { - switch (run.status) { - case 'completed': - case 'failed': - case 'cancelled': - return run.status; - default: - throw new Error( - `RuntimeEvent ended invocation ${run.runId} while its AgentRun is ${run.status}`, - ); - } + const outcome = runtimeInvocationOutcome(run); + if (outcome) return outcome; + throw new Error( + `RuntimeEvent ended invocation ${run.runId} while its ledger records no terminal fact`, + ); } function activationStatusAfterRecord( @@ -661,7 +662,7 @@ function assertRunStream(stream: AgentGraphRunStream): void { `Run ${stream.run.runId} belongs to ${stream.run.sessionId}, expected ${stream.operator.sessionId}`, ); } - if (!isSessionInlineRun(stream.run)) { + if (!isSessionInlineInvocation(stream.run.opening)) { throw new Error(`Graph activation ${stream.run.runId} must be a session-inline AgentRun`); } } @@ -681,7 +682,7 @@ function assertRuntimeEventIdentity(stream: AgentGraphRunStream, event: RuntimeE function compareOrderedRuntimeEvents(a: OrderedRuntimeEvent, b: OrderedRuntimeEvent): number { return ( a.event.ts - b.event.ts || - a.run.createdAt - b.run.createdAt || + a.run.openedAt - b.run.openedAt || compareAgentGraphIdentity(a.operator.operatorId, b.operator.operatorId) || compareAgentGraphIdentity(a.run.runId, b.run.runId) || a.committedEventOrdinal - b.committedEventOrdinal || diff --git a/packages/runtime/src/terminal-run-commit.ts b/packages/runtime/src/terminal-run-commit.ts index 8cc0335cab..ad9aff9cf3 100644 --- a/packages/runtime/src/terminal-run-commit.ts +++ b/packages/runtime/src/terminal-run-commit.ts @@ -18,23 +18,23 @@ */ import { isPartialRuntimeEvent, isTerminalRuntimeEvent } from '@maka/core/runtime-event'; -import type { - AgentRunEvent, - AgentRunHeader, - AgentRunEventType, - AgentRunStore, -} from '@maka/core/agent-run'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import type { RuntimeEventStore } from '@maka/core/runtime-event-store'; +import { + buildSyntheticTerminalRuntimeEvent, + type RuntimeInvocationOutcome, +} from '@maka/core/runtime-invocation'; import { classifyRuntimeEventTerminalFact, type RuntimeEventTerminalFact, } from './runtime-event-read-model.js'; -export type TerminalAgentRunStatus = Extract< - AgentRunHeader['status'], - 'completed' | 'failed' | 'cancelled' ->; +/** The three ids every RuntimeEvent of one run carries. */ +export interface RunIdentity { + sessionId: string; + runId: string; + turnId: string; +} export type TerminalRuntimeLedgerClassification = | { @@ -47,17 +47,17 @@ export type TerminalRuntimeLedgerClassification = terminalEvents: readonly RuntimeEvent[]; } | { - kind: 'incomplete_single_terminal'; - terminalEvent: RuntimeEvent; - terminalEvents: readonly RuntimeEvent[]; - } - | { - kind: 'ambiguous'; + /** + * More than one terminal event. Nothing ambiguous about it: a store seals + * a run on its first terminal, so a second one means the ledger was + * written around that seal and is corrupt. + */ + kind: 'corrupt'; terminalEvents: readonly RuntimeEvent[]; }; export function classifyTerminalRuntimeLedger( - run: AgentRunHeader, + run: RunIdentity, events: readonly RuntimeEvent[], ): TerminalRuntimeLedgerClassification { const terminalEvents = matchingTerminalRuntimeEvents(run, events); @@ -65,116 +65,54 @@ export function classifyTerminalRuntimeLedger( return { kind: 'none', terminalEvents }; } if (terminalEvents.length > 1) { - return { kind: 'ambiguous', terminalEvents }; + return { kind: 'corrupt', terminalEvents }; } const fact = classifyRuntimeEventTerminalFact(run, events).fact; if (fact) { return { kind: 'fact', fact, terminalEvents }; } - return { - kind: 'incomplete_single_terminal', - terminalEvent: terminalEvents[0]!, - terminalEvents, - }; + // The one terminal event carries no terminal status, so it ends the stream + // without ending the run. + return { kind: 'none', terminalEvents }; } -export interface CommitTerminalRunWithRuntimeFactInput { - runStore: AgentRunStore; +export interface CommitTerminalRunWithRuntimeFactInput extends RunIdentity { runtimeEventStore: RuntimeEventStore; newId: () => string; - sessionId: string; - runId: string; - turnId: string; - status: TerminalAgentRunStatus; + status: RuntimeInvocationOutcome; ts: number; terminalEvent: RuntimeEvent; failureClass?: string; failureMessage?: string; - traceWriteError?: string; abortSource?: string; - runEventData?: Record; - runEventMessage?: string; - existingEvents?: readonly Pick[]; } +/** + * Put one run's ending beyond doubt: the terminal RuntimeEvent, on stable + * storage, and nothing else. + * + * There is no projection to commit alongside it any more. The event states the + * outcome, the failure class and the abort source, so a second record could only + * ever disagree with it. + */ export async function commitTerminalRunWithRuntimeFact( input: CommitTerminalRunWithRuntimeFactInput, ): Promise { - if (isPartialRuntimeEvent(input.terminalEvent)) { - throw new Error('terminal RuntimeEvent must be final before terminal run header'); - } - const terminalStatus = terminalRunStatusFromRuntimeEvent(input.terminalEvent); - if (!terminalStatus) { - throw new Error('terminal RuntimeEvent must carry a terminal status'); - } - if (terminalStatus !== input.status) { - throw new Error( - `terminal RuntimeEvent status ${input.terminalEvent.status} cannot commit ${input.status} run header`, - ); - } - if ( - input.terminalEvent.sessionId !== input.sessionId || - input.terminalEvent.runId !== input.runId || - input.terminalEvent.turnId !== input.turnId - ) { - throw new Error('terminal RuntimeEvent identity does not match run header commit'); - } + assertCommittableTerminalEvent(input.terminalEvent, input, input.status); await input.runtimeEventStore.ensureTerminalRuntimeEventDurable( input.sessionId, input.runId, input.terminalEvent, ); - - await commitTerminalRunProjection(input); -} - -async function commitTerminalRunProjection( - input: CommitTerminalRunWithRuntimeFactInput, -): Promise { - const failureClass = input.status === 'failed' ? (input.failureClass ?? 'unknown') : undefined; - const abortSource = input.status === 'cancelled' ? input.abortSource : undefined; - await input.runStore.updateRun( - input.sessionId, - input.runId, - { - status: input.status, - updatedAt: input.ts, - completedAt: input.ts, - ...(failureClass ? { failureClass } : {}), - ...(input.failureMessage ? { failureMessage: input.failureMessage } : {}), - ...(input.traceWriteError ? { traceWriteError: input.traceWriteError } : {}), - ...(abortSource ? { abortSource } : {}), - }, - { durable: true }, - ); - - if (hasTerminalAgentRunEvent(input.existingEvents ?? [])) return; - const data = terminalRunEventData(input.status, failureClass, input.runEventData); - await input.runStore.appendEvent( - input.sessionId, - input.runId, - { - type: terminalAgentRunEventType(input.status), - id: input.newId(), - runId: input.runId, - sessionId: input.sessionId, - turnId: input.turnId, - ts: input.ts, - ...(input.runEventMessage ? { message: input.runEventMessage } : {}), - ...(Object.keys(data).length > 0 ? { data } : {}), - }, - { durable: true }, - ); } export interface CommitOrCreateTerminalRunFactInput extends Omit { - /** Runs after the terminal durability barrier, before the header commit. */ + /** Runs after the terminal durability barrier. */ afterTerminalDurable?: () => Promise; terminalEvent?: RuntimeEvent; - allowHeaderCommitFailure?: boolean; - fallbackStatus: TerminalAgentRunStatus; + fallbackStatus: RuntimeInvocationOutcome; fallbackInvocationId: string; fallbackFailureClass?: string; fallbackFailureMessage?: string; @@ -182,11 +120,9 @@ export interface CommitOrCreateTerminalRunFactInput export interface CommitOrCreateTerminalRunFactResult { terminalEvent: RuntimeEvent; - status: TerminalAgentRunStatus; + status: RuntimeInvocationOutcome; failureClass?: string; createdTerminalEvent: boolean; - headerCommitted: boolean; - headerCommitError?: unknown; } export async function commitOrCreateTerminalRunFact( @@ -200,11 +136,7 @@ export async function commitOrCreateTerminalRunFact( buildSyntheticTerminalRuntimeEvent({ id: input.newId(), invocationId: input.fallbackInvocationId, - run: { - sessionId: input.sessionId, - runId: input.runId, - turnId: input.turnId, - }, + run: input, status: input.fallbackStatus, ts: input.ts, ...(input.fallbackFailureClass ? { failureClass: input.fallbackFailureClass } : {}), @@ -213,20 +145,7 @@ export async function commitOrCreateTerminalRunFact( ? { message: input.fallbackFailureMessage ?? input.failureMessage } : {}), }); - const status = terminalRunStatusFromRuntimeEvent(terminalEvent); - if (!status) { - throw new Error('terminal RuntimeEvent must carry a terminal status'); - } - if (isPartialRuntimeEvent(terminalEvent)) { - throw new Error('terminal RuntimeEvent must be final before terminal run header'); - } - if ( - terminalEvent.sessionId !== input.sessionId || - terminalEvent.runId !== input.runId || - terminalEvent.turnId !== input.turnId - ) { - throw new Error('terminal RuntimeEvent identity does not match run header commit'); - } + const status = assertCommittableTerminalEvent(terminalEvent, input); const failureClass = status === 'failed' ? (runtimeEventFailureClass(terminalEvent) ?? input.failureClass ?? 'unknown') @@ -236,92 +155,48 @@ export async function commitOrCreateTerminalRunFact( input.runId, terminalEvent, ); - // Between the terminal durability barrier and the header commit: the one - // point where "the terminal fact is durable" is true and nothing else has - // been projected yet. Callers that must order a crash boundary against - // the barrier itself hang it here (#2313 corruption recovery, where the - // claimed event's own write never ran). + // The one point where "the terminal fact is durable" is true and nothing has + // read it yet. Callers that must order a crash boundary against the barrier + // itself hang it here (#2313 corruption recovery, where the claimed event's + // own write never ran). await input.afterTerminalDurable?.(); - let headerCommitted = false; - let headerCommitError: unknown; - try { - await commitTerminalRunProjection({ - ...input, - terminalEvent, - status, - ...(failureClass ? { failureClass } : {}), - ...(effectiveAbortSource ? { abortSource: effectiveAbortSource } : {}), - }); - headerCommitted = true; - } catch (error) { - if (!input.allowHeaderCommitFailure) throw error; - headerCommitError = error; - } return { terminalEvent, status, ...(failureClass ? { failureClass } : {}), createdTerminalEvent, - headerCommitted, - ...(headerCommitError !== undefined ? { headerCommitError } : {}), }; } -export interface BuildSyntheticTerminalRuntimeEventInput { - id: string; - invocationId: string; - run: Pick; - status: TerminalAgentRunStatus; - ts: number; - failureClass?: string; - abortSource?: string; - recoveryReason?: string; - diagnostic?: Record; - message?: string; -} - -export function buildSyntheticTerminalRuntimeEvent( - input: BuildSyntheticTerminalRuntimeEventInput, -): RuntimeEvent { - const failureClass = input.status === 'failed' ? (input.failureClass ?? 'unknown') : undefined; - const abortSource = input.status === 'cancelled' ? input.abortSource : undefined; - return { - id: input.id, - invocationId: input.invocationId, - runId: input.run.runId, - sessionId: input.run.sessionId, - turnId: input.run.turnId, - ts: input.ts, - partial: false, - role: 'system', - author: 'system', - status: input.status === 'cancelled' ? 'aborted' : input.status, - ...(failureClass - ? { - content: { - kind: 'error', - code: failureClass, - reason: failureClass, - message: input.message ?? failureClass, - }, - } - : {}), - actions: { - endInvocation: true, - stateDelta: { - ...(input.recoveryReason ? { recovered: true, recoveryReason: input.recoveryReason } : {}), - ...(input.diagnostic ?? {}), - ...(failureClass ? { failureClass } : {}), - ...(abortSource ? { abortSource } : {}), - }, - }, - }; +function assertCommittableTerminalEvent( + event: RuntimeEvent, + identity: RunIdentity, + expected?: RuntimeInvocationOutcome, +): RuntimeInvocationOutcome { + if (isPartialRuntimeEvent(event)) { + throw new Error('terminal RuntimeEvent must be final before it is committed'); + } + const status = terminalRunStatusFromRuntimeEvent(event); + if (!status) { + throw new Error('terminal RuntimeEvent must carry a terminal status'); + } + if (expected !== undefined && status !== expected) { + throw new Error(`terminal RuntimeEvent status ${event.status} cannot commit a ${expected} run`); + } + if ( + event.sessionId !== identity.sessionId || + event.runId !== identity.runId || + event.turnId !== identity.turnId + ) { + throw new Error('terminal RuntimeEvent identity does not match the run it ends'); + } + return status; } export interface BuildRecoveredTerminalRuntimeEventInput { id: string; - run: Pick; - status: TerminalAgentRunStatus; + run: RunIdentity & { invocationId?: string }; + status: RuntimeInvocationOutcome; ts: number; invocationId?: string; failureClass?: string; @@ -348,32 +223,6 @@ export function buildRecoveredTerminalRuntimeEvent( }); } -export function hasTerminalAgentRunEvent(events: readonly Pick[]): boolean { - return events.some( - (event) => - event.type === 'run_completed' || - event.type === 'run_failed' || - event.type === 'run_cancelled', - ); -} - -function terminalAgentRunEventType(status: TerminalAgentRunStatus): AgentRunEventType { - if (status === 'cancelled') return 'run_cancelled'; - if (status === 'failed') return 'run_failed'; - return 'run_completed'; -} - -function terminalRunEventData( - status: TerminalAgentRunStatus, - failureClass: string | undefined, - runEventData: Record | undefined, -): Record { - return { - ...(status === 'failed' && failureClass ? { failureClass } : {}), - ...(runEventData ?? {}), - }; -} - function runtimeEventFailureClass(event: RuntimeEvent): string | undefined { const stateDelta = event.actions?.stateDelta; if (typeof stateDelta?.failureClass === 'string' && stateDelta.failureClass.length > 0) { @@ -387,51 +236,15 @@ function runtimeEventFailureClass(event: RuntimeEvent): string | undefined { export function terminalRunStatusFromRuntimeEvent( event: RuntimeEvent, -): TerminalAgentRunStatus | undefined { +): RuntimeInvocationOutcome | undefined { if (event.status === 'completed') return 'completed'; if (event.status === 'failed') return 'failed'; if (event.status === 'aborted' || event.status === 'cancelled') return 'cancelled'; return undefined; } -export function effectiveRunHeaderFromTerminalFact( - run: AgentRunHeader, - fact: RuntimeEventTerminalFact, -): AgentRunHeader { - const completedAt = run.completedAt ?? fact.terminalEvent.ts; - const base = { ...run }; - delete base.failureClass; - delete base.failureMessage; - delete base.abortSource; - return { - ...base, - status: fact.runStatus, - updatedAt: Math.max(run.updatedAt, completedAt), - completedAt, - ...(fact.runStatus === 'failed' && fact.failureClass - ? { failureClass: fact.failureClass } - : {}), - ...(fact.runStatus === 'failed' && run.failureMessage - ? { failureMessage: run.failureMessage } - : {}), - ...(fact.runStatus === 'cancelled' && fact.abortSource - ? { abortSource: fact.abortSource } - : {}), - }; -} - -export function terminalRunHeaderMatchesFact( - run: AgentRunHeader, - fact: RuntimeEventTerminalFact, -): boolean { - if (run.status !== fact.runStatus) return false; - if (fact.runStatus === 'failed' && run.failureClass !== fact.failureClass) return false; - if (fact.runStatus === 'cancelled' && run.abortSource !== fact.abortSource) return false; - return true; -} - export function matchingTerminalRuntimeEvents( - run: AgentRunHeader, + run: RunIdentity, events: readonly RuntimeEvent[], ): RuntimeEvent[] { return events.filter( diff --git a/packages/storage/src/__tests__/agent-graph-supervisor-root-admission.test.ts b/packages/storage/src/__tests__/agent-graph-supervisor-root-admission.test.ts index 3cc98eac4c..d52f9143df 100644 --- a/packages/storage/src/__tests__/agent-graph-supervisor-root-admission.test.ts +++ b/packages/storage/src/__tests__/agent-graph-supervisor-root-admission.test.ts @@ -22,7 +22,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; -import type { RootExecutionDescriptor } from '@maka/core/agent-run'; +import type { RootExecutionDescriptor } from '@maka/core/runtime-invocation'; import { createSqliteAgentRunStore, type AdmitRootTurnInput } from '../agent-run-store.js'; test('Agent Graph supervisor admission durably binds wake identity and Graph orchestration', async () => { diff --git a/packages/storage/src/__tests__/claimed-agent-graph-root-admission.test.ts b/packages/storage/src/__tests__/claimed-agent-graph-root-admission.test.ts index 9b3e1c54cd..b39eccaa31 100644 --- a/packages/storage/src/__tests__/claimed-agent-graph-root-admission.test.ts +++ b/packages/storage/src/__tests__/claimed-agent-graph-root-admission.test.ts @@ -23,7 +23,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, test } from 'node:test'; import { AGENT_GRAPH_INTENT_CLAIM_SCHEMA_VERSION } from '@maka/core/agent-graph-control'; -import type { RootExecutionDescriptor } from '@maka/core/agent-run'; +import type { RootExecutionDescriptor } from '@maka/core/runtime-invocation'; import { createSqliteAgentRunStore, type AdmitRootTurnInput } from '../agent-run-store.js'; describe('claimed agent graph root admission', () => { diff --git a/packages/storage/src/__tests__/fixtures/invocation-opening.ts b/packages/storage/src/__tests__/fixtures/invocation-opening.ts new file mode 100644 index 0000000000..9d27552b39 --- /dev/null +++ b/packages/storage/src/__tests__/fixtures/invocation-opening.ts @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { encodeCanonicalRuntimeEvent } from '@maka/core/canonical-runtime-event'; +import type { RuntimeEventInvocationOpenedContent } from '@maka/core/runtime-event'; +import { buildInvocationOpenedEvent } from '@maka/core/runtime-invocation'; +import { DEFAULT_TOOL_MODE } from '@maka/core/tool-mode'; +import { createWorkspaceRuntimeStore } from '../../runtime-event-persistence.js'; + +export interface InvocationIdentity { + sessionId: string; + invocationId?: string; + runId: string; + turnId: string; + openedAt?: number; +} + +export function invocationOpening( + overrides: Partial = {}, +): RuntimeEventInvocationOpenedContent { + return { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: { + provenance: 'runtime', + backendKind: 'fake', + llmConnectionId: 'connection-1', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + }, + configuration: { + cwd: '/tmp/cwd', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: DEFAULT_TOOL_MODE, + agentSwarmAuthorization: 'none', + }, + root: { kind: 'user' }, + source: { kind: 'fresh' }, + ...overrides, + }; +} + +/** + * Commit the one fact that makes an invocation exist, the way the Runtime Host + * does, so the AgentRunEvent ledger has an anchor to hang its events on. + */ +export async function openInvocation( + workspaceRoot: string, + identity: InvocationIdentity, + content: RuntimeEventInvocationOpenedContent = invocationOpening(), +): Promise { + const invocationId = identity.invocationId ?? identity.runId; + const { event } = encodeCanonicalRuntimeEvent( + buildInvocationOpenedEvent({ + id: `invocation_opened:${invocationId}`, + run: { + sessionId: identity.sessionId, + invocationId, + runId: identity.runId, + turnId: identity.turnId, + }, + openedAt: identity.openedAt ?? 1, + opening: content, + }), + ); + const store = createWorkspaceRuntimeStore(workspaceRoot); + try { + await store.appendRuntimeEvent(identity.sessionId, identity.runId, event); + } finally { + store.close(); + } +} diff --git a/packages/storage/src/__tests__/fixtures/sqlite-recovery-concurrency-child.ts b/packages/storage/src/__tests__/fixtures/sqlite-recovery-concurrency-child.ts index 8370226c09..f481536b5f 100644 --- a/packages/storage/src/__tests__/fixtures/sqlite-recovery-concurrency-child.ts +++ b/packages/storage/src/__tests__/fixtures/sqlite-recovery-concurrency-child.ts @@ -24,6 +24,7 @@ import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; import { type ToolRecoveryFactEnvelope } from '@maka/core/tool-recovery-fact'; import { type WorkspaceBaselineAuthorityInput } from '@maka/core/workspace-version-authority'; import { createRuntimeBoundaryCursor, runtimePrefixSegment } from '@maka/core/runtime-boundary'; +import { DEFAULT_TOOL_MODE } from '@maka/core/tool-mode'; import { createSqliteRuntimeStore } from '../../sqlite-runtime-store.js'; import { acquireOperationalStateDatabase } from '../../operational-state-store.js'; import { @@ -128,32 +129,37 @@ try { providerProjectionVersion: 1, providerReplayDigest: `sha256:${'a'.repeat(64)}`, target, - targetRunHeader: { - ...target, - status: 'created', - backendKind: 'fake', - llmConnectionSlug: 'connection-1', - modelId: 'model-1', - cwd: '/workspace/repo', - permissionMode: 'ask', - collaborationMode: 'agent', - orchestrationMode: 'default', - orchestrationSource: 'session', - agentSwarmAuthorization: 'none', - createdAt: process.pid, - updatedAt: process.pid, - parentRunId: source.identity.runId, - parentTurnId: source.identity.turnId, - continuationSource: { - protocol: 'continuation_source_v2', - claimId: `claim-${process.pid}`, - boundaryDigest: boundary.manifestDigest, + targetOpening: { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: { + provenance: 'unknown', + backendKind: 'fake', + llmConnectionSlug: 'connection-1', + modelId: 'model-1', + }, + configuration: { + cwd: '/workspace/repo', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: DEFAULT_TOOL_MODE, + agentSwarmAuthorization: 'none', + }, + root: { kind: 'user' }, + source: { + kind: 'continuation', sourceInvocationId: source.identity.invocationId, sourceRunId: source.identity.runId, sourceTurnId: source.identity.turnId, sourceRuntimeEventHighWater: source.position.lastEventSeq, - sourcePrefixDigest: source.prefixDigest, - replayManifestDigest: boundary.manifestDigest, + claimId: `claim-${process.pid}`, + boundaryDigest: boundary.manifestDigest, + }, + lineage: { + parentRunId: source.identity.runId, + parentTurnId: source.identity.turnId, }, }, claimedAt: process.pid, diff --git a/packages/storage/src/__tests__/invocation-opening-backfill.test.ts b/packages/storage/src/__tests__/invocation-opening-backfill.test.ts new file mode 100644 index 0000000000..3e8a5d1610 --- /dev/null +++ b/packages/storage/src/__tests__/invocation-opening-backfill.test.ts @@ -0,0 +1,569 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, test } from 'node:test'; +import { DatabaseSync } from 'node:sqlite'; +import { encodeCanonicalRuntimeEvent } from '@maka/core/canonical-runtime-event'; +import { createRunCompositionSnapshot } from '@maka/core/run-composition'; +import { decodeRuntimeEvent } from '@maka/core/runtime-event'; +import type { LegacyRunHeader } from '../legacy-run-header.js'; +import { OPERATIONAL_STATE_DATABASE_NAME } from '../operational-state-store.js'; +import { migrateSqliteCoreExecutionDatabase } from '../sqlite-core-execution-schema.js'; +import { createSqliteRuntimeStore } from '../sqlite-runtime-store.js'; +import { + migrateSqliteRuntimeDatabase, + SQLITE_RUNTIME_SCHEMA_VERSION, +} from '../sqlite-runtime-schema.js'; +describe('invocation opening fact backfill', () => { + test('gives every header-only run the opening fact it never wrote', async () => { + await withHeaderOnlyRuns(async (databasePath) => { + const db = new DatabaseSync(databasePath); + try { + // One run already owns an immutable sequence; the backfill must leave it + // alone rather than rewrite its position one. + db.prepare(` + INSERT INTO runtime_events ( + event_id, session_id, invocation_id, run_id, turn_id, event_seq, + event_kind, payload_json, committed_at + ) VALUES ('existing-1', 'session-1', 'run-with-events', 'run-with-events', + 'turn-with-events', 1, 'text', '{}', 1) + `).run(); + migrateSqliteRuntimeDatabase(db); + assert.equal(readUserVersion(db), SQLITE_RUNTIME_SCHEMA_VERSION); + + const rows = db + .prepare(` + SELECT event_id, invocation_id, run_id, turn_id, event_seq, payload_json + FROM runtime_events + WHERE event_kind = 'invocation_opened' + ORDER BY run_id ASC + `) + .all() as Array<{ + event_id: string; + invocation_id: string; + run_id: string; + turn_id: string; + event_seq: number; + payload_json: string; + }>; + + assert.deepEqual( + rows.map((row) => row.run_id), + ['run-legacy-route', 'run-scheduled'], + 'only the header-only runs are backfilled', + ); + assert.deepEqual( + rows.map((row) => row.event_seq), + [1, 1], + 'a synthesized opening fact is event one of an otherwise empty invocation', + ); + + const legacy = decodeRuntimeEvent(JSON.parse(rows[0]!.payload_json)); + assert.equal(legacy.content?.kind, 'invocation_opened'); + if (legacy.content?.kind !== 'invocation_opened') throw new Error('unreachable'); + assert.equal( + legacy.content.route.provenance, + 'unknown', + 'a header with no Connection identity must not claim an authenticated route', + ); + assert.equal(legacy.content.route.modelId, 'legacy-model'); + assert.equal(legacy.content.source.kind, 'fresh'); + assert.equal(legacy.invocationId, 'run-legacy-route'); + + const scheduled = decodeRuntimeEvent(JSON.parse(rows[1]!.payload_json)); + if (scheduled.content?.kind !== 'invocation_opened') throw new Error('unreachable'); + assert.deepEqual(scheduled.content.root, { + kind: 'scheduled_task', + scheduledTaskId: 'task-9', + }); + assert.equal(scheduled.content.route.provenance, 'runtime'); + + // Both header-only runs were marked completed, so each gets the ending + // its header recorded, right after its opening. + const backfilled = db + .prepare(` + SELECT run_id, event_seq, event_kind FROM runtime_events + WHERE run_id IN ('run-legacy-route', 'run-scheduled') + ORDER BY run_id ASC, event_seq ASC + `) + .all() as Array<{ run_id: string; event_seq: number; event_kind: string }>; + assert.deepEqual( + backfilled.map(({ run_id, event_seq, event_kind }) => ({ + run_id, + event_seq, + event_kind, + })), + [ + { run_id: 'run-legacy-route', event_seq: 1, event_kind: 'invocation_opened' }, + { run_id: 'run-legacy-route', event_seq: 2, event_kind: 'completed' }, + { run_id: 'run-scheduled', event_seq: 1, event_kind: 'invocation_opened' }, + { run_id: 'run-scheduled', event_seq: 2, event_kind: 'completed' }, + ], + ); + const ordinals = db + .prepare('SELECT COUNT(*) AS total FROM runtime_session_event_ordinals') + .get() as { total: number }; + assert.equal( + ordinals.total, + backfilled.length, + 'every backfilled event joins the Session ordinal stream', + ); + + // The run that already owns an immutable sequence keeps it untouched: + // rewriting its position one would break digests other facts signed. + const withEvents = db + .prepare( + "SELECT event_id FROM runtime_events WHERE run_id = 'run-with-events' ORDER BY event_seq", + ) + .all() as Array<{ event_id: string }>; + assert.deepEqual( + withEvents.map((row) => row.event_id), + ['existing-1'], + ); + + // Its opening is not lost, though: it goes on the legacy shelf, keyed by + // the invocation id its own events already carry. + const legacyRows = db + .prepare(` + SELECT invocation_id, session_id, run_id, turn_id, opened_at, opening_json, + anchor_event_id + FROM runtime_legacy_invocation_openings + ORDER BY invocation_id + `) + .all() as Array<{ + invocation_id: string; + session_id: string; + run_id: string; + turn_id: string; + opened_at: number; + opening_json: string; + anchor_event_id: string; + }>; + assert.deepEqual( + legacyRows.map((row) => row.invocation_id), + ['run-with-events'], + 'only a run whose sequence is already immutable takes the legacy shelf', + ); + assert.equal(legacyRows[0]!.run_id, 'run-with-events'); + assert.equal(legacyRows[0]!.turn_id, 'turn-with-events'); + assert.equal(legacyRows[0]!.opened_at, 1); + assert.equal( + (JSON.parse(legacyRows[0]!.opening_json) as { kind: string }).kind, + 'invocation_opened', + ); + // The shelved opening describes that ledger, so it is anchored to the + // ledger's first event and cannot outlive it. + assert.equal( + legacyRows[0]!.anchor_event_id, + 'existing-1', + 'the shelved opening is anchored to the first event of the run it describes', + ); + } finally { + db.close(); + } + }); + }); + + test('enumerates event openings and migrated ones as one inventory', async () => { + await withHeaderOnlyRuns(async (databasePath) => { + const db = new DatabaseSync(databasePath); + try { + const { json } = encodeCanonicalRuntimeEvent({ + id: 'existing-1', + invocationId: 'run-with-events', + runId: 'run-with-events', + sessionId: 'session-1', + turnId: 'turn-with-events', + ts: 1, + partial: false, + role: 'user', + author: 'user', + modelVisibility: 'visible', + content: { kind: 'text', text: 'already immutable' }, + }); + db.prepare(` + INSERT INTO runtime_events ( + event_id, session_id, invocation_id, run_id, turn_id, event_seq, + event_kind, payload_json, committed_at + ) VALUES ('existing-1', 'session-1', 'run-with-events', 'run-with-events', + 'turn-with-events', 1, 'text', ?, 1) + `).run(json); + migrateSqliteRuntimeDatabase(db); + } finally { + db.close(); + } + + const store = createSqliteRuntimeStore(databasePath); + try { + const invocations = await store.listSessionInvocations('session-1'); + assert.deepEqual( + invocations.map((invocation) => invocation.invocationId), + ['run-legacy-route', 'run-scheduled', 'run-with-events'], + 'a migrated opening is enumerated beside the ones the events carry', + ); + for (const invocation of invocations) { + assert.equal(invocation.opening.kind, 'invocation_opened'); + assert.equal(invocation.sessionId, 'session-1'); + } + const migrated = invocations.find( + (invocation) => invocation.invocationId === 'run-with-events', + ); + assert.equal(migrated?.turnId, 'turn-with-events'); + assert.equal(migrated?.terminalEvent, undefined); + } finally { + store.close(); + } + }); + }); + + test('purging a migrated Session takes its shelved openings with it', async () => { + await withHeaderOnlyRuns(async (databasePath) => { + const db = new DatabaseSync(databasePath); + try { + const { json } = encodeCanonicalRuntimeEvent({ + id: 'existing-1', + invocationId: 'run-with-events', + runId: 'run-with-events', + sessionId: 'session-1', + turnId: 'turn-with-events', + ts: 1, + partial: false, + role: 'user', + author: 'user', + modelVisibility: 'visible', + content: { kind: 'text', text: 'already immutable' }, + }); + db.prepare(` + INSERT INTO runtime_events ( + event_id, session_id, invocation_id, run_id, turn_id, event_seq, + event_kind, payload_json, committed_at + ) VALUES ('existing-1', 'session-1', 'run-with-events', 'run-with-events', + 'turn-with-events', 1, 'text', ?, 1) + `).run(json); + migrateSqliteRuntimeDatabase(db); + } finally { + db.close(); + } + + // What purging a conversation does to this database: delete the Session's + // events. `conversation-operational-state.ts` runs exactly this statement + // on a lease that has `PRAGMA foreign_keys = ON`, which is also how + // `runtime_session_event_ordinals` is cleaned up today. + const purge = new DatabaseSync(databasePath); + try { + purge.exec('PRAGMA foreign_keys = ON'); + purge.prepare('DELETE FROM runtime_events WHERE session_id = ?').run('session-1'); + } finally { + purge.close(); + } + + // The shelved opening is only read when its invocation has no opening + // event, so a purge that deleted the events but left the shelf would make + // a completed run reappear as an active one. + const store = createSqliteRuntimeStore(databasePath); + try { + assert.deepEqual(await store.listSessionInvocations('session-1'), []); + assert.equal(await store.readRunInvocation('session-1', 'run-with-events'), undefined); + } finally { + store.close(); + } + + const check = new DatabaseSync(databasePath); + try { + assert.equal( + ( + check + .prepare('SELECT COUNT(*) AS count FROM runtime_legacy_invocation_openings') + .get() as { count: number } + ).count, + 0, + 'the shelf is empty, not merely unreadable', + ); + } finally { + check.close(); + } + }); + }); + + test('bounds, pages and addresses the same inventory', async () => { + await withHeaderOnlyRuns(async (databasePath) => { + const db = new DatabaseSync(databasePath); + try { + migrateSqliteRuntimeDatabase(db); + } finally { + db.close(); + } + + const store = createSqliteRuntimeStore(databasePath); + try { + const bounded = await store.listSessionInvocationsBounded('session-1', 2); + assert.deepEqual( + bounded.invocations.map((invocation) => invocation.invocationId), + ['run-legacy-route', 'run-scheduled'], + ); + assert.equal(bounded.truncated, true, 'the extra row read past the limit reports the rest'); + + const first = await store.listSessionInvocationsPage('session-1', { limit: 2 }); + assert.deepEqual( + first.invocations.map((invocation) => invocation.invocationId), + ['run-with-events', 'run-scheduled'], + 'a page runs newest first', + ); + const second = await store.listSessionInvocationsPage('session-1', { + limit: 2, + ...(first.nextCursor ? { before: first.nextCursor } : {}), + }); + assert.deepEqual( + second.invocations.map((invocation) => invocation.invocationId), + ['run-legacy-route'], + 'the cursor resumes without repeating or skipping a tied opening time', + ); + assert.equal(second.nextCursor, null); + + const one = await store.readInvocation('session-1', 'run-scheduled'); + assert.equal(one.turnId, 'turn-scheduled'); + assert.deepEqual(one.opening.root, { kind: 'scheduled_task', scheduledTaskId: 'task-9' }); + + await assert.rejects( + () => store.listSessionInvocationsPage('session-1', { limit: 0 }), + /between 1 and 256/, + ); + await assert.rejects( + () => + store.listSessionInvocationsPage('session-1', { + limit: 1, + before: { openedAt: Number.NaN, invocationId: 'run-scheduled' }, + }), + /Invalid invocation page cursor/, + ); + } finally { + store.close(); + } + }); + }); + + // Built from what the header era actually wrote, not from what the decoder + // accepts: every run that reached a provider carried the composition snapshot. + test('migrates a header exactly as the header era wrote it, composition included', async () => { + await withHeaderOnlyRuns(async (databasePath) => { + const db = new DatabaseSync(databasePath); + try { + const insert = db.prepare( + 'INSERT INTO core_agent_runs(session_id, run_id, created_at, record_json) VALUES (?, ?, ?, ?)', + ); + for (const record of [ + header({ + runId: 'run-composed', + turnId: 'turn-composed', + status: 'failed', + failureClass: 'provider_error', + failureMessage: 'the provider said no', + completedAt: 7, + runComposition: headerEraComposition(), + }), + header({ + runId: 'run-composed-events', + turnId: 'turn-composed-events', + runComposition: headerEraComposition(), + }), + ]) { + insert.run(record.sessionId, record.runId, record.createdAt, JSON.stringify(record)); + } + const { json } = encodeCanonicalRuntimeEvent({ + id: 'composed-1', + invocationId: 'run-composed-events', + runId: 'run-composed-events', + sessionId: 'session-1', + turnId: 'turn-composed-events', + ts: 1, + partial: false, + role: 'user', + author: 'user', + modelVisibility: 'visible', + content: { kind: 'text', text: 'already immutable' }, + }); + db.prepare(` + INSERT INTO runtime_events ( + event_id, session_id, invocation_id, run_id, turn_id, event_seq, + event_kind, payload_json, committed_at + ) VALUES ('composed-1', 'session-1', 'run-composed-events', 'run-composed-events', + 'turn-composed-events', 1, 'text', ?, 1) + `).run(json); + migrateSqliteRuntimeDatabase(db); + migrateSqliteCoreExecutionDatabase(db); + } finally { + db.close(); + } + + const store = createSqliteRuntimeStore(databasePath); + try { + const invocations = await store.listSessionInvocations('session-1'); + assert.deepEqual( + invocations.map((invocation) => invocation.invocationId).sort(), + [ + 'run-composed', + 'run-composed-events', + 'run-legacy-route', + 'run-scheduled', + 'run-with-events', + ], + 'a run whose header carried a composition snapshot is still a run', + ); + const composed = await store.readInvocation('session-1', 'run-composed'); + assert.equal(composed.terminalEvent?.status, 'failed'); + assert.equal(composed.terminalEvent?.ts, 7); + assert.equal(composed.terminalEvent?.actions?.stateDelta?.failureClass, 'provider_error'); + assert.equal( + composed.terminalEvent?.content?.kind === 'error' + ? composed.terminalEvent.content.message + : undefined, + 'the provider said no', + ); + } finally { + store.close(); + } + }); + }); + + test('refuses to migrate a header it cannot read, and drops nothing', async () => { + await withHeaderOnlyRuns(async (databasePath) => { + const db = new DatabaseSync(databasePath); + try { + // A graph wake with no delivery attempt is corruption. Inventing a root + // authority for it would be worse than refusing, and dropping the header + // would be worse still: the migration stops, and the database stays as + // the header era left it. + const corrupt = header({ + runId: 'run-corrupt-root', + turnId: 'turn-corrupt', + agentGraphWakeId: 'wake-1', + }); + db.prepare( + 'INSERT INTO core_agent_runs(session_id, run_id, created_at, record_json) VALUES (?, ?, ?, ?)', + ).run(corrupt.sessionId, corrupt.runId, corrupt.createdAt, JSON.stringify(corrupt)); + assert.throws(() => migrateSqliteRuntimeDatabase(db), /session-1\/run-corrupt-root/); + assert.equal(readUserVersion(db), SQLITE_RUNTIME_SCHEMA_VERSION - 1); + const openings = db + .prepare( + "SELECT COUNT(*) AS total FROM runtime_events WHERE event_kind = 'invocation_opened'", + ) + .get() as { total: number }; + assert.equal(openings.total, 0, 'the transaction rolled every other run back too'); + const headers = db + .prepare('SELECT COUNT(*) AS total FROM core_agent_runs WHERE record_json IS NOT NULL') + .get() as { total: number }; + assert.equal(headers.total, 4, 'every header is still there to be read by a fixed build'); + } finally { + db.close(); + } + }); + }); +}); + +function headerEraComposition() { + return createRunCompositionSnapshot({ + composerId: 'maka.default', + composerRevision: '1', + sourceRevisions: [{ id: 'system-prompt', revision: '1' }], + baseSystemPromptHash: `sha256:${'a'.repeat(64)}`, + toolCatalogHash: `sha256:${'b'.repeat(64)}`, + toolAvailabilityHash: `sha256:${'c'.repeat(64)}`, + baseProviderOptionsHash: `sha256:${'d'.repeat(64)}`, + toolNames: ['read_file'], + contextWindow: 200_000, + }); +} + +/** + * Put the database back the way the header era left it: runtime schema one step + * behind, no opening facts, and a `core_agent_runs` row that still carries the + * header the migration under test has to read. + */ +function rewindToHeaderEra(db: DatabaseSync): void { + db.exec('DROP INDEX IF EXISTS runtime_events_by_session_kind'); + db.exec('DROP INDEX IF EXISTS runtime_events_one_opening_per_invocation'); + db.exec('DROP INDEX IF EXISTS runtime_legacy_invocation_openings_by_session'); + db.exec('DROP TABLE IF EXISTS runtime_legacy_invocation_openings'); + db.exec("DELETE FROM runtime_events WHERE event_kind = 'invocation_opened'"); + db.exec( + 'ALTER TABLE runtime_continuation_claims RENAME COLUMN target_opening_json TO target_run_header_json', + ); + db.exec('ALTER TABLE core_agent_runs ADD COLUMN record_json TEXT'); + db.exec(`PRAGMA user_version = ${SQLITE_RUNTIME_SCHEMA_VERSION - 1}`); +} + +function readUserVersion(db: DatabaseSync): number { + return (db.prepare('PRAGMA user_version').get() as { user_version: number }).user_version; +} + +async function withHeaderOnlyRuns(run: (databasePath: string) => Promise): Promise { + const root = await mkdtemp(join(tmpdir(), 'maka-opening-backfill-')); + try { + const databasePath = join(root, OPERATIONAL_STATE_DATABASE_NAME); + const db = new DatabaseSync(databasePath); + try { + migrateSqliteRuntimeDatabase(db); + migrateSqliteCoreExecutionDatabase(db); + rewindToHeaderEra(db); + const insert = db.prepare( + 'INSERT INTO core_agent_runs(session_id, run_id, created_at, record_json) VALUES (?, ?, ?, ?)', + ); + for (const record of [ + header({ runId: 'run-legacy-route', turnId: 'turn-legacy', modelId: 'legacy-model' }), + header({ + runId: 'run-scheduled', + turnId: 'turn-scheduled', + llmConnectionId: 'connection-1', + scheduledTaskId: 'task-9', + }), + header({ runId: 'run-with-events', turnId: 'turn-with-events' }), + ]) { + insert.run(record.sessionId, record.runId, record.createdAt, JSON.stringify(record)); + } + } finally { + db.close(); + } + + await run(databasePath); + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +function header(overrides: Partial): LegacyRunHeader { + return { + runId: 'run-1', + invocationId: overrides.runId ?? 'run-1', + sessionId: 'session-1', + turnId: 'turn-1', + status: 'completed', + backendKind: 'ai-sdk', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + cwd: '/tmp/cwd', + permissionMode: 'ask', + createdAt: 1, + updatedAt: 2, + ...overrides, + }; +} diff --git a/packages/storage/src/__tests__/legacy-run-header.test.ts b/packages/storage/src/__tests__/legacy-run-header.test.ts new file mode 100644 index 0000000000..4617f90f40 --- /dev/null +++ b/packages/storage/src/__tests__/legacy-run-header.test.ts @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { describe, test } from 'node:test'; +import { + decodePersistedLegacyRunHeader, + invocationOpeningFromLegacyRunHeader, + type LegacyRunHeader, +} from '../legacy-run-header.js'; + +describe('legacy Run header decoding', () => { + test('rejects a header with multiple hosted root authorities', () => { + assert.throws( + () => + decodePersistedLegacyRunHeader({ + ...runHeader(), + scheduledTaskId: 'scheduled-task-1', + goalId: 'goal-1', + }), + /Invalid AgentRun header schema/, + ); + }); + + test('folds every retired persisted value', () => { + const decoded = decodePersistedLegacyRunHeader({ + ...runHeader(), + status: 'waiting_permission', + permissionMode: 'execute', + automationId: 'automation-1', + }); + assert.equal(decoded.status, 'waiting_for_user'); + assert.equal(decoded.permissionMode, 'ask'); + assert.equal(decoded.legacyAutomationId, 'automation-1'); + assert.equal(Object.hasOwn(decoded, 'automationId'), false); + }); + + test('accepts both bound and legacy connection identity', () => { + assert.equal(decodePersistedLegacyRunHeader(runHeader()).llmConnectionId, undefined); + const bound = decodePersistedLegacyRunHeader({ + ...runHeader(), + llmConnectionId: '11111111-1111-4111-8111-111111111111', + }); + assert.equal(bound.llmConnectionId, '11111111-1111-4111-8111-111111111111'); + assert.throws( + () => decodePersistedLegacyRunHeader({ ...runHeader(), llmConnectionId: '' }), + /Invalid AgentRun header schema/, + ); + }); + + test('projects an unbound connection as an unauthenticated route', () => { + const opening = invocationOpeningFromLegacyRunHeader( + decodePersistedLegacyRunHeader(runHeader()), + ); + assert.equal(opening.route.provenance, 'unknown'); + assert.equal(opening.source.kind, 'fresh'); + }); +}); + +describe('legacy continuation source decoding', () => { + test('rejects a V2 replay manifest that does not identify its boundary', () => { + assert.throws( + () => + decodePersistedLegacyRunHeader( + headerWithContinuation({ + ...validV2ContinuationSource(), + replayManifestDigest: `sha256:${'c'.repeat(64)}`, + }), + ), + /Invalid AgentRun header schema/, + ); + }); + + test('projects a V2 source onto the opening fact', () => { + const header = decodePersistedLegacyRunHeader( + headerWithContinuation(validV2ContinuationSource()), + ); + const source = invocationOpeningFromLegacyRunHeader(header).source; + assert.equal(source.kind, 'continuation'); + if (source.kind !== 'continuation') throw new Error('unreachable'); + assert.equal(source.claimId, 'claim-1'); + assert.equal(source.sourceRunId, 'source-run'); + }); +}); + +function runHeader(): Record { + return { + runId: 'run-1', + sessionId: 'session-1', + turnId: 'turn-1', + status: 'created', + backendKind: 'fake', + llmConnectionSlug: 'fake', + modelId: 'fake-model', + cwd: '/workspace', + permissionMode: 'ask', + createdAt: 1, + updatedAt: 1, + }; +} + +function headerWithContinuation( + continuationSource: LegacyRunHeader['continuationSource'], +): Record { + return { + ...runHeader(), + runId: 'target-run', + invocationId: 'target-invocation', + turnId: 'target-turn', + continuationSource, + }; +} + +function validV2ContinuationSource(): Extract< + NonNullable, + { protocol: 'continuation_source_v2' } +> { + return { + protocol: 'continuation_source_v2', + claimId: 'claim-1', + boundaryDigest: `sha256:${'a'.repeat(64)}`, + sourceInvocationId: 'source-invocation', + sourceRunId: 'source-run', + sourceTurnId: 'source-turn', + sourceRuntimeEventHighWater: 1, + sourcePrefixDigest: `sha256:${'b'.repeat(64)}`, + replayManifestDigest: `sha256:${'a'.repeat(64)}`, + }; +} diff --git a/packages/storage/src/__tests__/model-call-ledger.test.ts b/packages/storage/src/__tests__/model-call-ledger.test.ts index 1a4190d309..0fe026cf79 100644 --- a/packages/storage/src/__tests__/model-call-ledger.test.ts +++ b/packages/storage/src/__tests__/model-call-ledger.test.ts @@ -23,7 +23,6 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; import { describe, test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core/agent-run'; import { MODEL_CALL_ATTEMPT_EVENT_TYPE, MODEL_CALL_ATTEMPT_SCHEMA_VERSION, @@ -37,6 +36,7 @@ import { } from '../model-call-ledger.js'; import { acquireOperationalStateDatabase } from '../operational-state-store.js'; import { createSqliteAgentRunStore } from '../agent-run-store.js'; +import { openInvocation } from './fixtures/invocation-opening.js'; const NOW = 1_750_000_000_000; @@ -90,8 +90,8 @@ function appendAuthorityEvent( lease.transaction('write', () => { lease.database .prepare(` - INSERT OR IGNORE INTO core_agent_runs(session_id, run_id, created_at, record_json) - VALUES (?, ?, ?, '{}') + INSERT OR IGNORE INTO core_agent_runs(session_id, run_id, created_at) + VALUES (?, ?, ?) `) .run(sessionId, runId, NOW - 1_000); lease.database @@ -356,21 +356,8 @@ describe('canonical model call ledger', () => { describe('catching the read model up from the AgentRun authority', () => { test('consumes the high-water published by the real AgentRun append path', async () => { await withLedger(async (ledger, root) => { + await openInvocation(root, { sessionId: 'session-1', runId: 'run-1', turnId: 'turn-1' }); const runStore = createSqliteAgentRunStore(root); - const header: AgentRunHeader = { - runId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - status: 'created', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/cwd', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 1, - }; - await runStore.createRun(header); await runStore.appendEvent('session-1', 'run-1', { id: 'attempt-real-append', type: MODEL_CALL_ATTEMPT_EVENT_TYPE, diff --git a/packages/storage/src/__tests__/recovery-persistence-authority.test.ts b/packages/storage/src/__tests__/recovery-persistence-authority.test.ts index fefea71ba0..1f97d01030 100644 --- a/packages/storage/src/__tests__/recovery-persistence-authority.test.ts +++ b/packages/storage/src/__tests__/recovery-persistence-authority.test.ts @@ -108,7 +108,7 @@ describe('SQLite recovery persistence authority', () => { dispatch.ts, ); db.exec( - 'DROP TABLE runtime_session_event_ordinals; DROP TABLE runtime_partial_segments; DROP TABLE runtime_storage_root_binding; DROP TABLE runtime_managed_mutation_reservations; DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; DROP TABLE runtime_continuation_claims; DROP TABLE runtime_capabilities; PRAGMA user_version = 4;', + 'DROP INDEX runtime_events_by_session_kind; DROP INDEX runtime_events_one_opening_per_invocation; DROP TABLE runtime_legacy_invocation_openings; DROP TABLE runtime_session_event_ordinals; DROP TABLE runtime_partial_segments; DROP TABLE runtime_storage_root_binding; DROP TABLE runtime_managed_mutation_reservations; DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; DROP TABLE runtime_continuation_claims; DROP TABLE runtime_capabilities; PRAGMA user_version = 4;', ); db.close(); @@ -204,7 +204,7 @@ describe('SQLite recovery persistence authority', () => { 2, ); db.exec( - 'DROP TABLE runtime_session_event_ordinals; DROP TABLE runtime_partial_segments; DROP TABLE runtime_storage_root_binding; DROP TABLE runtime_managed_mutation_reservations; DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; DROP TABLE runtime_continuation_claims; DROP TABLE runtime_capabilities; PRAGMA user_version = 4;', + 'DROP INDEX runtime_events_by_session_kind; DROP INDEX runtime_events_one_opening_per_invocation; DROP TABLE runtime_legacy_invocation_openings; DROP TABLE runtime_session_event_ordinals; DROP TABLE runtime_partial_segments; DROP TABLE runtime_storage_root_binding; DROP TABLE runtime_managed_mutation_reservations; DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; DROP TABLE runtime_continuation_claims; DROP TABLE runtime_capabilities; PRAGMA user_version = 4;', ); db.close(); diff --git a/packages/storage/src/__tests__/regenerate-root-admission.test.ts b/packages/storage/src/__tests__/regenerate-root-admission.test.ts index 4429d04de6..10a228f0a4 100644 --- a/packages/storage/src/__tests__/regenerate-root-admission.test.ts +++ b/packages/storage/src/__tests__/regenerate-root-admission.test.ts @@ -22,7 +22,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; -import type { RootExecutionDescriptor } from '@maka/core/agent-run'; +import type { RootExecutionDescriptor } from '@maka/core/runtime-invocation'; import { createSqliteAgentRunStore, type AdmitRootTurnInput } from '../agent-run-store.js'; test('regenerate admission durably binds the immutable source Turn', async () => { diff --git a/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts b/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts index 8ef650997b..0890db5907 100644 --- a/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts @@ -23,7 +23,9 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { after, describe, test } from 'node:test'; import { DatabaseSync } from 'node:sqlite'; -import type { AgentRunHeader, EmittedAgentRunEvent } from '@maka/core/agent-run'; +import type { EmittedAgentRunEvent } from '@maka/core/agent-run'; +import { agentRunCompositionFromEvents } from '@maka/core/agent-run'; +import type { RunCompositionSnapshot } from '@maka/core/run-composition'; import { MODEL_CALL_ATTEMPT_SCHEMA_VERSION, decodeModelCallAttempt, @@ -43,22 +45,22 @@ import { removeTrackedControlDirectories, trackControlDirectory, } from './fixtures/control-directory-hygiene.js'; +import { openInvocation } from './fixtures/invocation-opening.js'; // The control directory of each resolved root lives outside that root, so a // temporary root's removal leaves it behind; reclaim the recorded rootIds here. after(removeTrackedControlDirectories); describe('SQLite core execution stores', () => { - test('persists AgentRun header and events', async () => { + test('persists AgentRun events against the invocation that opened them', async () => { await withRoot(async (root) => { + await openRun(root); const store = createSqliteAgentRunStore(root); - await store.createRun(runHeader()); await store.appendEvent('session-1', 'run-1', runEvent()); store.close?.(); const reopened = createSqliteAgentRunStore(root); try { - assert.equal((await reopened.readRun('session-1', 'run-1')).runId, 'run-1'); assert.equal((await reopened.readEvents('session-1', 'run-1'))[0]?.id, 'event-1'); } finally { reopened.close?.(); @@ -66,53 +68,23 @@ describe('SQLite core execution stores', () => { }); }); - test('folds retired AgentRun values only when reading persisted rows', async () => { + test('refuses to hang an event on a run no invocation ever opened', async () => { await withRoot(async (root) => { const store = createSqliteAgentRunStore(root); - await store.createRun(runHeader()); - await assert.rejects( - () => - store.createRun({ - ...runHeader({ runId: 'run-retired', turnId: 'turn-retired' }), - permissionMode: 'execute', - } as unknown as AgentRunHeader), - /Invalid AgentRun header schema/, - ); - store.close?.(); - - const database = new DatabaseSync(join(root, 'runtime.sqlite')); - try { - const row = database - .prepare("SELECT record_json AS recordJson FROM core_agent_runs WHERE run_id = 'run-1'") - .get() as { recordJson: string }; - const retired = JSON.parse(row.recordJson) as Record; - retired.status = 'waiting_permission'; - retired.permissionMode = 'execute'; - retired.automationId = 'automation-1'; - database - .prepare("UPDATE core_agent_runs SET record_json = ? WHERE run_id = 'run-1'") - .run(JSON.stringify(retired)); - } finally { - database.close(); - } - - const reopened = createSqliteAgentRunStore(root); try { - const decoded = await reopened.readRun('session-1', 'run-1'); - assert.equal(decoded.status, 'waiting_for_user'); - assert.equal(decoded.permissionMode, 'ask'); - assert.equal(decoded.legacyAutomationId, 'automation-1'); - assert.equal(Object.hasOwn(decoded, 'automationId'), false); + await assert.rejects(store.appendEvent('session-1', 'run-missing', runEvent()), { + code: 'ENOENT', + }); } finally { - reopened.close?.(); + store.close?.(); } }); }); test('advances the model-call high-water index with the authority append', async () => { await withRoot(async (root) => { + await openRun(root); const store = createSqliteAgentRunStore(root); - await store.createRun(runHeader()); await store.appendEvent('session-1', 'run-1', runEvent()); await store.appendEvent('session-1', 'run-1', { ...runEvent(), @@ -142,8 +114,8 @@ describe('SQLite core execution stores', () => { test('commits canonical authority without guessing a malformed projection order', async () => { await withRoot(async (root) => { + await openRun(root); const store = createSqliteAgentRunStore(root); - await store.createRun(runHeader()); await store.appendEvent( 'session-1', 'run-1', @@ -254,8 +226,8 @@ describe('SQLite core execution stores', () => { test('does not repair a malformed projection from a stale ledger revision', async () => { await withRoot(async (root) => { + await openRun(root); const store = createSqliteAgentRunStore(root); - await store.createRun(runHeader()); await store.appendEvent('session-1', 'run-1', runEvent()); await store.repairEventProjection( 'session-1', @@ -321,8 +293,8 @@ describe('SQLite core execution stores', () => { test('rejects a projection repair without a canonical ledger revision', async () => { await withRoot(async (root) => { + await openRun(root); const store = createSqliteAgentRunStore(root); - await store.createRun(runHeader()); await store.appendEvent('session-1', 'run-1', runEvent()); const before = await store.readEventProjection( 'session-1', @@ -349,8 +321,8 @@ describe('SQLite core execution stores', () => { test('backfills the model-call high-water when upgrading existing AgentRun rows', async () => { await withRoot(async (root) => { + await openRun(root); const store = createSqliteAgentRunStore(root); - await store.createRun(runHeader()); await store.appendEvent('session-1', 'run-1', { ...runEvent(), id: 'legacy-model-call-event', @@ -456,58 +428,10 @@ describe('SQLite core execution stores', () => { }); }); - test('pages AgentRuns by stable creation and run identity order', async () => { - await withRoot(async (root) => { - const store = createSqliteAgentRunStore(root); - try { - await store.createRun(runHeader({ runId: 'run-a', turnId: 'turn-a', createdAt: 1 })); - await store.createRun(runHeader({ runId: 'run-b', turnId: 'turn-b', createdAt: 2 })); - await store.createRun(runHeader({ runId: 'run-c', turnId: 'turn-c', createdAt: 2 })); - - const first = await store.listSessionRunsPage('session-1', { limit: 2 }); - assert.deepEqual( - first.runs.map((run) => run.runId), - ['run-c', 'run-b'], - ); - assert.deepEqual(first.nextCursor, { createdAt: 2, runId: 'run-b' }); - - await store.createRun(runHeader({ runId: 'run-d', turnId: 'turn-d', createdAt: 3 })); - const older = await store.listSessionRunsPage('session-1', { - limit: 2, - before: first.nextCursor ?? undefined, - }); - assert.deepEqual( - older.runs.map((run) => run.runId), - ['run-a'], - ); - assert.equal(older.nextCursor, null); - } finally { - store.close?.(); - } - }); - }); - - test('rejects a non-finite AgentRun page cursor', async () => { - await withRoot(async (root) => { - const store = createSqliteAgentRunStore(root); - try { - await assert.rejects( - store.listSessionRunsPage('session-1', { - limit: 1, - before: { createdAt: Number.NaN, runId: 'run-1' }, - }), - /Invalid AgentRun page cursor/u, - ); - } finally { - store.close?.(); - } - }); - }); - test('preserves provider failure diagnostics in the AgentRun authority after reopen', async () => { await withRoot(async (root) => { + await openRun(root); const store = createSqliteAgentRunStore(root); - await store.createRun(runHeader()); await store.appendEvent('session-1', 'run-1', { type: 'model_call_attempt_recorded', id: 'attempt-1', @@ -555,15 +479,21 @@ describe('SQLite core execution stores', () => { test('commits one immutable Run Composition snapshot', async () => { await withRoot(async (root) => { + await openRun(root); const store = createSqliteAgentRunStore(root); try { - await store.createRun(runHeader()); const composition = runComposition('1'); - await store.updateRun('session-1', 'run-1', { runComposition: composition }); - await store.updateRun('session-1', 'run-1', { runComposition: composition }); - assert.deepEqual((await store.readRun('session-1', 'run-1')).runComposition, composition); + await store.appendEvent('session-1', 'run-1', compositionEvent('event-1', composition)); + await store.appendEvent('session-1', 'run-1', compositionEvent('event-2', composition)); + const events = await store.readEvents('session-1', 'run-1'); + assert.deepEqual(agentRunCompositionFromEvents(events), composition); + assert.equal( + events.filter((event) => event.type === 'run_composition_recorded').length, + 1, + 'an identical re-append is the writer retrying, not a second composition', + ); await assert.rejects( - store.updateRun('session-1', 'run-1', { runComposition: runComposition('2') }), + store.appendEvent('session-1', 'run-1', compositionEvent('event-3', runComposition('2'))), /AgentRun Run Composition is immutable/u, ); } finally { @@ -574,8 +504,8 @@ describe('SQLite core execution stores', () => { test('reads an AgentRun event type this build does not write', async () => { await withRoot(async (root) => { + await openRun(root); const store = createSqliteAgentRunStore(root); - await store.createRun(runHeader()); await store.appendEvent('session-1', 'run-1', runEvent()); store.close?.(); @@ -675,26 +605,13 @@ async function withRoot(run: (root: string) => Promise): Promise { } } -function runHeader(overrides: Partial = {}): AgentRunHeader { - return { - runId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - status: 'created', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/cwd', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 1, - ...overrides, - }; +function openRun(root: string): Promise { + return openInvocation(root, { sessionId: 'session-1', runId: 'run-1', turnId: 'turn-1' }); } function runEvent(): EmittedAgentRunEvent { return { - type: 'run_started', + type: 'turn_started', id: 'event-1', runId: 'run-1', sessionId: 'session-1', @@ -730,7 +647,19 @@ function modelCallAttempt(overrides: Partial = {}): ModelCallA }; } -function runComposition(seed: string): NonNullable { +function compositionEvent(id: string, composition: RunCompositionSnapshot): EmittedAgentRunEvent { + return { + type: 'run_composition_recorded', + id, + runId: 'run-1', + sessionId: 'session-1', + turnId: 'turn-1', + ts: 5, + data: { runComposition: composition }, + }; +} + +function runComposition(seed: string): RunCompositionSnapshot { return { schemaVersion: 1, composerId: 'maka.interactive', diff --git a/packages/storage/src/__tests__/sqlite-recovery-concurrency.test.ts b/packages/storage/src/__tests__/sqlite-recovery-concurrency.test.ts index 0f13e96866..874ab86290 100644 --- a/packages/storage/src/__tests__/sqlite-recovery-concurrency.test.ts +++ b/packages/storage/src/__tests__/sqlite-recovery-concurrency.test.ts @@ -340,6 +340,9 @@ describe('SQLite recovery authority multi-process races', () => { try { db.exec(` DROP TABLE runtime_managed_mutation_reservations; + DROP INDEX runtime_events_by_session_kind; + DROP INDEX runtime_events_one_opening_per_invocation; + DROP TABLE runtime_legacy_invocation_openings; DROP TABLE runtime_session_event_ordinals; PRAGMA user_version = 10; UPDATE operational_schema_migrations SET version = 10 WHERE scope = 'runtime'; diff --git a/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts b/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts index e8d45e944e..5bdd3984b7 100644 --- a/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts +++ b/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts @@ -108,7 +108,17 @@ describe('SQLite runtime schema migration', () => { try { db.exec('PRAGMA foreign_keys = ON'); db.exec(` - CREATE TABLE runtime_events (event_id TEXT PRIMARY KEY); + CREATE TABLE runtime_events ( + event_id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + invocation_id TEXT NOT NULL, + run_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + event_seq INTEGER NOT NULL, + event_kind TEXT NOT NULL, + payload_json TEXT NOT NULL, + committed_at INTEGER NOT NULL + ); CREATE TABLE runtime_continuation_claims ( claim_id TEXT PRIMARY KEY, source_session_id TEXT NOT NULL, @@ -136,7 +146,8 @@ describe('SQLite runtime schema migration', () => { INSERT INTO runtime_continuation_claims VALUES ( 'claim-v1', 'session', 'source-invocation', 'source-run', 'source-turn', 1, 'sha256:source', 'sha256:boundary-v1', '{}', 1, 'sha256:replay-v1', - 'session', 'target-invocation-v1', 'target-run-v1', 'target-turn-v1', '{}', + 'session', 'target-invocation-v1', 'target-run-v1', 'target-turn-v1', + '{"runId": "target-run-v1", "invocationId": "target-invocation-v1", "sessionId": "session", "turnId": "target-turn-v1", "status": "created", "backendKind": "fake", "llmConnectionSlug": "connection-1", "modelId": "model-1", "cwd": "/workspace", "permissionMode": "ask", "createdAt": 1, "updatedAt": 1}', 1, NULL, NULL, 1 ); PRAGMA user_version = 14; @@ -144,7 +155,7 @@ describe('SQLite runtime schema migration', () => { migrateSqliteRuntimeDatabase(db); - assert.equal(SQLITE_RUNTIME_SCHEMA_VERSION, 15); + assert.equal(SQLITE_RUNTIME_SCHEMA_VERSION, 16); assert.equal( ( db @@ -155,6 +166,19 @@ describe('SQLite runtime schema migration', () => { ).version, 1, ); + assert.equal( + JSON.parse( + ( + db + .prepare( + "SELECT target_opening_json AS opening FROM runtime_continuation_claims WHERE claim_id = 'claim-v1'", + ) + .get() as { opening: string } + ).opening, + ).kind, + 'invocation_opened', + 'an open claim carries the opening it always implied, not a copy of the Run header', + ); db.exec(` INSERT INTO runtime_continuation_claims VALUES ( 'claim-v2', 'session', 'source-invocation', 'source-run', 'source-turn', 2, diff --git a/packages/storage/src/__tests__/sqlite-runtime-store.test.ts b/packages/storage/src/__tests__/sqlite-runtime-store.test.ts index 3fa3060403..6b99977949 100644 --- a/packages/storage/src/__tests__/sqlite-runtime-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-runtime-store.test.ts @@ -23,6 +23,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; import { describe, it } from 'node:test'; +import { DEFAULT_TOOL_MODE } from '@maka/core/tool-mode'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { RunSealedError } from '@maka/core/runtime-event-store'; import { canonicalToolArgsHash } from '@maka/core/tool-args-identity'; @@ -823,7 +824,7 @@ describe('SqliteRuntimeStore', () => { }); }); - it('decodes a persisted continuation target without widening new claims', async () => { + it('refuses a continuation target opening it cannot read, stored or submitted', async () => { await withStore(async (store, dbPath) => { const claim = continuationClaim(); await persistImmutablePrefix(store, continuationSourcePrefix()); @@ -832,13 +833,16 @@ describe('SqliteRuntimeStore', () => { store.claimContinuation({ claim: { ...claim, - targetRunHeader: { - ...claim.targetRunHeader, - permissionMode: 'execute', - } as unknown as ContinuationClaimV1['targetRunHeader'], + targetOpening: { + ...claim.targetOpening, + configuration: { + ...claim.targetOpening.configuration, + permissionMode: 'execute', + }, + } as unknown as ContinuationClaimV1['targetOpening'], }, }), - /Invalid AgentRun header schema/, + /Invalid RuntimeEvent invocation_opened schema/, ); assert.equal((await store.claimContinuation({ claim })).kind, 'acquired'); @@ -846,9 +850,9 @@ describe('SqliteRuntimeStore', () => { try { database.exec(` UPDATE runtime_continuation_claims - SET target_run_header_json = json_set( - target_run_header_json, - '$.permissionMode', + SET target_opening_json = json_set( + target_opening_json, + '$.configuration.permissionMode', 'execute' ) WHERE claim_id = 'claim-1'; @@ -857,8 +861,15 @@ describe('SqliteRuntimeStore', () => { database.close(); } - const persisted = await store.readContinuationClaimByBoundary(claim.boundaryDigest); - assert.equal(persisted?.targetRunHeader.permissionMode, 'ask'); + // A persisted Run header used to be widened on read. The opening fact has + // no legacy layer and none is wanted: a claim whose frozen opening cannot + // be read cannot authenticate the start event it exists to authenticate, + // and admitting one against a guessed opening would be the failure this + // record is meant to prevent. + await assert.rejects( + store.readContinuationClaimByBoundary(claim.boundaryDigest), + /Invalid RuntimeEvent invocation_opened schema/, + ); }); }); @@ -1076,6 +1087,47 @@ describe('SqliteRuntimeStore', () => { }); }); + it('lets a started continuation target be purged instead of refusing the delete', async () => { + await withStore(async (store, dbPath) => { + const claim = continuationClaim(); + await persistImmutablePrefix(store, continuationSourcePrefix()); + assert.equal((await store.claimContinuation({ claim })).kind, 'acquired'); + + const db = new DatabaseSync(dbPath); + try { + db.exec('PRAGMA foreign_keys = ON'); + // Stand the claim up the way starting a continuation does: its start + // event is event one of the target Session's run. + const start = db + .prepare('SELECT event_id, session_id FROM runtime_events ORDER BY event_seq ASC LIMIT 1') + .get() as { event_id: string; session_id: string }; + db.prepare( + "UPDATE runtime_continuation_claims SET start_event_id = ?, start_kind = 'runtime_admission' WHERE claim_id = ?", + ).run(start.event_id, claim.claimId); + + // Purging a conversation deletes its events. The claim used to have no + // ON DELETE clause, so the constraint refused this and rolled the whole + // purge back — for the user's delete, a copy rollback, an import + // discard and Session retirement alike. + db.prepare('DELETE FROM runtime_events WHERE session_id = ?').run(start.session_id); + + assert.equal( + ( + db + .prepare( + 'SELECT COUNT(*) AS count FROM runtime_continuation_claims WHERE claim_id = ?', + ) + .get(claim.claimId) as { count: number } + ).count, + 0, + 'a continuation whose target was deleted no longer names anything, so it goes too', + ); + } finally { + db.close(); + } + }); + }); + it('fails closed when continuation claim columns disagree with canonical payload', async () => { await withStore(async (store, dbPath) => { const claim = continuationClaim(); @@ -1913,32 +1965,37 @@ function continuationClaimForBoundary( providerProjectionVersion: 1, providerReplayDigest: 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', target, - targetRunHeader: { - ...target, - status: 'created', - backendKind: 'fake', - llmConnectionSlug: 'connection-1', - modelId: 'model-1', - cwd: '/workspace/repo', - permissionMode: 'ask', - collaborationMode: 'agent', - orchestrationMode: 'default', - orchestrationSource: 'session', - agentSwarmAuthorization: 'none', - createdAt: claimedAt, - updatedAt: claimedAt, - parentRunId: source.identity.runId, - parentTurnId: source.identity.turnId, - continuationSource: { - protocol: 'continuation_source_v2', - claimId, - boundaryDigest: boundary.manifestDigest, + targetOpening: { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: { + provenance: 'unknown', + backendKind: 'fake', + llmConnectionSlug: 'connection-1', + modelId: 'model-1', + }, + configuration: { + cwd: '/workspace/repo', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'default', + orchestrationSource: 'session', + toolMode: DEFAULT_TOOL_MODE, + agentSwarmAuthorization: 'none', + }, + root: { kind: 'user' }, + source: { + kind: 'continuation', sourceInvocationId: source.identity.invocationId, sourceRunId: source.identity.runId, sourceTurnId: source.identity.turnId, sourceRuntimeEventHighWater: source.position.lastEventSeq, - sourcePrefixDigest: source.prefixDigest, - replayManifestDigest: boundary.manifestDigest, + claimId, + boundaryDigest: boundary.manifestDigest, + }, + lineage: { + parentRunId: source.identity.runId, + parentTurnId: source.identity.turnId, }, }, claimedAt, @@ -2021,6 +2078,8 @@ function continuationStartEvent( partial: false, role: 'system', author: 'system', + modelVisibility: 'hidden', + content: claim.targetOpening, actions: { ...(overrides.toolBoundaryProtocol ? { runtimeProtocol: { toolBoundary: overrides.toolBoundaryProtocol } } diff --git a/packages/storage/src/__tests__/usage-stores.test.ts b/packages/storage/src/__tests__/usage-stores.test.ts index f8b1512538..3eb88bba2e 100644 --- a/packages/storage/src/__tests__/usage-stores.test.ts +++ b/packages/storage/src/__tests__/usage-stores.test.ts @@ -735,8 +735,8 @@ function appendModelCallAuthorityEvent( lease.transaction('write', () => { lease.database .prepare(` - INSERT INTO core_agent_runs(session_id, run_id, created_at, record_json) - VALUES (?, ?, 0, '{}') + INSERT INTO core_agent_runs(session_id, run_id, created_at) + VALUES (?, ?, 0) `) .run(value.sessionId, value.runId); lease.database diff --git a/packages/storage/src/__tests__/workhub-coordination-root-admission.test.ts b/packages/storage/src/__tests__/workhub-coordination-root-admission.test.ts index 8da5ea98f3..7eca36f5a6 100644 --- a/packages/storage/src/__tests__/workhub-coordination-root-admission.test.ts +++ b/packages/storage/src/__tests__/workhub-coordination-root-admission.test.ts @@ -22,7 +22,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; -import type { RootExecutionDescriptor } from '@maka/core/agent-run'; +import type { RootExecutionDescriptor } from '@maka/core/runtime-invocation'; import { createSqliteAgentRunStore } from '../agent-run-store.js'; test('WorkHub Coordination admission preserves its bounded content identity across restart', async () => { diff --git a/packages/storage/src/__tests__/workspace-version-authority-persistence.test.ts b/packages/storage/src/__tests__/workspace-version-authority-persistence.test.ts index ecbfa62598..e0ee5b154d 100644 --- a/packages/storage/src/__tests__/workspace-version-authority-persistence.test.ts +++ b/packages/storage/src/__tests__/workspace-version-authority-persistence.test.ts @@ -1346,6 +1346,9 @@ function recreateWorkspaceTablesAsSchema12(database: DatabaseSync): void { DROP TABLE runtime_workspace_heads_schema_13; DROP TABLE runtime_workspace_versions_schema_13; DROP TABLE runtime_managed_mutation_reservations; + DROP INDEX runtime_events_by_session_kind; + DROP INDEX runtime_events_one_opening_per_invocation; + DROP TABLE runtime_legacy_invocation_openings; PRAGMA user_version = 12; COMMIT; PRAGMA foreign_keys = ON; diff --git a/packages/storage/src/agent-run-store.ts b/packages/storage/src/agent-run-store.ts index b227cc2d75..dd855c9b3f 100644 --- a/packages/storage/src/agent-run-store.ts +++ b/packages/storage/src/agent-run-store.ts @@ -21,12 +21,7 @@ import { createHash } from 'node:crypto'; import { resolve } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import type { DatabaseSync } from 'node:sqlite'; -import { - decodeAgentRunEvent, - decodeAgentRunHeader, - decodeCurrentAgentRunHeader, - decodeRuntimeEvent, -} from './execution-record-codec.js'; +import { decodeAgentRunEvent, decodeRuntimeEvent } from './execution-record-codec.js'; import { immutableSteeringMessageId } from './runtime-event-invariants.js'; import { normalizeSubmittedTurnIntent, @@ -49,6 +44,12 @@ import { type SkillInvocationResult, } from '@maka/core/skill-invocation'; import { DurableStoreWriteError, type RuntimeEventStore } from '@maka/core/runtime-event-store'; +import type { + RuntimeInvocationPageInput, + RuntimeInvocationPageResult, + RuntimeInvocationRecord, + RuntimeInvocationSearchResult, +} from '@maka/core/runtime-invocation'; import { aggregateMessageContents, decodeMessageContent, @@ -63,6 +64,7 @@ import { MAX_ATTACHMENT_BYTES, MAX_ATTACHMENT_COUNT } from '@maka/core/attachmen import { MODEL_CALL_ATTEMPT_EVENT_TYPE } from '@maka/core/model-call-attempt'; import { LATEST_CONTEXT_PROJECTION_TYPE, + RUN_COMPOSITION_RECORDED_EVENT_TYPE, supersedesLatestContext, type LatestContextOrder, type AgentRunProjectionKey, @@ -70,12 +72,17 @@ import { type LatestContextProjectionInput, type AgentRunEvent, type AgentRunEventType, - type AgentRunHeader, type AgentRunStore, type EmittedAgentRunEvent, - type RootExecutionDescriptor, - isSessionInlineRun, } from '@maka/core/agent-run'; +import { + isSessionInlineInvocation, + type RootExecutionDescriptor, +} from '@maka/core/runtime-invocation'; +import { + decodeRuntimeInvocationOpened, + runtimeEventInvocationOpening, +} from '@maka/core/runtime-event'; import { encodeCanonicalRuntimeEvent } from '@maka/core/canonical-runtime-event'; import { isOrchestrationMode, @@ -216,8 +223,6 @@ export interface DurableAgentRunStore extends AgentRunStore, RootTurnAdmissionStore, RootTurnStartRejectionStore { - listSessionRunsBounded(sessionId: string, limit: number): Promise; - listSessionRunsPage(sessionId: string, input: AgentRunPageInput): Promise; readEventsBounded( sessionId: string, runId: string, @@ -229,7 +234,6 @@ export interface DurableAgentRunStore type: AgentRunEventType, budget: EvidenceReadBudget, ): Promise>; - listSessionRunsForRecovery(sessionId: string): Promise; readEventsForRecovery(sessionId: string, runId: string): Promise; readEventsForEvidence(sessionId: string, runId: string): Promise; readEventProjection( @@ -247,26 +251,6 @@ export interface DurableAgentRunStore close?(): void; } -export interface AgentRunIdentitySearchResult { - readonly runs: readonly AgentRunHeader[]; - readonly truncated: boolean; -} - -export interface AgentRunPageCursor { - readonly createdAt: number; - readonly runId: string; -} - -export interface AgentRunPageInput { - readonly before?: AgentRunPageCursor; - readonly limit: number; -} - -export interface AgentRunPageResult { - readonly runs: readonly AgentRunHeader[]; - readonly nextCursor: AgentRunPageCursor | null; -} - export type { BoundedEvidenceReadResult, EvidenceReadBudget } from './bounded-evidence.js'; export interface ConversationCopyRuntimeEventBatch { @@ -286,6 +270,17 @@ export interface RuntimeEventScanBudget { export type RuntimeEventScanResult = { readonly status: 'complete' | 'limit_exceeded' }; export interface DurableRuntimeEventStore extends RuntimeEventStore { + listSessionInvocations(sessionId: string): Promise; + readRunInvocation(sessionId: string, runId: string): Promise; + listSessionInvocationsBounded( + sessionId: string, + limit: number, + ): Promise; + listSessionInvocationsPage( + sessionId: string, + input: RuntimeInvocationPageInput, + ): Promise; + readInvocation(sessionId: string, invocationId: string): Promise; /** Visit one ordered, bounded SQLite snapshot without retaining the immutable ledger. */ scanRuntimeEvents( sessionId: string, @@ -342,207 +337,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { return Promise.resolve(); } - async createRun( - header: AgentRunHeader, - _options: { durable?: boolean } = {}, - ): Promise { - const normalized = normalizeCurrentAgentRunHeader(header, header.sessionId, header.runId); - this.#lease.transaction('write', () => { - const inserted = this.#lease.database - .prepare(` - INSERT OR IGNORE INTO core_agent_runs( - session_id, run_id, created_at, record_json - ) VALUES (?, ?, ?, ?) - `) - .run( - normalized.sessionId, - normalized.runId, - normalized.createdAt, - JSON.stringify(normalized, sanitizeJson), - ); - if (inserted.changes !== 1) { - throw new Error(`Agent run already exists: ${normalized.runId}`); - } - const count = this.#lease.database - .prepare('SELECT COUNT(*) AS count FROM core_agent_runs WHERE session_id = ?') - .get(normalized.sessionId) as { count?: unknown }; - const projection = this.#lease.database - .prepare(` - SELECT 1 AS present - FROM core_agent_run_projections - WHERE session_id = ? AND event_type = 'history_compact_checkpoint_recorded' - `) - .get(normalized.sessionId); - if (count.count === 1 && !projection) { - this.#lease.database - .prepare(` - INSERT INTO core_agent_run_projections(session_id, event_type, event_json) - VALUES (?, 'history_compact_checkpoint_recorded', NULL) - `) - .run(normalized.sessionId); - } - }); - return normalized; - } - - async updateRun( - sessionId: string, - runId: string, - patch: Partial, - _options: { durable?: boolean } = {}, - ): Promise { - assertMutableRunHeaderPatch(patch); - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - return this.#lease.transaction('write', () => { - const current = readSqliteAgentRun(this.#lease.database, sessionId, runId); - if (Object.hasOwn(patch, 'runComposition')) { - if (!patch.runComposition) { - throw new Error('AgentRun Run Composition cannot be cleared'); - } - if ( - current.runComposition && - !isDeepStrictEqual(current.runComposition, patch.runComposition) - ) { - throw new Error('AgentRun Run Composition is immutable'); - } - } - const next = normalizeCurrentAgentRunHeader( - { ...current, ...patch, sessionId, runId }, - sessionId, - runId, - ); - const result = this.#lease.database - .prepare(` - UPDATE core_agent_runs - SET created_at = ?, record_json = ? - WHERE session_id = ? AND run_id = ? - `) - .run(next.createdAt, JSON.stringify(next, sanitizeJson), sessionId, runId); - if (result.changes !== 1) throw new Error(`Failed to update run ${runId}`); - return next; - }); - } - - async readRun(sessionId: string, runId: string): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - return readSqliteAgentRun(this.#lease.database, sessionId, runId); - } - - async listSessionRuns(sessionId: string): Promise { - return this.listSessionRunsForRecovery(sessionId); - } - - async listSessionRunsBounded( - sessionId: string, - limit: number, - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertIdentitySearchLimit(limit); - const rows = this.#lease.database - .prepare(` - SELECT run_id, record_json - FROM core_agent_runs - WHERE session_id = ? - ORDER BY created_at, run_id - LIMIT ? - `) - .all(sessionId, limit + 1) as Array<{ run_id?: unknown; record_json?: unknown }>; - const truncated = rows.length > limit; - const runs = rows.slice(0, limit).map((row) => { - if (typeof row.run_id !== 'string' || typeof row.record_json !== 'string') { - throw new Error('Invalid SQLite AgentRun row'); - } - return decodePersistedAgentRunHeader(JSON.parse(row.record_json), sessionId, row.run_id); - }); - return { runs, truncated }; - } - - async listSessionRunsPage( - sessionId: string, - input: AgentRunPageInput, - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertIdentitySearchLimit(input.limit); - if (input.before) { - assertSafeId(input.before.runId, 'Invalid AgentRun page cursor'); - if (!Number.isFinite(input.before.createdAt)) { - throw new Error('Invalid AgentRun page cursor'); - } - } - const rows = this.#lease.database - .prepare( - input.before - ? ` - SELECT run_id, created_at, record_json - FROM core_agent_runs - WHERE session_id = ? - AND (created_at < ? OR (created_at = ? AND run_id < ?)) - ORDER BY created_at DESC, run_id DESC - LIMIT ? - ` - : ` - SELECT run_id, created_at, record_json - FROM core_agent_runs - WHERE session_id = ? - ORDER BY created_at DESC, run_id DESC - LIMIT ? - `, - ) - .all( - ...(input.before - ? [ - sessionId, - input.before.createdAt, - input.before.createdAt, - input.before.runId, - input.limit + 1, - ] - : [sessionId, input.limit + 1]), - ) as Array<{ run_id?: unknown; created_at?: unknown; record_json?: unknown }>; - const pageRows = rows.slice(0, input.limit); - const runs = pageRows.map((row) => { - if ( - typeof row.run_id !== 'string' || - typeof row.created_at !== 'number' || - typeof row.record_json !== 'string' - ) { - throw new Error('Invalid SQLite AgentRun page row'); - } - return decodePersistedAgentRunHeader(JSON.parse(row.record_json), sessionId, row.run_id); - }); - const last = pageRows.at(-1); - return { - runs, - nextCursor: - rows.length > input.limit && - last && - typeof last.run_id === 'string' && - typeof last.created_at === 'number' - ? { createdAt: last.created_at, runId: last.run_id } - : null, - }; - } - - async listSessionRunsForRecovery(sessionId: string): Promise { - assertSafeId(sessionId, 'Invalid session id'); - const rows = this.#lease.database - .prepare(` - SELECT run_id, record_json - FROM core_agent_runs - WHERE session_id = ? - ORDER BY created_at, run_id - `) - .all(sessionId) as Array<{ run_id?: unknown; record_json?: unknown }>; - return rows.map((row) => { - if (typeof row.run_id !== 'string' || typeof row.record_json !== 'string') { - throw new Error('Invalid SQLite AgentRun row'); - } - return decodePersistedAgentRunHeader(JSON.parse(row.record_json), sessionId, row.run_id); - }); - } - async appendEvent( sessionId: string, runId: string, @@ -552,13 +346,28 @@ class SqliteAgentRunStore implements DurableAgentRunStore { assertSafeId(sessionId, 'Invalid session id'); assertSafeId(runId, 'Invalid run id'); this.#lease.transaction('write', () => { - const header = readSqliteAgentRun(this.#lease.database, sessionId, runId); + const anchor = readSqliteRunAnchor(this.#lease.database, sessionId, runId); + this.#openLedgerStream(sessionId, runId, anchor.openedAt); const normalized = decodeAgentRunEvent(JSON.parse(JSON.stringify(event, sanitizeJson)), { sessionId, runId, - turnId: header.turnId, + turnId: anchor.turnId, }); const type = normalized.type as AgentRunEventType; + if (type === RUN_COMPOSITION_RECORDED_EVENT_TYPE) { + // Write-once, enforced where the record lives. The composition is what + // the run was dispatched against; a second, different one would claim + // the run ran on a prompt and tool surface it never saw. An identical + // re-append is the writer retrying, so it is absorbed rather than + // refused. + const recorded = readSqliteRunCompositionEvent(this.#lease.database, sessionId, runId); + if (recorded) { + if (!isDeepStrictEqual(recorded.data, normalized.data)) { + throw new Error('AgentRun Run Composition is immutable'); + } + return; + } + } const projectsCheckpoint = type === 'history_compact_checkpoint_recorded'; const projection = projectsCheckpoint ? inspectSqliteAgentRunProjection(this.#lease.database, sessionId, type) @@ -577,14 +386,54 @@ class SqliteAgentRunStore implements DurableAgentRunStore { // // Skipped for a subagent's run: those requests are real, but presenting // one as the SESSION's latest context attributes another agent's prompt - // to this one. The header is already loaded here, so the check is free. + // to this one. The opening fact is already loaded here, so the check is + // free. const latestContext = options.latestContext; - if (latestContext && isSessionInlineRun(header)) { + if (latestContext && anchor.sessionInline) { this.#writeLatestContextProjection(sessionId, normalized, latestContext); } }); } + /** + * Give this run's ledger its stream row, and the Session its first one. + * + * The row carries no semantic state: it is the parent `core_agent_run_events` + * hangs off and the place the model-call high water lives. Creating it on the + * first append is what stops it from being a second record of the run's + * existence — the opening fact already is that. + * + * The Session's first stream also initialises the compaction-checkpoint + * projection to an explicit empty, which is how a reader tells "no checkpoint + * yet" from "projection never built". + */ + #openLedgerStream(sessionId: string, runId: string, createdAt: number): void { + const inserted = this.#lease.database + .prepare( + 'INSERT OR IGNORE INTO core_agent_runs(session_id, run_id, created_at) VALUES (?, ?, ?)', + ) + .run(sessionId, runId, createdAt); + if (inserted.changes !== 1) return; + const count = this.#lease.database + .prepare('SELECT COUNT(*) AS count FROM core_agent_runs WHERE session_id = ?') + .get(sessionId) as { count?: unknown }; + if (count.count !== 1) return; + const projection = this.#lease.database + .prepare(` + SELECT 1 AS present + FROM core_agent_run_projections + WHERE session_id = ? AND event_type = 'history_compact_checkpoint_recorded' + `) + .get(sessionId); + if (projection) return; + this.#lease.database + .prepare(` + INSERT INTO core_agent_run_projections(session_id, event_type, event_json) + VALUES (?, 'history_compact_checkpoint_recorded', NULL) + `) + .run(sessionId); + } + /** * Monotonic by the request's own completion, not by arrival. * @@ -922,44 +771,72 @@ function readSqliteAgentRunLedgerRevision(db: DatabaseSync, sessionId: string): ); } -function normalizeCurrentAgentRunHeader( - value: unknown, - sessionId: string, - runId: string, -): AgentRunHeader { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - return decodeCurrentAgentRunHeader(JSON.parse(JSON.stringify(value, sanitizeJson)), { - sessionId, - runId, - }); -} - -function decodePersistedAgentRunHeader( - value: unknown, - sessionId: string, - runId: string, -): AgentRunHeader { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - return decodeAgentRunHeader(value, { sessionId, runId }); +/** + * What the operational ledger needs to know about the run it belongs to. + * + * All of it is read off the event spine rather than kept beside the ledger: the + * turn the records must agree with, when the invocation opened, and whether its + * output is the owning Session's own conversation. Copying any of it into a + * second row is what made the Run header a rival authority. + * + * An invocation whose opening the migration could not project keeps a readable + * ledger: its turn and clock come from the events it does have, and it fails + * closed on the one judgement the opening was needed for. + */ +interface LedgerRunAnchor { + turnId: string; + openedAt: number; + sessionInline: boolean; } -function readSqliteAgentRun(db: DatabaseSync, sessionId: string, runId: string): AgentRunHeader { - const row = db +function readSqliteRunAnchor(db: DatabaseSync, sessionId: string, runId: string): LedgerRunAnchor { + const opening = db .prepare(` - SELECT record_json - FROM core_agent_runs + SELECT turn_id, committed_at, payload_json + FROM runtime_events + WHERE session_id = ? AND run_id = ? AND event_kind = 'invocation_opened' + LIMIT 1 + `) + .get(sessionId, runId) as + | { turn_id: string; committed_at: number; payload_json: string } + | undefined; + if (opening) { + const content = runtimeEventInvocationOpening( + decodeRuntimeEvent(JSON.parse(opening.payload_json), { + sessionId, + runId, + turnId: opening.turn_id, + }), + ); + if (!content) throw new Error(`RuntimeEvent for run ${runId} is not an opening fact`); + return { + turnId: opening.turn_id, + openedAt: opening.committed_at, + sessionInline: isSessionInlineInvocation(content), + }; + } + const legacy = db + .prepare(` + SELECT turn_id, opened_at, opening_json + FROM runtime_legacy_invocation_openings WHERE session_id = ? AND run_id = ? + LIMIT 1 `) - .get(sessionId, runId) as { record_json?: unknown } | undefined; - if (!row) { - const error = new Error(`Agent run does not exist: ${runId}`) as NodeJS.ErrnoException; - error.code = 'ENOENT'; - throw error; + .get(sessionId, runId) as + | { turn_id: string; opened_at: number; opening_json: string } + | undefined; + if (legacy) { + return { + turnId: legacy.turn_id, + openedAt: legacy.opened_at, + sessionInline: isSessionInlineInvocation( + decodeRuntimeInvocationOpened(JSON.parse(legacy.opening_json)), + ), + }; } - if (typeof row.record_json !== 'string') throw new Error('Invalid SQLite AgentRun row'); - return decodePersistedAgentRunHeader(JSON.parse(row.record_json), sessionId, runId); + const error = new Error(`Agent run does not exist: ${runId}`) as NodeJS.ErrnoException; + error.code = 'ENOENT'; + throw error; } function readSqliteAgentRunEvents( @@ -976,7 +853,7 @@ function readSqliteAgentRunEvents( `) .all(sessionId, runId) as Array<{ record_json?: unknown }>; if (rows.length === 0) return []; - const header = readSqliteAgentRun(db, sessionId, runId); + const anchor = readSqliteRunAnchor(db, sessionId, runId); return rows.map((row) => { if (typeof row.record_json !== 'string') { throw new Error('Invalid SQLite AgentRun event row'); @@ -984,11 +861,40 @@ function readSqliteAgentRunEvents( return decodeAgentRunEvent(JSON.parse(row.record_json), { sessionId, runId, - turnId: header.turnId, + turnId: anchor.turnId, }); }); } +/** The run's one composition row, or nothing if it has not been dispatched yet. */ +function readSqliteRunCompositionEvent( + db: DatabaseSync, + sessionId: string, + runId: string, +): AgentRunEvent | undefined { + const row = db + .prepare(` + SELECT record_json + FROM core_agent_run_events + WHERE session_id = ? AND run_id = ? AND event_type = ? + ORDER BY sequence + LIMIT 1 + `) + .get(sessionId, runId, RUN_COMPOSITION_RECORDED_EVENT_TYPE) as + | { record_json?: unknown } + | undefined; + if (!row) return undefined; + if (typeof row.record_json !== 'string') { + throw new Error('Invalid SQLite AgentRun event row'); + } + const anchor = readSqliteRunAnchor(db, sessionId, runId); + return decodeAgentRunEvent(JSON.parse(row.record_json), { + sessionId, + runId, + turnId: anchor.turnId, + }); +} + function readSqliteAgentRunEventsForEvidence( db: DatabaseSync, sessionId: string, @@ -1015,7 +921,7 @@ function readSqliteAgentRunEventsForEvidence( .all(sessionId, runId, type) ) as Array<{ sequence?: unknown; record_json?: unknown }>; if (rows.length === 0) return []; - const header = readSqliteAgentRun(db, sessionId, runId); + const anchor = readSqliteRunAnchor(db, sessionId, runId); return rows.map((row) => { const lineNumber = typeof row.sequence === 'number' && Number.isSafeInteger(row.sequence) ? row.sequence + 1 : 0; @@ -1026,7 +932,7 @@ function readSqliteAgentRunEventsForEvidence( return decodeAgentRunEvent(JSON.parse(row.record_json), { sessionId, runId, - turnId: header.turnId, + turnId: anchor.turnId, }); } catch (error) { return { @@ -1034,8 +940,8 @@ function readSqliteAgentRunEventsForEvidence( id: `run-event-corrupt-${lineNumber}`, runId, sessionId, - turnId: header.turnId, - ts: header.updatedAt, + turnId: anchor.turnId, + ts: anchor.openedAt, message: error instanceof Error ? error.message : 'Invalid SQLite AgentRun event row', data: { lineNumber }, }; @@ -1347,26 +1253,6 @@ export function rootTurnAdmissionRecordFits(input: AdmitRootTurnInput): boolean } } -const MUTABLE_AGENT_RUN_HEADER_FIELDS = new Set([ - 'status', - 'updatedAt', - 'completedAt', - 'runComposition', - 'failureClass', - 'failureMessage', - 'abortSource', - 'traceWriteError', -]); - -function assertMutableRunHeaderPatch(patch: Partial): void { - const immutable = Object.keys(patch).filter( - (key) => !MUTABLE_AGENT_RUN_HEADER_FIELDS.has(key as keyof AgentRunHeader), - ); - if (immutable.length > 0) { - throw new Error(`AgentRun admission identity is immutable: ${immutable.sort().join(', ')}`); - } -} - function shouldPreserveCheckpointProjectionDuringAppend( current: AgentRunEvent | null | undefined, candidate: AgentRunEvent, diff --git a/packages/storage/src/execution-record-codec.ts b/packages/storage/src/execution-record-codec.ts index d12f76046f..c5aa7c9048 100644 --- a/packages/storage/src/execution-record-codec.ts +++ b/packages/storage/src/execution-record-codec.ts @@ -19,10 +19,7 @@ import { decodeAgentRunEvent as decodeCanonicalAgentRunEvent, - decodeAgentRunHeader as decodeCanonicalAgentRunHeader, - decodePersistedAgentRunHeader, type AgentRunEvent, - type AgentRunHeader, } from '@maka/core/agent-run'; import { @@ -40,34 +37,6 @@ export function decodeStoredMessage(value: unknown): StoredMessage { return decodePersistedStoredMessage(markPersisted(value)); } -export function decodeAgentRunHeader( - value: unknown, - expected: { sessionId: string; runId: string }, -): AgentRunHeader { - try { - const header = decodePersistedAgentRunHeader(markPersisted(value)); - if (header.sessionId !== expected.sessionId || header.runId !== expected.runId) { - throw new Error('AgentRun header identity does not match its path'); - } - return header; - } catch (error) { - throw new Error(`Invalid AgentRun header for run ${expected.runId}: malformed fields`, { - cause: error, - }); - } -} - -export function decodeCurrentAgentRunHeader( - value: unknown, - expected: { sessionId: string; runId: string }, -): AgentRunHeader { - const header = decodeCanonicalAgentRunHeader(value); - if (header.sessionId !== expected.sessionId || header.runId !== expected.runId) { - throw new Error('AgentRun header identity does not match its path'); - } - return header; -} - export function decodeAgentRunEvent( value: unknown, expected: { sessionId: string; runId: string; turnId: string }, @@ -85,7 +54,7 @@ export function decodeAgentRunEvent( export function decodeRuntimeEvent( value: unknown, - expected: Pick, + expected: { sessionId: string; runId: string; turnId: string; invocationId?: string }, ): RuntimeEvent { const event = decodeCanonicalRuntimeEvent(value); if ( diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index ed9714e275..29e0679225 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -17,21 +17,19 @@ * under the License. */ -import type { - AgentRunEvent, - AgentRunEventType, - AgentRunHeader, - AgentRunProjectionKey, -} from '@maka/core/agent-run'; +import type { AgentRunEvent, AgentRunEventType, AgentRunProjectionKey } from '@maka/core/agent-run'; import type { RuntimeEvent, ToolBoundaryProtocol } from '@maka/core/runtime-event'; import type { RuntimeContinuationAuthorityStore } from '@maka/core/runtime-event-store'; +import type { + RuntimeInvocationPageInput, + RuntimeInvocationPageResult, + RuntimeInvocationRecord, + RuntimeInvocationSearchResult, +} from '@maka/core/runtime-invocation'; import type { SessionHeader, SessionSummary, StoredMessage, TurnRecord } from '@maka/core/session'; import type { SessionListFilter } from '@maka/core/runtime-inputs'; import { createSqliteAgentRunStore, - type AgentRunIdentitySearchResult, - type AgentRunPageInput, - type AgentRunPageResult, type AdmitRootTurnInput, type AdmitRootTurnResult, type CommitRootTurnStartRejectionInput, @@ -95,9 +93,6 @@ export { } from './sqlite-session-metadata-store.js'; export type { - AgentRunIdentitySearchResult, - AgentRunPageInput, - AgentRunPageResult, AdmitRootTurnInput, AdmitRootTurnResult, CommitRootTurnStartRejectionInput, @@ -181,10 +176,6 @@ export interface ExecutionSessionReader { } export interface ExecutionAgentRunReader { - readRun(sessionId: string, runId: string): Promise; - listSessionRuns(sessionId: string): Promise; - listSessionRunsBounded(sessionId: string, limit: number): Promise; - listSessionRunsPage(sessionId: string, input: AgentRunPageInput): Promise; readEvents(sessionId: string, runId: string): Promise; readEventsBounded( sessionId: string, @@ -209,6 +200,22 @@ export interface ExecutionAgentRunReader { } export interface ExecutionRuntimeEventReader { + /** + * A Session's run inventory, read from its canonical events. This is the + * definition of the inventory, not a cache of it, so nothing writes or + * repairs it. + */ + listSessionInvocations(sessionId: string): Promise; + readRunInvocation(sessionId: string, runId: string): Promise; + listSessionInvocationsBounded( + sessionId: string, + limit: number, + ): Promise; + listSessionInvocationsPage( + sessionId: string, + input: RuntimeInvocationPageInput, + ): Promise; + readInvocation(sessionId: string, invocationId: string): Promise; readRuntimeEvents(sessionId: string, runId: string): Promise; readRuntimeEventsBounded( sessionId: string, @@ -492,17 +499,6 @@ async function createExecutionStoresForWrite run(() => agentRunStore.createRun(header, options)), - updateRun: (sessionId, runId, patch, options) => - run(() => agentRunStore.updateRun(sessionId, runId, patch, options)), - readRun: (sessionId, runId) => run(() => agentRunStore.readRun(sessionId, runId)), - listSessionRuns: (sessionId) => run(() => agentRunStore.listSessionRuns(sessionId)), - listSessionRunsBounded: (sessionId, limit) => - run(() => agentRunStore.listSessionRunsBounded(sessionId, limit)), - listSessionRunsPage: (sessionId, input) => - run(() => agentRunStore.listSessionRunsPage(sessionId, input)), - listSessionRunsForRecovery: (sessionId) => - run(() => agentRunStore.listSessionRunsForRecovery(sessionId)), appendEvent: (sessionId, runId, event, options) => run(() => agentRunStore.appendEvent(sessionId, runId, event, options)), readEvents: (sessionId, runId) => run(() => agentRunStore.readEvents(sessionId, runId)), @@ -555,6 +551,16 @@ async function createExecutionStoresForWrite runtimeEventStore.readImmutableRuntimeEvents(sessionId, runId)), readImmutableRuntimePrefix: (input) => run(() => runtimeEventStore.readImmutableRuntimePrefix(input)), + listSessionInvocations: (sessionId) => + run(() => runtimeEventStore.listSessionInvocations(sessionId)), + readRunInvocation: (sessionId, runId) => + run(() => runtimeEventStore.readRunInvocation(sessionId, runId)), + listSessionInvocationsBounded: (sessionId, limit) => + run(() => runtimeEventStore.listSessionInvocationsBounded(sessionId, limit)), + listSessionInvocationsPage: (sessionId, input) => + run(() => runtimeEventStore.listSessionInvocationsPage(sessionId, input)), + readInvocation: (sessionId, invocationId) => + run(() => runtimeEventStore.readInvocation(sessionId, invocationId)), readSessionRuntimeEvents: (sessionId) => run(() => runtimeEventStore.readSessionRuntimeEvents(sessionId)), readSessionRuntimeEventEntries: (sessionId) => @@ -641,12 +647,6 @@ async function openExecutionStoresForRead run(() => agentRunStore.readRun(sessionId, runId)), - listSessionRuns: (sessionId) => run(() => agentRunStore.listSessionRuns(sessionId)), - listSessionRunsBounded: (sessionId, limit) => - run(() => agentRunStore.listSessionRunsBounded(sessionId, limit)), - listSessionRunsPage: (sessionId, input) => - run(() => agentRunStore.listSessionRunsPage(sessionId, input)), readEvents: (sessionId, runId) => run(() => agentRunStore.readEvents(sessionId, runId)), readEventsBounded: (sessionId, runId, budget) => run(() => agentRunStore.readEventsBounded(sessionId, runId, budget)), @@ -666,6 +666,16 @@ async function openExecutionStoresForRead runtimeEventStore.readRuntimeEventsBounded(sessionId, runId, budget)), readImmutableRuntimeEvents: (sessionId, runId) => run(() => runtimeEventStore.readImmutableRuntimeEvents(sessionId, runId)), + listSessionInvocations: (sessionId) => + run(() => runtimeEventStore.listSessionInvocations(sessionId)), + readRunInvocation: (sessionId, runId) => + run(() => runtimeEventStore.readRunInvocation(sessionId, runId)), + listSessionInvocationsBounded: (sessionId, limit) => + run(() => runtimeEventStore.listSessionInvocationsBounded(sessionId, limit)), + listSessionInvocationsPage: (sessionId, input) => + run(() => runtimeEventStore.listSessionInvocationsPage(sessionId, input)), + readInvocation: (sessionId, invocationId) => + run(() => runtimeEventStore.readInvocation(sessionId, invocationId)), readSessionRuntimeEvents: (sessionId) => run(() => runtimeEventStore.readSessionRuntimeEvents(sessionId)), }, diff --git a/packages/storage/src/legacy-run-header.ts b/packages/storage/src/legacy-run-header.ts new file mode 100644 index 0000000000..7136b7df95 --- /dev/null +++ b/packages/storage/src/legacy-run-header.ts @@ -0,0 +1,454 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * The Run header as builds before the invocation opening fact wrote it. + * + * Nothing writes this shape any more and no live code reads it. It lives here, + * beside the migration that consumes it, because a persisted row still carries + * it: reading old data is the only remaining reason the shape exists, and + * keeping it out of `@maka/core` is what stops it from being a second live + * authority again. + */ + +import { + decodePersistedPermissionMode, + isPermissionMode, + type PermissionMode, +} from '@maka/core/permission'; +import { isCollaborationMode, type CollaborationMode } from '@maka/core/collaboration'; +import { + isAgentSwarmAuthorizationSource, + isEffectiveOrchestrationSource, + isOrchestrationMode, + type AgentSwarmAuthorizationSource, + type EffectiveOrchestrationSource, + type OrchestrationMode, +} from '@maka/core/orchestration'; +import type { PersistedBackendKind } from '@maka/core/session'; +import { + defineObjectShape, + hasExactShape, + isFiniteNumber, + isOptionalString, + isRecord, +} from '@maka/core/record-schema'; +import { DEFAULT_TOOL_MODE, isToolMode, type ToolMode } from '@maka/core/tool-mode'; +import type { + RuntimeEventInvocationOpenedContent, + RuntimeInvocationLineage, + RuntimeInvocationOpenSource, + RuntimeInvocationRootAuthority, + RuntimeInvocationRoute, +} from '@maka/core/runtime-event'; + +const LEGACY_RUN_STATUSES = [ + 'created', + 'running', + 'waiting_for_user', + 'completed', + 'failed', + 'cancelled', +] as const; + +type LegacyRunStatus = (typeof LEGACY_RUN_STATUSES)[number]; + +interface LegacyContinuationSourceV1 { + sourceInvocationId: string; + sourceRunId: string; + sourceTurnId: string; + sourceRuntimeEventHighWater: number; +} + +interface LegacyContinuationSourceV2 extends LegacyContinuationSourceV1 { + protocol: 'continuation_source_v2'; + claimId: string; + boundaryDigest: `sha256:${string}`; + sourcePrefixDigest: `sha256:${string}`; + replayManifestDigest: `sha256:${string}`; +} + +type LegacyContinuationSource = LegacyContinuationSourceV1 | LegacyContinuationSourceV2; + +export interface LegacyRunHeader { + runId: string; + invocationId?: string; + sessionId: string; + turnId: string; + status: LegacyRunStatus; + backendKind: PersistedBackendKind; + llmConnectionId?: string; + providerStateIdentity?: `sha256:${string}`; + llmConnectionSlug: string; + modelId: string; + cwd: string; + workspaceIdentity?: string; + permissionMode: PermissionMode; + collaborationMode?: CollaborationMode; + orchestrationMode?: OrchestrationMode; + orchestrationSource?: EffectiveOrchestrationSource; + agentSwarmAuthorization?: AgentSwarmAuthorizationSource; + toolMode?: ToolMode; + createdAt: number; + updatedAt: number; + completedAt?: number; + parentRunId?: string; + resumedFromRunId?: string; + retriedFromRunId?: string; + agentId?: string; + agentName?: string; + parentTurnId?: string; + retriedFromTurnId?: string; + regeneratedFromTurnId?: string; + branchOfTurnId?: string; + parentSessionId?: string; + continuationSource?: LegacyContinuationSource; + scheduledTaskId?: string; + legacyAutomationId?: string; + goalId?: string; + agentGraphWakeId?: string; + agentGraphWakeAttemptId?: string; + rootExecutionKind?: 'context_compact'; + failureClass?: string; + failureMessage?: string; + abortSource?: string; + traceWriteError?: string; + /** + * The provider-dispatch snapshot the header era attached to every run that + * reached a provider. Nothing on the spine reads it back, so the migration + * only has to know it is there: a header carrying it is a well-formed + * header, not a corrupt one. + */ + runComposition?: object; +} + +const LEGACY_RUN_HEADER_SHAPE = defineObjectShape()( + [ + 'runId', + 'sessionId', + 'turnId', + 'status', + 'backendKind', + 'llmConnectionSlug', + 'modelId', + 'cwd', + 'permissionMode', + 'createdAt', + 'updatedAt', + ], + [ + 'invocationId', + 'llmConnectionId', + 'providerStateIdentity', + 'completedAt', + 'parentRunId', + 'resumedFromRunId', + 'retriedFromRunId', + 'agentId', + 'agentName', + 'parentTurnId', + 'retriedFromTurnId', + 'regeneratedFromTurnId', + 'branchOfTurnId', + 'parentSessionId', + 'workspaceIdentity', + 'continuationSource', + 'scheduledTaskId', + 'legacyAutomationId', + 'goalId', + 'agentGraphWakeId', + 'agentGraphWakeAttemptId', + 'rootExecutionKind', + 'failureClass', + 'failureMessage', + 'abortSource', + 'traceWriteError', + 'collaborationMode', + 'orchestrationMode', + 'orchestrationSource', + 'agentSwarmAuthorization', + 'toolMode', + 'runComposition', + ], +); + +const LEGACY_CONTINUATION_SOURCE_V1_SHAPE = defineObjectShape()( + ['sourceInvocationId', 'sourceRunId', 'sourceTurnId', 'sourceRuntimeEventHighWater'], + [], +); + +const LEGACY_CONTINUATION_SOURCE_V2_SHAPE = defineObjectShape()( + [ + 'protocol', + 'sourceInvocationId', + 'sourceRunId', + 'sourceTurnId', + 'sourceRuntimeEventHighWater', + 'claimId', + 'boundaryDigest', + 'sourcePrefixDigest', + 'replayManifestDigest', + ], + [], +); + +const RETIRED_RUN_STATUSES: Readonly> = { + waiting_permission: 'waiting_for_user', +}; + +export function decodePersistedLegacyRunHeader(persisted: unknown): LegacyRunHeader { + let value = persisted; + if ( + isRecord(value) && + value.automationId !== undefined && + value.legacyAutomationId === undefined + ) { + const { automationId, ...current } = value; + value = { ...current, legacyAutomationId: automationId }; + } + if (isRecord(value)) { + const status = + typeof value.status === 'string' + ? (RETIRED_RUN_STATUSES[value.status] ?? value.status) + : value.status; + const permissionMode = decodePersistedPermissionMode(value.permissionMode); + if (status !== value.status || permissionMode !== value.permissionMode) { + value = { ...value, status, permissionMode }; + } + } + return decodeLegacyRunHeader(value); +} + +function decodeLegacyRunHeader(value: unknown): LegacyRunHeader { + if (!isRecord(value) || !hasExactShape(value, LEGACY_RUN_HEADER_SHAPE)) { + throw new Error('Invalid AgentRun header schema'); + } + const valid = + typeof value.runId === 'string' && + typeof value.sessionId === 'string' && + typeof value.turnId === 'string' && + (LEGACY_RUN_STATUSES as readonly unknown[]).includes(value.status) && + isPersistedBackendKind(value.backendKind) && + (value.llmConnectionId === undefined || + (typeof value.llmConnectionId === 'string' && value.llmConnectionId.length > 0)) && + (value.providerStateIdentity === undefined || isSha256Digest(value.providerStateIdentity)) && + typeof value.llmConnectionSlug === 'string' && + typeof value.modelId === 'string' && + typeof value.cwd === 'string' && + isPermissionMode(value.permissionMode) && + (value.collaborationMode === undefined || isCollaborationMode(value.collaborationMode)) && + (value.orchestrationMode === undefined || isOrchestrationMode(value.orchestrationMode)) && + (value.orchestrationSource === undefined || + isEffectiveOrchestrationSource(value.orchestrationSource)) && + (value.agentSwarmAuthorization === undefined || + isAgentSwarmAuthorizationSource(value.agentSwarmAuthorization)) && + (value.rootExecutionKind === undefined || value.rootExecutionKind === 'context_compact') && + Number(value.scheduledTaskId !== undefined) + + Number(value.legacyAutomationId !== undefined) + + Number(value.goalId !== undefined) + + Number(value.agentGraphWakeId !== undefined) <= + 1 && + (value.toolMode === undefined || isToolMode(value.toolMode)) && + isFiniteNumber(value.createdAt) && + isFiniteNumber(value.updatedAt) && + isOptionalString(value.invocationId) && + (value.completedAt === undefined || isFiniteNumber(value.completedAt)) && + [ + value.parentRunId, + value.resumedFromRunId, + value.retriedFromRunId, + value.agentId, + value.agentName, + value.parentTurnId, + value.retriedFromTurnId, + value.regeneratedFromTurnId, + value.branchOfTurnId, + value.parentSessionId, + value.workspaceIdentity, + value.scheduledTaskId, + value.legacyAutomationId, + value.goalId, + value.agentGraphWakeId, + value.agentGraphWakeAttemptId, + value.failureClass, + value.failureMessage, + value.abortSource, + value.traceWriteError, + ].every(isOptionalString) && + (value.runComposition === undefined || isRecord(value.runComposition)) && + (value.continuationSource === undefined || + isLegacyContinuationSource(value.continuationSource)); + if (!valid) throw new Error('Invalid AgentRun header schema'); + return value as unknown as LegacyRunHeader; +} + +/** + * Project one legacy Run header onto its invocation opening fact. + * + * Route provenance fails closed. A header with no Connection identity cannot + * prove which endpoint and credential owned the run, so it projects as + * `unknown` rather than as an authenticated route; its transcript and tool + * evidence stay readable either way. + * + * Throws when a root authority marker is present but incomplete — that is + * corruption, and inventing a root would be worse than refusing one. + */ +export function invocationOpeningFromLegacyRunHeader( + header: LegacyRunHeader, +): RuntimeEventInvocationOpenedContent { + const lineage: RuntimeInvocationLineage = { + ...(header.parentRunId !== undefined ? { parentRunId: header.parentRunId } : {}), + ...(header.resumedFromRunId !== undefined ? { resumedFromRunId: header.resumedFromRunId } : {}), + ...(header.retriedFromRunId !== undefined ? { retriedFromRunId: header.retriedFromRunId } : {}), + ...(header.parentTurnId !== undefined ? { parentTurnId: header.parentTurnId } : {}), + ...(header.parentSessionId !== undefined ? { parentSessionId: header.parentSessionId } : {}), + ...(header.retriedFromTurnId !== undefined + ? { retriedFromTurnId: header.retriedFromTurnId } + : {}), + ...(header.regeneratedFromTurnId !== undefined + ? { regeneratedFromTurnId: header.regeneratedFromTurnId } + : {}), + ...(header.branchOfTurnId !== undefined ? { branchOfTurnId: header.branchOfTurnId } : {}), + ...(header.agentId !== undefined ? { agentId: header.agentId } : {}), + ...(header.agentName !== undefined ? { agentName: header.agentName } : {}), + }; + return { + kind: 'invocation_opened', + protocol: 'invocation_opened_v1', + route: invocationRouteFromLegacyRunHeader(header), + configuration: { + cwd: header.cwd, + permissionMode: header.permissionMode, + collaborationMode: header.collaborationMode ?? 'agent', + orchestrationMode: header.orchestrationMode ?? 'default', + orchestrationSource: header.orchestrationSource ?? 'session', + toolMode: header.toolMode ?? DEFAULT_TOOL_MODE, + ...(header.agentSwarmAuthorization !== undefined + ? { agentSwarmAuthorization: header.agentSwarmAuthorization } + : {}), + ...(header.workspaceIdentity !== undefined + ? { workspaceIdentity: header.workspaceIdentity } + : {}), + }, + root: invocationRootFromLegacyRunHeader(header), + source: invocationOpenSourceFromLegacyRunHeader(header), + ...(Object.keys(lineage).length > 0 ? { lineage } : {}), + }; +} + +function invocationRouteFromLegacyRunHeader(header: LegacyRunHeader): RuntimeInvocationRoute { + if (header.llmConnectionId === undefined) { + return { + provenance: 'unknown', + backendKind: header.backendKind, + llmConnectionSlug: header.llmConnectionSlug, + modelId: header.modelId, + }; + } + return { + provenance: 'runtime', + backendKind: header.backendKind, + llmConnectionId: header.llmConnectionId, + llmConnectionSlug: header.llmConnectionSlug, + modelId: header.modelId, + ...(header.providerStateIdentity !== undefined + ? { providerStateIdentity: header.providerStateIdentity } + : {}), + }; +} + +function invocationRootFromLegacyRunHeader( + header: LegacyRunHeader, +): RuntimeInvocationRootAuthority { + if (header.scheduledTaskId !== undefined) { + return { kind: 'scheduled_task', scheduledTaskId: header.scheduledTaskId }; + } + if (header.goalId !== undefined) return { kind: 'goal', goalId: header.goalId }; + if (header.legacyAutomationId !== undefined) { + return { kind: 'legacy_automation', legacyAutomationId: header.legacyAutomationId }; + } + if (header.agentGraphWakeId !== undefined) { + if (header.agentGraphWakeAttemptId === undefined) { + throw new Error(`AgentRun ${header.runId} has a graph wake with no delivery attempt`); + } + return { + kind: 'agent_graph_supervisor_wake', + wakeId: header.agentGraphWakeId, + attemptId: header.agentGraphWakeAttemptId, + }; + } + if (header.rootExecutionKind === 'context_compact') return { kind: 'context_compact' }; + return { kind: 'user' }; +} + +function invocationOpenSourceFromLegacyRunHeader( + header: LegacyRunHeader, +): RuntimeInvocationOpenSource { + const source = header.continuationSource; + if (!source) return { kind: 'fresh' }; + const v2 = 'protocol' in source ? source : undefined; + return { + kind: 'continuation', + sourceInvocationId: source.sourceInvocationId, + sourceRunId: source.sourceRunId, + sourceTurnId: source.sourceTurnId, + sourceRuntimeEventHighWater: source.sourceRuntimeEventHighWater, + ...(v2 ? { claimId: v2.claimId, boundaryDigest: v2.boundaryDigest } : {}), + }; +} + +function isLegacyContinuationSource(value: unknown): value is LegacyContinuationSource { + if (!isRecord(value)) return false; + const common = + typeof value.sourceInvocationId === 'string' && + typeof value.sourceRunId === 'string' && + typeof value.sourceTurnId === 'string' && + typeof value.sourceRuntimeEventHighWater === 'number' && + Number.isSafeInteger(value.sourceRuntimeEventHighWater) && + value.sourceRuntimeEventHighWater >= 0; + if (!common) return false; + if (hasExactShape(value, LEGACY_CONTINUATION_SOURCE_V1_SHAPE)) return true; + return ( + hasExactShape(value, LEGACY_CONTINUATION_SOURCE_V2_SHAPE) && + value.protocol === 'continuation_source_v2' && + typeof value.claimId === 'string' && + value.claimId.length > 0 && + typeof value.sourceInvocationId === 'string' && + value.sourceInvocationId.length > 0 && + typeof value.sourceRunId === 'string' && + value.sourceRunId.length > 0 && + typeof value.sourceTurnId === 'string' && + value.sourceTurnId.length > 0 && + typeof value.sourceRuntimeEventHighWater === 'number' && + value.sourceRuntimeEventHighWater > 0 && + isSha256Digest(value.boundaryDigest) && + isSha256Digest(value.sourcePrefixDigest) && + isSha256Digest(value.replayManifestDigest) && + value.replayManifestDigest === value.boundaryDigest + ); +} + +/** `'fake'` stays accepted: runs written by builds that shipped FakeBackend must keep decoding (#3211). */ +function isPersistedBackendKind(value: unknown): value is PersistedBackendKind { + return value === 'ai-sdk' || value === 'fake'; +} + +function isSha256Digest(value: unknown): value is `sha256:${string}` { + return typeof value === 'string' && /^sha256:[0-9a-f]{64}$/.test(value); +} diff --git a/packages/storage/src/runtime-event-persistence.ts b/packages/storage/src/runtime-event-persistence.ts index 26b11fc6f0..893e734e50 100644 --- a/packages/storage/src/runtime-event-persistence.ts +++ b/packages/storage/src/runtime-event-persistence.ts @@ -19,6 +19,12 @@ import { join } from 'node:path'; import type { RuntimeEvent } from '@maka/core/runtime-event'; +import type { + RuntimeInvocationPageInput, + RuntimeInvocationPageResult, + RuntimeInvocationRecord, + RuntimeInvocationSearchResult, +} from '@maka/core/runtime-invocation'; import type { BoundedEvidenceReadResult, EvidenceReadBudget } from './agent-run-store.js'; import { createSqliteRuntimeStore, type SqliteRuntimeStore } from './sqlite-runtime-store.js'; import { @@ -40,6 +46,17 @@ export type RuntimeEventReadPersistence = { }; export interface RuntimeEventReadStore { + listSessionInvocations(sessionId: string): Promise; + readRunInvocation(sessionId: string, runId: string): Promise; + listSessionInvocationsBounded( + sessionId: string, + limit: number, + ): Promise; + listSessionInvocationsPage( + sessionId: string, + input: RuntimeInvocationPageInput, + ): Promise; + readInvocation(sessionId: string, invocationId: string): Promise; readRuntimeEvents(sessionId: string, runId: string): Promise; readRuntimeEventsBounded( sessionId: string, @@ -79,6 +96,15 @@ export async function openRuntimeEventReadPersistence(input: { return { kind: 'sqlite', runtimeEventStore: Object.freeze({ + listSessionInvocations: (sessionId: string) => store.listSessionInvocations(sessionId), + readRunInvocation: (sessionId: string, runId: string) => + store.readRunInvocation(sessionId, runId), + listSessionInvocationsBounded: (sessionId: string, limit: number) => + store.listSessionInvocationsBounded(sessionId, limit), + listSessionInvocationsPage: (sessionId: string, input: RuntimeInvocationPageInput) => + store.listSessionInvocationsPage(sessionId, input), + readInvocation: (sessionId: string, invocationId: string) => + store.readInvocation(sessionId, invocationId), readRuntimeEvents: (sessionId: string, runId: string) => store.readRuntimeEvents(sessionId, runId), readRuntimeEventsBounded: (sessionId: string, runId: string, budget: EvidenceReadBudget) => diff --git a/packages/storage/src/sqlite-core-execution-schema.ts b/packages/storage/src/sqlite-core-execution-schema.ts index 9f49a23c19..caefc73fca 100644 --- a/packages/storage/src/sqlite-core-execution-schema.ts +++ b/packages/storage/src/sqlite-core-execution-schema.ts @@ -19,7 +19,7 @@ import type { DatabaseSync } from 'node:sqlite'; -export const SQLITE_CORE_EXECUTION_SCHEMA_VERSION = 6; +export const SQLITE_CORE_EXECUTION_SCHEMA_VERSION = 7; export function migrateSqliteCoreExecutionDatabase(db: DatabaseSync): void { db.exec(` @@ -27,7 +27,6 @@ export function migrateSqliteCoreExecutionDatabase(db: DatabaseSync): void { session_id TEXT NOT NULL, run_id TEXT NOT NULL, created_at INTEGER NOT NULL, - record_json TEXT NOT NULL, latest_model_call_sequence INTEGER CHECK (latest_model_call_sequence >= 0), PRIMARY KEY (session_id, run_id) ); @@ -148,6 +147,9 @@ export function migrateSqliteCoreExecutionDatabase(db: DatabaseSync): void { 'latest_model_call_sequence', 'INTEGER CHECK (latest_model_call_sequence >= 0)', ); + // The runtime migration runs first and has already turned every stored Run header into an + // invocation opening fact, so the row keeps only what the ledger needs to hang its events on. + dropColumn(db, 'core_agent_runs', 'record_json'); db.exec(` UPDATE core_agent_runs SET latest_model_call_sequence = ( @@ -182,3 +184,9 @@ function ensureColumn(db: DatabaseSync, table: string, column: string, definitio if (columns.some((candidate) => candidate.name === column)) return; db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`); } + +function dropColumn(db: DatabaseSync, table: string, column: string): void { + const columns = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name?: unknown }>; + if (!columns.some((candidate) => candidate.name === column)) return; + db.exec(`ALTER TABLE ${table} DROP COLUMN ${column}`); +} diff --git a/packages/storage/src/sqlite-runtime-schema.ts b/packages/storage/src/sqlite-runtime-schema.ts index d9ecaee2cd..a10532f026 100644 --- a/packages/storage/src/sqlite-runtime-schema.ts +++ b/packages/storage/src/sqlite-runtime-schema.ts @@ -18,8 +18,19 @@ */ import type { DatabaseSync } from 'node:sqlite'; +import { + decodePersistedLegacyRunHeader, + invocationOpeningFromLegacyRunHeader, + type LegacyRunHeader, +} from './legacy-run-header.js'; +import type { RuntimeEvent } from '@maka/core/runtime-event'; +import { encodeCanonicalRuntimeEvent } from '@maka/core/canonical-runtime-event'; +import { + buildInvocationOpenedEvent, + buildSyntheticTerminalRuntimeEvent, +} from '@maka/core/runtime-invocation'; -export const SQLITE_RUNTIME_SCHEMA_VERSION = 15; +export const SQLITE_RUNTIME_SCHEMA_VERSION = 16; export const RUNTIME_RECOVERY_AUTHORITY_CAPABILITY = 'runtime_recovery_authority'; export const RUNTIME_RECOVERY_AUTHORITY_CAPABILITY_VERSION = 1; export const RUNTIME_CONTINUATION_AUTHORITY_CAPABILITY = 'runtime_continuation_authority'; @@ -484,8 +495,321 @@ const MIGRATIONS: ReadonlyMap = new Map([ ALTER TABLE runtime_continuation_claims_v15 RENAME TO runtime_continuation_claims; `, ], + [ + 16, + ` + CREATE INDEX runtime_events_by_session_kind + ON runtime_events(session_id, event_kind, invocation_id); + + CREATE UNIQUE INDEX runtime_events_one_opening_per_invocation + ON runtime_events(invocation_id) + WHERE event_kind = 'invocation_opened'; + + CREATE TABLE runtime_legacy_invocation_openings ( + invocation_id TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + run_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + opened_at INTEGER NOT NULL, + opening_json TEXT NOT NULL, + -- UNIQUE is what indexes this side of the foreign key. SQLite indexes only + -- the parent, so without it every deleted RuntimeEvent scans this whole + -- table looking for rows to cascade. + anchor_event_id TEXT NOT NULL UNIQUE + REFERENCES runtime_events(event_id) ON DELETE CASCADE + ) WITHOUT ROWID; + + CREATE INDEX runtime_legacy_invocation_openings_by_session + ON runtime_legacy_invocation_openings(session_id, opened_at, invocation_id); + + -- Rebuilt rather than renamed in place, because the column rename is not + -- the only thing this claim needs. Its start event belongs to the target + -- Session, and the foreign key had no ON DELETE clause, so purging that + -- Session was refused outright by the constraint and the whole purge rolled + -- back. A continuation whose target has been deleted no longer names + -- anything, so the claim goes with it and the source boundary it held is + -- free again. + CREATE TABLE runtime_continuation_claims_v16 ( + claim_id TEXT PRIMARY KEY, + source_session_id TEXT NOT NULL, + source_invocation_id TEXT NOT NULL, + source_run_id TEXT NOT NULL, + source_turn_id TEXT NOT NULL, + source_event_high_water INTEGER NOT NULL CHECK (source_event_high_water > 0), + source_prefix_digest TEXT NOT NULL, + boundary_digest TEXT NOT NULL UNIQUE, + boundary_json TEXT NOT NULL, + provider_projection_version INTEGER NOT NULL CHECK (provider_projection_version IN (1, 2)), + provider_replay_digest TEXT NOT NULL, + target_session_id TEXT NOT NULL, + target_invocation_id TEXT NOT NULL UNIQUE, + target_run_id TEXT NOT NULL UNIQUE, + target_turn_id TEXT NOT NULL, + target_opening_json TEXT NOT NULL, + claimed_at INTEGER NOT NULL, + start_event_id TEXT UNIQUE REFERENCES runtime_events(event_id) ON DELETE CASCADE, + start_kind TEXT CHECK ( + start_kind IS NULL OR start_kind IN ('runtime_admission', 'claim_repair') + ), + protocol_version INTEGER NOT NULL CHECK (protocol_version = 1), + UNIQUE ( + source_session_id, + source_run_id, + source_event_high_water, + source_prefix_digest + ), + UNIQUE (target_session_id, target_turn_id) + ); + + INSERT INTO runtime_continuation_claims_v16 ( + claim_id, source_session_id, source_invocation_id, source_run_id, source_turn_id, + source_event_high_water, source_prefix_digest, boundary_digest, boundary_json, + provider_projection_version, provider_replay_digest, target_session_id, + target_invocation_id, target_run_id, target_turn_id, target_opening_json, + claimed_at, start_event_id, start_kind, protocol_version + ) + SELECT + claim_id, source_session_id, source_invocation_id, source_run_id, source_turn_id, + source_event_high_water, source_prefix_digest, boundary_digest, boundary_json, + provider_projection_version, provider_replay_digest, target_session_id, + target_invocation_id, target_run_id, target_turn_id, target_run_header_json, + claimed_at, start_event_id, start_kind, protocol_version + FROM runtime_continuation_claims; + DROP TABLE runtime_continuation_claims; + ALTER TABLE runtime_continuation_claims_v16 RENAME TO runtime_continuation_claims; + `, + ], ]); +/** + * Data migrations that a SQL statement cannot express, applied inside the same + * transaction as their schema step. They project persisted records through the + * one TypeScript mapping that owns that projection, so a migration and the live + * writer can never classify a field two different ways. + */ +const DATA_MIGRATIONS: ReadonlyMap void> = new Map([ + [ + 16, + (db) => { + backfillInvocationOpeningFacts(db); + projectContinuationClaimOpenings(db); + }, + ], +]); + +/** + * Replace each open claim's embedded target Run header with the opening fact it + * always implied. + * + * The header was only ever there so the start event could be checked against + * it, and the check went through the projection anyway. Projecting once, here, + * leaves one representation instead of a copy plus a derivation. + * + * A row this cannot project is dropped rather than left half-migrated: an + * undecodable claim could not have admitted a start event before this migration + * either, and keeping it would only block the boundary it holds. + */ +function projectContinuationClaimOpenings(db: DatabaseSync): void { + const rows = db + .prepare('SELECT claim_id, target_opening_json FROM runtime_continuation_claims') + .all() as Array<{ claim_id: string; target_opening_json: string }>; + const update = db.prepare( + 'UPDATE runtime_continuation_claims SET target_opening_json = ? WHERE claim_id = ?', + ); + const remove = db.prepare('DELETE FROM runtime_continuation_claims WHERE claim_id = ?'); + for (const row of rows) { + try { + const header = decodePersistedLegacyRunHeader(JSON.parse(row.target_opening_json)); + update.run(JSON.stringify(invocationOpeningFromLegacyRunHeader(header)), row.claim_id); + } catch { + remove.run(row.claim_id); + } + } +} + +/** + * Give every Run header its opening fact, so that after this migration the + * opening lives in the runtime database rather than on the header. + * + * A run that never wrote a RuntimeEvent gets the real thing: the opening fact + * as event one of its own invocation. A run that already has events cannot, + * because it owns an immutable sequence whose position 1, digests and coverage + * other facts already point at; inserting into it would rewrite signed history. + * Its opening is recorded in `runtime_legacy_invocation_openings` instead, + * which only this migration ever writes. Readers merge the two, so nothing + * downstream has to know which shelf a given opening came off. + * + * A shelved opening describes a ledger that already exists, so it is anchored to + * that ledger's first event and dies with it. Without the anchor, deleting a + * Session's events would leave the opening behind, and the inventory would + * report the run again with no ending — a completed run coming back as an + * active one. + * + * A header this cannot project fails closed: it is skipped, and its transcript + * and tool evidence stay exactly as readable as before. + */ +function backfillInvocationOpeningFacts(db: DatabaseSync): void { + if (!hasTable(db, 'core_agent_runs')) return; + // The header column is dropped by the core-execution migration that follows + // this one, so its absence means every header it held is already an opening + // fact. Nothing left to project, and the two scopes stay independently + // replayable. + if (!hasColumn(db, 'core_agent_runs', 'record_json')) return; + const rows = db + .prepare(` + SELECT + r.session_id, + r.run_id, + r.record_json, + first_event.invocation_id AS existing_invocation_id, + first_event.event_id AS anchor_event_id + FROM core_agent_runs r + LEFT JOIN runtime_events first_event ON first_event.event_id = ( + SELECT e.event_id FROM runtime_events e + WHERE e.session_id = r.session_id AND e.run_id = r.run_id + ORDER BY e.event_seq ASC LIMIT 1 + ) + ORDER BY r.created_at ASC, r.run_id ASC + `) + .all() as Array<{ + session_id: string; + run_id: string; + record_json: string; + existing_invocation_id: string | null; + anchor_event_id: string | null; + }>; + const insertEvent = db.prepare(` + INSERT INTO runtime_events ( + event_id, session_id, invocation_id, run_id, turn_id, event_seq, + event_kind, payload_json, committed_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + const insertOrdinal = db.prepare(` + INSERT INTO runtime_session_event_ordinals(session_id, ordinal, event_id) + SELECT ?, COALESCE(MAX(ordinal), 0) + 1, ? + FROM runtime_session_event_ordinals WHERE session_id = ? + `); + const insertLegacyOpening = db.prepare(` + INSERT OR IGNORE INTO runtime_legacy_invocation_openings ( + invocation_id, session_id, run_id, turn_id, opened_at, opening_json, + anchor_event_id + ) VALUES (?, ?, ?, ?, ?, ?, ?) + `); + // The header column is dropped right after this, so a header this cannot read + // is a run that would silently cease to exist. Refusing the whole migration + // keeps the database as it was, and the failure names the row instead of + // hiding it. + const unreadable = (row: { session_id: string; run_id: string }, cause: unknown): Error => + new Error( + `Cannot migrate the AgentRun header of ${row.session_id}/${row.run_id}: ${ + cause instanceof Error ? cause.message : String(cause) + }`, + { cause }, + ); + for (const row of rows) { + let header: LegacyRunHeader; + let opening: string; + try { + header = decodePersistedLegacyRunHeader(JSON.parse(row.record_json)); + opening = JSON.stringify(invocationOpeningFromLegacyRunHeader(header)); + } catch (error) { + throw unreadable(row, error); + } + if (row.existing_invocation_id !== null && row.anchor_event_id !== null) { + // The invocation id its own events already carry is the one every reader + // joins on, so the legacy row is keyed by that rather than by the header's + // copy, which older builds minted independently. + insertLegacyOpening.run( + row.existing_invocation_id, + header.sessionId, + header.runId, + header.turnId, + header.createdAt, + opening, + row.anchor_event_id, + ); + continue; + } + // A run with no events of its own gets the facts its header held, where + // facts live now: the opening, and the ending if the header recorded one. + // A header still marked in flight stays open; recovery settles it the way it + // settles any run the process died holding. + const run = { + sessionId: header.sessionId, + invocationId: header.invocationId ?? header.runId, + runId: header.runId, + turnId: header.turnId, + }; + const events = [ + buildInvocationOpenedEvent({ + id: `invocation_opened:${header.runId}`, + run, + openedAt: header.createdAt, + opening: invocationOpeningFromLegacyRunHeader(header), + }), + ...(header.status === 'completed' || + header.status === 'failed' || + header.status === 'cancelled' + ? [ + buildSyntheticTerminalRuntimeEvent({ + id: `invocation_terminal:${header.runId}`, + invocationId: run.invocationId, + run, + status: header.status, + ts: header.completedAt ?? header.updatedAt, + ...(header.failureClass !== undefined ? { failureClass: header.failureClass } : {}), + ...(header.failureMessage !== undefined ? { message: header.failureMessage } : {}), + ...(header.abortSource !== undefined ? { abortSource: header.abortSource } : {}), + }), + ] + : []), + ]; + events.forEach((event, index) => { + let encoded: { event: RuntimeEvent; json: string }; + try { + encoded = encodeCanonicalRuntimeEvent(event); + } catch (error) { + throw unreadable(row, error); + } + insertEvent.run( + event.id, + event.sessionId, + event.invocationId, + event.runId, + event.turnId, + index + 1, + runtimeEventKind(event), + encoded.json, + event.ts, + ); + insertOrdinal.run(event.sessionId, event.id, event.sessionId); + }); + } +} + +/** The `event_kind` column: the one coarse label every reader indexes events by. */ +export function runtimeEventKind(event: RuntimeEvent): string { + return ( + event.content?.kind ?? + event.status ?? + (event.actions?.workspaceFact ? 'workspace_fact' : undefined) ?? + (event.actions?.toolDispatch ? 'tool_dispatch' : undefined) ?? + (event.actions?.endInvocation ? 'invocation_end' : 'runtime_fact') + ); +} + +function hasColumn(db: DatabaseSync, table: string, column: string): boolean { + const columns = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name?: unknown }>; + return columns.some((candidate) => candidate.name === column); +} + +function hasTable(db: DatabaseSync, name: string): boolean { + const row = db + .prepare("SELECT 1 AS present FROM sqlite_master WHERE type = 'table' AND name = ?") + .get(name) as { present?: unknown } | undefined; + return row?.present === 1; +} + export function configureSqliteRuntimeDatabase(db: DatabaseSync): void { // Bound lock acquisition before touching persistent journal state. WAL mode is // database-persistent, so established workspaces only need to verify it rather @@ -530,6 +854,7 @@ export function migrateSqliteRuntimeDatabase( const sql = MIGRATIONS.get(version); if (!sql) throw new Error(`Missing SQLite runtime migration ${version}`); db.exec(sql); + DATA_MIGRATIONS.get(version)?.(db); db.exec(`PRAGMA user_version = ${version}`); } if (ownsTransaction) db.exec('COMMIT'); diff --git a/packages/storage/src/sqlite-runtime-store.ts b/packages/storage/src/sqlite-runtime-store.ts index a25a0b7cb3..c71dcbfb01 100644 --- a/packages/storage/src/sqlite-runtime-store.ts +++ b/packages/storage/src/sqlite-runtime-store.ts @@ -43,8 +43,10 @@ import { } from '@maka/core/workspace-version-authority'; import { decodeRuntimeEvent, + decodeRuntimeInvocationOpened, isPartialRuntimeEvent, isTerminalRuntimeEvent, + runtimeEventInvocationOpening, TOOL_BOUNDARY_PROTOCOL_V1, type RuntimeEvent, type RuntimeEventManagedWorkspaceMutationV2, @@ -61,11 +63,16 @@ import { type RuntimeRecoveryBundleStore, type RuntimeWorkspaceVersionAuthorityStore, } from '@maka/core/runtime-event-store'; +import type { + RuntimeInvocationPageCursor, + RuntimeInvocationPageInput, + RuntimeInvocationPageResult, + RuntimeInvocationRecord, + RuntimeInvocationSearchResult, +} from '@maka/core/runtime-invocation'; import { type ToolRecoveryDecisionFact } from '@maka/core/tool-recovery-fact'; import { canonicalToolArgsHash, stableJsonStringify } from '@maka/core/tool-args-identity'; import { encodeCanonicalRuntimeEvent } from '@maka/core/canonical-runtime-event'; -import { decodePersistedAgentRunHeader, type AgentRunHeader } from '@maka/core/agent-run'; -import { markPersisted } from '@maka/core/persisted-value'; import { scanToolLedger, ToolLedgerCorruptionError, @@ -76,6 +83,7 @@ import { } from '@maka/core/tool-ledger-scanner'; import { buildImmutableRuntimePrefix, + continuationStartEventMatchesClaim, decodeContinuationClaim, type ContinuationClaimV1, type ImmutableRuntimePrefixV1, @@ -95,6 +103,7 @@ import { RUNTIME_CONTINUATION_AUTHORITY_CAPABILITY_VERSION, RUNTIME_WORKSPACE_VERSION_AUTHORITY_CAPABILITY, RUNTIME_WORKSPACE_VERSION_AUTHORITY_CAPABILITY_VERSION, + runtimeEventKind, SQLITE_RUNTIME_SCHEMA_VERSION, } from './sqlite-runtime-schema.js'; import { @@ -125,6 +134,19 @@ export { SQLITE_RUNTIME_SCHEMA_VERSION } from './sqlite-runtime-schema.js'; export type { ToolRecoveryMode } from '@maka/core/runtime-event'; +/** + * `isTerminalRuntimeEvent` asked in SQL. + * + * The TypeScript predicate stays the authority; this only lets a query find the + * terminal event without decoding every row it passes over. Both have to say the + * same thing, so the SQL half is written once here instead of at each query. + */ +const TERMINAL_RUNTIME_EVENT_SQL = `( + json_extract(payload_json, '$.actions.endInvocation') = 1 + OR json_extract(payload_json, '$.status') + IN ('completed', 'failed', 'aborted', 'cancelled') + )`; + const RUNTIME_EVENT_SCAN_BATCH_SIZE = 128; const RUNTIME_PARTIAL_SEGMENT_TARGET_BYTES = 64 * 1024; @@ -526,6 +548,255 @@ export class SqliteRuntimeStore return this.readRuntimeEventsSync(sessionId, runId); } + /** + * Enumerate a Session's invocations: the opening fact names each one, and its + * highest-sequence event says whether it ended. + * + * Invocations that predate the opening fact could not be given one without + * rewriting an immutable sequence, so the migration parked their openings in + * `runtime_legacy_invocation_openings`. Both shelves are merged here and the + * result says nothing about which one a record came from: an opening is an + * opening, and a consumer that branched on its storage would be encoding the + * migration window into its own logic. + */ + async listSessionInvocations(sessionId: string): Promise { + assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); + return this.readTransaction(() => + this.readInvocationOpeningsSync(sessionId, { direction: 'asc' }).map((row) => + this.completeInvocationRecordSync(row), + ), + ); + } + + async readRunInvocation( + sessionId: string, + runId: string, + ): Promise { + assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); + assertRuntimeStorageSafeId(runId, 'Invalid run id'); + return this.readTransaction(() => { + const row = this.readInvocationOpeningsSync(sessionId, { direction: 'asc', runId }).at(0); + return row ? this.completeInvocationRecordSync(row) : undefined; + }); + } + + /** + * The first page of a Session's invocations, plus whether more exist. + * + * The extra row this reads past the limit is the whole truncation signal, so a + * caller never has to count a Session it declined to load. + */ + async listSessionInvocationsBounded( + sessionId: string, + limit: number, + ): Promise { + assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); + assertInvocationSearchLimit(limit); + return this.readTransaction(() => { + const rows = this.readInvocationOpeningsSync(sessionId, { + direction: 'asc', + limit: limit + 1, + }); + return { + invocations: rows.slice(0, limit).map((row) => this.completeInvocationRecordSync(row)), + truncated: rows.length > limit, + }; + }); + } + + /** One newest-first page of a Session's invocations. */ + async listSessionInvocationsPage( + sessionId: string, + input: RuntimeInvocationPageInput, + ): Promise { + assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); + assertInvocationSearchLimit(input.limit); + if (input.before) { + assertRuntimeStorageSafeId(input.before.invocationId, 'Invalid invocation page cursor'); + if (!Number.isFinite(input.before.openedAt)) { + throw new Error('Invalid invocation page cursor'); + } + } + return this.readTransaction(() => { + const rows = this.readInvocationOpeningsSync(sessionId, { + direction: 'desc', + limit: input.limit + 1, + ...(input.before ? { before: input.before } : {}), + }); + const page = rows.slice(0, input.limit); + const last = page.at(-1); + return { + invocations: page.map((row) => this.completeInvocationRecordSync(row)), + nextCursor: + rows.length > input.limit && last + ? { openedAt: last.openedAt, invocationId: last.invocationId } + : null, + }; + }); + } + + /** + * One invocation named by its own identity. + * + * Absence throws rather than returning `undefined`: every caller here holds an + * invocation id that some durable fact already handed it, so a missing opening + * is corruption and not a branch a reader should be asked to handle. + */ + async readInvocation(sessionId: string, invocationId: string): Promise { + assertRuntimeStorageSafeId(sessionId, 'Invalid session id'); + assertRuntimeStorageSafeId(invocationId, 'Invalid invocation id'); + return this.readTransaction(() => { + const row = this.readInvocationOpeningsSync(sessionId, { + direction: 'asc', + invocationId, + }).at(0); + if (!row) throw new Error(`Runtime invocation not found: ${invocationId}`); + return this.completeInvocationRecordSync(row); + }); + } + + /** + * Read invocation openings off both shelves as one ordered sequence. + * + * Every writer of an opening event stamps `committed_at` with the event's own + * timestamp, so that column orders the event shelf by the same value the + * record reports as `openedAt` and the legacy shelf keeps under `opened_at`. + * Ordering and paging therefore happen in SQL, and a bounded caller decodes + * only the openings it asked for. + */ + private readInvocationOpeningsSync( + sessionId: string, + options: { + direction: 'asc' | 'desc'; + limit?: number; + before?: RuntimeInvocationPageCursor; + invocationId?: string; + runId?: string; + }, + ): Omit[] { + const order = options.direction === 'desc' ? 'DESC' : 'ASC'; + const rows = this.db + .prepare(` + SELECT * FROM ( + SELECT + event_id AS event_id, + invocation_id AS invocation_id, + run_id AS run_id, + turn_id AS turn_id, + committed_at AS opened_at, + payload_json AS opening_json, + 1 AS from_events + FROM runtime_events + WHERE session_id = :sessionId AND event_kind = 'invocation_opened' + UNION ALL + SELECT + NULL, + legacy.invocation_id, + legacy.run_id, + legacy.turn_id, + legacy.opened_at, + legacy.opening_json, + 0 + FROM runtime_legacy_invocation_openings AS legacy + WHERE legacy.session_id = :sessionId + AND NOT EXISTS ( + SELECT 1 FROM runtime_events + WHERE runtime_events.invocation_id = legacy.invocation_id + AND runtime_events.event_kind = 'invocation_opened' + ) + ) + WHERE (:invocationId IS NULL OR invocation_id = :invocationId) + AND (:runId IS NULL OR run_id = :runId) + AND ( + :beforeOpenedAt IS NULL + OR opened_at < :beforeOpenedAt + OR (opened_at = :beforeOpenedAt AND invocation_id < :beforeInvocationId) + ) + ORDER BY opened_at ${order}, invocation_id ${order} + LIMIT :limit + `) + .all({ + sessionId, + invocationId: options.invocationId ?? null, + runId: options.runId ?? null, + beforeOpenedAt: options.before?.openedAt ?? null, + beforeInvocationId: options.before?.invocationId ?? null, + limit: options.limit ?? -1, + }) as unknown as Array<{ + event_id: string | null; + invocation_id: string; + run_id: string; + turn_id: string; + opened_at: number; + opening_json: string; + from_events: number; + }>; + return rows.map((row) => { + if (row.from_events !== 1) { + return { + sessionId, + invocationId: row.invocation_id, + runId: row.run_id, + turnId: row.turn_id, + openedAt: row.opened_at, + opening: decodeRuntimeInvocationOpened(JSON.parse(row.opening_json)), + }; + } + const event = decodeRuntimeEventStorageRow({ + event_id: row.event_id ?? '', + session_id: sessionId, + invocation_id: row.invocation_id, + run_id: row.run_id, + turn_id: row.turn_id, + payload_json: row.opening_json, + }); + const opening = runtimeEventInvocationOpening(event); + if (!opening) { + throw new Error(`RuntimeEvent ${event.id} is indexed as an opening fact but is not one`); + } + return { + sessionId: event.sessionId, + invocationId: event.invocationId, + runId: event.runId, + turnId: event.turnId, + openedAt: event.ts, + opening, + }; + }); + } + + /** + * An invocation's ending is its first terminal event, wherever it sits. + * + * The store seals a run on that event, so for anything it wrote itself the + * first terminal is also the only one and the last event. Ledgers written + * before the seal existed can carry a straggler after the terminal, and + * reading those as unfinished would contradict every other reader of the same + * rule: recovery, the read model and continuation resume all take the first + * terminal. A ledger that somehow holds two is corrupt, and saying so is the + * job of those readers — this inventory feeds Session lists, so it reports the + * ending it can see rather than poisoning the whole Session over one run. + */ + private completeInvocationRecordSync( + record: Omit, + ): RuntimeInvocationRecord { + const terminalRow = this.db + .prepare(` + SELECT event_id, session_id, invocation_id, run_id, turn_id, payload_json + FROM runtime_events + WHERE invocation_id = ? + AND ${TERMINAL_RUNTIME_EVENT_SQL} + ORDER BY event_seq ASC + LIMIT 1 + `) + .get(record.invocationId) as unknown as RuntimeEventStorageRow | undefined; + const terminal = terminalRow ? decodeRuntimeEventStorageRow(terminalRow) : undefined; + return { + ...record, + ...(terminal && isTerminalRuntimeEvent(terminal) ? { terminalEvent: terminal } : {}), + }; + } + async scanRuntimeEvents( sessionId: string, runId: string, @@ -935,7 +1206,7 @@ export class SqliteRuntimeStore target_invocation_id, target_run_id, target_turn_id, - target_run_header_json, + target_opening_json, claimed_at, protocol_version ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1) @@ -956,7 +1227,7 @@ export class SqliteRuntimeStore claim.target.invocationId, claim.target.runId, claim.target.turnId, - stableJsonStringify(claim.targetRunHeader), + stableJsonStringify(claim.targetOpening), claim.claimedAt, ); } catch (error) { @@ -1027,7 +1298,7 @@ export class SqliteRuntimeStore target_invocation_id, target_run_id, target_turn_id, - target_run_header_json, + target_opening_json, claimed_at, start_event_id, start_kind, @@ -3017,7 +3288,7 @@ export class SqliteRuntimeStore target_invocation_id, target_run_id, target_turn_id, - target_run_header_json, + target_opening_json, claimed_at, start_event_id, start_kind, @@ -3048,7 +3319,7 @@ export class SqliteRuntimeStore target_invocation_id, target_run_id, target_turn_id, - target_run_header_json, + target_opening_json, claimed_at, start_event_id, start_kind, @@ -3370,11 +3641,7 @@ export class SqliteRuntimeStore SELECT event_id, session_id, invocation_id, run_id, turn_id, payload_json FROM runtime_events WHERE session_id = ? AND run_id = ? - AND ( - json_extract(payload_json, '$.actions.endInvocation') = 1 - OR json_extract(payload_json, '$.status') - IN ('completed', 'failed', 'aborted', 'cancelled') - ) + AND ${TERMINAL_RUNTIME_EVENT_SQL} ORDER BY event_seq ASC `) .all(event.sessionId, event.runId) as unknown as RuntimeEventStorageRow[]; @@ -4001,49 +4268,7 @@ function assertContinuationStartEvent( event: RuntimeEvent, startKind: 'runtime_admission' | 'claim_repair', ): void { - const start = event.actions?.continuationStart; - const runtimeProtocol = event.actions?.runtimeProtocol; - const actionKeys = event.actions ? Object.keys(event.actions) : []; - const validActionShape = - actionKeys.includes('continuationStart') && - actionKeys.every((key) => key === 'continuationStart' || key === 'runtimeProtocol') && - actionKeys.length === (runtimeProtocol === undefined ? 1 : 2); - const validRuntimeProtocol = - runtimeProtocol === undefined || - (startKind === 'runtime_admission' && - runtimeProtocol.toolBoundary === TOOL_BOUNDARY_PROTOCOL_V1); - const source = claim.boundary.segments.at(-1)!; - if ( - event.sessionId !== claim.target.sessionId || - event.invocationId !== claim.target.invocationId || - event.runId !== claim.target.runId || - event.turnId !== claim.target.turnId || - event.ts < claim.claimedAt || - event.partial || - event.role !== 'system' || - event.author !== 'system' || - event.status !== undefined || - event.content !== undefined || - !event.actions || - !validActionShape || - !validRuntimeProtocol || - !start || - start.protocol !== 'continuation_start_v2' || - start.provenance !== startKind || - start.claimId !== claim.claimId || - start.boundaryDigest !== claim.boundaryDigest || - start.replayManifestDigest !== claim.boundary.manifestDigest || - start.providerProjectionVersion !== claim.providerProjectionVersion || - start.providerReplayDigest !== claim.providerReplayDigest || - !isDeepStrictEqual(start.immediateSource, { - sessionId: source.identity.sessionId, - invocationId: source.identity.invocationId, - runId: source.identity.runId, - turnId: source.identity.turnId, - highWater: source.position.lastEventSeq, - prefixDigest: source.prefixDigest, - }) - ) { + if (!continuationStartEventMatchesClaim(event, claim, startKind)) { throw new Error('Invalid continuation-start authority event'); } } @@ -4063,6 +4288,12 @@ function assertRuntimeStorageSafeId(value: string, message: string): void { if (!isRuntimeStorageSafeId(value)) throw new Error(message); } +function assertInvocationSearchLimit(limit: number): void { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 256) { + throw new RangeError('Runtime invocation search limit must be an integer between 1 and 256'); + } +} + interface RuntimeEventStorageRow { event_id: string; session_id: string; @@ -4399,7 +4630,7 @@ interface ContinuationClaimStorageRow { target_invocation_id: string; target_run_id: string; target_turn_id: string; - target_run_header_json: string; + target_opening_json: string; claimed_at: number; start_event_id: string | null; start_kind: 'runtime_admission' | 'claim_repair' | null; @@ -4449,16 +4680,6 @@ function decodeStoredRuntimeEvent(storedJson: string): RuntimeEvent { return decodeRuntimeEvent(JSON.parse(storedJson)); } -function runtimeEventKind(event: RuntimeEvent): string { - return ( - event.content?.kind ?? - event.status ?? - (event.actions?.workspaceFact ? 'workspace_fact' : undefined) ?? - (event.actions?.toolDispatch ? 'tool_dispatch' : undefined) ?? - (event.actions?.endInvocation ? 'invocation_end' : 'runtime_fact') - ); -} - interface RuntimePartialSnapshot { event: RuntimeEvent; afterEventId?: string; @@ -4586,9 +4807,7 @@ function decodeContinuationClaimRow(row: ContinuationClaimStorageRow): Continuat throw new Error(`Unsupported continuation claim protocol ${row.protocol_version}`); } const boundary = JSON.parse(row.boundary_json) as unknown; - const targetRunHeader = decodePersistedAgentRunHeader( - markPersisted(JSON.parse(row.target_run_header_json)), - ); + const targetOpening = JSON.parse(row.target_opening_json) as unknown; const claim = decodeContinuationClaim({ protocol: 'continuation_claim_v1', claimId: row.claim_id, @@ -4602,7 +4821,7 @@ function decodeContinuationClaimRow(row: ContinuationClaimStorageRow): Continuat runId: row.target_run_id, turnId: row.target_turn_id, }, - targetRunHeader, + targetOpening, claimedAt: row.claimed_at, }); const source = claim.boundary.segments.at(-1)!; diff --git a/scripts/release-cli-file-policy.test.mjs b/scripts/release-cli-file-policy.test.mjs index 2766b4c2b6..84e7e8e371 100644 --- a/scripts/release-cli-file-policy.test.mjs +++ b/scripts/release-cli-file-policy.test.mjs @@ -61,7 +61,11 @@ describe('CLI release file policy', () => { './test-only/client-capability-host', './test-only/execution-candidate-e2e-main', ], - runtime: ['./test-only/fake-backend', './test-only/observation-text-reader'], + runtime: [ + './test-only/fake-backend', + './test-only/observation-text-reader', + './test-only/invocation-fixture', + ], })) { const manifestPath = join(repoRoot, 'packages', directory, 'package.json'); const source = JSON.parse(readFileSync(manifestPath, 'utf8'));