diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 7bbea49f..0e0b2c4b 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -10945,6 +10945,13 @@ Generic environment provider executor config. External packages implement > `optional` **registry?**: [`AgentEnvironmentProviderRegistry`](runtime/environment-provider.md#agentenvironmentproviderregistry) +##### steering? + +> `optional` **steering?**: [`SandboxSteeringOptions`](#sandboxsteeringoptions) + +Compose the provider through the existing steerable sandbox session. +The provider still owns environment creation and session semantics. + *** ### RouterToolsSeam diff --git a/docs/api/runtime/environment-provider.md b/docs/api/runtime/environment-provider.md index 438fa85d..197fb4b4 100644 --- a/docs/api/runtime/environment-provider.md +++ b/docs/api/runtime/environment-provider.md @@ -144,6 +144,14 @@ Options for exposing an `AgentEnvironmentProvider` through the legacy sandbox cl **`Experimental`** +##### requireSession? + +> `optional` **requireSession?**: `boolean` + +**`Experimental`** + +Require declared live continuation plus concrete session controls. + ##### mapCreateOptions? > `optional` **mapCreateOptions?**: (`options`) => `Partial`\<`CreateAgentEnvironmentInput`\> diff --git a/src/runtime/environment-provider.test.ts b/src/runtime/environment-provider.test.ts index 86d5f5ba..d10b5b65 100644 --- a/src/runtime/environment-provider.test.ts +++ b/src/runtime/environment-provider.test.ts @@ -76,7 +76,15 @@ describe('environment provider adapters', () => { yield { type: 'result', data: { finalText: `ok:${input.prompt}` }, - usage: { inputTokens: 2, outputTokens: 3, cost: 0.01 }, + usage: { + inputTokens: 2, + outputTokens: 3, + totalTokens: 7, + cacheReadInputTokens: 4, + cacheCreationInputTokens: 1, + reasoningTokens: 2, + cost: 0.01, + }, } }, }) @@ -130,11 +138,22 @@ describe('environment provider adapters', () => { expect(sessionPrompt).toMatchObject({ prompt: 'continue' }) await resumed.interrupt() expect(cancelled).toBe(1) + expect(events).toHaveLength(1) expect(events[0]).toMatchObject({ - type: 'llm_call', - data: { inputTokens: 2, outputTokens: 3, totalCostUsd: 0.01 }, + type: 'result', + data: { + finalText: 'ok:hello', + usage: { + inputTokens: 2, + outputTokens: 3, + totalTokens: 7, + cacheReadInputTokens: 4, + cacheCreationInputTokens: 1, + reasoningTokens: 2, + totalCostUsd: 0.01, + }, + }, }) - expect(events.at(-1)).toMatchObject({ type: 'result', data: { finalText: 'ok:hello' } }) }) it('adapts a SandboxClient to a neutral provider with create/stream/workspace methods', async () => { @@ -505,6 +524,63 @@ describe('environment provider adapters', () => { await expect(box.prompt('hello')).rejects.toThrow(/terminal result/) }) + it('destroys an environment that cannot satisfy a required session', async () => { + let destroyed = 0 + const provider: AgentEnvironmentProvider = { + name: 'stream-only-provider', + capabilities: () => fakeCapabilities(), + async create() { + return fakeEnvironment({ + stream: async function* (): AsyncIterable {}, + async destroy() { + destroyed += 1 + }, + }) + }, + } + + await expect( + providerAsSandboxClient(provider, { requireSession: true }).create({ + backend: { type: 'pi', profile: { name: 'worker' } }, + }), + ).rejects.toThrow(/session\(\) is required/) + expect(destroyed).toBe(1) + }) + + it('rejects providers that cannot support live continuation before creating an environment', async () => { + for (const disabled of ['continuation', 'live-streaming'] as const) { + const capabilities = fakeCapabilities() + if (disabled === 'continuation') capabilities.sessions.continue = false + else capabilities.streaming.live = false + let createCalls = 0 + const provider: AgentEnvironmentProvider = { + name: `no-${disabled}`, + capabilities: () => capabilities, + async create() { + createCalls += 1 + return fakeEnvironment({ + session: () => ({ + id: 'session', + status: async () => 'running', + events: async function* (): AsyncIterable {}, + result: async () => ({ text: '', success: true }), + prompt: async () => ({ text: '', success: true }), + cancel: async () => {}, + }), + stream: async function* (): AsyncIterable {}, + }) + }, + } + + await expect( + providerAsSandboxClient(provider, { requireSession: true }).create({ + backend: { type: 'pi', profile: { name: 'worker' } }, + }), + ).rejects.toThrow(/live session continuation is required/) + expect(createCalls).toBe(0) + } + }) + it('adapts a provider to an ExecutorFactory and reports real usage', async () => { const provider: AgentEnvironmentProvider = { name: 'fake-provider', @@ -516,7 +592,7 @@ describe('environment provider adapters', () => { yield { type: 'result', data: { finalText: 'hello world' }, - usage: { inputTokens: 7, outputTokens: 11, cost: 0.03 }, + usage: { inputTokens: 7, outputTokens: 11, reasoningTokens: 5, cost: 0.03 }, } }, }) @@ -531,18 +607,319 @@ describe('environment provider adapters', () => { const artifact = executor.resultArtifact() expect(usage).toEqual([ - { kind: 'tokens', input: 7, output: 11 }, + { kind: 'tokens', input: 7, output: 16 }, { kind: 'cost', usd: 0.03 }, { kind: 'iteration' }, ]) expect(artifact.out).toMatchObject({ content: 'hello world' }) expect(artifact.spent).toMatchObject({ iterations: 1, - tokens: { input: 7, output: 11 }, + tokens: { input: 7, output: 16 }, usd: 0.03, }) }) + it('composes a fully profiled provider through the existing steerable session', async () => { + const firstTurnStreaming = deferred() + const finishFirstTurn = deferred() + const secondTurnStreaming = deferred() + const secondTurnCancelled = deferred() + const turns: AgentTurnInput[] = [] + let created: Parameters[0] | undefined + let cancellations = 0 + let destroyed = 0 + const session: AgentSession = { + id: 'provider-session', + async status() { + return 'running' + }, + async *events(): AsyncIterable {}, + async result() { + return { text: '', success: true } + }, + async prompt() { + return { text: '', success: true } + }, + async cancel() { + cancellations += 1 + secondTurnCancelled.resolve() + throw new Error('provider reported cancellation after stopping the turn') + }, + } + const provider: AgentEnvironmentProvider = { + name: 'session-provider', + capabilities: () => fakeCapabilities(), + async create(input) { + created = input + return fakeEnvironment({ + session(id) { + if (id !== session.id && id !== turns[0]?.sessionId) { + throw new Error(`unexpected session ${id}`) + } + return session + }, + async *stream(input): AsyncIterable { + turns.push(input) + if (turns.length === 1) { + yield { + type: 'message.part.updated', + data: { + part: { + type: 'tool', + tool: 'read', + callID: 'call-1', + state: { + status: 'completed', + input: { path: 'src/index.ts' }, + output: 'source', + }, + }, + }, + } + firstTurnStreaming.resolve() + await finishFirstTurn.promise + yield { + type: 'result', + data: { finalText: 'first answer' }, + usage: { + inputTokens: 2, + outputTokens: 3, + reasoningTokens: 4, + cacheReadInputTokens: 11, + cost: 0.1, + }, + } + return + } + if (turns.length === 2) { + secondTurnStreaming.resolve() + await secondTurnCancelled.promise + throw new Error('provider-specific interrupted turn') + } + yield { + type: 'usage', + data: {}, + usage: { + inputTokens: 5, + outputTokens: 7, + reasoningTokens: 6, + cacheCreationInputTokens: 2, + cost: 0.2, + }, + } + yield { type: 'result', data: { finalText: 'changed direction' } } + }, + async destroy() { + destroyed += 1 + }, + }) + }, + } + const profile = { + name: 'full-worker', + prompt: { + systemPrompt: 'Lead the investigation.', + instructions: ['Keep exact evidence.'], + }, + model: { + default: 'zai/glm-5.2', + reasoningEffort: 'high', + }, + harness: 'pi', + permissions: { shell: 'allow' }, + tools: { shell: true, web: true }, + mcp: { + papers: { transport: 'http', url: 'https://papers.example.test/mcp' }, + }, + subagents: { critic: { prompt: 'Find the strongest counterexample.' } }, + resources: { + instructions: 'Use the attached protocol.', + skills: [{ kind: 'inline', name: 'falsify', content: 'Try to disprove the claim.' }], + }, + hooks: { afterTool: [{ command: './capture-result' }] }, + modes: { adversarial: { prompt: 'Try the opposite mechanism.' } }, + metadata: { lineage: 'materials-v1' }, + extensions: { pi: { autoApprove: true } }, + } as unknown as AgentProfile + const factory = createExecutor({ + backend: 'provider', + provider, + defaults: { + workspace: { cwd: '/repo' }, + providerOptions: { region: 'us-west', tenancy: 'team-a' }, + }, + steering: {}, + }) + const spec: AgentSpec = { profile, harness: null } + const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} } + const executor = factory(spec, ctx) + + const running = collect( + executor.execute('investigate', ctx.signal) as AsyncIterable, + ) + await firstTurnStreaming.promise + expect(executor.progress?.()).toMatchObject({ + turns: 0, + pendingMessages: 0, + recentActivity: [ + { + kind: 'tool', + label: 'read', + status: 'ok', + detail: 'src/index.ts', + }, + ], + }) + executor.deliver?.({ steer: 'Test the opposite mechanism.', interrupt: false }) + finishFirstTurn.resolve() + await secondTurnStreaming.promise + executor.deliver?.({ steer: 'Stop and change direction.', interrupt: true }) + const usage = await running + + expect(executor.runtime).toBe('session-provider') + expect(created?.profile).toBe(profile) + expect(created).toMatchObject({ + backend: 'pi', + workspace: { cwd: '/repo' }, + signal: ctx.signal, + providerOptions: { + region: 'us-west', + tenancy: 'team-a', + sandboxCreateOptions: { backend: { type: 'pi' } }, + }, + }) + expect(usage).toEqual([ + { kind: 'tokens', input: 2, output: 7 }, + { kind: 'cost', usd: 0.1 }, + { kind: 'iteration' }, + { kind: 'tokens', input: 5, output: 13 }, + { kind: 'cost', usd: 0.2 }, + { kind: 'iteration' }, + ]) + expect(turns).toHaveLength(3) + expect(turns[0]).toMatchObject({ prompt: 'investigate' }) + expect(turns[1]?.prompt).toContain('Test the opposite mechanism.') + expect(turns[2]?.prompt).toContain('Stop and change direction.') + expect(new Set(turns.map((turn) => turn.sessionId)).size).toBe(1) + expect(turns[0]?.sessionId).toBeTruthy() + expect(cancellations).toBe(1) + expect(destroyed).toBe(1) + await expect(executor.traceSource?.()?.collect()).resolves.toMatchObject([ + { toolName: 'read', args: { path: 'src/index.ts' }, status: 'ok' }, + ]) + const artifact = executor.resultArtifact() + expect(artifact).toMatchObject({ + out: { + content: 'changed direction', + turns: 2, + toolCalls: ['read'], + }, + spent: { + iterations: 2, + tokens: { input: 7, output: 20 }, + }, + }) + expect(artifact.spent.usd).toBeCloseTo(0.3) + }) + + it('leaves backend selection to the provider when no caller or profile preference exists', async () => { + let created: Parameters[0] | undefined + const provider: AgentEnvironmentProvider = { + name: 'native-default-provider', + capabilities: () => fakeCapabilities(), + async create(input) { + created = input + return fakeEnvironment({ + session: (id) => ({ + id, + status: async () => 'running', + events: async function* (): AsyncIterable {}, + result: async () => ({ text: '', success: true }), + prompt: async () => ({ text: '', success: true }), + cancel: async () => {}, + }), + stream: async function* (): AsyncIterable { + yield { + type: 'vendor.tool.finished', + data: { opaque: true }, + normalized: { + type: 'message.part.updated', + part: { + id: 'tool-part', + sessionID: 'session', + messageID: 'message', + type: 'tool', + tool: 'read', + callID: 'normalized-call', + state: { + status: 'completed', + input: { path: 'README.md' }, + output: 'source', + }, + }, + }, + } + yield { + type: 'vendor.text.delta', + data: { opaque: true }, + normalized: { + type: 'message.part.updated', + part: { + id: 'text-part', + sessionID: 'session', + messageID: 'message', + type: 'text', + text: 'provider default', + }, + delta: 'provider default', + }, + } + yield { + type: 'vendor.turn.finished', + data: { opaque: true }, + normalized: { type: 'status', status: 'completed' }, + } + }, + }) + }, + } + const factory = createExecutor({ backend: 'provider', provider, steering: {} }) + const spec: AgentSpec = { + profile: { name: 'provider-default-worker' } as AgentProfile, + harness: null, + } + const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} } + const executor = factory(spec, ctx) + + await collect(executor.execute('task', ctx.signal) as AsyncIterable) + + expect(created).not.toHaveProperty('backend') + expect(created?.providerOptions?.sandboxCreateOptions).not.toHaveProperty('backend') + expect(executor.progress?.()).toMatchObject({ + recentActivity: [ + { + at: expect.any(Number), + kind: 'tool', + label: 'read', + status: 'ok', + detail: 'README.md', + }, + { + at: expect.any(Number), + kind: 'turn', + label: 'turn 0', + }, + ], + }) + await expect(executor.traceSource?.()?.collect()).resolves.toMatchObject([ + { toolName: 'read', args: { path: 'README.md' }, status: 'ok' }, + ]) + expect(executor.resultArtifact().out).toMatchObject({ + content: 'provider default', + toolCalls: ['read'], + }) + }) + it('plugs a provider into createExecutor as backend data', async () => { const provider: AgentEnvironmentProvider = { name: 'package-provider', @@ -651,3 +1028,11 @@ function fakeCapabilities() { confidential: true, } } + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} diff --git a/src/runtime/environment-provider.ts b/src/runtime/environment-provider.ts index 362c0bd8..4a0dc686 100644 --- a/src/runtime/environment-provider.ts +++ b/src/runtime/environment-provider.ts @@ -174,6 +174,8 @@ export function resolveAgentEnvironmentProvider( export interface ProviderAsSandboxClientOptions { defaults?: Partial requireTerminalEvent?: boolean + /** Require declared live continuation plus concrete session controls. */ + requireSession?: boolean mapCreateOptions?: ( options: CreateSandboxOptions | undefined, ) => Partial @@ -187,17 +189,40 @@ export function providerAsSandboxClient( ): SandboxClient { return { async create(createOptions?: CreateSandboxOptions): Promise { + const defaults = options.defaults ?? {} + const sandboxInput = createInputFromSandboxOptions(createOptions) + const customInput = options.mapCreateOptions?.(createOptions) ?? {} const mapped = { - ...(options.defaults ?? {}), - ...createInputFromSandboxOptions(createOptions), - ...(options.mapCreateOptions?.(createOptions) ?? {}), + ...defaults, + ...sandboxInput, + ...customInput, + providerOptions: { + ...(defaults.providerOptions ?? {}), + ...(sandboxInput.providerOptions ?? {}), + ...(customInput.providerOptions ?? {}), + }, } + if (mapped.backend === undefined) delete mapped.backend if (mapped.profile === undefined) { throw new ValidationError( `providerAsSandboxClient(${provider.name}): profile required in defaults or CreateSandboxOptions.backend.profile`, ) } + if (options.requireSession) { + const capabilities = await provider.capabilities() + if (!capabilities.streaming.live || !capabilities.sessions.continue) { + throw new ValidationError( + `providerAsSandboxClient(${provider.name}): live session continuation is required`, + ) + } + } const environment = await provider.create(mapped as CreateAgentEnvironmentInput) + if (options.requireSession && !environment.session) { + await environment.destroy?.() + throw new ValidationError( + `providerAsSandboxClient(${provider.name}): session() is required`, + ) + } return environmentAsSandboxInstance(environment, { requireTerminalEvent: options.requireTerminalEvent ?? true, }) @@ -420,14 +445,15 @@ function createInputFromSandboxOptions( ): Partial { const profile = options?.backend?.profile const backend = options?.backend?.type + const workspace = { + ...(options?.environment ? { environment: options.environment } : {}), + ...(options?.git?.url ? { repoUrl: options.git.url } : {}), + ...(options?.git?.ref ? { gitRef: options.git.ref } : {}), + } return { ...(profile !== undefined ? { profile } : {}), ...(backend ? { backend } : {}), - workspace: { - ...(options?.environment ? { environment: options.environment } : {}), - ...(options?.git?.url ? { repoUrl: options.git.url } : {}), - ...(options?.git?.ref ? { gitRef: options.git.ref } : {}), - }, + ...(Object.keys(workspace).length > 0 ? { workspace } : {}), ...(options?.resources ? { resources: options.resources as ResourceRequest } : {}), ...(options?.env ? { env: options.env } : {}), ...(options?.secrets ? { secrets: options.secrets } : {}), @@ -530,12 +556,65 @@ function environmentAsSandboxInstance( ): AsyncGenerator { let terminal = false const input = turnInputFromPrompt(message, promptOptions) - for await (const event of environment.stream(input)) { - if (isTerminalEnvironmentEvent(event)) terminal = true - const usageEvent = usageSandboxEvent(event) - if (usageEvent) yield usageEvent - yield sandboxEventFromEnvironmentEvent(event) + let cancellation: Promise | undefined + let cancellationStarted = false + let cancellationFailed = false + let cancellationError: unknown + const cancel = () => { + if (cancellationStarted || !input.sessionId || !environment.session) return + cancellationStarted = true + try { + cancellation = environment + .session(input.sessionId) + .cancel() + .catch((error: unknown) => { + cancellationFailed = true + cancellationError = error + }) + } catch (error) { + cancellationFailed = true + cancellationError = error + } } + input.signal?.addEventListener('abort', cancel, { once: true }) + if (input.signal?.aborted) cancel() + let streamFailed = false + let streamError: unknown + try { + for await (const event of environment.stream(input)) { + if (isTerminalEnvironmentEvent(event)) terminal = true + const usageEvent = usageSandboxEvent(event) + if (usageEvent) yield usageEvent + yield sandboxEventFromEnvironmentEvent(event) + } + } catch (error) { + streamFailed = true + streamError = error + } finally { + input.signal?.removeEventListener('abort', cancel) + if (input.signal?.aborted) { + cancel() + await cancellation + } + } + if (input.signal?.aborted) { + const abortError = new DOMException('Provider session stream aborted', 'AbortError') + const causes = [ + ...(streamFailed ? [streamError] : []), + ...(cancellationFailed ? [cancellationError] : []), + ] + if (causes.length > 0) { + Object.defineProperty(abortError, 'cause', { + value: + causes.length === 1 + ? causes[0] + : new AggregateError(causes, 'Provider stream and cancellation failed'), + }) + } + throw abortError + } + if (streamFailed) throw streamError + if (cancellationFailed) throw cancellationError if (options.requireTerminalEvent && !terminal) { throw new ValidationError( `providerAsSandboxClient(${environment.provider}): stream ended without a terminal result/done/status event`, @@ -822,18 +901,42 @@ function environmentEventFromSandboxEvent(event: SandboxEvent): AgentEnvironment } function sandboxEventFromEnvironmentEvent(event: AgentEnvironmentEvent): SandboxEvent { + const normalized = event.normalized + const type = normalized?.type ?? event.type + const baseData = normalized ? sandboxDataFromNormalizedEvent(event.data, normalized) : event.data + const usage = event.usage ? tokenUsageData(event.usage) : undefined + const data = (() => { + if (!usage) return baseData + if (isUsageType(type)) + return { + ...baseData, + tokensIn: event.usage?.inputTokens, + tokensOut: event.usage?.outputTokens, + ...(event.usage?.cost !== undefined ? { costUsd: event.usage.cost } : {}), + ...usage, + } + if (isNestedUsageType(type)) return { ...baseData, usage } + if (type === 'done') { + return { + ...baseData, + tokenUsage: usage, + ...(usage.totalCostUsd !== undefined ? { totalCostUsd: usage.totalCostUsd } : {}), + } + } + return baseData + })() return { - type: event.type, - data: { - ...event.data, - ...(event.usage ? { usage: tokenUsageData(event.usage) } : {}), - }, + type, + data, ...(event.id ? { id: event.id } : {}), } } function usageSandboxEvent(event: AgentEnvironmentEvent): SandboxEvent | undefined { - if (!event.usage || isUsageType(event.type)) return undefined + const type = event.normalized?.type ?? event.type + if (!event.usage || isUsageType(type) || isNestedUsageType(type) || type === 'done') { + return undefined + } const usage = tokenUsageData(event.usage) if ( usage.inputTokens === undefined && @@ -845,11 +948,33 @@ function usageSandboxEvent(event: AgentEnvironmentEvent): SandboxEvent | undefin return { type: 'llm_call', data: usage } } -function tokenUsageData(usage: TokenUsage): Record { +function sandboxDataFromNormalizedEvent( + rawData: Record, + normalized: NonNullable, +): Record { + const { type: _type, ...normalizedData } = normalized + return { + ...rawData, + ...normalizedData, + ...(normalized.type === 'message.part.updated' && normalized.part.type === 'text' + ? { text: normalized.part.text } + : {}), + } +} + +function tokenUsageData(usage: TokenUsage): Record { return { inputTokens: usage.inputTokens, outputTokens: usage.outputTokens, - totalCostUsd: usage.cost, + ...(usage.totalTokens !== undefined ? { totalTokens: usage.totalTokens } : {}), + ...(usage.cacheReadInputTokens !== undefined + ? { cacheReadInputTokens: usage.cacheReadInputTokens } + : {}), + ...(usage.cacheCreationInputTokens !== undefined + ? { cacheCreationInputTokens: usage.cacheCreationInputTokens } + : {}), + ...(usage.reasoningTokens !== undefined ? { reasoningTokens: usage.reasoningTokens } : {}), + ...(usage.cost !== undefined ? { totalCostUsd: usage.cost } : {}), } } @@ -959,19 +1084,29 @@ function resultTextFromData(data: Record): string | undefined { } function isTerminalEnvironmentEvent(event: AgentEnvironmentEvent): boolean { - if (event.type === 'result' || event.type === 'done' || event.type === 'final') return true - if (event.type.endsWith('.completed') || event.type.endsWith('.failed')) return true - if (event.type === 'status') { - const status = event.data.status - return status === 'completed' || status === 'failed' || status === 'cancelled' - } - return false + if (isTerminalEventShape(event.type, event.data)) return true + const normalized = event.normalized + return ( + normalized?.type === 'status' && + (normalized.status === 'completed' || normalized.status === 'failed') + ) +} + +function isTerminalEventShape(type: string, data: Record): boolean { + if (type === 'result' || type === 'done' || type === 'final') return true + if (type.endsWith('.completed') || type.endsWith('.failed')) return true + if (type !== 'status') return false + return data.status === 'completed' || data.status === 'failed' || data.status === 'cancelled' } function isUsageType(type: string): boolean { return type === 'llm_call' || type === 'usage' || type === 'cost.usage' } +function isNestedUsageType(type: string): boolean { + return type === 'message.completed' || type === 'result' || type === 'final' +} + function usageFromEnvironmentEvent(event: AgentEnvironmentEvent): { input: number output: number @@ -980,7 +1115,7 @@ function usageFromEnvironmentEvent(event: AgentEnvironmentEvent): { const usage = event.usage ?? tokenUsageFromData(event.data) return { input: finiteNumber(usage?.inputTokens) ?? 0, - output: finiteNumber(usage?.outputTokens) ?? 0, + output: (finiteNumber(usage?.outputTokens) ?? 0) + (finiteNumber(usage?.reasoningTokens) ?? 0), usd: finiteNumber(usage?.cost) ?? finiteNumber(event.data.costUsd) ?? @@ -1004,17 +1139,33 @@ function tokenUsageFromData(data: Record): TokenUsage | undefin finiteNumber(usageRecord.outputTokens) ?? finiteNumber(usageRecord.tokensOut) ?? finiteNumber(usageRecord.completion_tokens) + const totalTokens = finiteNumber(usageRecord.totalTokens) + const cacheReadInputTokens = finiteNumber(usageRecord.cacheReadInputTokens) + const cacheCreationInputTokens = finiteNumber(usageRecord.cacheCreationInputTokens) + const reasoningTokens = finiteNumber(usageRecord.reasoningTokens) const cost = finiteNumber(usageRecord.cost) ?? finiteNumber(usageRecord.costUsd) ?? finiteNumber(usageRecord.totalCostUsd) ?? finiteNumber(data.costUsd) ?? finiteNumber(data.totalCostUsd) - if (inputTokens === undefined && outputTokens === undefined && cost === undefined) + if ( + inputTokens === undefined && + outputTokens === undefined && + totalTokens === undefined && + cacheReadInputTokens === undefined && + cacheCreationInputTokens === undefined && + reasoningTokens === undefined && + cost === undefined + ) return undefined return { inputTokens: inputTokens ?? 0, outputTokens: outputTokens ?? 0, + ...(totalTokens !== undefined ? { totalTokens } : {}), + ...(cacheReadInputTokens !== undefined ? { cacheReadInputTokens } : {}), + ...(cacheCreationInputTokens !== undefined ? { cacheCreationInputTokens } : {}), + ...(reasoningTokens !== undefined ? { reasoningTokens } : {}), ...(cost !== undefined ? { cost } : {}), } } @@ -1028,6 +1179,24 @@ function mergeTokenUsage( return { inputTokens: left.inputTokens + right.inputTokens, outputTokens: left.outputTokens + right.outputTokens, + ...(left.totalTokens !== undefined || right.totalTokens !== undefined + ? { totalTokens: (left.totalTokens ?? 0) + (right.totalTokens ?? 0) } + : {}), + ...(left.cacheReadInputTokens !== undefined || right.cacheReadInputTokens !== undefined + ? { + cacheReadInputTokens: + (left.cacheReadInputTokens ?? 0) + (right.cacheReadInputTokens ?? 0), + } + : {}), + ...(left.cacheCreationInputTokens !== undefined || right.cacheCreationInputTokens !== undefined + ? { + cacheCreationInputTokens: + (left.cacheCreationInputTokens ?? 0) + (right.cacheCreationInputTokens ?? 0), + } + : {}), + ...(left.reasoningTokens !== undefined || right.reasoningTokens !== undefined + ? { reasoningTokens: (left.reasoningTokens ?? 0) + (right.reasoningTokens ?? 0) } + : {}), ...(left.cost !== undefined || right.cost !== undefined ? { cost: (left.cost ?? 0) + (right.cost ?? 0) } : {}), diff --git a/src/runtime/sandbox-events.ts b/src/runtime/sandbox-events.ts index 67f67bf7..b2305d62 100644 --- a/src/runtime/sandbox-events.ts +++ b/src/runtime/sandbox-events.ts @@ -159,7 +159,12 @@ function buildLlmCall( agentRunName: string, ): (RuntimeStreamEvent & { type: 'llm_call' }) | undefined { const tokensIn = pickFiniteNumber(data, ['tokensIn', 'inputTokens', 'prompt_tokens']) - const tokensOut = pickFiniteNumber(data, ['tokensOut', 'outputTokens', 'completion_tokens']) + const outputTokens = pickFiniteNumber(data, ['tokensOut', 'outputTokens', 'completion_tokens']) + const reasoningTokens = pickFiniteNumber(data, ['reasoningTokens']) + const tokensOut = + outputTokens !== undefined || reasoningTokens !== undefined + ? (outputTokens ?? 0) + (reasoningTokens ?? 0) + : undefined const costUsd = pickFiniteNumber(data, ['costUsd', 'totalCostUsd', 'cost_usd', 'cost']) if (tokensIn === undefined && tokensOut === undefined && costUsd === undefined) { return undefined diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index 95fb0998..82b746f0 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -50,6 +50,7 @@ import { type AgentEnvironmentProviderRegistry, type ProviderExecutorOptions, providerAsExecutor, + providerAsSandboxClient, resolveAgentEnvironmentProvider, } from '../environment-provider' import { routerChatWithUsage, type ToolSpec } from '../router-client' @@ -217,6 +218,11 @@ export interface BridgeSeam { export interface ProviderSeam extends ProviderExecutorOptions { provider: AgentEnvironmentProvider | string registry?: AgentEnvironmentProviderRegistry + /** + * Compose the provider through the existing steerable sandbox session. + * The provider still owns environment creation and session semantics. + */ + steering?: SandboxSteeringOptions } const routerSeamKey = 'router' @@ -1635,6 +1641,55 @@ export function createExecutor(config: ExecutorConfig): ExecutorFactory providerSeam.provider, providerSeam.registry, ) + if (providerSeam.steering) { + if (providerSeam.taskToTurn) { + throw new ValidationError( + 'createExecutor(provider, steering): taskToTurn is not representable by the text-only steerable session', + ) + } + if (providerSeam.destroyOnSettle === false) { + throw new ValidationError( + 'createExecutor(provider, steering): destroyOnSettle=false conflicts with the session-owned environment lifecycle', + ) + } + const providerBackend = selectedProviderBackend(spec, providerSeam) + const sandboxClient = providerAsSandboxClient(provider, { + defaults: { + ...(providerSeam.defaults ?? {}), + signal: seamed.signal, + }, + requireTerminalEvent: providerSeam.requireTerminalEvent, + requireSession: true, + ...(providerBackend === undefined + ? { + mapCreateOptions: (createOptions) => { + const { backend: _internalBackend, ...sandboxCreateOptions } = + createOptions ?? {} + return { + backend: undefined, + providerOptions: { sandboxCreateOptions }, + } + }, + } + : {}), + }) + const harness = (providerBackend ?? 'opencode') as BackendType + const providerCtx: ExecutorContext = { + ...seamed, + seams: { + ...seamed.seams, + [sandboxSeamKey]: { + sandboxClient, + steering: providerSeam.steering, + } satisfies SandboxSeam, + }, + } + const executor = sandboxExecutor({ ...spec, harness }, providerCtx) + return { + ...executor, + runtime: providerSeam.runtime ?? (provider.name as Runtime), + } + } return providerAsExecutor(provider, providerSeam)(spec, seamed) } case 'sandbox': { @@ -1647,6 +1702,16 @@ export function createExecutor(config: ExecutorConfig): ExecutorFactory } } +function selectedProviderBackend(spec: AgentSpec, seam: ProviderSeam): string | undefined { + const profile = spec.profile as AgentSpec['profile'] & { harness?: unknown } + return ( + spec.harness ?? + seam.defaults?.backend ?? + (typeof profile.harness === 'string' ? profile.harness : undefined) ?? + (typeof profile.metadata?.backendType === 'string' ? profile.metadata.backendType : undefined) + ) +} + // ── The open registry ────────────────────────────────────────────────────────── /** diff --git a/tests/kernel/sandbox-events.test.ts b/tests/kernel/sandbox-events.test.ts index e1eef07f..60f1ed1f 100644 --- a/tests/kernel/sandbox-events.test.ts +++ b/tests/kernel/sandbox-events.test.ts @@ -120,6 +120,24 @@ describe('extractLlmCallEvent — strict numeric coercion', () => { ).toEqual({ type: 'llm_call', model: 'agent', tokensIn: 100, tokensOut: 20 }) }) + it.each([ + { + type: 'result', + data: { usage: { inputTokens: 2, outputTokens: 3, reasoningTokens: 7 } }, + }, + { + type: 'usage', + data: { inputTokens: 2, outputTokens: 3, reasoningTokens: 7 }, + }, + ] as const)('folds separate reasoning tokens into output for $type events', (event) => { + expect(extractLlmCallEvent(event, 'agent')).toEqual({ + type: 'llm_call', + model: 'agent', + tokensIn: 2, + tokensOut: 10, + }) + }) + // Regression: sandbox 0.4.0's terminal `done` event carries usage under // `tokenUsage` (not `usage`) with cost top-level — without this the in-process // loopDispatch ledger read {0,0} and the backend-integrity guard misreported a