diff --git a/.changeset/restore-turn-model-fallback.md b/.changeset/restore-turn-model-fallback.md new file mode 100644 index 000000000..088dc9ca9 --- /dev/null +++ b/.changeset/restore-turn-model-fallback.md @@ -0,0 +1,5 @@ +--- +"@pymodel/pythinker-code": patch +--- + +A session whose model was deleted or lost its provider switches to the best available model at the next turn, with a warning, instead of failing the turn. diff --git a/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts b/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts index 178248543..dd15f5a9b 100644 --- a/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts +++ b/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts @@ -709,7 +709,7 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService { let snapshot = this.turnConfigs.get(turnId); if (snapshot === undefined) { snapshot = { - resolved: this.profile.resolveModelContext(), + resolved: this.profile.resolveModelContext(turnId), params: this.profile.resolveRequestParams(), systemPrompt: this.profile.getSystemPrompt(), }; diff --git a/packages/agent-core-v2/src/agent/profile/profileOps.ts b/packages/agent-core-v2/src/agent/profile/profileOps.ts index ef2246283..1fab0559e 100644 --- a/packages/agent-core-v2/src/agent/profile/profileOps.ts +++ b/packages/agent-core-v2/src/agent/profile/profileOps.ts @@ -3,7 +3,7 @@ import { nothing, original } from 'immer'; import { z } from 'zod'; import type { EnvironmentDisclosureSnapshot } from '#/app/agentProfileCatalog/agentProfileCatalog'; -import { AgentEvent2 } from '#/app/event/event2'; +import { AgentEvent2, Event2 } from '#/app/event/event2'; import type { ThinkingEffort } from '#human/llm/thinking'; import { defineState } from '#/state/state'; @@ -127,6 +127,18 @@ export class WarningIssued extends AgentEvent2 { } export interface WarningIssued extends WarningIssuedPayload {} +export interface ModelFallbackSwitchedPayload { + readonly turnId?: number; + readonly fromModel: string; + readonly toModel: string; +} + +export class ModelFallbackSwitched extends Event2 { + static override readonly type = 'turn.model_fallback.switched'; + static override readonly observable = true; +} +export interface ModelFallbackSwitched extends ModelFallbackSwitchedPayload {} + export const profileKey = defineState( 'profile', (): ProfileModelState => ({ diff --git a/packages/agent-core-v2/src/agent/profile/profileService.ts b/packages/agent-core-v2/src/agent/profile/profileService.ts index 887907d30..79332bf19 100644 --- a/packages/agent-core-v2/src/agent/profile/profileService.ts +++ b/packages/agent-core-v2/src/agent/profile/profileService.ts @@ -8,6 +8,10 @@ import { IModelCatalog, type Model } from '#/llm-adapter/model/catalog'; import { type ModelOverrides } from '#/llm-adapter/model/model.types'; import { type ModelRequestParams, type SamplingOptions } from '#/llm-adapter/model/model-requester'; import { IProtocolAdapterRegistry } from '#/llm-adapter/protocol/protocol'; +import { IModelService } from '#/llm-adapter/model/model'; +import { rankDefaultModelCandidates } from '#/llm-adapter/model/default-model-policy'; +import { resolveModelForReady } from '#/llm-adapter/model/model-auth'; +import { IProviderService } from '#/llm-adapter/provider/provider'; import { drivesThinkingThroughTraits, modelSupportsThinkingEffort, @@ -74,6 +78,7 @@ import { profileKey, ToolsResetActiveTools, ToolsSetActiveTools, + ModelFallbackSwitched, WarningIssued, type ActiveToolsState, type ConfigUpdatePayload, @@ -162,6 +167,8 @@ export class AgentProfileService extends Disposable implements IAgentProfileServ @IPluginService private readonly plugins: IPluginService, @IAgentIdentity private readonly identity: IAgentIdentity, @IAgentAgentsMdReminderService private readonly agentsMdReminder: IAgentAgentsMdReminderService, + @IModelService private readonly models: IModelService, + @IProviderService private readonly providers: IProviderService, ) { super(); this.states.contributeState(profileKey); @@ -442,7 +449,8 @@ export class AgentProfileService extends Disposable implements IAgentProfileServ return this.resolveThinkingState(this.tryResolveRawModel()).effective; } - resolveModelContext(): ProfileModelContext { + resolveModelContext(turnId?: number): ProfileModelContext { + if (turnId !== undefined) this.ensureResolvableModel(turnId); const modelAlias = this.model; const model = this.modelCatalog.get(modelAlias); const loopControl = this.config.get('loopControl'); @@ -458,6 +466,36 @@ export class AgentProfileService extends Disposable implements IAgentProfileServ }; } + private ensureResolvableModel(turnId: number): void { + const alias = this.modelAlias; + if (alias === undefined) return; + const models = this.models.list(); + const providers = this.providers.list(); + const defaultProvider = this.providers.getDefaultProvider(); + const isReady = (id: string): boolean => + resolveModelForReady(id, models, providers, defaultProvider).resolved; + const resolution = resolveModelForReady(alias, models, providers, defaultProvider); + if (resolution.resolved) return; + const resolvedTo = rankDefaultModelCandidates(models).find(isReady); + if (resolvedTo === undefined || resolvedTo === alias) return; + this.update({ modelAlias: resolvedTo }); + this.telemetry.track2('model_fallback_triggered', { + turn_id: turnId, + from_model: alias, + to_model: resolvedTo, + }); + void this.dispatcher.dispatch( + new ModelFallbackSwitched({ turnId, fromModel: alias, toModel: resolvedTo }), + ); + void this.dispatcher.dispatch( + new WarningIssued({ + agentId: this.scopeContext.agentId, + code: 'model-fallback', + message: `Model "${alias}" is no longer available (${resolution.reason}); switched to "${resolvedTo}".`, + }), + ); + } + resolveRequestParams(): ModelRequestParams { const model = this.tryResolveRawModel(); const thinking = this.resolveThinkingState(model); diff --git a/packages/agent-core-v2/src/app/telemetry/events.ts b/packages/agent-core-v2/src/app/telemetry/events.ts index d428eccf5..df31d32a0 100644 --- a/packages/agent-core-v2/src/app/telemetry/events.ts +++ b/packages/agent-core-v2/src/app/telemetry/events.ts @@ -85,6 +85,12 @@ export interface TurnEndedEvent { enabled_plugins?: string; } +export interface ModelFallbackEvent { + turn_id: number; + from_model: string; + to_model: string; +} + export interface PromptCacheProbeEvent { source: 'fork'; turn_id: number; @@ -652,6 +658,16 @@ export const telemetryEventDefinitions = { 'Comma-separated sorted ids of enabled, loaded plugins when the turn ends; empty string for a known empty set, absent when no plugin snapshot is available', }, }), + model_fallback_triggered: defineAgentTelemetryEvent({ + owner: 'pythinker-code', + comment: + 'At turn start the bound model no longer resolved, so the agent switched to the best-ranked ready model.', + properties: { + turn_id: 'Per-agent turn index (main or subagent); pair with agent_id to locate a turn within a session', + from_model: 'Bound model alias that no longer resolved', + to_model: 'Ready model alias the agent switched to', + }, + }), prompt_cache_probe: defineAgentTelemetryEvent({ owner: 'pythinker-code', comment: diff --git a/packages/agent-core-v2/src/llm-adapter/model/default-model-policy.ts b/packages/agent-core-v2/src/llm-adapter/model/default-model-policy.ts new file mode 100644 index 000000000..cb5a80f02 --- /dev/null +++ b/packages/agent-core-v2/src/llm-adapter/model/default-model-policy.ts @@ -0,0 +1,56 @@ +import type { ModelRecord, ModelsSection } from './model'; + +const TOOL_USE_CAPABILITY = 'tool_use'; + +export function rankDefaultModelCandidates(models: ModelsSection): string[] { + return Object.entries(models) + .filter(([, record]) => isEligibleDefaultModel(record)) + .map(([id, record]) => ({ + id, + declaresToolUse: declaresToolUse(record), + context: effectiveContextSize(record) ?? 0, + })) + .toSorted((a, b) => { + if (a.declaresToolUse !== b.declaresToolUse) return a.declaresToolUse ? -1 : 1; + if (a.context !== b.context) return b.context - a.context; + const aParts = splitModelId(a.id); + const bParts = splitModelId(b.id); + if (aParts.providerId !== bParts.providerId) { + return aParts.providerId.localeCompare(bParts.providerId); + } + const byModelPart = bParts.modelPart.localeCompare(aParts.modelPart, 'en', { + numeric: true, + }); + if (byModelPart !== 0) return byModelPart; + return a.id.localeCompare(b.id); + }) + .map((candidate) => candidate.id); +} + +function isEligibleDefaultModel(record: ModelRecord): boolean { + const context = effectiveContextSize(record); + if (context !== undefined && context <= 0) return false; + const capabilities = effectiveCapabilities(record); + if (capabilities === undefined || capabilities.length === 0) return true; + return capabilities.some((entry) => entry.trim().toLowerCase() === TOOL_USE_CAPABILITY); +} + +function splitModelId(id: string): { readonly providerId: string; readonly modelPart: string } { + const slash = id.indexOf('/'); + if (slash === -1) return { providerId: '', modelPart: id }; + return { providerId: id.slice(0, slash), modelPart: id.slice(slash + 1) }; +} + +function effectiveCapabilities(record: ModelRecord): readonly string[] | undefined { + return record.overrides?.capabilities ?? record.capabilities; +} + +function effectiveContextSize(record: ModelRecord): number | undefined { + return record.overrides?.maxContextSize ?? record.maxContextSize; +} + +function declaresToolUse(record: ModelRecord): boolean { + const capabilities = effectiveCapabilities(record); + if (capabilities === undefined) return false; + return capabilities.some((entry) => entry.trim().toLowerCase() === TOOL_USE_CAPABILITY); +} diff --git a/packages/agent-core-v2/test/agent/profile/modelFallback.test.ts b/packages/agent-core-v2/test/agent/profile/modelFallback.test.ts new file mode 100644 index 000000000..6f993b6c2 --- /dev/null +++ b/packages/agent-core-v2/test/agent/profile/modelFallback.test.ts @@ -0,0 +1,157 @@ +import { emptyUsage } from '#human/llm/usage'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { IEventBus } from '#/app/event/eventBus'; +import { IModelService } from '#/llm-adapter/model/model'; +import { IAgentLLMRequesterService } from '#/agent/llmRequester/llmRequester'; +import { IAgentProfileService } from '#/agent/profile/profile'; +import { ModelFallbackSwitched, WarningIssued } from '#/agent/profile/profileOps'; + +import { recordingTelemetry, type TelemetryRecord } from '../../app/telemetry/stubs'; +import { + createTestAgent, + llmGenerateServices, + requesterFromGenerateFn, + telemetryServices, + type TestAgentContext, +} from '../../harness'; + +function okGenerate() { + let calls = 0; + return { + services: llmGenerateServices( + requesterFromGenerateFn(async () => { + calls += 1; + return { + id: `mf-${String(calls)}`, + message: { + role: 'assistant' as const, + content: [{ type: 'text' as const, text: 'ok' }], + toolCalls: [], + }, + usage: emptyUsage(), + finishReason: 'completed' as const, + rawFinishReason: 'stop', + }; + }), + ), + calls: () => calls, + }; +} + +describe('turn-start model fallback', () => { + let ctx: TestAgentContext; + + afterEach(async () => { + try { + await ctx.expectResumeMatches(); + } finally { + await ctx.dispose(); + } + }); + + function requestTurn(turnId: number): Promise { + return ctx.get(IAgentLLMRequesterService).request({ source: { type: 'turn', turnId, step: 1 } }); + } + + function captureSwitches(): ModelFallbackSwitched[] { + const switched: ModelFallbackSwitched[] = []; + ctx.get(IEventBus).subscribe(ModelFallbackSwitched, (event) => switched.push(event)); + return switched; + } + + it('switches a dead bound model to the best-ranked ready model', async () => { + const generate = okGenerate(); + const records: TelemetryRecord[] = []; + ctx = createTestAgent(generate.services, telemetryServices(recordingTelemetry(records))); + const switched = captureSwitches(); + const notices: WarningIssued[] = []; + ctx.get(IEventBus).subscribe(WarningIssued, (event) => notices.push(event)); + ctx.get(IAgentProfileService).update({ modelAlias: 'dead-model' }); + + await requestTurn(1); + + expect(generate.calls()).toBe(1); + expect(ctx.get(IAgentProfileService).getModel()).toBe('mock-model'); + expect(switched).toHaveLength(1); + expect(switched[0]).toMatchObject({ turnId: 1, fromModel: 'dead-model', toModel: 'mock-model' }); + expect(notices).toHaveLength(1); + expect(notices[0]?.code).toBe('model-fallback'); + expect(notices[0]?.message).toContain('dead-model'); + expect(notices[0]?.message).toContain('mock-model'); + const fallbackEvents = records.filter((record) => record.event === 'model_fallback_triggered'); + expect(fallbackEvents).toHaveLength(1); + expect(fallbackEvents[0]?.properties).toMatchObject({ + turn_id: 1, + from_model: 'dead-model', + to_model: 'mock-model', + }); + }); + + it('switches a model whose provider was deleted', async () => { + ctx = createTestAgent(okGenerate().services, { + initialConfig: { + models: { + 'orphan-model': { provider: 'no-such-provider', model: 'orphan', maxContextSize: 1_000 }, + }, + }, + }); + const switched = captureSwitches(); + ctx.get(IAgentProfileService).update({ modelAlias: 'orphan-model' }); + + await requestTurn(1); + + expect(ctx.get(IAgentProfileService).getModel()).toBe('mock-model'); + expect(switched).toHaveLength(1); + expect(switched[0]).toMatchObject({ fromModel: 'orphan-model', toModel: 'mock-model' }); + }); + + it('skips an unready candidate that would win the ranking', async () => { + ctx = createTestAgent(okGenerate().services, { + initialConfig: { + models: { + 'zombie-model': { provider: 'no-such-provider', model: 'zombie', maxContextSize: 100_000_000 }, + }, + }, + }); + const switched = captureSwitches(); + ctx.get(IAgentProfileService).update({ modelAlias: 'dead-model' }); + + await requestTurn(1); + + expect(ctx.get(IAgentProfileService).getModel()).toBe('mock-model'); + expect(switched).toHaveLength(1); + expect(switched[0]).toMatchObject({ fromModel: 'dead-model', toModel: 'mock-model' }); + }); + + it('keeps the dead alias when no candidate is ready', async () => { + ctx = createTestAgent(okGenerate().services, { + initialConfig: { + models: { + 'aaa-model': { provider: 'no-such-provider', model: 'aaa', maxContextSize: 5_000 }, + 'orphan-model': { provider: 'no-such-provider', model: 'orphan', maxContextSize: 1_000 }, + }, + }, + }); + const switched = captureSwitches(); + await ctx.get(IModelService).delete('mock-model'); + ctx.get(IAgentProfileService).update({ modelAlias: 'orphan-model' }); + + await requestTurn(1).catch(() => undefined); + + expect(ctx.get(IAgentProfileService).getModel()).toBe('orphan-model'); + expect(switched).toHaveLength(0); + }); + + it('leaves a healthy bound model untouched', async () => { + const records: TelemetryRecord[] = []; + ctx = createTestAgent(okGenerate().services, telemetryServices(recordingTelemetry(records))); + const switched = captureSwitches(); + + await requestTurn(1); + + expect(ctx.get(IAgentProfileService).getModel()).toBe('mock-model'); + expect(switched).toHaveLength(0); + expect(records.filter((record) => record.event === 'model_fallback_triggered')).toHaveLength(0); + }); +}); diff --git a/packages/agent-core-v2/test/session/agentLifecycle/agentLifecycle.test.ts b/packages/agent-core-v2/test/session/agentLifecycle/agentLifecycle.test.ts index 9dd570207..338d5a6a1 100644 --- a/packages/agent-core-v2/test/session/agentLifecycle/agentLifecycle.test.ts +++ b/packages/agent-core-v2/test/session/agentLifecycle/agentLifecycle.test.ts @@ -36,6 +36,8 @@ import { agentContextOf, IAgentScopeContext } from '#/agent/scopeContext/scopeCo import { IAgentIdentity } from '#/app/agentIdentity/agentIdentity'; import { IBuiltinAgentProfileLoader } from '#/app/agentProfileCatalog/builtinAgentProfileLoader'; import { IModelCatalog } from '#/llm-adapter/model/catalog'; +import { IModelService } from '#/llm-adapter/model/model'; +import { IProviderService } from '#/llm-adapter/provider/provider'; import type { ToolCall } from '#human/llm/message'; import { IProtocolAdapterRegistry } from '#/llm-adapter/protocol/protocol'; import { IHostClock } from '#/os/interface/hostClock'; @@ -325,6 +327,11 @@ describe('AgentLifecycleService', () => { cwd: '/tmp/pythinker-agentLifecycle-home', getEnv: () => undefined, } as unknown as IBootstrapService); + ix.stub(IModelService, { list: () => ({}) } as unknown as IModelService); + ix.stub(IProviderService, { + list: () => ({}), + getDefaultProvider: () => undefined, + } as unknown as IProviderService); ix.stub(IFlagService, { _serviceBrand: undefined, enabled: () => false, diff --git a/packages/agent-core-v2/test/session/subagent/subagentScopeCache.test.ts b/packages/agent-core-v2/test/session/subagent/subagentScopeCache.test.ts index 17f313db5..47a1a48f4 100644 --- a/packages/agent-core-v2/test/session/subagent/subagentScopeCache.test.ts +++ b/packages/agent-core-v2/test/session/subagent/subagentScopeCache.test.ts @@ -52,6 +52,8 @@ import { AgentReminderService, IAgentReminderService } from '#/features/reminder import { ISessionSkillCatalog } from '#/features/skill/session/skillCatalog'; import { SubagentSuspended } from '#/features/dynamic_workflow/session/sessionDynamicWorkflowService'; import { IModelCatalog } from '#/llm-adapter/model/catalog'; +import { IModelService } from '#/llm-adapter/model/model'; +import { IProviderService } from '#/llm-adapter/provider/provider'; import { IProtocolAdapterRegistry } from '#/llm-adapter/protocol/protocol'; import { McpConnectionManager } from '#/mcpCore/connection-manager'; import { McpOAuthService } from '#/mcpCore/oauth/service'; @@ -392,6 +394,11 @@ describe('SessionSubagentScopeCacheService', () => { cwd: '/tmp/pythinker-subagentScopeCache-home', getEnv: () => undefined, } as unknown as IBootstrapService); + ix.stub(IModelService, { list: () => ({}) } as unknown as IModelService); + ix.stub(IProviderService, { + list: () => ({}), + getDefaultProvider: () => undefined, + } as unknown as IProviderService); ix.stub(IFlagService, { _serviceBrand: undefined, enabled: () => false,