diff --git a/apps/desktop/src/main/cindy-brain/__tests__/cindySlot.test.ts b/apps/desktop/src/main/cindy-brain/__tests__/cindySlot.test.ts index 99857220b9..1be490d421 100644 --- a/apps/desktop/src/main/cindy-brain/__tests__/cindySlot.test.ts +++ b/apps/desktop/src/main/cindy-brain/__tests__/cindySlot.test.ts @@ -748,6 +748,25 @@ describe('目录空清单 = 能力暂不可用', () => { }); describe('意识专属后端覆盖(解析表第②层)', () => { + it('新版 Provider-aware 覆盖保留精确来源到最终派发', async () => { + const { slot, generateImage } = makeSlot({ + getMediaOverride: vi.fn(() => ({ + modelId: 'gpt-image-2', + providerId: 'openai', + label: 'GPT Image 2', + })), + }); + + const result = await slot.handleModelRequest('art', REQ); + + expect(result).toMatchObject({ ok: true, model: 'gpt-image-2', modelLabel: 'GPT Image 2' }); + expect(generateImage).toHaveBeenCalledWith({ + prompt: '一只猫', + model: 'gpt-image-2', + providerId: 'openai', + }); + }); + it('覆盖压过档位;调用显式点名仍压过覆盖;下架型号的覆盖静默落回', async () => { const pinned = makeSlot({ getOverride: vi.fn(() => 'gemini-3-pro-image') as unknown as CindySlotDeps['getOverride'], diff --git a/apps/desktop/src/main/cindy-brain/cindySlot.ts b/apps/desktop/src/main/cindy-brain/cindySlot.ts index 3e0b3977fb..9cc57ba773 100644 --- a/apps/desktop/src/main/cindy-brain/cindySlot.ts +++ b/apps/desktop/src/main/cindy-brain/cindySlot.ts @@ -136,6 +136,12 @@ export interface CindyImageCapabilities { maxEditImages?: number; } +export interface CindyMediaOverrideSelection { + modelId: string; + providerId: string; + label?: string; +} + export interface CindySlotDeps { getGhost(id: string): InstalledGhost | null; /** 当前账号作用域;跨 await 的媒体任务必须捕获并持续复核。 */ @@ -147,6 +153,7 @@ export interface CindySlotDeps { generateImage(params: { prompt: string; model: string; + providerId?: string; aspectRatio?: GhostImageAspectRatio; }): Promise<{ buffer: Uint8Array; mimeType: string }>; /** 主机统一图片通道·改图;源图以磁盘路径喂给网关(意识摸不到路径)。 @@ -154,6 +161,7 @@ export interface CindySlotDeps { editImage(params: { prompt: string; model: string; + providerId?: string; imagePaths: string[]; aspectRatio?: GhostImageAspectRatio; }): Promise<{ buffer: Uint8Array; mimeType: string }>; @@ -161,7 +169,7 @@ export interface CindySlotDeps { * 该图像型号的 provider 实际能力。缺席/查无 = 只执行通用 1–4 图粗筛; * provider 上限更低时,slot 在读源图与出网前给出型号级明确拒绝。 */ - imageCapabilities?(model: string): CindyImageCapabilities | null; + imageCapabilities?(model: string, providerId?: string): CindyImageCapabilities | null; /** * 主机统一视频通道·文生视频(art 视频 provider 层复用,submit→ * 轮询→下载一条龙在注入实现里完成);返回视频字节与 mime,外加实际 @@ -169,7 +177,7 @@ export interface CindySlotDeps { * 分钟级才 resolve,在途名额在整个等待期占用。 */ generateVideo( - params: { prompt: string; model: string } & CindyVideoParams, + params: { prompt: string; model: string; providerId?: string } & CindyVideoParams, ): Promise<{ buffer: Uint8Array; mimeType: string; videoParams?: GhostVideoResultParams }>; /** * 主机统一视频通道·参考图生视频(源图以磁盘路径注入)。`refMode` 决定这 @@ -181,6 +189,7 @@ export interface CindySlotDeps { params: { prompt: string; model: string; + providerId?: string; imagePaths: string[]; refMode: GhostVideoRefMode; } & CindyVideoParams, @@ -190,7 +199,7 @@ export interface CindySlotDeps { * 该型号 → null)。可选依赖:不注入 = 跳过按型号校验,只做协议层粗筛 * (值仍会被 provider 层自己的校验拦下,只是话术不如这里友好)。 */ - videoCapabilities?(model: string): CindyVideoCapabilities | null; + videoCapabilities?(model: string, providerId?: string): CindyVideoCapabilities | null; /** * 指纹 → 磁盘路径,且仅当该媒体在此意识名下(出生或画廊,查账本); * 不属于它 / 查无此账 / 文件缺失一律 null(不区分,不给探测空间)。 @@ -204,6 +213,14 @@ export interface CindySlotDeps { * 白名单校验(型号可能已随主机演进下架)。 */ getOverride(ghostId: string, capability: string): string | null; + /** + * Provider-aware 媒体覆盖。新版 Host 偏好按 providerId + modelId 保存;存在时 + * 必须贯穿能力校验与最终派发,不能降回裸 modelId 后走 first-wins。 + */ + getMediaOverride?( + ghostId: string, + capability: string, + ): CindyMediaOverrideSelection | null; /** * 当前图像能力配置——真身是 providers.json 运行时目录(与会话模型列表 * 同一获取来源),每单现读跟随热更。models = 白名单与显示名;defaults = @@ -830,6 +847,8 @@ export class GhostCindySlot { const defaults = cfg.defaults; const whitelist = new Set(cfg.models.map((m) => m.id)); let model = defaults.standard; + let providerId: string | undefined; + let providerModelLabel: string | undefined; if (p.tier !== undefined) { if (typeof p.tier !== 'string' || !(GHOST_MODEL_TIERS as readonly string[]).includes(p.tier)) { return { ok: false, message: `未知档位(可用:${GHOST_MODEL_TIERS.join(' / ')})` }; @@ -837,13 +856,20 @@ export class GhostCindySlot { model = defaults[p.tier as GhostModelTier]; } const capability = `${info.category}.${info.action}`; - const override = this.deps.getOverride(ghostId, capability); - if (override !== null) { - if (whitelist.has(override)) { - model = override; - } else { - // 钉的型号已随白名单演进下架:落回上面的档位/默认,不让老配置卡死能力。 - this.deps.log?.warn('ghost cindy override no longer whitelisted, ignored', { ghostId, override }); + const mediaOverride = this.deps.getMediaOverride?.(ghostId, capability) ?? null; + if (mediaOverride) { + model = mediaOverride.modelId; + providerId = mediaOverride.providerId; + providerModelLabel = mediaOverride.label; + } else { + const override = this.deps.getOverride(ghostId, capability); + if (override !== null) { + if (whitelist.has(override)) { + model = override; + } else { + // 钉的型号已随白名单演进下架:落回上面的档位/默认,不让老配置卡死能力。 + this.deps.log?.warn('ghost cindy override no longer whitelisted, ignored', { ghostId, override }); + } } } if (p.model !== undefined) { @@ -855,6 +881,10 @@ export class GhostCindySlot { message: `不支持的模型(不在主机白名单内)。当前可用:${cfg.models.length > 0 ? cfg.models.map((m) => m.id).join(' / ') : '(暂无可用型号)'}`, }; } + if (p.model !== model) { + providerId = undefined; + providerModelLabel = undefined; + } model = p.model; } @@ -864,7 +894,7 @@ export class GhostCindySlot { // 明拒并列出该型号的可用值,不做最近似降级——静默改成别的档位会让意识 // 以为自己的参数生效了。 if (info.category === 'video' && presentVideoKeys.length > 0) { - const caps = this.deps.videoCapabilities?.(model) ?? null; + const caps = this.deps.videoCapabilities?.(model, providerId) ?? null; if (caps) { const unsupported = describeUnsupportedVideoParams(videoParams, caps); if (unsupported) { @@ -897,7 +927,7 @@ export class GhostCindySlot { } if (kind === 'edit_image') { - const perModelMax = this.deps.imageCapabilities?.(model)?.maxEditImages; + const perModelMax = this.deps.imageCapabilities?.(model, providerId)?.maxEditImages; if (perModelMax !== undefined && hashes.length > perModelMax) { const label = cfg.models.find((m) => m.id === model)?.label ?? model; return { @@ -912,7 +942,7 @@ export class GhostCindySlot { // 张数上限都不一样。不支持即明拒,不降级成另一种用法——降级会出一条 // 用户没要的片子,还照样计费。 if (kind === 'edit_video') { - const caps = this.deps.videoCapabilities?.(model) ?? null; + const caps = this.deps.videoCapabilities?.(model, providerId) ?? null; const perModelMax = caps?.maxImagesByRefMode?.[refMode]; if (caps?.maxImagesByRefMode !== undefined) { const label = cfg.models.find((m) => m.id === model)?.label ?? model; @@ -961,6 +991,7 @@ export class GhostCindySlot { this.deps.log?.info(`ghost cindy-request ${kind} start`, { ghostId, model, + ...(providerId ? { providerId } : {}), callId, ...(p.mode === 'submit' ? { mode: 'submit' } : {}), }); @@ -997,6 +1028,7 @@ export class GhostCindySlot { generated = await this.deps.editImage({ prompt, model, + ...(providerId ? { providerId } : {}), imagePaths, ...(aspectRatio !== undefined ? { aspectRatio } : {}), }); @@ -1004,12 +1036,25 @@ export class GhostCindySlot { generated = await this.deps.generateImage({ prompt, model, + ...(providerId ? { providerId } : {}), ...(aspectRatio !== undefined ? { aspectRatio } : {}), }); } else if (kind === 'edit_video') { - generated = await this.deps.editVideo({ prompt, model, imagePaths, refMode, ...videoParams }); + generated = await this.deps.editVideo({ + prompt, + model, + ...(providerId ? { providerId } : {}), + imagePaths, + refMode, + ...videoParams, + }); } else { - generated = await this.deps.generateVideo({ prompt, model, ...videoParams }); + generated = await this.deps.generateVideo({ + prompt, + model, + ...(providerId ? { providerId } : {}), + ...videoParams, + }); } assertOwnerScopeCurrent(); @@ -1027,13 +1072,15 @@ export class GhostCindySlot { this.deps.log?.info(`ghost cindy-request ${kind} done`, { ghostId, model, + ...(providerId ? { providerId } : {}), callId, hash: saved.hash, bytes: generated.buffer.byteLength, }); // 实际选型随结果回传(主机权威信息):意识交卷 note、会话里的 AI 与 // 用户由此看得见"这单是谁画的"。 - const modelLabel = cfg.models.find((m) => m.id === model)?.label ?? model; + const modelLabel = + providerModelLabel ?? cfg.models.find((m) => m.id === model)?.label ?? model; // 图片代办附带像素宽高(字节头解析,best-effort):意识供聊天卡片时 // 据此精确声明卡高,首帧零跳动;解析不出就缺省,意识回退估计值。 const dims = diff --git a/apps/desktop/src/main/cindy-brain/codexImageClient.ts b/apps/desktop/src/main/cindy-brain/codexImageClient.ts index 527ccc13da..5cf4883e41 100644 --- a/apps/desktop/src/main/cindy-brain/codexImageClient.ts +++ b/apps/desktop/src/main/cindy-brain/codexImageClient.ts @@ -7,9 +7,11 @@ * SSE stream because image-generation events may be newer than SDK typings. */ +import { randomUUID } from 'node:crypto'; import fs from 'node:fs/promises'; import type { ImageChannel, ImageChannelResult } from './imageChannelRegistry.js'; +import { mediaRequestParamsForLog, mediaRequestUrlForLog } from '../cindy-media/mediaRequestLog.js'; import { sniffMediaMime } from '../cindy-media/sniffMediaMime.js'; import { createLogger } from '../logger.js'; @@ -143,6 +145,7 @@ export function createCodexImageChannel(opts: CreateCodexImageChannelOptions): I prompt: string; imagePaths?: string[]; aspectRatio?: '1:1' | '3:2' | '2:3'; + signal?: AbortSignal; }): Promise { if (params.model !== `openai/${IMAGE_MODEL}`) { throw new Error(`Codex 图像通道不支持模型:${params.model}`); @@ -159,36 +162,66 @@ export function createCodexImageChannel(opts: CreateCodexImageChannelOptions): I { type: 'input_text', text: params.prompt }, ...images, ]; - const response = await doFetch(CODEX_RESPONSES_URL, { + const body = { + model: HOST_MODEL, + store: false, + stream: true, + instructions: 'Use the image_generation tool to fulfill this image request.', + input: [{ type: 'message', role: 'user', content }], + tools: [ + { + type: 'image_generation', + model: IMAGE_MODEL, + ...(params.aspectRatio ? { size: SIZE_BY_ASPECT[params.aspectRatio] } : {}), + quality: 'medium', + output_format: 'png', + background: 'opaque', + partial_images: 1, + }, + ], + }; + const requestId = randomUUID(); + const startedAt = Date.now(); + const requestLog = { + requestId, + providerId: 'openai', + modelId: params.model, method: 'POST', - headers: { - Authorization: `Bearer ${auth.accessToken}`, - 'Content-Type': 'application/json', - Accept: 'text/event-stream', - 'OpenAI-Beta': 'responses=experimental', - originator: 'codex_cli_rs', - 'User-Agent': USER_AGENT, - ...(auth.accountId ? { 'ChatGPT-Account-Id': auth.accountId } : {}), - }, - body: JSON.stringify({ - model: HOST_MODEL, - store: false, - stream: true, - instructions: 'Use the image_generation tool to fulfill this image request.', - input: [{ type: 'message', role: 'user', content }], - tools: [ - { - type: 'image_generation', - model: IMAGE_MODEL, - ...(params.aspectRatio ? { size: SIZE_BY_ASPECT[params.aspectRatio] } : {}), - quality: 'medium', - output_format: 'png', - background: 'opaque', - partial_images: 1, - }, - ], - }), + url: mediaRequestUrlForLog(CODEX_RESPONSES_URL), + }; + log.info('media request dispatch', { + ...requestLog, + params: mediaRequestParamsForLog(body), }); + let response: Response; + try { + response = await doFetch(CODEX_RESPONSES_URL, { + method: 'POST', + headers: { + Authorization: `Bearer ${auth.accessToken}`, + 'Content-Type': 'application/json', + Accept: 'text/event-stream', + 'OpenAI-Beta': 'responses=experimental', + originator: 'codex_cli_rs', + 'User-Agent': USER_AGENT, + ...(auth.accountId ? { 'ChatGPT-Account-Id': auth.accountId } : {}), + }, + body: JSON.stringify(body), + signal: params.signal, + }); + log.info('media request response', { + ...requestLog, + status: response.status, + durationMs: Date.now() - startedAt, + }); + } catch (error) { + log.warn('media request failed', { + ...requestLog, + durationMs: Date.now() - startedAt, + error: mediaRequestParamsForLog(error instanceof Error ? error.message : String(error)), + }); + throw error; + } if (!response.ok) await httpError(response, auth.accessToken, opts.onAuthFailure); const b64 = await collectImageB64(response); if (!b64) throw new Error('Codex 返回中没有图片,请重试或改用 OpenAI Platform API key'); @@ -197,8 +230,9 @@ export function createCodexImageChannel(opts: CreateCodexImageChannelOptions): I return { ready: opts.hasOAuthLogin, - generateImage: ({ model, prompt, aspectRatio }) => generate({ model, prompt, aspectRatio }), - editImage: ({ model, prompt, imagePaths, aspectRatio }) => - generate({ model, prompt, imagePaths, aspectRatio }), + generateImage: ({ model, prompt, aspectRatio, signal }) => + generate({ model, prompt, aspectRatio, signal }), + editImage: ({ model, prompt, imagePaths, aspectRatio, signal }) => + generate({ model, prompt, imagePaths, aspectRatio, signal }), }; } diff --git a/apps/desktop/src/main/cindy-brain/forge.ts b/apps/desktop/src/main/cindy-brain/forge.ts index caf1549d43..ba5f20c972 100644 --- a/apps/desktop/src/main/cindy-brain/forge.ts +++ b/apps/desktop/src/main/cindy-brain/forge.ts @@ -2242,20 +2242,23 @@ const result = await (await fetch('/media-models?type=image')).json(); // models:[{ // id, // name, +// providerId, // modalities:{ input:['text','image'], output:['image'] } // }], -// defaultModelId:string|null +// defaultModelId:string|null, +// defaultProviderId:string|null // } \`\`\` -\`type\` 只接受 \`image\` / \`video\`。Host 按 Gateway \`mode\` 切大类,并结合插件在 -\`cindy.image/video\` 声明的动作、Gateway \`modalities\`、Guide operation 与当前客户端 +\`type\` 只接受 \`image\` / \`video\`。Host 按模型 \`mode\` 切大类,并结合插件在 +\`cindy.image/video\` 声明的动作、模型 \`modalities\`、Guide operation 与当前客户端 协议支持度,只返回当前真正可执行的模型。单个模型的 Guide 缺失、损坏或版本过新只隔离 该模型,不拖垮整个目录。 -响应仍只把 Gateway \`architecture\` 已归一化后的 \`modalities.input/output\` 原样交给插件, +响应只把已归一化的 \`modalities.input/output\` 与来源 \`providerId\` 交给插件, 不下发 Guide、endpoint、凭证或 Host 内部兼容判定。插件可把用户选择的模型 id 存进自己的 -\`/kv\`,再通过工具结果或插件说明交给当前 Agent;付费请求前 Core 会再次校验。 +\`/kv\`,但必须同时保存 \`providerId\`,并把这对精确选择交给当前 Agent;同一个模型 id +可由多个 Provider 提供,不能按 id 去重或自行改换来源。付费请求前 Core 会再次校验。 插件与 Agent 不需要新的媒体协议,继续使用现有工具调用链: diff --git a/apps/desktop/src/main/cindy-brain/imageChannelRegistry.ts b/apps/desktop/src/main/cindy-brain/imageChannelRegistry.ts index 4f8ad470da..99e566a6fe 100644 --- a/apps/desktop/src/main/cindy-brain/imageChannelRegistry.ts +++ b/apps/desktop/src/main/cindy-brain/imageChannelRegistry.ts @@ -68,12 +68,14 @@ export interface ImageChannel { model: string; prompt: string; aspectRatio?: GhostImageAspectRatio; + signal?: AbortSignal; }): Promise; editImage(params: { model: string; prompt: string; imagePaths: string[]; aspectRatio?: GhostImageAspectRatio; + signal?: AbortSignal; }): Promise; } diff --git a/apps/desktop/src/main/cindy-brain/index.ts b/apps/desktop/src/main/cindy-brain/index.ts index 067056251f..57011956aa 100644 --- a/apps/desktop/src/main/cindy-brain/index.ts +++ b/apps/desktop/src/main/cindy-brain/index.ts @@ -361,6 +361,10 @@ import { getCodexImageAuthBinding } from './codexImageAuthBinding.js'; import { createGatewayImageClient } from '../cindy-proxy-media/api/gatewayImageClient.js'; import { createXaiVideoProvider } from '../cindy-proxy-media/video/providers/xai.js'; import * as blobStore from '../cindy-media/blobStore.js'; +import { + configureProviderMediaRuntime, + listProviderMediaModels, +} from '../cindy-media/providerMediaRuntime.js'; import * as ledger from '../cindy-media/ledger.js'; import { ingestMedia, supportedMime } from '../cindy-media/ingest.js'; import { captureMediaRefCompensationScope } from '../cindy-media/refCompensationJournal.js'; @@ -374,6 +378,7 @@ import { filterEnabledGatewayMediaModels, isMediaModelExecutable, listExecutableMediaModels, + supportsMediaCapability, } from '../model-access/mediaModels.js'; // ⚠️ 下面三个依赖必须保持模块顶层静态 import,禁止改回函数内 await import(): // 运行时 import() 会被 Rollup 编译成跨 chunk 的 require(尤其 drizzle-orm 会拆独立 @@ -3148,38 +3153,118 @@ const getCatalogVideoConfig = (): ReturnType => const getCatalogEmbedConfig = (): ReturnType => getCatalogMediaConfig('embed'); -/** - * Art 等插件的媒体偏好只认 Gateway `/models` 快照,并叠加客户端现有停用准入。 - * 不合并 providers.json 的 OpenAI/Gemini/自定义来源;第三方媒体模型后续单独接入。 - */ -function getGatewayMediaPreferenceConfig( +const MEDIA_PREFERENCE_PREFIX = 'media:'; + +function encodeMediaPreference(providerId: string, modelId: string): string { + return `${MEDIA_PREFERENCE_PREFIX}${encodeURIComponent(providerId)}:${encodeURIComponent(modelId)}`; +} + +function decodeMediaPreference(value: string): { providerId: string; modelId: string } | null { + if (!value.startsWith(MEDIA_PREFERENCE_PREFIX)) return null; + const encoded = value.slice(MEDIA_PREFERENCE_PREFIX.length); + const separator = encoded.indexOf(':'); + if (separator <= 0 || separator === encoded.length - 1) return null; + try { + const providerId = decodeURIComponent(encoded.slice(0, separator)); + const modelId = decodeURIComponent(encoded.slice(separator + 1)); + return providerId && modelId ? { providerId, modelId } : null; + } catch { + return null; + } +} + +type CindyMediaPreferenceModel = CindyMediaCatalogConfig['models'][number] & { + modelId: string; + modelName: string; + providerName: string; + group: string; + routing?: import('@cindy/model-providers').Provider['routing']; +}; + +interface CindyMediaPreferenceConfig { + models: CindyMediaPreferenceModel[]; + defaults: { standard: string; draft: string; best: string } | null; +} + +/** Art 等插件的媒体偏好统一合并已就绪 Provider 与 Gateway 可执行模型。 */ +function getMediaPreferenceConfig( capability: GhostMediaCapability, -): CindyMediaCatalogConfig { +): CindyMediaPreferenceConfig { const kind = capability.startsWith('image.') ? 'image' : 'video'; const coreCapability: MediaCapability = capability === 'video.edit' ? 'video.image_to_video' : capability; - const models = filterEnabledGatewayMediaModels( + const providers = new Map( + getActiveCatalog().providers.map((provider) => [provider.id, provider] as const), + ); + const providerModels: CindyMediaPreferenceModel[] = listProviderMediaModels() + .filter( + (model) => + model.mode === (kind === 'image' ? 'image_generation' : 'video_generation') && + supportsMediaCapability(model.modalities, coreCapability), + ) + .map((model) => { + const provider = providers.get(model.providerId); + const providerName = provider?.name ?? model.providerId; + return { + id: encodeMediaPreference(model.providerId, model.id), + modelId: model.id, + label: model.name, + modelName: model.name, + providerId: model.providerId, + providerName, + group: providerName, + ...(provider?.routing ? { routing: provider.routing } : {}), + supportsEdit: supportsMediaCapability(model.modalities, 'image.edit'), + }; + }); + const gatewayModels: CindyMediaPreferenceModel[] = filterEnabledGatewayMediaModels( getXdGatewayModels(), coreCapability, readModelDisableOverrides(), ) .filter((model) => isMediaModelExecutable(model.id, coreCapability)) - .map((model) => ({ - id: model.id, - label: model.name ?? model.id, - providerId: 'xd', - supportsEdit: isMediaModelExecutable( - model.id, - kind === 'image' ? 'image.edit' : 'video.image_to_video', - ), - })); - const standard = models[0]?.id; + .map((model) => { + const provider = providers.get('xd'); + const providerName = provider?.name ?? 'Cindy AI'; + const modelName = model.name ?? model.id; + return { + id: encodeMediaPreference('xd', model.id), + modelId: model.id, + label: modelName, + modelName, + providerId: 'xd', + providerName, + group: providerName, + ...(provider?.routing ? { routing: provider.routing } : {}), + supportsEdit: isMediaModelExecutable( + model.id, + kind === 'image' ? 'image.edit' : 'video.image_to_video', + ), + }; + }); + const models = [...gatewayModels, ...providerModels]; + const standard = gatewayModels[0]?.id ?? providerModels[0]?.id; return { models, defaults: standard ? { standard, draft: standard, best: standard } : null, }; } +function resolveMediaPreferenceModel( + config: CindyMediaPreferenceConfig, + preference: string | undefined, +): CindyMediaPreferenceModel | null { + if (!preference) return null; + const exact = config.models.find((model) => model.id === preference); + if (exact) return exact; + if (decodeMediaPreference(preference)) return null; + return ( + config.models.find((model) => model.providerId === 'xd' && model.modelId === preference) ?? + config.models.find((model) => model.modelId === preference) ?? + null + ); +} + /** * 插件配置页的模型目录;这里只读目录,不提供任何生成入口。Host 只按 Gateway mode * 切图片/视频大类并透传 modalities,具体动作支持度由插件自行解释。 @@ -3229,6 +3314,7 @@ async function getGhostConfigurableMediaModels( models: models.map((model) => ({ id: model.id, name: model.name ?? model.id, + providerId: model.providerId, ...(model.modalities ? { modalities: { @@ -3238,7 +3324,11 @@ async function getGhostConfigurableMediaModels( } : {}), })), - defaultModelId: models[0]?.id ?? null, + defaultModelId: models.find((model) => model.providerId === 'xd')?.id ?? models[0]?.id ?? null, + defaultProviderId: + models.find((model) => model.providerId === 'xd')?.providerId ?? + models[0]?.providerId ?? + null, }; } catch (error) { log.warn('read plugin media model catalog failed', { @@ -3252,7 +3342,8 @@ async function getGhostConfigurableMediaModels( /** * 读取插件在 Host「Cindy 能力」中的现有媒体选型。 - * 这里只投影当前有效值,不复制或迁移配置;override 失效时与设置 UI 一样回落目录默认。 + * 这里只投影当前有效值,不复制或迁移配置;用户已明确配置的精确来源失效时直接报错, + * 不能静默回落到另一个 Provider 或默认模型。 */ function getGhostConfiguredMediaModel( ghostId: string, @@ -3279,14 +3370,24 @@ function getGhostConfiguredMediaModel( }; } - const config = getGatewayMediaPreferenceConfig(mediaCapability); - const available = new Set(config.models.map((model) => model.id)); + const config = getMediaPreferenceConfig(mediaCapability); const override = readGhostCindyOverrides(ghostId)[mediaCapability]; - const modelId = override && available.has(override) ? override : config.defaults?.standard; - if (!modelId) { - return { ok: false, errorCode: 'NOT_AVAILABLE', message: '当前没有可用的媒体模型' }; + const selected = override + ? resolveMediaPreferenceModel(config, override) + : resolveMediaPreferenceModel(config, config.defaults?.standard); + if (!selected) { + return { + ok: false, + errorCode: 'NOT_AVAILABLE', + message: override ? '已配置的媒体模型当前不可用' : '当前没有可用的媒体模型', + }; } - return { ok: true, capability: mediaCapability, modelId }; + return { + ok: true, + capability: mediaCapability, + modelId: selected.modelId, + providerId: selected.providerId, + }; } /** @@ -3295,8 +3396,17 @@ function getGhostConfiguredMediaModel( * generateImage / editImage / 视频提交边界按**当前** override 重算启用候选再验一次, * 不在册即拒,这次付费请求不发出(与 scheduler 派发前重裁决同语义)。 */ -function assertMediaModelStillEnabled(kind: 'image' | 'video', model: string): void { - if (!getCatalogMediaConfig(kind).models.some((m) => m.id === model)) { +function assertMediaModelStillEnabled( + kind: 'image' | 'video', + model: string, + providerId?: string, +): void { + const available = providerId + ? getMediaPreferenceConfig(kind === 'image' ? 'image.generate' : 'video.generate').models.some( + (candidate) => candidate.providerId === providerId && candidate.modelId === model, + ) + : getCatalogMediaConfig(kind).models.some((candidate) => candidate.id === model); + if (!available) { throw new Error( kind === 'image' ? '图像模型不可用(可能已停用或来源凭证未就绪),本次生成已取消' @@ -3343,6 +3453,7 @@ async function readImageFileAsDataUri(absPath: string): Promise { async function runGhostVideo( params: { alias: string; + providerId?: string; prompt: string; imageDataUris?: string[]; /** 参考图用法(仅图生视频有);不传 = 执行器缺省的首尾帧。 */ @@ -3354,7 +3465,7 @@ async function runGhostVideo( throw new Error('视频能力不可用:主机未配置视频通道'); } // 提交紧前重查(第二十一轮):参考图 data URI 准备是 await,窗口内被停用即拒。 - assertMediaModelStillEnabled('video', params.alias); + assertMediaModelStillEnabled('video', params.alias, params.providerId); const r = await submitAndAwaitVideo(registry, params); return { buffer: r.buffer, @@ -3376,8 +3487,17 @@ async function runGhostVideo( * 某视频型号的画面参数支持集(cindySlot 按型号二次校验用)。registry 缺席 * 或 alias 查无 → null,cindySlot 据此跳过按型号校验(值仍会被执行器兜底拦下)。 */ -function getGhostVideoCapabilities(model: string): CindyVideoCapabilities | null { +function getGhostVideoCapabilities( + model: string, + providerId?: string, +): CindyVideoCapabilities | null { try { + if (providerId) { + const available = getMediaPreferenceConfig('video.generate').models.some( + (candidate) => candidate.providerId === providerId && candidate.modelId === model, + ); + if (!available) return null; + } const registry = getVideoProviderRegistry(); if (!registry || !registry.hasAny()) return null; const caps = registry.resolveByAlias(model).provider.capabilities; @@ -3395,9 +3515,14 @@ function getGhostVideoCapabilities(model: string): CindyVideoCapabilities | null } /** 图像 provider 的型号级编辑上限;slot 用它在文件 IO / 凭证读取前早拒。 */ -function getGhostImageCapabilities(model: string): CindyImageCapabilities | null { +function getGhostImageCapabilities( + model: string, + providerId?: string, +): CindyImageCapabilities | null { try { - return { maxEditImages: resolveImageChannelForModel(model, 'edit').maxEditImages }; + return { + maxEditImages: resolveImageChannelForModel(model, 'edit', providerId).maxEditImages, + }; } catch { return null; } @@ -3474,6 +3599,7 @@ function getImageChannelRegistry(): ImageChannelRegistry { // 通道;目录 id 带 openai/ 前缀,public API 适配层剥前缀。 const openaiImagesClient = createGatewayImageClient({ getApiKey: () => getProviderSecretStore().get('openai-images'), + logger: log, // 境外端点吃系统代理(outboundFetch):main 的裸 fetch 不读系统代理设置, // 代理软件非 TUN 模式下会直连失败(2026-07 review;xd 网关通道不注 —— // 网关域名境内直连,与现状一致)。 @@ -3485,7 +3611,8 @@ function getImageChannelRegistry(): ImageChannelRegistry { }, brandLabel: 'OpenAI', missingKeyMessage: 'OpenAI 图像 API key 未配置,请到「设置 → 模型供应商 → OpenAI」填入后重试', - beforeDispatch: (model) => assertMediaModelStillEnabled('image', `openai/${model}`), + beforeDispatch: (model) => + assertMediaModelStillEnabled('image', `openai/${model}`, 'openai'), }); const stripOpenaiPrefix = (id: string) => id.startsWith('openai/') ? id.slice('openai/'.length) : id; @@ -3498,7 +3625,7 @@ function getImageChannelRegistry(): ImageChannelRegistry { await getCodexImageAuthBinding().onAuthFailure(failure); }, fetchImplementation: ((url, init) => outboundFetch(url as string, init)) as typeof fetch, - beforeDispatch: (model) => assertMediaModelStillEnabled('image', model), + beforeDispatch: (model) => assertMediaModelStillEnabled('image', model, 'openai'), }); registry.register('openai', { // 用户明确配置 Platform key 时优先走确定性的 public Images API;否则复用 @@ -3506,24 +3633,30 @@ function getImageChannelRegistry(): ImageChannelRegistry { ready: () => hasOpenaiPlatformKey() || codexImagesClient.ready(), generateImage: (params) => hasOpenaiPlatformKey() - ? openaiImagesClient.generateImage({ - model: stripOpenaiPrefix(params.model), - prompt: params.prompt, - ...(params.aspectRatio - ? { size: GHOST_ASPECT_TO_GATEWAY_SIZE[params.aspectRatio] } - : {}), - }) + ? openaiImagesClient.generateImage( + { + model: stripOpenaiPrefix(params.model), + prompt: params.prompt, + ...(params.aspectRatio + ? { size: GHOST_ASPECT_TO_GATEWAY_SIZE[params.aspectRatio] } + : {}), + }, + params.signal, + ) : codexImagesClient.generateImage(params), editImage: (params) => hasOpenaiPlatformKey() - ? openaiImagesClient.editImage({ - model: stripOpenaiPrefix(params.model), - prompt: params.prompt, - imagePaths: params.imagePaths, - ...(params.aspectRatio - ? { size: GHOST_ASPECT_TO_GATEWAY_SIZE[params.aspectRatio] } - : {}), - }) + ? openaiImagesClient.editImage( + { + model: stripOpenaiPrefix(params.model), + prompt: params.prompt, + imagePaths: params.imagePaths, + ...(params.aspectRatio + ? { size: GHOST_ASPECT_TO_GATEWAY_SIZE[params.aspectRatio] } + : {}), + }, + params.signal, + ) : codexImagesClient.editImage(params), }); imageChannelRegistrySingleton = registry; @@ -3536,7 +3669,22 @@ function getImageChannelRegistry(): ImageChannelRegistry { * (cindyMediaCatalog first-wins 定格);白名单查无该模型时视同已停用 * (assertMediaModelStillEnabled 同窗口语义)。 */ -function resolveImageChannelForModel(model: string, operation: 'generate' | 'edit' = 'generate') { +function resolveImageChannelForModel( + model: string, + operation: 'generate' | 'edit' = 'generate', + providerId?: string, +) { + if (providerId) { + const capability: MediaCapability = operation === 'edit' ? 'image.edit' : 'image.generate'; + const exact = getMediaPreferenceConfig(capability).models.find( + (candidate) => candidate.providerId === providerId && candidate.modelId === model, + ); + if (!exact) throw new Error('图像模型或来源不可用,本次生成已取消'); + if (operation === 'edit' && !exact.supportsEdit) { + throw new Error(`图像来源 ${providerId} 不支持图像编辑,请在设置中选择支持编辑的来源`); + } + return getImageChannelRegistry().resolve(providerId); + } const entry = getCatalogMediaConfig('image').models.find((m) => m.id === model); if (!entry) { const slash = model.indexOf('/'); @@ -3549,6 +3697,76 @@ function resolveImageChannelForModel(model: string, operation: 'generate' | 'edi return getImageChannelRegistry().resolve(entry.providerId); } +function listLocalProviderMediaModels() { + const access = readModelDisableOverrides(); + return getActiveCatalog().providers.flatMap((provider) => { + if ( + provider.id === 'xd' || + isProviderDisabled(access, provider.id) || + !getImageChannelRegistry().isProviderReady(provider.id) + ) { + return []; + } + const supportsEdit = getImageChannelRegistry().isProviderEditReady(provider.id); + return (provider.imageModels ?? []).flatMap((model) => { + if ( + !model.modalities || + isModelDisabled(access, provider.id, model.id) || + !model.modalities.output.includes('image') + ) { + return []; + } + const input = supportsEdit + ? [...model.modalities.input] + : model.modalities.input.filter((modality) => modality !== 'image'); + return [ + { + id: model.id, + name: model.name, + providerId: provider.id, + mode: 'image_generation' as const, + modalities: { input, output: [...model.modalities.output] }, + ...(model.officialDocs ? { officialDocs: model.officialDocs } : {}), + }, + ]; + }); + }); +} + +configureProviderMediaRuntime({ + listModels: listLocalProviderMediaModels, + invoke: async (request) => { + if (request.capability !== 'image.generate' && request.capability !== 'image.edit') { + throw new Error('当前第三方 Provider 执行通道不支持该媒体能力'); + } + const operation = request.capability === 'image.edit' ? 'edit' : 'generate'; + const channel = resolveImageChannelForModel(request.modelId, operation, request.providerId); + if ( + operation === 'edit' && + channel.maxEditImages !== undefined && + request.imagePaths.length > channel.maxEditImages + ) { + throw new Error(`当前图像来源最多支持 ${channel.maxEditImages} 张参考图`); + } + const response = + operation === 'edit' + ? await channel.editImage({ + model: request.modelId, + prompt: request.prompt, + imagePaths: request.imagePaths, + ...(request.aspectRatio ? { aspectRatio: request.aspectRatio } : {}), + signal: request.signal, + }) + : await channel.generateImage({ + model: request.modelId, + prompt: request.prompt, + ...(request.aspectRatio ? { aspectRatio: request.aspectRatio } : {}), + signal: request.signal, + }); + return decodeImageResponse(response); + }, +}); + /** * Plugin media and storage use the same process-local AppSession owner * boundary as the rest of the owner-scoped runtime. @@ -3565,10 +3783,10 @@ export function getGhostCindySlot(): GhostCindySlot { isOwnerBoundaryPending: () => isGhostBoundaryPending(), // model 已在 modelSlot 按白名单校验;归属来源(providerId)按白名单条目 // 定位,经 imageChannelRegistry 取对应执行通道(2026-07 图像多来源)。 - generateImage: async ({ prompt, model, aspectRatio }) => { + generateImage: async ({ prompt, model, providerId, aspectRatio }) => { try { - assertMediaModelStillEnabled('image', model); - const channel = resolveImageChannelForModel(model); + assertMediaModelStillEnabled('image', model, providerId); + const channel = resolveImageChannelForModel(model, 'generate', providerId); return decodeImageResponse( await channel.generateImage({ model, @@ -3580,10 +3798,10 @@ export function getGhostCindySlot(): GhostCindySlot { humanizeImageChannelError(err); } }, - editImage: async ({ prompt, model, imagePaths, aspectRatio }) => { + editImage: async ({ prompt, model, providerId, imagePaths, aspectRatio }) => { try { - assertMediaModelStillEnabled('image', model); - const channel = resolveImageChannelForModel(model, 'edit'); + assertMediaModelStillEnabled('image', model, providerId); + const channel = resolveImageChannelForModel(model, 'edit', providerId); return decodeImageResponse( await channel.editImage({ model, @@ -3596,17 +3814,17 @@ export function getGhostCindySlot(): GhostCindySlot { humanizeImageChannelError(err); } }, - generateVideo: async ({ prompt, model, ...videoParams }) => { + generateVideo: async ({ prompt, model, providerId, ...videoParams }) => { try { - assertMediaModelStillEnabled('video', model); - return await runGhostVideo({ alias: model, prompt, ...videoParams }); + assertMediaModelStillEnabled('video', model, providerId); + return await runGhostVideo({ alias: model, providerId, prompt, ...videoParams }); } catch (err) { humanizeImageChannelError(err); } }, - editVideo: async ({ prompt, model, imagePaths, refMode, ...videoParams }) => { + editVideo: async ({ prompt, model, providerId, imagePaths, refMode, ...videoParams }) => { try { - assertMediaModelStillEnabled('video', model); + assertMediaModelStillEnabled('video', model, providerId); // 先算总量再读(闸按 refMode 分档:存量首尾帧不设闸,原样)。闸与 // 读取绑在一个入口里,顺序是那边的结构保证、不是这里的约定;结果 // 保序——顺序即语义:首/尾帧,或提示词里 [Image 1]… 的序号。 @@ -3617,6 +3835,7 @@ export function getGhostCindySlot(): GhostCindySlot { ); return await runGhostVideo({ alias: model, + providerId, prompt, imageDataUris, refMode, @@ -3629,8 +3848,23 @@ export function getGhostCindySlot(): GhostCindySlot { // 画面参数按型号二次校验的数据源(registry capabilities)。 imageCapabilities: getGhostImageCapabilities, videoCapabilities: getGhostVideoCapabilities, + getMediaOverride: (ghostId, capability) => { + const value = readGhostCindyOverrides(ghostId)[capability as CindyCapabilityKey] ?? null; + if (!value) return null; + const decoded = decodeMediaPreference(value); + if (!decoded) return null; + const selected = getMediaPreferenceConfig(capability as GhostMediaCapability).models.find( + (candidate) => + candidate.providerId === decoded.providerId && candidate.modelId === decoded.modelId, + ); + return { + ...decoded, + ...(selected ? { label: selected.label } : {}), + }; + }, getOverride: (ghostId, capability) => { - return readGhostCindyOverrides(ghostId)[capability as CindyCapabilityKey] ?? null; + const value = readGhostCindyOverrides(ghostId)[capability as CindyCapabilityKey] ?? null; + return value && !decodeMediaPreference(value) ? value : null; }, getImageConfig: getCatalogImageConfig, getVideoConfig: getCatalogVideoConfig, @@ -5823,7 +6057,7 @@ export function registerGhostIpc(): void { // 能力键的类目取对应清单)。defaultModel:目录默认选型的展示信息 // ("默认(GPT Image 2)"),让用户看得见"跟随"当下跟的是谁; // null = 目录没有该类目的模型(能力暂不可用),渲染层据此显示灰字而非下拉。 - const byKind = (cfg: CindyMediaCatalogConfig) => { + const byKind = (cfg: CindyMediaCatalogConfig | CindyMediaPreferenceConfig) => { const standard = cfg.defaults?.standard; return { options: cfg.models, @@ -5863,10 +6097,10 @@ export function registerGhostIpc(): void { : null; event.returnValue = { overrides, - image: byKind(getGatewayMediaPreferenceConfig('image.generate')), - imageEdit: byKind(getGatewayMediaPreferenceConfig('image.edit')), - video: byKind(getGatewayMediaPreferenceConfig('video.generate')), - videoEdit: byKind(getGatewayMediaPreferenceConfig('video.edit')), + image: byKind(getMediaPreferenceConfig('image.generate')), + imageEdit: byKind(getMediaPreferenceConfig('image.edit')), + video: byKind(getMediaPreferenceConfig('video.generate')), + videoEdit: byKind(getMediaPreferenceConfig('video.edit')), text: { options: textOptions, defaultModel: @@ -5939,10 +6173,10 @@ export function registerGhostIpc(): void { // fail-closed 兜底,不在这层把「暂时没配 key」当成非法值。 if ( !isCindyOverrideModelAllowed(capability as string, model, { - image: getGatewayMediaPreferenceConfig( + image: getMediaPreferenceConfig( capability === 'image.edit' ? 'image.edit' : 'image.generate', ).models, - video: getGatewayMediaPreferenceConfig( + video: getMediaPreferenceConfig( capability === 'video.edit' ? 'video.edit' : 'video.generate', ).models, embed: getCatalogEmbedConfig().models, diff --git a/apps/desktop/src/main/cindy-media/__tests__/invocationService.test.ts b/apps/desktop/src/main/cindy-media/__tests__/invocationService.test.ts index 281feb41e3..5013079979 100644 --- a/apps/desktop/src/main/cindy-media/__tests__/invocationService.test.ts +++ b/apps/desktop/src/main/cindy-media/__tests__/invocationService.test.ts @@ -20,6 +20,8 @@ const mocks = vi.hoisted(() => ({ ownerGeneration: 1, models: vi.fn(), executableModels: vi.fn(), + providerModel: vi.fn(), + providerInvoke: vi.fn(), guide: vi.fn(), outboundFetch: vi.fn(), ingestMedia: vi.fn(), @@ -83,6 +85,10 @@ vi.mock('../../model-access/mediaModels.js', () => ({ } }, })); +vi.mock('../providerMediaRuntime.js', () => ({ + resolveProviderMediaModel: mocks.providerModel, + invokeProviderMedia: mocks.providerInvoke, +})); vi.mock('../ingest.js', () => ({ ingestMedia: mocks.ingestMedia })); vi.mock('../blobStore.js', () => ({ readFile: mocks.readBlob, @@ -216,13 +222,17 @@ describe('Cindy Core media invocation state and security boundary', () => { mocks.failTransitionTo = null; mocks.rows.clear(); mocks.models.mockReset().mockResolvedValue([ - { id: 'image-model', name: 'Image Model', mode: 'image_generation' }, + { id: 'image-model', name: 'Image Model', providerId: 'xd', mode: 'image_generation' }, ]); mocks.executableModels.mockReset().mockResolvedValue({ - models: [{ id: 'image-model', name: 'Image Model', mode: 'image_generation' }], + models: [ + { id: 'image-model', name: 'Image Model', providerId: 'xd', mode: 'image_generation' }, + ], unavailable: [], candidateCount: 1, }); + mocks.providerModel.mockReset(); + mocks.providerInvoke.mockReset(); mocks.guide.mockReset(); mocks.outboundFetch.mockReset(); mocks.ingestMedia.mockReset().mockResolvedValue({ @@ -274,6 +284,85 @@ describe('Cindy Core media invocation state and security boundary', () => { expect(mocks.ingestMedia).toHaveBeenCalledTimes(1); }); + it('同名媒体模型按 providerId 精确准备并调用第三方来源', async () => { + const providerModel = { + id: 'openai/gpt-image-2', + name: 'GPT Image 2', + providerId: 'openai', + mode: 'image_generation', + modalities: { input: ['text', 'image'], output: ['image'] }, + officialDocs: 'https://platform.openai.com/docs/guides/image-generation', + }; + mocks.models.mockResolvedValue([ + { ...providerModel, providerId: 'xd' }, + providerModel, + ]); + mocks.providerModel.mockReturnValue(providerModel); + mocks.providerInvoke.mockResolvedValue({ buffer: PNG, mimeType: 'image/png' }); + + const prepared = await callCindyMedia({ + action: 'prepare', + providerId: 'openai', + modelId: providerModel.id, + capability: 'image.generate', + }); + + expect(prepared).toMatchObject({ + ok: true, + status: 'prepared', + provider_id: 'openai', + model_id: providerModel.id, + }); + expect(mocks.guide).not.toHaveBeenCalled(); + + await expect( + callCindyMedia({ + action: 'request', + invocationId: prepared.invocation_id as string, + body: { prompt: 'cat' }, + }), + ).resolves.toMatchObject({ + ok: true, + status: 'complete', + xdt_image_urls: [`cindy-media://blobs/${'a'.repeat(64)}.png`], + }); + expect(mocks.providerInvoke).toHaveBeenCalledWith({ + providerId: 'openai', + modelId: providerModel.id, + capability: 'image.generate', + prompt: 'cat', + imagePaths: [], + signal: expect.any(AbortSignal), + }); + }); + + it('旧调用未传 providerId 时不在同名 Provider 中静默选择来源', async () => { + const model = { + id: 'openai/gpt-image-2', + name: 'GPT Image 2', + providerId: 'openai', + mode: 'image_generation', + modalities: { input: ['text', 'image'], output: ['image'] }, + officialDocs: 'https://platform.openai.com/docs/guides/image-generation', + }; + mocks.models.mockResolvedValue([{ ...model, providerId: 'xd' }, model]); + + await expect( + callCindyMedia({ + action: 'prepare', + modelId: model.id, + capability: 'image.generate', + }), + ).resolves.toMatchObject({ + ok: false, + errorCode: 'MODEL_NOT_AVAILABLE', + retryable: false, + message: expect.stringContaining('provider_id'), + }); + expect(mocks.guide).not.toHaveBeenCalled(); + expect(mocks.providerModel).not.toHaveBeenCalled(); + }); + it('终态历史不占用在途 invocation 上限', async () => { mocks.guide.mockResolvedValue( resolvedGuide( @@ -626,7 +715,9 @@ describe('Cindy Core media invocation state and security boundary', () => { owner: mocks.currentUserId, drizzle: { owner: mocks.currentUserId }, }; - return [{ id: 'image-model', name: 'Image Model', mode: 'image_generation' }]; + return [ + { id: 'image-model', name: 'Image Model', providerId: 'xd', mode: 'image_generation' }, + ]; }); await expect( @@ -844,7 +935,7 @@ describe('Cindy Core media invocation state and security boundary', () => { capability: 'video.generate', }; mocks.models.mockResolvedValue([ - { id: 'image-model', name: 'Video Model', mode: 'video_generation' }, + { id: 'image-model', name: 'Video Model', providerId: 'xd', mode: 'video_generation' }, ]); mocks.guide.mockResolvedValue(resolvedGuide(asyncOperation)); mocks.outboundFetch @@ -905,7 +996,7 @@ describe('Cindy Core media invocation state and security boundary', () => { capability: 'video.generate', }; mocks.models.mockResolvedValue([ - { id: 'image-model', name: 'Video Model', mode: 'video_generation' }, + { id: 'image-model', name: 'Video Model', providerId: 'xd', mode: 'video_generation' }, ]); mocks.guide.mockResolvedValue(resolvedGuide(asyncOperation)); const successPayload = { @@ -978,7 +1069,7 @@ describe('Cindy Core media invocation state and security boundary', () => { capability: 'video.generate', }; mocks.models.mockResolvedValue([ - { id: 'image-model', name: 'Video Model', mode: 'video_generation' }, + { id: 'image-model', name: 'Video Model', providerId: 'xd', mode: 'video_generation' }, ]); mocks.guide.mockResolvedValue(resolvedGuide(asyncOperation)); mocks.outboundFetch @@ -1037,7 +1128,7 @@ describe('Cindy Core media invocation state and security boundary', () => { capability: 'video.generate', }; mocks.models.mockResolvedValue([ - { id: 'image-model', name: 'Video Model', mode: 'video_generation' }, + { id: 'image-model', name: 'Video Model', providerId: 'xd', mode: 'video_generation' }, ]); mocks.guide.mockResolvedValue(resolvedGuide(asyncOperation)); mocks.outboundFetch diff --git a/apps/desktop/src/main/cindy-media/__tests__/mediaRequestLog.test.ts b/apps/desktop/src/main/cindy-media/__tests__/mediaRequestLog.test.ts new file mode 100644 index 0000000000..69d6e5a0da --- /dev/null +++ b/apps/desktop/src/main/cindy-media/__tests__/mediaRequestLog.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest'; + +import { mediaRequestParamsForLog, mediaRequestUrlForLog } from '../mediaRequestLog.js'; + +describe('media request log redaction', () => { + it('保留实际 URL 并脱敏 query 凭证', () => { + expect( + mediaRequestUrlForLog( + 'https://user:pass@example.test/v1/images?model=gpt-image-2&api_key=secret#local', + ), + ).toBe( + 'https://%5BREDACTED%5D:%5BREDACTED%5D@example.test/v1/images?model=gpt-image-2&api_key=%5BREDACTED%5D', + ); + }); + + it('保留参数结构并收敛凭证和媒体正文', () => { + expect( + mediaRequestParamsForLog({ + model: 'openai/gpt-image-2', + prompt: '生成一张图', + apiKey: 'secret', + image: 'data:image/png;base64,aGk=', + source: 'https://example.test/input.png?token=secret#frame', + }), + ).toEqual({ + model: 'openai/gpt-image-2', + prompt: '生成一张图', + apiKey: '[REDACTED]', + image: '[data URL mime=image/png bytes=2]', + source: 'https://example.test/input.png?token=%5BREDACTED%5D#frame', + }); + }); +}); diff --git a/apps/desktop/src/main/cindy-media/invocationService.ts b/apps/desktop/src/main/cindy-media/invocationService.ts index 6fb4940bf6..cf15f1dc8a 100644 --- a/apps/desktop/src/main/cindy-media/invocationService.ts +++ b/apps/desktop/src/main/cindy-media/invocationService.ts @@ -10,6 +10,8 @@ import type { MediaResultKind, ResolvedMediaInvocationGuide, } from '../../shared/mediaInvocation.js'; +import { MODEL_ACCESS_INVOCATION_GUIDE_SCHEMA_VERSION } from '../../shared/mediaInvocation.js'; +import { GHOST_IMAGE_ASPECT_RATIOS, type GhostImageAspectRatio } from '../../shared/ghost.js'; import { getAppCapabilities } from '../appCapabilities.js'; import * as authManager from '../authManager.js'; import * as imageCacheStore from '../imageCacheStore.js'; @@ -29,6 +31,12 @@ import { import { getProviderSecretStore } from '../secrets/providerSecretStore.js'; import * as blobStore from './blobStore.js'; import { ingestMedia } from './ingest.js'; +import { mediaRequestParamsForLog, mediaRequestUrlForLog } from './mediaRequestLog.js'; +import { + invokeProviderMedia, + resolveProviderMediaModel, + type ProviderMediaRuntimeModel, +} from './providerMediaRuntime.js'; import { sniffMediaMime } from './sniffMediaMime.js'; import { countMediaInvocations, @@ -59,6 +67,7 @@ const TERMINAL_MEDIA_RESULT_ERRORS = new Set([ 'MEDIA_RESULT_TOO_LARGE', 'RESPONSE_TOO_LARGE', ]); +const CLIENT_PROVIDER_IMAGE_GUIDE_ID = 'cindy-provider-image-v1'; interface MediaConnection { baseUrl: string; @@ -176,6 +185,65 @@ function submissionOutcomeUnknown(message: string): Record { }); } +function providerImageGuide( + model: ProviderMediaRuntimeModel, + capability: 'image.generate' | 'image.edit', +): PreparedMediaInvocationGuide { + const edit = capability === 'image.edit'; + return { + schemaVersion: MODEL_ACCESS_INVOCATION_GUIDE_SCHEMA_VERSION, + guideId: CLIENT_PROVIDER_IMAGE_GUIDE_ID, + modelId: model.id, + revision: '1', + connection: { providerId: model.providerId }, + capability, + request: { + method: 'POST', + path: '/client-provider-media', + bodyEncoding: 'json', + bodyModelPath: ['model'], + timeoutMs: 600_000, + maxRequestBytes: MAX_LOCAL_MEDIA_INPUT_TOTAL_BYTES + 1024 * 1024, + maxResponseBytes: MAX_IMAGE_RESULT_BYTES, + }, + response: { + mode: 'sync', + media: [{ path: ['image'], encoding: 'base64', kind: 'image' }], + }, + instructions: edit + ? '必填 prompt 和 image。image 可传一条 Cindy 本地媒体引用或引用数组;可选 aspect_ratio。' + : '必填 prompt;可选 aspect_ratio。model 与凭证由 Cindy 注入。', + exampleBody: { + prompt: edit ? '描述希望如何修改图片' : '描述希望生成的图片', + ...(edit ? { image: 'cindy-media://blobs/.png' } : {}), + }, + inputSchema: { + type: 'object', + additionalProperties: false, + required: edit ? ['prompt', 'image'] : ['prompt'], + properties: { + prompt: { type: 'string', minLength: 1 }, + ...(edit + ? { + image: { + oneOf: [ + { type: 'string', minLength: 1 }, + { type: 'array', minItems: 1, items: { type: 'string', minLength: 1 } }, + ], + }, + } + : {}), + aspect_ratio: { type: 'string', enum: [...GHOST_IMAGE_ASPECT_RATIOS] }, + }, + }, + officialDocs: model.officialDocs ?? 'https://platform.openai.com/docs/guides/images', + }; +} + +function isClientProviderInvocation(invocation: StoredMediaInvocation): boolean { + return invocation.guide.guideId === CLIENT_PROVIDER_IMAGE_GUIDE_ID; +} + function resolveConnection(providerId: string): MediaConnection { if (providerId !== 'xd') { throw new MediaInvocationError( @@ -342,6 +410,10 @@ function multipartRequestBody( } async function dispatchRequest(input: { + invocationId: string; + providerId: string; + modelId: string; + capability: MediaCapability; connection: MediaConnection; method: 'GET' | 'POST'; path: string; @@ -356,13 +428,29 @@ async function dispatchRequest(input: { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), input.timeoutMs); timeout.unref?.(); + const url = requestUrl(input.connection.baseUrl, input.path); + const startedAt = Date.now(); + const requestLog = { + invocationId: input.invocationId, + providerId: input.providerId, + modelId: input.modelId, + capability: input.capability, + operation: input.operation, + method: input.method, + url: mediaRequestUrlForLog(url), + }; + let responseStatus: number | undefined; try { const requestBody = input.body ? input.bodyEncoding === 'multipart' ? multipartRequestBody(input.body, input.multipartFiles ?? []) : JSON.stringify(input.body) : undefined; - const response = await outboundFetch(requestUrl(input.connection.baseUrl, input.path), { + log.info('media request dispatch', { + ...requestLog, + params: mediaRequestParamsForLog(input.body ?? {}), + }); + const response = await outboundFetch(url, { method: input.method, headers: { Accept: 'application/json', @@ -376,6 +464,12 @@ async function dispatchRequest(input: { redirect: 'error', signal: controller.signal, }); + responseStatus = response.status; + log.info('media request response', { + ...requestLog, + status: response.status, + durationMs: Date.now() - startedAt, + }); const buffer = await readBoundedResponse(response, input.maxResponseBytes); if (!response.ok) { const message = providerErrorMessage(buffer); @@ -410,6 +504,12 @@ async function dispatchRequest(input: { throw new MediaInvocationError('UPSTREAM_RESPONSE_INVALID', '上游成功响应不是合法 JSON'); } } catch (error) { + log.warn('media request failed', { + ...requestLog, + ...(responseStatus !== undefined ? { status: responseStatus } : {}), + durationMs: Date.now() - startedAt, + error: mediaRequestParamsForLog(error instanceof Error ? error.message : String(error)), + }); if (error instanceof MediaInvocationError) throw error; const aborted = error instanceof Error && error.name === 'AbortError'; if (input.operation === 'poll') { @@ -562,6 +662,81 @@ async function prepareRequestBody( return output; } +async function localImagePath( + ref: string, + state: { localInputs: number; localBytes: number }, +): Promise { + let resolved: { absPath: string; mimeType: string }; + try { + if (ref.startsWith('cindy-media://')) resolved = blobStore.resolveSafe(ref); + else if (ref.startsWith('xdt-image://')) resolved = imageCacheStore.resolveSafe(ref); + else throw new Error('unsupported local media reference'); + } catch { + throw new MediaInvocationError( + 'MEDIA_INPUT_INVALID', + '第三方 Provider 参考图必须是 Cindy 本地媒体引用', + ); + } + const stat = await fs.stat(resolved.absPath); + state.localInputs += 1; + state.localBytes += stat.size; + if ( + state.localInputs > MAX_LOCAL_MEDIA_INPUTS || + state.localBytes > MAX_LOCAL_MEDIA_INPUT_TOTAL_BYTES || + stat.size <= 0 || + stat.size > MAX_LOCAL_MEDIA_INPUT_BYTES || + !resolved.mimeType.startsWith('image/') + ) { + throw new MediaInvocationError('MEDIA_INPUT_INVALID', '参考图数量、大小或格式不受支持'); + } + return resolved.absPath; +} + +async function providerImageRequest( + invocation: StoredMediaInvocation, + body: Record, +): Promise<{ + prompt: string; + imagePaths: string[]; + aspectRatio?: GhostImageAspectRatio; +}> { + const prompt = body.prompt; + if (typeof prompt !== 'string' || prompt.trim().length === 0 || prompt.length > 100_000) { + throw new MediaInvocationError('REQUEST_INVALID', 'prompt 必须是非空字符串'); + } + let aspectRatio: GhostImageAspectRatio | undefined; + if (body.aspect_ratio !== undefined) { + if ( + typeof body.aspect_ratio !== 'string' || + !(GHOST_IMAGE_ASPECT_RATIOS as readonly string[]).includes(body.aspect_ratio) + ) { + throw new MediaInvocationError( + 'REQUEST_INVALID', + `aspect_ratio 只支持 ${GHOST_IMAGE_ASPECT_RATIOS.join(' / ')}`, + ); + } + aspectRatio = body.aspect_ratio as GhostImageAspectRatio; + } + const imagePaths: string[] = []; + if (invocation.capability === 'image.edit') { + const raw = body.image; + const refs = typeof raw === 'string' ? [raw] : Array.isArray(raw) ? raw : []; + if ( + refs.length === 0 || + refs.some((value) => typeof value !== 'string' || value.length === 0) + ) { + throw new MediaInvocationError('REQUEST_INVALID', 'image.edit 必须提供本地图片引用'); + } + const state = { localInputs: 0, localBytes: 0 }; + for (const ref of refs as string[]) imagePaths.push(await localImagePath(ref, state)); + } + return { + prompt, + imagePaths, + ...(aspectRatio ? { aspectRatio } : {}), + }; +} + function maxResultBytes(kind: MediaResultKind): number { if (kind === 'image') return MAX_IMAGE_RESULT_BYTES; if (kind === 'audio') return MAX_AUDIO_RESULT_BYTES; @@ -776,6 +951,122 @@ async function persistCompletedInvocation( return failure('INTERNAL', '媒体结果已生成,但本地未能保存最终结果'); } +async function submitProviderInvocation( + invocation: StoredMediaInvocation, + body: Record, + scope: MediaAuthScope, + db: DbClient, +): Promise> { + const providerId = invocation.guide.connection.providerId; + const providerModel = resolveProviderMediaModel( + providerId, + invocation.modelId, + invocation.capability, + ); + if (!providerModel) { + return failure('MODEL_NOT_AVAILABLE', '该第三方媒体模型或执行来源已不可用,本次生成未发出'); + } + const input = await providerImageRequest(invocation, body); + assertAuthScope(scope, invocation.owner); + const claimed = await transitionMediaInvocation( + { + id: invocation.id, + owner: invocation.owner, + from: 'prepared', + to: 'submitting', + }, + db, + ); + if (!claimed) { + const current = await getMediaInvocation(invocation.id, invocation.owner, db); + assertAuthScope(scope, invocation.owner); + return failure( + 'INVOCATION_ALREADY_USED', + `该 invocation 当前状态为 ${current?.state ?? 'unknown'};付费提交不可重复执行`, + ); + } + assertAuthScope(scope, invocation.owner); + try { + const result = await invokeProviderMedia({ + providerId, + modelId: providerModel.id, + capability: invocation.capability, + ...input, + signal: AbortSignal.timeout(invocation.guide.request.timeoutMs), + }); + assertAuthScope(scope, invocation.owner); + if ( + result.buffer.byteLength === 0 || + result.buffer.byteLength > MAX_IMAGE_RESULT_BYTES || + !result.mimeType.startsWith('image/') || + !blobStore.supportedMime(result.mimeType) + ) { + throw new MediaInvocationError('MEDIA_RESULT_INVALID', '第三方 Provider 返回了无效图片'); + } + const stored = await ingestMedia( + { + buffer: result.buffer, + mimeType: result.mimeType, + refs: [], + assertStillValid: () => assertAuthScope(scope, invocation.owner), + }, + db.drizzle, + ); + assertAuthScope(scope, invocation.owner); + const media = { xdt_image_urls: [stored.url] }; + const responseJson = JSON.stringify(media); + const persisted = await transitionMediaInvocation( + { + id: invocation.id, + owner: invocation.owner, + from: 'submitting', + to: 'pending', + responseJson, + }, + db, + ); + assertAuthScope(scope, invocation.owner); + if (!persisted) { + await transitionMediaInvocation( + { + id: invocation.id, + owner: invocation.owner, + from: 'submitting', + to: 'unknown', + }, + db, + ).catch(() => false); + return submissionOutcomeUnknown('第三方媒体已生成,但本地未能保存调用结果;不要自动重提'); + } + return persistCompletedInvocation( + { ...invocation, state: 'pending', responseJson }, + media, + scope, + db, + ); + } catch (error) { + await transitionMediaInvocation( + { + id: invocation.id, + owner: invocation.owner, + from: 'submitting', + to: 'unknown', + }, + db, + ).catch(() => false); + log.warn('provider media submission failed after claim', { + providerId, + modelId: providerModel.id, + error: error instanceof Error ? error.message : String(error), + }); + return submissionOutcomeUnknown( + error instanceof Error + ? `第三方媒体请求未能确认结果:${error.message}` + : '第三方媒体请求未能确认结果;不要自动重提', + ); + } +} + async function materializeSyncInvocation( invocation: StoredMediaInvocation, response: unknown, @@ -825,6 +1116,7 @@ async function materializeSyncInvocation( } async function prepareInvocation( + providerId: string | undefined, modelId: string, capability: MediaCapability, ): Promise> { @@ -835,61 +1127,82 @@ async function prepareInvocation( assertAuthScope(scope); const models = await listAvailableMediaModels(capability); assertAuthScope(scope); - const model = models.find((candidate) => candidate.id === modelId); + const matchingModels = models.filter( + (candidate) => candidate.id === modelId && (!providerId || candidate.providerId === providerId), + ); + if (!providerId && matchingModels.length > 1) { + return failure( + 'MODEL_NOT_AVAILABLE', + '该模型同时来自多个 Provider,请从模型目录选择精确来源并在 prepare 时传入 provider_id', + ); + } + const model = matchingModels[0]; if (!model) { - return failure('MODEL_NOT_AVAILABLE', '该模型当前不可见,或不是请求的媒体类型'); + return failure('MODEL_NOT_AVAILABLE', '该模型或指定 Provider 当前不可见,或不是请求的媒体类型'); } - let resolvedGuide: ResolvedMediaInvocationGuide; - try { - resolvedGuide = await fetchMediaInvocationGuide(modelId); - assertAuthScope(scope); - } catch (error) { - if (error instanceof ServerApiError && error.code === 'MEDIA_INVOCATION_GUIDE_NOT_FOUND') { - return failure('GUIDE_NOT_AVAILABLE', '该模型当前没有可用的调用说明', false, { - outcomeKnown: true, - allowedActions: GUIDE_FALLBACK_ACTIONS, - }); + let preparedGuide: PreparedMediaInvocationGuide; + if (model.providerId !== 'xd') { + if (capability !== 'image.generate' && capability !== 'image.edit') { + return failure('CAPABILITY_NOT_SUPPORTED', '该第三方 Provider 当前不支持请求的媒体能力'); } - if (error instanceof MediaGuideCompatibilityError) { - log.warn('media Guide rejected by current client', { - modelId, - code: error.code, - detail: error.detail, - }); - return failure(error.code, error.message, false, { - outcomeKnown: true, - allowedActions: GUIDE_FALLBACK_ACTIONS, - }); + const providerModel = resolveProviderMediaModel(model.providerId, modelId, capability); + if (!providerModel) { + return failure('MODEL_NOT_AVAILABLE', '该第三方媒体模型或执行来源当前不可用'); } - if (error instanceof ServerApiError) { - return failure('GUIDE_SERVICE_UNAVAILABLE', '媒体调用说明暂时无法读取,请稍后重试', true, { - outcomeKnown: true, - allowedActions: GUIDE_FALLBACK_ACTIONS, - }); + preparedGuide = providerImageGuide(providerModel, capability); + } else { + let resolvedGuide: ResolvedMediaInvocationGuide; + try { + resolvedGuide = await fetchMediaInvocationGuide(modelId); + assertAuthScope(scope); + } catch (error) { + if (error instanceof ServerApiError && error.code === 'MEDIA_INVOCATION_GUIDE_NOT_FOUND') { + return failure('GUIDE_NOT_AVAILABLE', '该模型当前没有可用的调用说明', false, { + outcomeKnown: true, + allowedActions: GUIDE_FALLBACK_ACTIONS, + }); + } + if (error instanceof MediaGuideCompatibilityError) { + log.warn('media Guide rejected by current client', { + modelId, + code: error.code, + detail: error.detail, + }); + return failure(error.code, error.message, false, { + outcomeKnown: true, + allowedActions: GUIDE_FALLBACK_ACTIONS, + }); + } + if (error instanceof ServerApiError) { + return failure('GUIDE_SERVICE_UNAVAILABLE', '媒体调用说明暂时无法读取,请稍后重试', true, { + outcomeKnown: true, + allowedActions: GUIDE_FALLBACK_ACTIONS, + }); + } + throw error; } - throw error; - } - const operation = resolvedGuide.guide.operations.find( - (candidate) => candidate.capability === capability, - ); - if (!operation) { - return failure( - 'CAPABILITY_NOT_SUPPORTED', - '该模型的调用协议当前不支持请求的媒体能力', - false, - { - outcomeKnown: true, - allowedActions: GUIDE_FALLBACK_ACTIONS, - }, + const operation = resolvedGuide.guide.operations.find( + (candidate) => candidate.capability === capability, ); + if (!operation) { + return failure( + 'CAPABILITY_NOT_SUPPORTED', + '该模型的调用协议当前不支持请求的媒体能力', + false, + { + outcomeKnown: true, + allowedActions: GUIDE_FALLBACK_ACTIONS, + }, + ); + } + const { operations: _operations, ...guideProtocol } = resolvedGuide.guide; + void _operations; + preparedGuide = { + modelId: resolvedGuide.modelId, + ...guideProtocol, + ...operation, + }; } - const { operations: _operations, ...guideProtocol } = resolvedGuide.guide; - void _operations; - const preparedGuide: PreparedMediaInvocationGuide = { - modelId: resolvedGuide.modelId, - ...guideProtocol, - ...operation, - }; await pruneInvocations(owner, db); assertAuthScope(scope); if ((await countMediaInvocations(owner, db)) >= MAX_INVOCATIONS) { @@ -912,8 +1225,9 @@ async function prepareInvocation( ok: true, status: 'prepared', invocation_id: id, + provider_id: model.providerId, model_id: modelId, - model_name: model.name, + model_name: model.name ?? model.id, capability, guide_revision: preparedGuide.revision, instructions: preparedGuide.instructions, @@ -949,6 +1263,17 @@ async function submitInvocation( if (invocation.state === 'complete') { return completedInvocationResult(invocation); } + if ( + invocation.state === 'pending' && + isClientProviderInvocation(invocation) && + invocation.responseJson + ) { + const media = persistedResponse(invocation); + if (!media || typeof media !== 'object' || Array.isArray(media)) { + return failure('MEDIA_RESULT_INVALID', '第三方媒体调用保存的结果不合法'); + } + return persistCompletedInvocation(invocation, media as Record, scope, db); + } if ( invocation.state === 'pending' && invocation.guide.response.mode === 'sync' && @@ -975,11 +1300,14 @@ async function submitInvocation( assertAuthScope(scope, invocation.owner); return failure('INVOCATION_EXPIRED', '调用准备已超过 5 分钟,请重新查询模型并 prepare'); } + if (isClientProviderInvocation(invocation)) { + return submitProviderInvocation(invocation, body, scope, db); + } // prepare 与实际付费提交之间可能隔着 Agent 组装参数的时间;提交边界重新读取 // Gateway 清单和客户端停用状态,避免模型/供应商刚被停用后仍发出新请求。 const models = await listAvailableMediaModels(invocation.capability); assertAuthScope(scope, invocation.owner); - if (!models.some((model) => model.id === invocation.modelId)) { + if (!models.some((model) => model.providerId === 'xd' && model.id === invocation.modelId)) { return failure('MODEL_NOT_AVAILABLE', '该模型已下架或被停用,本次生成未发出'); } const requestBody = await prepareRequestBody(body, invocation.guide); @@ -1017,6 +1345,10 @@ async function submitInvocation( assertAuthScope(scope, invocation.owner); try { const response = await dispatchRequest({ + invocationId: invocation.id, + providerId: invocation.guide.connection.providerId, + modelId: invocation.modelId, + capability: invocation.capability, connection, method: invocation.guide.request.method, path: invocation.guide.request.path, @@ -1243,6 +1575,10 @@ async function pollInvocation(invocation: StoredMediaInvocation): Promise ({ id: model.id, + provider_id: model.providerId, ...(model.name ? { name: model.name } : {}), ...(model.mode ? { mode: model.mode } : {}), })), @@ -1374,7 +1711,11 @@ export async function callCindyMedia( }; } if (request.action === 'prepare') { - return prepareInvocation(request.modelId, request.capability as MediaCapability); + return prepareInvocation( + request.providerId, + request.modelId, + request.capability as MediaCapability, + ); } const invocation = await requireInvocation(request.invocationId); return await (request.action === 'request' diff --git a/apps/desktop/src/main/cindy-media/mediaCapabilities.ts b/apps/desktop/src/main/cindy-media/mediaCapabilities.ts new file mode 100644 index 0000000000..af4caf2d8c --- /dev/null +++ b/apps/desktop/src/main/cindy-media/mediaCapabilities.ts @@ -0,0 +1,21 @@ +import type { MediaCapability } from '@cindy/model-providers'; + +const MEDIA_CAPABILITY_REQUIREMENTS: Record = { + 'image.generate': { input: ['text'], output: 'image' }, + 'image.edit': { input: ['text', 'image'], output: 'image' }, + 'video.generate': { input: ['text'], output: 'video' }, + 'video.image_to_video': { input: ['text', 'image'], output: 'video' }, +}; + +export function supportsMediaCapability( + modalities: { input: string[]; output: string[] } | undefined, + capability: MediaCapability, +): boolean { + if (!modalities) return false; + const requirement = MEDIA_CAPABILITY_REQUIREMENTS[capability]; + const inputs = new Set(modalities.input); + return ( + modalities.output.includes(requirement.output) && + requirement.input.every((input) => inputs.has(input)) + ); +} diff --git a/apps/desktop/src/main/cindy-media/mediaRequestLog.ts b/apps/desktop/src/main/cindy-media/mediaRequestLog.ts new file mode 100644 index 0000000000..0a68c057d4 --- /dev/null +++ b/apps/desktop/src/main/cindy-media/mediaRequestLog.ts @@ -0,0 +1,98 @@ +import { redactSensitiveText } from '@cindy/maker-shared/error-redaction'; + +const MAX_LOG_STRING_CHARS = 20_000; +const MAX_LOG_DEPTH = 24; +const SENSITIVE_PARAM_NAME = + /(?:^|[-_.])(authorization|proxy[-_]?authorization|api[-_]?key|access[-_]?key(?:[-_]?id)?|private[-_]?key|key|access[-_]?token|refresh[-_]?token|id[-_]?token|token|secret|password|passwd|signature|credential|cookie|session)(?:$|[-_.])/i; + +function boundedText(value: string): string { + const redacted = redactSensitiveText(value); + if (redacted.length <= MAX_LOG_STRING_CHARS) return redacted; + return `${redacted.slice(0, MAX_LOG_STRING_CHARS)}...[truncated ${redacted.length - MAX_LOG_STRING_CHARS} chars]`; +} + +function dataUrlSummary(value: string): string | null { + const match = /^data:([^;,]+)(;base64)?,([\s\S]*)$/i.exec(value); + if (!match) return null; + const mimeType = match[1].toLowerCase(); + const encoded = match[3].replace(/\s/g, ''); + const bytes = match[2] + ? Math.max( + 0, + Math.floor((encoded.length * 3) / 4) - + (encoded.endsWith('==') ? 2 : encoded.endsWith('=') ? 1 : 0), + ) + : Buffer.byteLength(encoded, 'utf8'); + return `[data URL mime=${mimeType} bytes=${bytes}]`; +} + +function urlForLog(rawUrl: string, stripFragment: boolean): string { + try { + const url = new URL(rawUrl); + if (url.username) url.username = '[REDACTED]'; + if (url.password) url.password = '[REDACTED]'; + for (const key of [...url.searchParams.keys()]) { + if (SENSITIVE_PARAM_NAME.test(key)) url.searchParams.set(key, '[REDACTED]'); + } + if (stripFragment) url.hash = ''; + else if (url.hash) url.hash = boundedText(url.hash); + return url.toString(); + } catch { + return boundedText(rawUrl); + } +} + +/** 保留实际请求 URL,只脱敏可能携带凭证的 query 值;fragment 不会发给上游。 */ +export function mediaRequestUrlForLog(rawUrl: string): string { + return urlForLog(rawUrl, true); +} + +/** + * 请求参数日志保留真实结构和值;凭证、data URL、二进制与本地文件内容只留描述, + * 避免日志泄露密钥或被图片/视频 base64 撑爆。 + */ +export function mediaRequestParamsForLog(value: unknown): unknown { + const seen = new WeakSet(); + + const visit = (item: unknown, key: string | null, depth: number): unknown => { + if (key && SENSITIVE_PARAM_NAME.test(key)) return '[REDACTED]'; + if (typeof item === 'string') { + const dataSummary = dataUrlSummary(item); + if (dataSummary) return dataSummary; + if (/^https?:\/\/\S+$/i.test(item)) return urlForLog(item, false); + return boundedText(item); + } + if ( + item === null || + typeof item === 'number' || + typeof item === 'boolean' || + typeof item === 'undefined' + ) { + return item; + } + if (typeof item === 'bigint') return item.toString(); + if (Buffer.isBuffer(item) || item instanceof Uint8Array) { + return `[binary bytes=${item.byteLength}]`; + } + if (typeof Blob !== 'undefined' && item instanceof Blob) { + return `[blob mime=${item.type || 'unknown'} bytes=${item.size}]`; + } + if (typeof item !== 'object') return boundedText(String(item)); + if (depth >= MAX_LOG_DEPTH) return '[max depth]'; + if (seen.has(item)) return '[circular]'; + seen.add(item); + if (Array.isArray(item)) { + const result = item.map((child) => visit(child, null, depth + 1)); + seen.delete(item); + return result; + } + const result: Record = {}; + for (const [childKey, child] of Object.entries(item as Record)) { + result[childKey] = visit(child, childKey, depth + 1); + } + seen.delete(item); + return result; + }; + + return visit(value, null, 0); +} diff --git a/apps/desktop/src/main/cindy-media/providerMediaRuntime.ts b/apps/desktop/src/main/cindy-media/providerMediaRuntime.ts new file mode 100644 index 0000000000..4f8de9dc50 --- /dev/null +++ b/apps/desktop/src/main/cindy-media/providerMediaRuntime.ts @@ -0,0 +1,69 @@ +import type { MediaCapability } from '@cindy/model-providers'; +import type { GhostImageAspectRatio } from '../../shared/ghost.js'; +import { supportsMediaCapability } from './mediaCapabilities.js'; + +export interface ProviderMediaRuntimeModel { + id: string; + name: string; + providerId: string; + mode: 'image_generation' | 'video_generation'; + modalities: { input: string[]; output: string[] }; + officialDocs?: string; +} + +export interface ProviderMediaRuntimeRequest { + providerId: string; + modelId: string; + capability: MediaCapability; + prompt: string; + imagePaths: string[]; + aspectRatio?: GhostImageAspectRatio; + signal?: AbortSignal; +} + +export interface ProviderMediaRuntimeResult { + buffer: Buffer; + mimeType: string; +} + +interface ProviderMediaRuntime { + listModels(): ProviderMediaRuntimeModel[]; + invoke(request: ProviderMediaRuntimeRequest): Promise; +} + +let runtime: ProviderMediaRuntime | null = null; + +export function configureProviderMediaRuntime(next: ProviderMediaRuntime): void { + runtime = next; +} + +export function listProviderMediaModels(): ProviderMediaRuntimeModel[] { + return runtime?.listModels() ?? []; +} + +export function resolveProviderMediaModel( + providerId: string, + modelId: string, + capability: MediaCapability, +): ProviderMediaRuntimeModel | null { + return ( + listProviderMediaModels().find( + (model) => + model.providerId === providerId && + model.id === modelId && + supportsMediaCapability(model.modalities, capability), + ) ?? null + ); +} + +export async function invokeProviderMedia( + request: ProviderMediaRuntimeRequest, +): Promise { + const active = resolveProviderMediaModel( + request.providerId, + request.modelId, + request.capability, + ); + if (!active || !runtime) throw new Error('第三方媒体模型或执行来源当前不可用'); + return runtime.invoke(request); +} diff --git a/apps/desktop/src/main/cindy-proxy-media/api/gatewayImageClient.ts b/apps/desktop/src/main/cindy-proxy-media/api/gatewayImageClient.ts index 9e6761f3b1..61464ab731 100644 --- a/apps/desktop/src/main/cindy-proxy-media/api/gatewayImageClient.ts +++ b/apps/desktop/src/main/cindy-proxy-media/api/gatewayImageClient.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'node:crypto'; import fs from 'node:fs/promises'; import path from 'node:path'; import type { @@ -7,6 +8,10 @@ import type { } from '../types.js'; import type { LiziMcpLogger } from '@cindy/mcps'; import type { CindyProxyMediaMaybePromise, CindyProxyMediaProxyConfig } from '../types.js'; +import { + mediaRequestParamsForLog, + mediaRequestUrlForLog, +} from '../../cindy-media/mediaRequestLog.js'; export class GatewayImageError extends Error { constructor( @@ -151,6 +156,43 @@ export function createGatewayImageClient(opts: CreateGatewayImageClientOptions): const editUrl = joinProxyUrl(baseUrl, opts.proxy.editPath); const doFetch = opts.fetchImplementation ?? fetch; + async function loggedFetch( + url: string, + init: RequestInit, + model: string, + params: unknown, + ): Promise { + const requestId = randomUUID(); + const startedAt = Date.now(); + const requestLog = { + requestId, + provider: brandLabel, + modelId: model, + method: init.method ?? 'GET', + url: mediaRequestUrlForLog(url), + }; + opts.logger?.info('media request dispatch', { + ...requestLog, + params: mediaRequestParamsForLog(params), + }); + try { + const response = await doFetch(url, init); + opts.logger?.info('media request response', { + ...requestLog, + status: response.status, + durationMs: Date.now() - startedAt, + }); + return response; + } catch (error) { + opts.logger?.warn('media request failed', { + ...requestLog, + durationMs: Date.now() - startedAt, + error: mediaRequestParamsForLog(error instanceof Error ? error.message : String(error)), + }); + throw error; + } + } + async function requireApiKey(): Promise { const key = await Promise.resolve(opts.getApiKey()); if (!key) { @@ -190,15 +232,20 @@ export function createGatewayImageClient(opts: CreateGatewayImageClientOptions): // 停用轴派发前重查(PR #744 review 第二十一轮):凭证获取是 await,期间该 // (供应商, 模型) 可能被用户停用 —— payload 就绪、请求发出的紧前再验一次。 beforeDispatch?.(params.model); - const res = await doFetch(generateUrl, { - method: 'POST', - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', + const res = await loggedFetch( + generateUrl, + { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + signal, }, - body: JSON.stringify(body), - signal, - }); + params.model, + body, + ); return parseResponse(res, params.model, brandLabel); } @@ -223,22 +270,37 @@ export function createGatewayImageClient(opts: CreateGatewayImageClientOptions): if (allowSizeQuality) form.append('size', params.size ?? 'auto'); if (params.quality) form.append('quality', params.quality); + const imageParams: Array<{ filename: string; mimeType: string; bytes: number }> = []; for (const p of params.imagePaths) { const buf = await fs.readFile(p); const filename = path.basename(p); - form.append('image[]', new Blob([buf], { type: mimeFromFilename(filename) }), filename); + const mimeType = mimeFromFilename(filename); + form.append('image[]', new Blob([buf], { type: mimeType }), filename); + imageParams.push({ filename, mimeType, bytes: buf.byteLength }); } // 同上:凭证获取 + 逐张 fs.readFile 都是 await,提交紧前重查。 beforeDispatch?.(params.model); - const res = await doFetch(editUrl, { - method: 'POST', - headers: { - Authorization: `Bearer ${apiKey}`, + const res = await loggedFetch( + editUrl, + { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + }, + body: form as unknown as BodyInit, + signal, }, - body: form as unknown as BodyInit, - signal, - }); + params.model, + { + model: params.model, + prompt: params.prompt, + n: params.n ?? 1, + ...(allowSizeQuality ? { size: params.size ?? 'auto' } : {}), + ...(params.quality ? { quality: params.quality } : {}), + images: imageParams, + }, + ); return parseResponse(res, params.model, brandLabel); } diff --git a/apps/desktop/src/main/model-access/__tests__/mediaModels.test.ts b/apps/desktop/src/main/model-access/__tests__/mediaModels.test.ts index 623284c125..534cac5f22 100644 --- a/apps/desktop/src/main/model-access/__tests__/mediaModels.test.ts +++ b/apps/desktop/src/main/model-access/__tests__/mediaModels.test.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; const serverApiFetchMock = vi.hoisted(() => vi.fn()); const readModelDisableOverridesMock = vi.hoisted(() => vi.fn()); +const listProviderMediaModelsMock = vi.hoisted(() => vi.fn()); vi.mock('../../serverApiClient.js', () => ({ serverApiFetch: serverApiFetchMock, @@ -21,6 +22,9 @@ vi.mock('../../clientEndpointsService.js', () => ({ vi.mock('../../maker-host/model-disable-store.js', () => ({ readModelDisableOverrides: readModelDisableOverridesMock, })); +vi.mock('../../cindy-media/providerMediaRuntime.js', () => ({ + listProviderMediaModels: listProviderMediaModelsMock, +})); import { fetchMediaInvocationGuide, @@ -73,6 +77,7 @@ describe('listAvailableMediaModels', () => { beforeEach(() => { serverApiFetchMock.mockReset().mockResolvedValue(payload); readModelDisableOverridesMock.mockReset().mockReturnValue({}); + listProviderMediaModelsMock.mockReset().mockReturnValue([]); }); it('不带操作筛选时按 Gateway mode 返回图片/视频模型', async () => { @@ -99,6 +104,25 @@ describe('listAvailableMediaModels', () => { ]); }); + it('同名媒体模型按 providerId 保留为两个可选来源', async () => { + listProviderMediaModelsMock.mockReturnValue([ + { + id: 'image-with-guide', + name: 'Provider Image', + providerId: 'openai', + mode: 'image_generation', + modalities: { input: ['text', 'image'], output: ['image'] }, + }, + ]); + + const models = await listAvailableMediaModels('image.generate'); + + expect(models.filter((model) => model.id === 'image-with-guide')).toMatchObject([ + { id: 'image-with-guide', providerId: 'xd' }, + { id: 'image-with-guide', providerId: 'openai' }, + ]); + }); + it('叠加客户端现有 XD provider/model 停用准入', async () => { readModelDisableOverridesMock.mockReturnValueOnce({ disabledModels: { 'xd:image-without-guide': true }, diff --git a/apps/desktop/src/main/model-access/mediaModels.ts b/apps/desktop/src/main/model-access/mediaModels.ts index 5485e450ee..c344487314 100644 --- a/apps/desktop/src/main/model-access/mediaModels.ts +++ b/apps/desktop/src/main/model-access/mediaModels.ts @@ -16,6 +16,8 @@ import { type ResolvedMediaInvocationGuide, } from '../../shared/mediaInvocation.js'; import { getClientEndpoint } from '../clientEndpointsService.js'; +import { supportsMediaCapability } from '../cindy-media/mediaCapabilities.js'; +import { listProviderMediaModels } from '../cindy-media/providerMediaRuntime.js'; import { readModelDisableOverrides } from '../maker-host/model-disable-store.js'; import { serverApiFetch, ServerApiError } from '../serverApiClient.js'; @@ -62,11 +64,13 @@ export interface UnavailableMediaModel { } export interface ExecutableMediaModelsResult { - models: ModelCatalogEntry[]; + models: ExecutableMediaModel[]; unavailable: UnavailableMediaModel[]; candidateCount: number; } +export type ExecutableMediaModel = ModelCatalogEntry & { providerId: string }; + interface ExecutableMediaSnapshot { models: ModelCatalogEntry[]; capabilitiesByModel: Map>; @@ -99,28 +103,38 @@ export function isMediaModelExecutable( return executableMediaSnapshot?.capabilitiesByModel.get(modelId)?.has(capability) === true; } -const MEDIA_CAPABILITY_REQUIREMENTS: Record< - MediaCapability, - { input: string[]; inputAny?: string[]; output: string } -> = { - 'image.generate': { input: ['text'], output: 'image' }, - 'image.edit': { input: ['text', 'image'], output: 'image' }, - 'video.generate': { input: ['text'], output: 'video' }, - 'video.image_to_video': { input: ['text', 'image'], output: 'video' }, -}; +export { supportsMediaCapability } from '../cindy-media/mediaCapabilities.js'; -export function supportsMediaCapability( - modalities: { input: string[]; output: string[] } | undefined, - capability: MediaCapability, -): boolean { - if (!modalities) return false; - const requirement = MEDIA_CAPABILITY_REQUIREMENTS[capability]; - const inputs = new Set(modalities.input); - return ( - modalities.output.includes(requirement.output) && - requirement.input.every((input) => inputs.has(input)) && - (requirement.inputAny === undefined || requirement.inputAny.some((input) => inputs.has(input))) - ); +function availableProviderMediaModels(capability?: MediaCapability): ExecutableMediaModel[] { + return listProviderMediaModels() + .filter( + (model) => capability === undefined || supportsMediaCapability(model.modalities, capability), + ) + .map((model) => ({ + id: model.id, + name: model.name, + providerId: model.providerId, + mode: model.mode, + modalities: { + input: [...model.modalities.input], + output: [...model.modalities.output], + }, + })); +} + +function mergeMediaModels( + preferred: readonly ExecutableMediaModel[], + fallback: readonly ExecutableMediaModel[], +): ExecutableMediaModel[] { + const seen = new Set(); + const models: ExecutableMediaModel[] = []; + for (const model of [...preferred, ...fallback]) { + const key = `${model.providerId}\u0000${model.id}`; + if (seen.has(key)) continue; + seen.add(key); + models.push(model); + } + return models; } /** @@ -184,12 +198,19 @@ async function fetchGatewayMediaModels(): Promise { export async function listAvailableMediaModels( capability?: MediaCapability, -): Promise { - return filterEnabledGatewayMediaModels( - await fetchGatewayMediaModels(), - capability, - readModelDisableOverrides(), - ); +): Promise { + const providerModels = availableProviderMediaModels(capability); + try { + const gatewayModels = filterEnabledGatewayMediaModels( + await fetchGatewayMediaModels(), + capability, + readModelDisableOverrides(), + ).map((model) => ({ ...model, providerId: CINDY_AI_PROVIDER_ID })); + return mergeMediaModels(gatewayModels, providerModels); + } catch (error) { + if (providerModels.length > 0) return providerModels; + throw error; + } } export async function fetchMediaInvocationGuide( @@ -417,7 +438,18 @@ export async function listExecutableMediaModels( capabilities: readonly MediaCapability[] = [], options: { includeDisabled?: boolean; forceRefresh?: boolean } = {}, ): Promise { - const snapshot = await getExecutableMediaSnapshot(options.forceRefresh === true); + const providerModels = availableProviderMediaModels().filter((model) => + capabilities.every((capability) => supportsMediaCapability(model.modalities, capability)), + ); + let snapshot: ExecutableMediaSnapshot; + try { + snapshot = await getExecutableMediaSnapshot(options.forceRefresh === true); + } catch (error) { + if (providerModels.length > 0) { + return { models: providerModels, unavailable: [], candidateCount: providerModels.length }; + } + throw error; + } const candidates = filterEnabledGatewayMediaModels( snapshot.models, undefined, @@ -425,12 +457,12 @@ export async function listExecutableMediaModels( ).filter((model) => capabilities.every((capability) => supportsMediaCapability(model.modalities, capability)), ); - const models: ModelCatalogEntry[] = []; + const gatewayModels: ExecutableMediaModel[] = []; const unavailable: UnavailableMediaModel[] = []; for (const model of candidates) { const supported = snapshot.capabilitiesByModel.get(model.id); if (supported && capabilities.every((capability) => supported.has(capability))) { - models.push(model); + gatewayModels.push({ ...model, providerId: CINDY_AI_PROVIDER_ID }); continue; } unavailable.push( @@ -442,5 +474,10 @@ export async function listExecutableMediaModels( }, ); } - return { models, unavailable, candidateCount: candidates.length }; + const models = mergeMediaModels(gatewayModels, providerModels); + return { + models, + unavailable, + candidateCount: candidates.length + providerModels.length, + }; } diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index a6424f7cc9..b6e5e973f1 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -846,6 +846,22 @@ const appearanceSettingsInfo = ipcRenderer.sendSync( 'appearance-settings:get-sync', ) as AppearanceSettings | null; +type CindyMediaPreferenceOption = { + id: string; + label: string; + group: string; + providerId: string; + providerName: string; + modelId: string; + modelName: string; + routing?: import('@cindy/model-providers').Provider['routing']; +}; + +type CindyMediaPreferenceKind = { + options: CindyMediaPreferenceOption[]; + defaultModel: CindyMediaPreferenceOption | null; +}; + contextBridge.exposeInMainWorld('electronAPI', { platform: process.platform, osRelease: ipcRenderer.sendSync('get-os-release') as string, @@ -1057,22 +1073,10 @@ contextBridge.exposeInMainWorld('electronAPI', { * 每类目一份下拉数据(能力键按类目取对应清单)。 * options 为空或 defaultModel 为 null = 目录没给该类目模型,能力暂不可用。 */ - image: { - options: Array<{ id: string; label: string }>; - defaultModel: { id: string; label: string } | null; - }; - imageEdit: { - options: Array<{ id: string; label: string }>; - defaultModel: { id: string; label: string } | null; - }; - video: { - options: Array<{ id: string; label: string }>; - defaultModel: { id: string; label: string } | null; - }; - videoEdit: { - options: Array<{ id: string; label: string }>; - defaultModel: { id: string; label: string } | null; - }; + image: CindyMediaPreferenceKind; + imageEdit: CindyMediaPreferenceKind; + video: CindyMediaPreferenceKind; + videoEdit: CindyMediaPreferenceKind; /** 文本类(快问快答):选项是当前供应商目录的全部文本模型(cat: 编码钉值, * 带供应商/模型/徽标等结构化字段供富列表渲染);declaredModel = 身份卡声明 * 的偏好模型;utilityProfiles = 存量轻量档位钉的展示名表(老钉值回显用)。 */ diff --git a/apps/desktop/src/renderer/cindy-brain/CindyCapabilityPrefs.tsx b/apps/desktop/src/renderer/cindy-brain/CindyCapabilityPrefs.tsx index 62b2863cca..fe44f0736c 100644 --- a/apps/desktop/src/renderer/cindy-brain/CindyCapabilityPrefs.tsx +++ b/apps/desktop/src/renderer/cindy-brain/CindyCapabilityPrefs.tsx @@ -5,18 +5,170 @@ import { useCallback, useState } from 'react'; import { useTranslation } from 'react-i18next'; -import { Sparkles } from 'lucide-react'; +import { Check, ChevronDown, Sparkles } from 'lucide-react'; import { toast } from '@/lib/toast'; import { cn } from '@/lib/utils'; -import { - OneshotModelPinPicker, - type OneshotPinOption, -} from '@/cindy-brain/OneshotModelPinPicker'; +import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover'; +import { ModelIconMark } from '@/components/new-chat/ModelSelector'; +import { OneshotModelPinPicker, type OneshotPinOption } from '@/cindy-brain/OneshotModelPinPicker'; /** 跟随默认在 select 里的哨兵值(覆盖表里"没有这项"= 跟随默认)。 */ const FOLLOW_DEFAULT_VALUE = '__default__'; +interface MediaModelOption { + id: string; + modelId: string; + label: string; + providerId: string; + providerName: string; + routing?: import('@cindy/model-providers').Provider['routing']; +} + +/** 媒体模型沿用原来的轻量下拉,只在模型名前补来源 Provider 图标。 */ +function MediaModelPicker({ + value, + defaultModel, + options, + onChange, + ariaLabel, + dense, +}: { + value?: string; + defaultModel: MediaModelOption; + options: readonly MediaModelOption[]; + onChange: (pin: string | null) => void; + ariaLabel: string; + dense?: boolean; +}) { + const { t } = useTranslation(); + const [open, setOpen] = useState(false); + const current = value + ? options.find((option) => option.id === value) ?? + options.find((option) => option.providerId === 'xd' && option.modelId === value) ?? + options.find((option) => option.modelId === value) + : undefined; + const staleValue = value && !current ? value : null; + const selected = current ?? defaultModel; + const selectable = options.filter((option) => option.id !== defaultModel.id); + const triggerLabel = + current?.label ?? + staleValue ?? + t('settings.ghosts.detail.cindyPrefs.defaultOption', { model: defaultModel.label }); + + const select = (pin: string | null): void => { + if (pin !== value) onChange(pin); + setOpen(false); + }; + + const optionClass = (active: boolean): string => + cn( + 'flex w-full items-center gap-2 rounded-lg px-3 py-2 text-left transition-colors', + 'hover:bg-[var(--model-item-hover)] focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[var(--focus-ring)]', + dense ? 'text-12 leading-5' : 'text-13 leading-5', + active && 'bg-[var(--model-item-hover)]', + ); + + return ( + + + + + +
+ + {selectable.map((option) => { + const active = value === option.id; + return ( + + ); + })} + {staleValue && ( + + )} +
+
+
+ ); +} + /** * Ghost 申请的每项 Cindy 能力一行,可钉后端(供应商×模型)。 * Settings 详情与 Plugin 详情共用这一份实现,避免两个入口产生不同配置口径。 @@ -89,17 +241,18 @@ export function CindyCapabilityPrefs({ // 按能力键的类目取对应清单。少一个分支的后果不是少个下拉,而是拿 // **图像模型清单**去填一个文本能力——用户存进去的值链路根本不认。 const isText = capability.startsWith('text.'); - const kind = capability === 'video.edit' - ? prefs.videoEdit - : capability.startsWith('video.') - ? prefs.video - : isText - ? prefs.text - : capability.startsWith('embed.') - ? prefs.embed - : capability === 'image.edit' - ? prefs.imageEdit - : prefs.image; + const kind = + capability === 'video.edit' + ? prefs.videoEdit + : capability.startsWith('video.') + ? prefs.video + : isText + ? prefs.text + : capability.startsWith('embed.') + ? prefs.embed + : capability === 'image.edit' + ? prefs.imageEdit + : prefs.image; // 目录没给这个类目任何模型 = 能力暂不可用:行照旧显示(插件确实申请了 // 这项能力),但右侧不给下拉,改一句不可点的灰字,不拿旧型号冒充可选。 const defaultModel = kind.defaultModel; @@ -114,10 +267,11 @@ export function CindyCapabilityPrefs({ // (插件声明优先于系统链;用户在下面的钉档永远最大)。 const declaredModel = isText ? (prefs.text?.declaredModel ?? null) : null; const current = overrides[capability]; - const selectValue = current && current !== defaultModel?.id ? current : FOLLOW_DEFAULT_VALUE; + const selectValue = + current && current !== defaultModel?.id ? current : FOLLOW_DEFAULT_VALUE; if (isText) { // 快问快答:目录全量文本模型的富列表选择器(图标/折扣与订阅徽标/分组/ - // 搜索,对齐新建对话的模型选择器);image/video 仍是原生 select。 + // 搜索,对齐新建对话的模型选择器)。 const textOptions: readonly OneshotPinOption[] = prefs.text?.options ?? []; // 存量轻量档位钉(目录扩展前钉下的合法值)回显友好名,不当 stale 露 id。 const legacyPinLabel = current @@ -160,7 +314,59 @@ export function CindyCapabilityPrefs({ ); } - // image/video 类目:原生 select(options 无分组信息,平铺)。 + + // image/video 保持原有轻量下拉,只用 Provider 图标区分同名模型的不同来源。 + if (!capability.startsWith('embed.')) { + const mediaOptions = kind.options as unknown as readonly MediaModelOption[]; + const mediaDefault = defaultModel as unknown as MediaModelOption; + return ( +
+ + {t(`settings.ghosts.detail.cindyPrefs.cap.${capability}`)} + + {unavailable && current === undefined ? ( + + {t('settings.ghosts.detail.cindyPrefs.noModels')} + + ) : unavailable ? ( + + ) : ( + void handleChange(capability, pin ?? FOLLOW_DEFAULT_VALUE)} + ariaLabel={t(`settings.ghosts.detail.cindyPrefs.cap.${capability}`)} + dense={appearance !== 'plugin'} + /> + )} +
+ ); + } + + // embed 类目继续使用原生 select。 const options: { id: string; label: string; group?: string }[] = kind.options; const selectable = options.filter((o) => o.id !== defaultModel?.id); const groupNames: string[] = []; @@ -170,7 +376,9 @@ export function CindyCapabilityPrefs({ } // 覆盖值已不在当前清单(目录演进/形态更替):如实显示原值,不假装跟随默认。 const staleOverride = - current && selectValue !== FOLLOW_DEFAULT_VALUE && !selectable.some((o) => o.id === current) + current && + selectValue !== FOLLOW_DEFAULT_VALUE && + !selectable.some((o) => o.id === current) ? current : null; return ( @@ -206,7 +414,9 @@ export function CindyCapabilityPrefs({ )} > {groupNames.map((groupName) => groupName === '' ? ( diff --git a/apps/desktop/src/renderer/features/plugin/__tests__/GhostPluginDetailSections.test.tsx b/apps/desktop/src/renderer/features/plugin/__tests__/GhostPluginDetailSections.test.tsx index 90c1b7f517..3df4df5e92 100644 --- a/apps/desktop/src/renderer/features/plugin/__tests__/GhostPluginDetailSections.test.tsx +++ b/apps/desktop/src/renderer/features/plugin/__tests__/GhostPluginDetailSections.test.tsx @@ -52,6 +52,7 @@ vi.mock('react-i18next', () => ({ 'settings.ghosts.detail.collapseInfoValue': `Collapse ${String(options?.label ?? '')}`, 'settings.ghosts.detail.panelNotDocked': 'Not docked', 'settings.ghosts.detail.cindyPrefs.noModels': 'No models available', + 'settings.defaults.restore': 'Restore default', 'settings.ghosts.detail.oauthScopeStale': 'This authorization does not include newly added permissions. Reconnect to enable them.', }; @@ -589,17 +590,47 @@ describe('Ghost plugin detail sections', () => { overrides: {}, image: { options: [ - { id: 'image-default', label: 'Image Default' }, - { id: 'image-option', label: 'Image Option' }, + { + id: 'image-default', + label: 'Image Default', + providerId: 'xd', + providerName: 'Cindy AI', + }, + { + id: 'image-option', + label: 'Image Option', + providerId: 'xd', + providerName: 'Cindy AI', + }, ], - defaultModel: { id: 'image-default', label: 'Image Default' }, + defaultModel: { + id: 'image-default', + label: 'Image Default', + providerId: 'xd', + providerName: 'Cindy AI', + }, }, imageEdit: { options: [ - { id: 'image-edit', label: 'Image Edit' }, - { id: 'image-edit-option', label: 'Image Edit Option' }, + { + id: 'image-edit', + label: 'Image Edit', + providerId: 'xd', + providerName: 'Cindy AI', + }, + { + id: 'image-edit-option', + label: 'Image Edit Option', + providerId: 'xd', + providerName: 'Cindy AI', + }, ], - defaultModel: { id: 'image-edit', label: 'Image Edit' }, + defaultModel: { + id: 'image-edit', + label: 'Image Edit', + providerId: 'xd', + providerName: 'Cindy AI', + }, }, video: { options: [{ id: 'video-default', label: 'Video Default' }], @@ -625,18 +656,16 @@ describe('Ghost plugin detail sections', () => { expect(container.querySelector('.cindy-capability-prefs')).toBeTruthy(); expect(container.querySelector('.cindy-capability-row')).toBeTruthy(); - const selects = screen.getAllByRole('combobox') as HTMLSelectElement[]; - expect(selects).toHaveLength(2); - expect(Array.from(selects[0]!.options, (option) => option.value)).toEqual([ - '__default__', - 'image-option', - ]); - expect(Array.from(selects[1]!.options, (option) => option.value)).toEqual([ - '__default__', - 'image-edit-option', - ]); - expect(selects[0]!.className).toContain('cindy-capability-select'); - expect(selects[0]!.className).toContain('max-w-[60%]'); + const pickers = screen.getAllByRole('combobox'); + expect(pickers).toHaveLength(2); + fireEvent.click(pickers[0]!); + expect(within(screen.getByRole('listbox')).getAllByRole('option')).toHaveLength(2); + expect(screen.getByText('Image Option')).toBeTruthy(); + fireEvent.click(pickers[0]!); + fireEvent.click(pickers[1]!); + expect(within(screen.getByRole('listbox')).getAllByRole('option')).toHaveLength(2); + expect(screen.getByText('Image Edit Option')).toBeTruthy(); + expect(pickers[0]!.className).toContain('max-w-[60%]'); }); // 2026-08-05:快问快答钉档扩展为目录全量文本模型——富列表选择器(供应商 @@ -851,7 +880,8 @@ describe('Ghost plugin detail sections', () => { vi.unstubAllEnvs(); }); - it('replaces the select with tertiary copy for ability categories the catalog has no models for', () => { + it('keeps a reset entry for a stale media override when the catalog has no models', async () => { + const setCindyPref = vi.fn(async () => ({ overrides: {} })); Object.defineProperty(window, 'electronAPI', { configurable: true, value: { @@ -859,8 +889,20 @@ describe('Ghost plugin detail sections', () => { cindyPrefsSync: () => ({ overrides: { 'video.generate': 'retired-video-model' }, image: { - options: [{ id: 'image-default', label: 'Image Default' }], - defaultModel: { id: 'image-default', label: 'Image Default' }, + options: [ + { + id: 'image-default', + label: 'Image Default', + providerId: 'xd', + providerName: 'Cindy AI', + }, + ], + defaultModel: { + id: 'image-default', + label: 'Image Default', + providerId: 'xd', + providerName: 'Cindy AI', + }, }, imageEdit: { options: [{ id: 'image-edit', label: 'Image Edit' }], @@ -870,7 +912,7 @@ describe('Ghost plugin detail sections', () => { video: { options: [], defaultModel: null }, videoEdit: { options: [], defaultModel: null }, }), - setCindyPref: vi.fn(), + setCindyPref, }, }, }); @@ -888,10 +930,13 @@ describe('Ghost plugin detail sections', () => { expect(screen.getAllByRole('combobox')).toHaveLength(1); const empties = container.querySelectorAll('.cindy-capability-empty'); - expect(empties).toHaveLength(2); - empties.forEach((node) => { - expect(node.textContent).toBe('No models available'); - expect(node.className).toContain('text-[var(--text-tertiary)]'); + expect(empties).toHaveLength(1); + expect(empties[0]!.textContent).toBe('No models available'); + expect(empties[0]!.className).toContain('text-[var(--text-tertiary)]'); + + fireEvent.click(screen.getByRole('button', { name: 'Restore default' })); + await waitFor(() => { + expect(setCindyPref).toHaveBeenCalledWith('builtin.example', 'video.generate', null); }); }); diff --git a/apps/desktop/src/renderer/vite-env.d.ts b/apps/desktop/src/renderer/vite-env.d.ts index f5d114d709..0123e96ba4 100644 --- a/apps/desktop/src/renderer/vite-env.d.ts +++ b/apps/desktop/src/renderer/vite-env.d.ts @@ -1075,6 +1075,22 @@ interface GoalStatusPayload { lastReason: string | null; } +type CindyMediaPreferenceOption = { + id: string; + label: string; + group: string; + providerId: string; + providerName: string; + modelId: string; + modelName: string; + routing?: import('@cindy/model-providers').Provider['routing']; +}; + +type CindyMediaPreferenceKind = { + options: CindyMediaPreferenceOption[]; + defaultModel: CindyMediaPreferenceOption | null; +}; + interface ElectronAPI { platform: string; osRelease: string; @@ -1227,22 +1243,10 @@ interface ElectronAPI { */ cindyPrefsSync: (id: string) => { overrides: Record; - image: { - options: Array<{ id: string; label: string }>; - defaultModel: { id: string; label: string } | null; - }; - imageEdit: { - options: Array<{ id: string; label: string }>; - defaultModel: { id: string; label: string } | null; - }; - video: { - options: Array<{ id: string; label: string }>; - defaultModel: { id: string; label: string } | null; - }; - videoEdit: { - options: Array<{ id: string; label: string }>; - defaultModel: { id: string; label: string } | null; - }; + image: CindyMediaPreferenceKind; + imageEdit: CindyMediaPreferenceKind; + video: CindyMediaPreferenceKind; + videoEdit: CindyMediaPreferenceKind; /** 文本类(快问快答):选项是当前供应商目录的全部文本模型(cat: 编码钉值, * 带供应商/模型/徽标等结构化字段供富列表渲染);declaredModel = 身份卡声明 * 的偏好模型(目录里解析得到才给,"跟随默认"行据此如实展示实际路由)。 */ diff --git a/apps/desktop/src/shared/ghost.ts b/apps/desktop/src/shared/ghost.ts index 82aedc795c..c5ee96a896 100644 --- a/apps/desktop/src/shared/ghost.ts +++ b/apps/desktop/src/shared/ghost.ts @@ -6055,7 +6055,7 @@ export type GhostMediaCapability = (typeof GHOST_MEDIA_CAPABILITIES)[number]; /** 插件读取自身某项 Cindy 媒体能力当前实际选型的只读结果。 */ export type GhostCindyPreferenceResult = - | { ok: true; capability: GhostMediaCapability; modelId: string } + | { ok: true; capability: GhostMediaCapability; modelId: string; providerId: string } | { ok: false; errorCode: 'INVALID_REQUEST' | 'PERMISSION_DENIED' | 'NOT_AVAILABLE'; @@ -6074,10 +6074,13 @@ export type GhostMediaModelsResult = models: Array<{ id: string; name: string; + /** 同名模型可由多个来源提供;与 id 一起构成精确选择。 */ + providerId: string; /** Gateway architecture 的归一化投影;缺省表示上游未声明,插件不得猜测。 */ modalities?: { input: string[]; output: string[] }; }>; defaultModelId: string | null; + defaultProviderId: string | null; } | { ok: false; diff --git a/packages/cindy-tools/src/__tests__/ghostMcp.test.ts b/packages/cindy-tools/src/__tests__/ghostMcp.test.ts index de05d2bb1e..9de39df692 100644 --- a/packages/cindy-tools/src/__tests__/ghostMcp.test.ts +++ b/packages/cindy-tools/src/__tests__/ghostMcp.test.ts @@ -1114,12 +1114,14 @@ describe("cindy · media MCP 边界", () => { const result = await handleMedia(fakeDeps({ callMedia }), { action: "prepare", capability: "image.generate", + provider_id: "openai", model_id: "vendor/image-model", }); expect(callMedia).toHaveBeenCalledWith({ action: "prepare", capability: "image.generate", + providerId: "openai", modelId: "vendor/image-model", }); expect(parsePayload(result)).toMatchObject({ ok: true, status: "prepared" }); diff --git a/packages/cindy-tools/src/ghost/mcpServer.ts b/packages/cindy-tools/src/ghost/mcpServer.ts index f06c4e9754..2d7a38cc45 100644 --- a/packages/cindy-tools/src/ghost/mcpServer.ts +++ b/packages/cindy-tools/src/ghost/mcpServer.ts @@ -96,7 +96,7 @@ const D_MEDIA = [ "如果插件需要消费最终结果,Agent 可在本工具成功后再调用插件声明的普通工具,并通过 ghost_call.attachments 显式交接;这不是生成请求的必经步骤。", "模型存在性来自当前账号的 Model Access model group;list_models 只投影同时满足 Gateway modalities、Guide operation 与当前客户端协议支持度的可执行模型。", "媒体生成必须由当前 Agent 通过本工具发起;插件面板和插件沙箱代码不得直接提交生成请求。", - "插件已返回用户配置的 model_id 时可直接用它和目标 capability 走 prepare → request;没有已配置 model_id 时,先用 list_models 查询可用媒体模型。prepare 会由 Server 根据 model_id 返回 Guide,并在 Guide 不存在或不支持该 capability 时明确报错。", + "插件已返回用户配置的 model_id/provider_id 时必须原样传给 prepare,再按目标 capability 走 prepare → request;provider_id 用于区分不同 Provider 下的同名模型。没有已配置模型时,先用 list_models 查询。Gateway 模型的 prepare 会由 Server 根据 model_id 返回 Guide,并在 Guide 不存在或不支持该 capability 时明确报错。", "异步任务的 request 返回 pending 时,再按 recommended_poll_after_ms 调 poll;同步任务会直接返回 xdt_image_urls / xdt_video_urls。", "模型 id、endpoint、Authorization 和 wire model 均由 Host 管理,不要写进 body,不要猜测或覆盖。", "Guide 缺失、能力不匹配或当前客户端不支持协议时,结果会带稳定 errorCode、retryable、outcomeKnown 和 allowedActions;可按 allowedActions 换模型、改用其它已授权工具或仍存在的旧链路,不要把 INTERNAL 当成协议能力结论。", @@ -508,6 +508,7 @@ export async function handleMedia( input: { action: "list_models" | "prepare" | "request" | "poll"; capability?: CindyMediaCapability; + provider_id?: string; model_id?: string; invocation_id?: string; body?: Record; @@ -549,6 +550,7 @@ export async function handleMedia( } result = await deps.callMedia({ action: "prepare", + ...(input.provider_id ? { providerId: input.provider_id } : {}), modelId: input.model_id, capability: input.capability, }); @@ -1152,6 +1154,11 @@ export function createCindyGhostsMcpServer( .max(256) .optional() .describe("prepare 时必填;使用插件返回的已配置 model_id,或来自本次 list_models"), + provider_id: z + .string() + .max(128) + .optional() + .describe("prepare 时可选;插件或 list_models 返回 provider_id 时必须原样传入,以区分同名模型的执行来源"), invocation_id: z .string() .max(128) diff --git a/packages/cindy-tools/src/types.ts b/packages/cindy-tools/src/types.ts index e1c7086798..c206a53ea3 100644 --- a/packages/cindy-tools/src/types.ts +++ b/packages/cindy-tools/src/types.ts @@ -277,7 +277,13 @@ export type CindyMediaCapability = /** 当前 Agent 专用的永久 media 工具与 Host 之间的稳定请求面。插件运行时代码不调用。 */ export type CindyMediaToolRequest = | { action: 'list_models'; capability?: CindyMediaCapability } - | { action: 'prepare'; modelId: string; capability: CindyMediaCapability } + | { + action: 'prepare'; + /** 精确执行来源;插件配置返回 providerId 时必须原样传入。 */ + providerId?: string; + modelId: string; + capability: CindyMediaCapability; + } | { action: 'request'; invocationId: string; body: Record } | { action: 'poll'; invocationId: string }; diff --git a/packages/model-providers/src/builtin.ts b/packages/model-providers/src/builtin.ts index 2d6e0fac5b..95fa6b25e8 100644 --- a/packages/model-providers/src/builtin.ts +++ b/packages/model-providers/src/builtin.ts @@ -145,7 +145,14 @@ const OPENAI_PROVIDER: Provider = { // image_generation tool;用户另配 `openai-images` Platform key 时优先走 public // Images API。id 带 openai/ 前缀(跨供应商数据契约,防 first-wins 归属漂移); // 不声明 imageDefaults(xd 默认地位不动)。 - imageModels: [{ id: 'openai/gpt-image-2', name: 'GPT Image 2' }], + imageModels: [ + { + id: 'openai/gpt-image-2', + name: 'GPT Image 2', + modalities: { input: ['text', 'image'], output: ['image'] }, + officialDocs: 'https://platform.openai.com/docs/guides/image-generation', + }, + ], routing: { codex: { upstream: 'https://chatgpt.com/backend-api/codex', diff --git a/packages/model-providers/src/catalog.ts b/packages/model-providers/src/catalog.ts index 234dbf9272..e42aabdb34 100644 --- a/packages/model-providers/src/catalog.ts +++ b/packages/model-providers/src/catalog.ts @@ -392,7 +392,12 @@ function validateProvider(p: Provider): void { function validateMediaModels( providerId: string, modelsField: string, - models: { id: string; name: string }[] | undefined, + models: Array<{ + id: string; + name: string; + modalities?: { input: string[]; output: string[] }; + officialDocs?: string; + }> | undefined, defaultsField: string, defaults: { standard: string; draft?: string; best?: string } | undefined, ): void { @@ -405,6 +410,44 @@ function validateMediaModels( assert(typeof m.name === 'string' && m.name.length > 0, `provider '${providerId}' ${modelsField} '${m.id}' missing name`); assert(!seen.has(m.id), `provider '${providerId}' ${modelsField} has duplicate id '${m.id}'`); seen.add(m.id); + if (m.modalities !== undefined) { + assert( + m.modalities && typeof m.modalities === 'object' && !Array.isArray(m.modalities), + `provider '${providerId}' ${modelsField} '${m.id}' modalities must be an object`, + ); + for (const key of ['input', 'output'] as const) { + const values = m.modalities[key]; + assert( + Array.isArray(values) && values.length > 0 && values.length <= 16, + `provider '${providerId}' ${modelsField} '${m.id}' modalities.${key} must be a non-empty bounded array`, + ); + assert( + values.every( + (value) => + typeof value === 'string' && + value.length > 0 && + value.length <= 64 && + value.trim() === value, + ) && new Set(values).size === values.length, + `provider '${providerId}' ${modelsField} '${m.id}' modalities.${key} contains invalid values`, + ); + } + } + if (m.officialDocs !== undefined) { + let valid = false; + if (typeof m.officialDocs === 'string' && m.officialDocs.length <= 2_048) { + try { + const url = new URL(m.officialDocs); + valid = url.protocol === 'https:' && !url.username && !url.password; + } catch { + valid = false; + } + } + assert( + valid, + `provider '${providerId}' ${modelsField} '${m.id}' officialDocs must be https`, + ); + } } } if (defaults !== undefined) { diff --git a/packages/model-providers/src/index.ts b/packages/model-providers/src/index.ts index c1c291b731..8898adf80c 100644 --- a/packages/model-providers/src/index.ts +++ b/packages/model-providers/src/index.ts @@ -19,6 +19,7 @@ export type { RoutingDescriptor, ModelCost, CatalogModel, + ProviderMediaModel, Provider, Catalog, CustomProviderConfig, diff --git a/packages/model-providers/src/types.ts b/packages/model-providers/src/types.ts index 0f7bf46f03..b980383502 100644 --- a/packages/model-providers/src/types.ts +++ b/packages/model-providers/src/types.ts @@ -421,6 +421,15 @@ export interface CatalogModel { disabled?: boolean; } +/** Provider 自己执行的媒体模型;modalities 是能力判断的唯一依据。 */ +export interface ProviderMediaModel { + id: string; + name: string; + modalities?: { input: string[]; output: string[] }; + officialDocs?: string; + disabled?: boolean; +} + /** 供应商定义。 */ export interface Provider { /** 'anthropic' | 'openai' | 'xd' | 未来自定义 id。 */ @@ -461,7 +470,7 @@ export interface Provider { * `disabled` 是视图层字段(与 CatalogModel.disabled 同语义):buildRegistry 按用户 * 停用 override 烘焙,设置页据此渲染专属媒体条目的停用状态;目录数据本身不携带。 */ - imageModels?: { id: string; name: string; disabled?: boolean }[]; + imageModels?: ProviderMediaModel[]; /** * 图像能力的默认选型(与 imageModels 配套;值必须是 imageModels 里的 id): * - standard:未指定任何偏好时的默认模型(意识 cindy 槽"默认"档的真身); @@ -476,7 +485,7 @@ export interface Provider { * 消费方为意识 cindy 槽(白名单 + 详情页下拉)。可选,additions-only。 * `disabled` 同 imageModels:视图层停用标志,buildRegistry 烘焙。 */ - videoModels?: { id: string; name: string; disabled?: boolean }[]; + videoModels?: ProviderMediaModel[]; /** * 视频能力的默认选型(与 videoModels 配套;值必须是 videoModels 里的 id; * 语义同 imageDefaults:standard 必填,draft/best 缺省回落 standard)。