diff --git a/readme.md b/readme.md index 51b3fa3d..ef039327 100644 --- a/readme.md +++ b/readme.md @@ -203,6 +203,12 @@ edit.registerAssetGenerator(async ({ clipId, asset, signal }) => { }); ``` +Pass a model catalogue as the registration's `catalogue` option to show model and option +controls. Entries must include their option schema; those without one are ignored. The +[Edit API](https://shotstack.io/docs/api/#shotstack-edit) returns this shape from +`GET /models?expand=options`. The SDK stores a snapshot; fetching and refreshing it remain +the host's responsibility. + The SDK writes the returned URL to the clip, so the change is undoable and autosaves like any other edit. It tracks whether a clip is generating or has failed, and renders those states; a rejection's message is shown as-is next to a retry action. Everything diff --git a/src/core/edit-session.ts b/src/core/edit-session.ts index 7e8140ec..707587fe 100644 --- a/src/core/edit-session.ts +++ b/src/core/edit-session.ts @@ -57,8 +57,14 @@ import * as pixi from "pixi.js"; import { CommandQueue } from "./commands/command-queue"; import { CommandNoop, type EditCommand, type CommandContext, type CommandResult } from "./commands/types"; import { EditDocument } from "./edit-document"; -import { AssetGenerator, type AssetGeneratorHandler, type ClipGenerationState } from "./generation/asset-generator"; +import { + AssetGenerator, + type AssetGeneratorHandler, + type AssetGeneratorOptions, + type ClipGenerationState +} from "./generation/asset-generator"; import { migrateLegacyGeneratedAsset } from "./generation/legacy-asset-migration"; +import type { GenerationAssetType, GenerationModelDefinition } from "./generation/model-catalogue"; import { PlayerReconciler } from "./player-reconciler"; import { resolve as resolveDocument, resolveClip as resolveClipById, type SingleClipContext } from "./resolver"; import { InvalidAssetUrlError, extractClipUrls, extractTrackUrls } from "./url-validation"; @@ -441,8 +447,9 @@ export class Edit { * URL back to the clip; the host owns how generation happens and what a failure * message says. Without a handler, no generate affordance is shown. */ - public registerAssetGenerator(handler: AssetGeneratorHandler): void { - this.assetGenerator.register(handler); + public registerAssetGenerator(handler: AssetGeneratorHandler, options?: AssetGeneratorOptions): void { + this.assetGenerator.register(handler, options); + this.internalEvents.emit(InternalEvent.AssetGeneratorChanged); } /** @internal */ @@ -450,6 +457,11 @@ export class Edit { return this.assetGenerator.hasHandler(); } + /** @internal */ + public getGenerationModels(type: GenerationAssetType): readonly GenerationModelDefinition[] | undefined { + return this.assetGenerator.getModels(type); + } + /** * Generate the asset for a prompt-bearing clip and write the result to it. * diff --git a/src/core/events/edit-events.ts b/src/core/events/edit-events.ts index 0b591484..44338faa 100644 --- a/src/core/events/edit-events.ts +++ b/src/core/events/edit-events.ts @@ -135,6 +135,7 @@ export const InternalEvent = { ClipBlurred: "clip:blurred", // Asset generation UI + AssetGeneratorChanged: "assetGenerator:changed", ClipGenerationStarted: "clip:generationStarted", ClipGenerationCompleted: "clip:generationCompleted", ClipGenerationFailed: "clip:generationFailed" @@ -232,6 +233,7 @@ export type InternalEventMap = { [InternalEvent.ClipBlurred]: void; // Asset generation UI + [InternalEvent.AssetGeneratorChanged]: void; [InternalEvent.ClipGenerationStarted]: { clipId: string }; [InternalEvent.ClipGenerationCompleted]: { clipId: string }; [InternalEvent.ClipGenerationFailed]: { clipId: string; error: string }; diff --git a/src/core/generation/asset-generator.ts b/src/core/generation/asset-generator.ts index f862fa51..c3dd8341 100644 --- a/src/core/generation/asset-generator.ts +++ b/src/core/generation/asset-generator.ts @@ -1,5 +1,12 @@ import { isAiAsset } from "@core/shared/ai-asset-utils"; +import { + type GenerationAssetType, + type GenerationModelCatalogueResponse, + type GenerationModelDefinition, + readGenerationModels +} from "./model-catalogue"; + /** Passed to the host handler for one generation. */ export interface AssetGenerationRequest { clipId: string; @@ -16,6 +23,11 @@ export interface AssetGenerationRequest { /** Resolves with the URL of the generated asset. */ export type AssetGeneratorHandler = (request: AssetGenerationRequest) => Promise<{ url: string }>; +export interface AssetGeneratorOptions { + /** Model catalogue with each entry's option schema included. Entries without one are ignored. */ + catalogue?: GenerationModelCatalogueResponse; +} + export interface ClipGenerationState { status: "generating" | "failed"; /** Failure message: the host's when generation failed, the SDK's when the result could not be applied. */ @@ -37,13 +49,19 @@ export interface AssetGeneratorDeps { */ export class AssetGenerator { private handler?: AssetGeneratorHandler; + private models?: readonly GenerationModelDefinition[]; private readonly states = new Map(); private readonly controllers = new Map(); constructor(private readonly deps: AssetGeneratorDeps) {} - public register(handler: AssetGeneratorHandler): void { + public register(handler: AssetGeneratorHandler, options?: AssetGeneratorOptions): void { this.handler = handler; + this.models = options?.catalogue === undefined ? undefined : readGenerationModels(options.catalogue); + } + + public getModels(type: GenerationAssetType): readonly GenerationModelDefinition[] | undefined { + return this.models?.filter(model => model.type === type); } public hasHandler(): boolean { diff --git a/src/core/generation/model-catalogue.ts b/src/core/generation/model-catalogue.ts new file mode 100644 index 00000000..b9228868 --- /dev/null +++ b/src/core/generation/model-catalogue.ts @@ -0,0 +1,151 @@ +import type { operations } from "@shotstack/schemas"; + +export type GenerationModelCatalogueResponse = operations["getModels"]["responses"][200]["content"]["application/json"]; +export type GenerationAssetType = "image" | "video" | "audio"; + +export type GenerationOptionDefinition = { + name: string; + title: string; + type: "string" | "boolean" | "integer"; + required: boolean; + values?: readonly string[]; + format?: "uri"; + minimum?: number; + maximum?: number; + hasDefault: boolean; + defaultValue?: unknown; +}; + +/** A published option the editor has no control for; shown read-only so its value is never a surprise. */ +export type GenerationUnsupportedOption = { + name: string; + title: string; +}; + +export type GenerationModelDefinition = { + model: string; + type: GenerationAssetType; + optionNames: readonly string[]; + options: readonly GenerationOptionDefinition[]; + unsupported: readonly GenerationUnsupportedOption[]; +}; + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const hasOwn = (value: Record, key: string): boolean => Object.prototype.hasOwnProperty.call(value, key); + +export const isGenerationOptionValueValid = (option: GenerationOptionDefinition, value: unknown): boolean => { + if (option.type === "string") { + if (typeof value !== "string" || value.length === 0) return false; + if (option.values && !option.values.includes(value)) return false; + if (option.format !== "uri") return true; + try { + return Boolean(new URL(value)); + } catch { + return false; + } + } + if (option.type === "boolean") return typeof value === "boolean"; + if (!Number.isInteger(value)) return false; + if (option.minimum !== undefined && (value as number) < option.minimum) return false; + if (option.maximum !== undefined && (value as number) > option.maximum) return false; + return true; +}; + +const readOption = (name: string, value: unknown, required: boolean): GenerationOptionDefinition | undefined => { + if (!isRecord(value) || !["string", "boolean", "integer"].includes(String(value["type"]))) return undefined; + if (value["enum"] !== undefined && (!Array.isArray(value["enum"]) || !value["enum"].every(item => typeof item === "string"))) { + return undefined; + } + if (value["format"] !== undefined && value["format"] !== "uri") return undefined; + + const option: GenerationOptionDefinition = { + name, + title: typeof value["title"] === "string" ? value["title"] : name, + type: value["type"] as GenerationOptionDefinition["type"], + required, + ...(Array.isArray(value["enum"]) ? { values: value["enum"] as string[] } : {}), + ...(value["format"] === "uri" ? { format: "uri" as const } : {}), + ...(typeof value["minimum"] === "number" ? { minimum: value["minimum"] } : {}), + ...(typeof value["maximum"] === "number" ? { maximum: value["maximum"] } : {}), + hasDefault: hasOwn(value, "default"), + ...(hasOwn(value, "default") ? { defaultValue: value["default"] } : {}) + }; + + if (option.hasDefault && !isGenerationOptionValueValid(option, option.defaultValue)) { + return { ...option, hasDefault: false, defaultValue: undefined }; + } + return option; +}; + +export const readGenerationModels = (catalogue: unknown): readonly GenerationModelDefinition[] => { + if (!isRecord(catalogue) || !Array.isArray(catalogue["models"])) return []; + + return catalogue["models"].flatMap(entry => { + if (!isRecord(entry) || typeof entry["model"] !== "string") return []; + if (!(["image", "video", "audio"] as const).includes(entry["type"] as GenerationAssetType)) return []; + + const { options: schema } = entry; + if (!isRecord(schema) || schema["type"] !== "object" || schema["additionalProperties"] !== false) return []; + const { properties } = schema; + if (!isRecord(properties)) return []; + const required = schema["required"] === undefined ? [] : schema["required"]; + if (!Array.isArray(required) || !required.every(name => typeof name === "string")) return []; + if (required.includes("inputSrc")) return []; + + const options: GenerationOptionDefinition[] = []; + const unsupported: GenerationUnsupportedOption[] = []; + for (const [name, value] of Object.entries(properties)) { + const option = readOption(name, value, required.includes(name)); + if (option) options.push(option); + else unsupported.push({ name, title: isRecord(value) && typeof value["title"] === "string" ? value["title"] : name }); + } + if (required.some(name => !options.some(option => option.name === name))) return []; + + return [ + { + model: entry["model"], + type: entry["type"] as GenerationAssetType, + optionNames: Object.keys(properties), + options, + unsupported + } + ]; + }); +}; + +export const reconcileGenerationOptions = ( + model: GenerationModelDefinition, + raw: Record, + resolved: Record +): Record => { + const next: Record = Object.fromEntries( + Object.keys(raw) + .filter(name => !model.optionNames.includes(name)) + .map(name => [name, undefined]) + ); + + for (const name of model.optionNames) { + const option = model.options.find(candidate => candidate.name === name); + const value = hasOwn(resolved, name) ? resolved[name] : raw[name]; + + if (!option) { + if (hasOwn(raw, name)) next[name] = raw[name]; + } else if (hasOwn(raw, name) && isGenerationOptionValueValid(option, value)) { + next[name] = raw[name]; + } else if (option.hasDefault) { + next[name] = option.defaultValue; + } else if (hasOwn(raw, name)) { + next[name] = undefined; + } + } + + return next; +}; + +export const missingGenerationOptions = ( + model: GenerationModelDefinition, + values: Record +): readonly string[] => + model.options.filter(option => option.required && !isGenerationOptionValueValid(option, values[option.name])).map(option => option.title); diff --git a/src/index.ts b/src/index.ts index 12f74134..e3b4b33c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,7 +9,7 @@ export { UIController } from "@core/ui/ui-controller"; export { WebGLUnsupportedError } from "@core/webgl-support"; export type { UIControllerOptions, ToolbarButtonConfig } from "@core/ui/ui-controller"; -export type { AssetGenerationRequest, AssetGeneratorHandler } from "@core/generation/asset-generator"; +export type { AssetGenerationRequest, AssetGeneratorHandler, AssetGeneratorOptions } from "@core/generation/asset-generator"; export type { EditConfig } from "@core/schemas"; export type { CommandResult } from "@core/commands/types"; diff --git a/test-package.js b/test-package.js index 85009934..67400ee3 100644 --- a/test-package.js +++ b/test-package.js @@ -60,7 +60,7 @@ const CONTRACT = { "clearSelection(", "registerClipRenderer(" ], - Edit: ["validateEdit(", "getTimelineFonts(", "getContentClipIdForLuma(", "getInternalEvents(", "pruneUnusedFonts("] + Edit: ["validateEdit(", "getTimelineFonts(", "getContentClipIdForLuma(", "getInternalEvents(", "getGenerationModels(", "pruneUnusedFonts("] }, dtsForbiddenTokens: [ "export declare class SelectionHandles", @@ -128,7 +128,7 @@ const CONTRACT = { "export declare type Seconds =" ], dtsPublicAnchors: [ - { className: "Edit", tokens: ["load(): Promise;"] }, + { className: "Edit", tokens: ["load(): Promise;", "registerAssetGenerator(handler: AssetGeneratorHandler, options?: AssetGeneratorOptions): void;"] }, { className: "Canvas", tokens: ["load(): Promise;"] }, { className: "UIController", tokens: ["registerButton(config: ToolbarButtonConfig): this;"] }, { className: "Timeline", tokens: ["load(): Promise;"] } diff --git a/tests/asset-generator.test.ts b/tests/asset-generator.test.ts index e6c057f6..3b0bc7d5 100644 --- a/tests/asset-generator.test.ts +++ b/tests/asset-generator.test.ts @@ -1,7 +1,12 @@ import { AssetGenerator, type AssetGeneratorDeps } from "@core/generation/asset-generator"; +import type { GenerationModelCatalogueResponse } from "@core/generation/model-catalogue"; const PROMPT_ASSET = { type: "image", prompt: "a red apple" }; +const imageCatalogue = (model: string): GenerationModelCatalogueResponse => ({ + models: [{ model, type: "image", options: { type: "object", properties: {}, additionalProperties: false } }] +}); + function makeDeps(overrides: Partial = {}) { const started: string[] = []; const completed: string[] = []; @@ -23,6 +28,34 @@ function makeDeps(overrides: Partial = {}) { } describe("AssetGenerator", () => { + it("stores a catalogue snapshot and replaces it on re-registration", () => { + const generator = new AssetGenerator(makeDeps().deps); + const source = imageCatalogue("first"); + generator.register(async () => ({ url: "https://cdn/out.png" }), { catalogue: source }); + (source.models[0] as { model: string }).model = "mutated"; + + expect(generator.getModels("image")?.map(({ model }) => model)).toEqual(["first"]); + + generator.register(async () => ({ url: "https://cdn/out.png" }), { catalogue: imageCatalogue("second") }); + expect(generator.getModels("image")?.map(({ model }) => model)).toEqual(["second"]); + + generator.register(async () => ({ url: "https://cdn/out.png" })); + expect(generator.getModels("image")).toBeUndefined(); + }); + + it("returns only models for the requested asset type", () => { + const generator = new AssetGenerator(makeDeps().deps); + const mixed = { + models: [ + ...imageCatalogue("image-model").models, + { model: "audio-model", type: "audio", options: { type: "object", properties: {}, additionalProperties: false } } + ] + } as GenerationModelCatalogueResponse; + generator.register(async () => ({ url: "https://cdn/out.png" }), { catalogue: mixed }); + + expect(generator.getModels("audio")?.map(({ model }) => model)).toEqual(["audio-model"]); + }); + it("writes the generated src back and reports completion", async () => { const { deps, started, completed, applied } = makeDeps(); const generator = new AssetGenerator(deps); diff --git a/tests/edit-clip-operations.test.ts b/tests/edit-clip-operations.test.ts index 41c9cca7..d4d386fa 100644 --- a/tests/edit-clip-operations.test.ts +++ b/tests/edit-clip-operations.test.ts @@ -764,6 +764,25 @@ describe("Edit Clip Operations", () => { }); describe("asset generation", () => { + it("notifies internal UI when generator configuration changes", () => { + const changed = jest.fn(); + edit.getInternalEvents().on(InternalEvent.AssetGeneratorChanged, changed); + + edit.registerAssetGenerator(async () => ({ url: "https://cdn.example.com/out.png" }), { + catalogue: { + models: [ + { + model: "flux-schnell", + type: "image", + options: { type: "object", properties: {}, additionalProperties: false } + } + ] + } + }); + + expect(changed).toHaveBeenCalledTimes(1); + }); + type DocLookup = { document: { getClipId(t: number, c: number): string | null } }; const clipIdAt = (target: Edit, t: number, c: number) => ((target as unknown as DocLookup).document.getClipId(t, c) as string); diff --git a/tests/model-catalogue.test.ts b/tests/model-catalogue.test.ts new file mode 100644 index 00000000..5aef9f56 --- /dev/null +++ b/tests/model-catalogue.test.ts @@ -0,0 +1,115 @@ +import { + type GenerationModelCatalogueResponse, + type GenerationOptionDefinition, + isGenerationOptionValueValid, + missingGenerationOptions, + readGenerationModels, + reconcileGenerationOptions +} from "@core/generation/model-catalogue"; + +const modelWithOptions = ( + properties: Record, + required: string[] = [], + model = "custom", + type: "image" | "video" | "audio" = "image" +) => ({ + model, + type, + options: { type: "object", properties, ...(required.length > 0 ? { required } : {}), additionalProperties: false } +}); + +const catalogue: GenerationModelCatalogueResponse = { + models: [ + modelWithOptions({}, [], "flux-schnell"), + modelWithOptions( + { + resolution: { + type: "string", + title: "Resolution", + enum: ["1K", "2K"], + default: "1K" + } + }, + [], + "nano-banana-2" + ), + modelWithOptions( + { inputSrc: { type: "string", format: "uri", title: "Start image" } }, + ["inputSrc"], + "needs-image", + "video" + ), + modelWithOptions( + { + inputSrc: { type: "string", format: "uri", title: "Start image" }, + generateAudio: { type: "boolean", title: "Generate audio" } + }, + [], + "seedance-2.0", + "video" + ), + { model: "unexpanded", type: "audio" } + ] +}; + +describe("generation model catalogue", () => { + it("keeps expanded models and excludes required input media", () => { + const models = readGenerationModels(catalogue); + + expect(models.map(({ model }) => model)).toEqual(["flux-schnell", "nano-banana-2", "seedance-2.0"]); + }); + + it("preserves raw merge fields when their resolved value is valid", () => { + const model = readGenerationModels(catalogue).find(entry => entry.model === "nano-banana-2"); + + expect(reconcileGenerationOptions(model!, { resolution: "{{ SIZE }}", old: true }, { resolution: "2K", old: true })).toEqual({ + resolution: "{{ SIZE }}", + old: undefined + }); + }); + + it("uses a destination default when the shared value is invalid", () => { + const model = readGenerationModels(catalogue).find(entry => entry.model === "nano-banana-2"); + + expect(reconcileGenerationOptions(model!, { resolution: "4K" }, { resolution: "4K" })).toEqual({ resolution: "1K" }); + }); + + it("records an unrenderable property with its published title", () => { + const [entry] = readGenerationModels({ + models: [ + modelWithOptions({ + forceInstrumental: { type: "boolean", title: "Instrumental only" }, + compositionPlan: { type: "object", title: "Composition plan", properties: {}, additionalProperties: false } + }) + ] + }); + expect(entry?.options.map(o => o.name)).toEqual(["forceInstrumental"]); + expect(entry?.unsupported).toEqual([{ name: "compositionPlan", title: "Composition plan" }]); + expect(entry?.optionNames).toEqual(["forceInstrumental", "compositionPlan"]); + }); + + it("keeps optional unsupported properties but rejects unsupported required ones", () => { + const optional = modelWithOptions({ seed: { type: "number" } }); + const required = modelWithOptions({ seed: { type: "number" } }, ["seed"]); + + expect(readGenerationModels({ models: [optional] })).toHaveLength(1); + expect(readGenerationModels({ models: [required] })).toHaveLength(0); + }); + + it.each([ + [{ type: "boolean", name: "enabled", title: "Enabled", required: true, hasDefault: false }, false, true], + [{ type: "integer", name: "length", title: "Length", required: true, minimum: 1, hasDefault: false }, 0, false], + [{ type: "string", format: "uri", name: "source", title: "Source", required: true, hasDefault: false }, "not a URL", false], + [{ type: "string", name: "voice", title: "Voice", required: true, hasDefault: false }, "", false] + ])("validates an option against its published constraints", (option, value, expected) => { + expect(isGenerationOptionValueValid(option as GenerationOptionDefinition, value)).toBe(expected); + }); + + it("reports an empty required field by title", () => { + const [model] = readGenerationModels({ + models: [modelWithOptions({ voice: { type: "string", title: "Voice" } }, ["voice"], "speech", "audio")] + }); + + expect(missingGenerationOptions(model!, { voice: "" })).toEqual(["Voice"]); + }); +});