[1/2] feat: accept generation model catalogues

readme.md

@@ -203,6 +203,12 @@ edit.registerAssetGenerator(async ({ clipId, asset, signal }) => {
 });
 ```
 
+Pass a model catalogue as the registration's `catalogue` option to show model and option
+controls. Entries must include their option schema; those without one are ignored. The
+[Edit API](https://shotstack.io/docs/api/#shotstack-edit) returns this shape from
+`GET /models?expand=options`. The SDK stores a snapshot; fetching and refreshing it remain
+the host's responsibility.
+
 The SDK writes the returned URL to the clip, so the change is undoable and autosaves
 like any other edit. It tracks whether a clip is generating or has failed, and renders
 those states; a rejection's message is shown as-is next to a retry action. Everything

src/core/edit-session.ts

@@ -57,8 +57,14 @@ import * as pixi from "pixi.js";
 import { CommandQueue } from "./commands/command-queue";
 import { CommandNoop, type EditCommand, type CommandContext, type CommandResult } from "./commands/types";
 import { EditDocument } from "./edit-document";
-import { AssetGenerator, type AssetGeneratorHandler, type ClipGenerationState } from "./generation/asset-generator";
+import {
+	AssetGenerator,
+	type AssetGeneratorHandler,
+	type AssetGeneratorOptions,
+	type ClipGenerationState
+} from "./generation/asset-generator";
 import { migrateLegacyGeneratedAsset } from "./generation/legacy-asset-migration";
+import type { GenerationAssetType, GenerationModelDefinition } from "./generation/model-catalogue";
 import { PlayerReconciler } from "./player-reconciler";
 import { resolve as resolveDocument, resolveClip as resolveClipById, type SingleClipContext } from "./resolver";
 import { InvalidAssetUrlError, extractClipUrls, extractTrackUrls } from "./url-validation";
@@ -441,15 +447,21 @@ export class Edit {
 	 * URL back to the clip; the host owns how generation happens and what a failure
 	 * message says. Without a handler, no generate affordance is shown.
 	 */
-	public registerAssetGenerator(handler: AssetGeneratorHandler): void {
-		this.assetGenerator.register(handler);
+	public registerAssetGenerator(handler: AssetGeneratorHandler, options?: AssetGeneratorOptions): void {
+		this.assetGenerator.register(handler, options);
+		this.internalEvents.emit(InternalEvent.AssetGeneratorChanged);
 	}
 
 	/** @internal */
 	public hasAssetGenerator(): boolean {
 		return this.assetGenerator.hasHandler();
 	}
 
+	/** @internal */
+	public getGenerationModels(type: GenerationAssetType): readonly GenerationModelDefinition[] | undefined {
+		return this.assetGenerator.getModels(type);
+	}
+
 	/**
 	 * Generate the asset for a prompt-bearing clip and write the result to it.
 	 *

src/core/events/edit-events.ts

@@ -135,6 +135,7 @@ export const InternalEvent = {
 	ClipBlurred: "clip:blurred",
 
 	// Asset generation UI
+	AssetGeneratorChanged: "assetGenerator:changed",
 	ClipGenerationStarted: "clip:generationStarted",
 	ClipGenerationCompleted: "clip:generationCompleted",
 	ClipGenerationFailed: "clip:generationFailed"
@@ -232,6 +233,7 @@ export type InternalEventMap = {
 	[InternalEvent.ClipBlurred]: void;
 
 	// Asset generation UI
+	[InternalEvent.AssetGeneratorChanged]: void;
 	[InternalEvent.ClipGenerationStarted]: { clipId: string };
 	[InternalEvent.ClipGenerationCompleted]: { clipId: string };
 	[InternalEvent.ClipGenerationFailed]: { clipId: string; error: string };

src/core/generation/asset-generator.ts

@@ -1,5 +1,12 @@
 import { isAiAsset } from "@core/shared/ai-asset-utils";
 
+import {
+	type GenerationAssetType,
+	type GenerationModelCatalogueResponse,
+	type GenerationModelDefinition,
+	readGenerationModels
+} from "./model-catalogue";
+
 /** Passed to the host handler for one generation. */
 export interface AssetGenerationRequest {
 	clipId: string;
@@ -16,6 +23,11 @@ export interface AssetGenerationRequest {
 /** Resolves with the URL of the generated asset. */
 export type AssetGeneratorHandler = (request: AssetGenerationRequest) => Promise<{ url: string }>;
 
+export interface AssetGeneratorOptions {
+	/** Model catalogue with each entry's option schema included. Entries without one are ignored. */
+	catalogue?: GenerationModelCatalogueResponse;
+}
+
 export interface ClipGenerationState {
 	status: "generating" | "failed";
 	/** Failure message: the host's when generation failed, the SDK's when the result could not be applied. */
@@ -37,13 +49,19 @@ export interface AssetGeneratorDeps {
  */
 export class AssetGenerator {
 	private handler?: AssetGeneratorHandler;
+	private models?: readonly GenerationModelDefinition[];
 	private readonly states = new Map<string, ClipGenerationState>();
 	private readonly controllers = new Map<string, AbortController>();
 
 	constructor(private readonly deps: AssetGeneratorDeps) {}
 
-	public register(handler: AssetGeneratorHandler): void {
+	public register(handler: AssetGeneratorHandler, options?: AssetGeneratorOptions): void {
 		this.handler = handler;
+		this.models = options?.catalogue === undefined ? undefined : readGenerationModels(options.catalogue);
+	}
+
+	public getModels(type: GenerationAssetType): readonly GenerationModelDefinition[] | undefined {
+		return this.models?.filter(model => model.type === type);
 	}
 
 	public hasHandler(): boolean {

src/core/generation/model-catalogue.ts

@@ -0,0 +1,151 @@
+import type { operations } from "@shotstack/schemas";
+
+export type GenerationModelCatalogueResponse = operations["getModels"]["responses"][200]["content"]["application/json"];
+export type GenerationAssetType = "image" | "video" | "audio";
+
+export type GenerationOptionDefinition = {
+	name: string;
+	title: string;
+	type: "string" | "boolean" | "integer";
+	required: boolean;
+	values?: readonly string[];
+	format?: "uri";
+	minimum?: number;
+	maximum?: number;
+	hasDefault: boolean;
+	defaultValue?: unknown;
+};
+
+/** A published option the editor has no control for; shown read-only so its value is never a surprise. */
+export type GenerationUnsupportedOption = {
+	name: string;
+	title: string;
+};
+
+export type GenerationModelDefinition = {
+	model: string;
+	type: GenerationAssetType;
+	optionNames: readonly string[];
+	options: readonly GenerationOptionDefinition[];
+	unsupported: readonly GenerationUnsupportedOption[];
+};
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+	typeof value === "object" && value !== null && !Array.isArray(value);
+
+const hasOwn = (value: Record<string, unknown>, key: string): boolean => Object.prototype.hasOwnProperty.call(value, key);
+
+export const isGenerationOptionValueValid = (option: GenerationOptionDefinition, value: unknown): boolean => {
+	if (option.type === "string") {
+		if (typeof value !== "string" || value.length === 0) return false;
+		if (option.values && !option.values.includes(value)) return false;
+		if (option.format !== "uri") return true;
+		try {
+			return Boolean(new URL(value));
+		} catch {
+			return false;
+		}
+	}
+	if (option.type === "boolean") return typeof value === "boolean";
+	if (!Number.isInteger(value)) return false;
+	if (option.minimum !== undefined && (value as number) < option.minimum) return false;
+	if (option.maximum !== undefined && (value as number) > option.maximum) return false;
+	return true;
+};
+
+const readOption = (name: string, value: unknown, required: boolean): GenerationOptionDefinition | undefined => {
+	if (!isRecord(value) || !["string", "boolean", "integer"].includes(String(value["type"]))) return undefined;
+	if (value["enum"] !== undefined && (!Array.isArray(value["enum"]) || !value["enum"].every(item => typeof item === "string"))) {
+		return undefined;
+	}
+	if (value["format"] !== undefined && value["format"] !== "uri") return undefined;
+
+	const option: GenerationOptionDefinition = {
+		name,
+		title: typeof value["title"] === "string" ? value["title"] : name,
+		type: value["type"] as GenerationOptionDefinition["type"],
+		required,
+		...(Array.isArray(value["enum"]) ? { values: value["enum"] as string[] } : {}),
+		...(value["format"] === "uri" ? { format: "uri" as const } : {}),
+		...(typeof value["minimum"] === "number" ? { minimum: value["minimum"] } : {}),
+		...(typeof value["maximum"] === "number" ? { maximum: value["maximum"] } : {}),
+		hasDefault: hasOwn(value, "default"),
+		...(hasOwn(value, "default") ? { defaultValue: value["default"] } : {})
+	};
+
+	if (option.hasDefault && !isGenerationOptionValueValid(option, option.defaultValue)) {
+		return { ...option, hasDefault: false, defaultValue: undefined };
+	}
+	return option;
+};
+
+export const readGenerationModels = (catalogue: unknown): readonly GenerationModelDefinition[] => {
+	if (!isRecord(catalogue) || !Array.isArray(catalogue["models"])) return [];
+
+	return catalogue["models"].flatMap(entry => {
+		if (!isRecord(entry) || typeof entry["model"] !== "string") return [];
+		if (!(["image", "video", "audio"] as const).includes(entry["type"] as GenerationAssetType)) return [];
+
+		const { options: schema } = entry;
+		if (!isRecord(schema) || schema["type"] !== "object" || schema["additionalProperties"] !== false) return [];
+		const { properties } = schema;
+		if (!isRecord(properties)) return [];
+		const required = schema["required"] === undefined ? [] : schema["required"];
+		if (!Array.isArray(required) || !required.every(name => typeof name === "string")) return [];
+		if (required.includes("inputSrc")) return [];
+
+		const options: GenerationOptionDefinition[] = [];
+		const unsupported: GenerationUnsupportedOption[] = [];
+		for (const [name, value] of Object.entries(properties)) {
+			const option = readOption(name, value, required.includes(name));
+			if (option) options.push(option);
+			else unsupported.push({ name, title: isRecord(value) && typeof value["title"] === "string" ? value["title"] : name });
+		}
+		if (required.some(name => !options.some(option => option.name === name))) return [];
+
+		return [
+			{
+				model: entry["model"],
+				type: entry["type"] as GenerationAssetType,
+				optionNames: Object.keys(properties),
+				options,
+				unsupported
+			}
+		];
+	});
+};
+
+export const reconcileGenerationOptions = (
+	model: GenerationModelDefinition,
+	raw: Record<string, unknown>,
+	resolved: Record<string, unknown>
+): Record<string, unknown> => {
+	const next: Record<string, unknown> = Object.fromEntries(
+		Object.keys(raw)
+			.filter(name => !model.optionNames.includes(name))
+			.map(name => [name, undefined])
+	);
+
+	for (const name of model.optionNames) {
+		const option = model.options.find(candidate => candidate.name === name);
+		const value = hasOwn(resolved, name) ? resolved[name] : raw[name];
+
+		if (!option) {
+			if (hasOwn(raw, name)) next[name] = raw[name];
+		} else if (hasOwn(raw, name) && isGenerationOptionValueValid(option, value)) {
+			next[name] = raw[name];
+		} else if (option.hasDefault) {
+			next[name] = option.defaultValue;
+		} else if (hasOwn(raw, name)) {
+			next[name] = undefined;
+		}
+	}
+
+	return next;
+};
+
+export const missingGenerationOptions = (
+	model: GenerationModelDefinition,
+	values: Record<string, unknown>
+): readonly string[] =>
+	model.options.filter(option => option.required && !isGenerationOptionValueValid(option, values[option.name])).map(option => option.title);

src/index.ts

@@ -9,7 +9,7 @@ export { UIController } from "@core/ui/ui-controller";
 export { WebGLUnsupportedError } from "@core/webgl-support";
 
 export type { UIControllerOptions, ToolbarButtonConfig } from "@core/ui/ui-controller";
-export type { AssetGenerationRequest, AssetGeneratorHandler } from "@core/generation/asset-generator";
+export type { AssetGenerationRequest, AssetGeneratorHandler, AssetGeneratorOptions } from "@core/generation/asset-generator";
 export type { EditConfig } from "@core/schemas";
 export type { CommandResult } from "@core/commands/types";
 

test-package.js

@@ -60,7 +60,7 @@ const CONTRACT = {
 			"clearSelection(",
 			"registerClipRenderer("
 		],
-		Edit: ["validateEdit(", "getTimelineFonts(", "getContentClipIdForLuma(", "getInternalEvents(", "pruneUnusedFonts("]
+		Edit: ["validateEdit(", "getTimelineFonts(", "getContentClipIdForLuma(", "getInternalEvents(", "getGenerationModels(", "pruneUnusedFonts("]
 	},
 	dtsForbiddenTokens: [
 		"export declare class SelectionHandles",
@@ -128,7 +128,7 @@ const CONTRACT = {
 		"export declare type Seconds ="
 	],
 	dtsPublicAnchors: [
-		{ className: "Edit", tokens: ["load(): Promise<void>;"] },
+		{ className: "Edit", tokens: ["load(): Promise<void>;", "registerAssetGenerator(handler: AssetGeneratorHandler, options?: AssetGeneratorOptions): void;"] },
 		{ className: "Canvas", tokens: ["load(): Promise<void>;"] },
 		{ className: "UIController", tokens: ["registerButton(config: ToolbarButtonConfig): this;"] },
 		{ className: "Timeline", tokens: ["load(): Promise<void>;"] }

tests/asset-generator.test.ts

@@ -1,7 +1,12 @@
 import { AssetGenerator, type AssetGeneratorDeps } from "@core/generation/asset-generator";
+import type { GenerationModelCatalogueResponse } from "@core/generation/model-catalogue";
 
 const PROMPT_ASSET = { type: "image", prompt: "a red apple" };
 
+const imageCatalogue = (model: string): GenerationModelCatalogueResponse => ({
+	models: [{ model, type: "image", options: { type: "object", properties: {}, additionalProperties: false } }]
+});
+
 function makeDeps(overrides: Partial<AssetGeneratorDeps> = {}) {
 	const started: string[] = [];
 	const completed: string[] = [];
@@ -23,6 +28,34 @@ function makeDeps(overrides: Partial<AssetGeneratorDeps> = {}) {
 }
 
 describe("AssetGenerator", () => {
+	it("stores a catalogue snapshot and replaces it on re-registration", () => {
+		const generator = new AssetGenerator(makeDeps().deps);
+		const source = imageCatalogue("first");
+		generator.register(async () => ({ url: "https://cdn/out.png" }), { catalogue: source });
+		(source.models[0] as { model: string }).model = "mutated";
+
+		expect(generator.getModels("image")?.map(({ model }) => model)).toEqual(["first"]);
+
+		generator.register(async () => ({ url: "https://cdn/out.png" }), { catalogue: imageCatalogue("second") });
+		expect(generator.getModels("image")?.map(({ model }) => model)).toEqual(["second"]);
+
+		generator.register(async () => ({ url: "https://cdn/out.png" }));
+		expect(generator.getModels("image")).toBeUndefined();
+	});
+
+	it("returns only models for the requested asset type", () => {
+		const generator = new AssetGenerator(makeDeps().deps);
+		const mixed = {
+			models: [
+				...imageCatalogue("image-model").models,
+				{ model: "audio-model", type: "audio", options: { type: "object", properties: {}, additionalProperties: false } }
+			]
+		} as GenerationModelCatalogueResponse;
+		generator.register(async () => ({ url: "https://cdn/out.png" }), { catalogue: mixed });
+
+		expect(generator.getModels("audio")?.map(({ model }) => model)).toEqual(["audio-model"]);
+	});
+
 	it("writes the generated src back and reports completion", async () => {
 		const { deps, started, completed, applied } = makeDeps();
 		const generator = new AssetGenerator(deps);

tests/edit-clip-operations.test.ts

@@ -764,6 +764,25 @@ describe("Edit Clip Operations", () => {
 	});
 
 	describe("asset generation", () => {
+		it("notifies internal UI when generator configuration changes", () => {
+			const changed = jest.fn();
+			edit.getInternalEvents().on(InternalEvent.AssetGeneratorChanged, changed);
+
+			edit.registerAssetGenerator(async () => ({ url: "https://cdn.example.com/out.png" }), {
+				catalogue: {
+					models: [
+						{
+							model: "flux-schnell",
+							type: "image",
+							options: { type: "object", properties: {}, additionalProperties: false }
+						}
+					]
+				}
+			});
+
+			expect(changed).toHaveBeenCalledTimes(1);
+		});
+
 		type DocLookup = { document: { getClipId(t: number, c: number): string | null } };
 		const clipIdAt = (target: Edit, t: number, c: number) =>
 			((target as unknown as DocLookup).document.getClipId(t, c) as string);