feat(model): enhance model configuration and script handling for audio and video capabilities

web/src/services/api/audio.ts

@@ -2,7 +2,8 @@ import axios from "axios";
 
 import { audioMimeType, normalizeAudioFormatValue, normalizeAudioSpeedValue, normalizeAudioVoiceValue } from "@/lib/audio-generation";
 import { uploadMediaFile, type UploadedFile } from "@/services/file-storage";
-import { buildApiUrl, resolveModelRequestConfig, type AiConfig } from "@/stores/use-config-store";
+import { buildApiUrl, resolveModelRequestConfig, resolveModelScript, type AiConfig } from "@/stores/use-config-store";
+import { runModelPlugin } from "./model-plugin";
 
 type RequestOptions = { signal?: AbortSignal };
 
@@ -20,8 +21,26 @@ function aiHeaders(config: AiConfig) {
 export async function requestAudioGeneration(config: AiConfig, prompt: string, options?: RequestOptions): Promise<Blob> {
     const requestConfig = resolveModelRequestConfig(config, config.model || config.audioModel);
     const model = requestConfig.model.trim();
-    assertAudioConfig(requestConfig, model);
     const format = normalizeAudioFormatValue(config.audioFormat);
+    const script = resolveModelScript(config, config.model || config.audioModel);
+    if (script) {
+        if (!model) throw new Error("请先配置音频模型");
+        if (!requestConfig.baseUrl.trim()) throw new Error("请先配置 Base URL");
+        if (!requestConfig.apiKey.trim()) throw new Error("请先配置 API Key");
+        try {
+            const result = await runModelPlugin({
+                capability: "audio",
+                script,
+                config: requestConfig,
+                input: { prompt, params: { voice: normalizeAudioVoiceValue(config.audioVoice), format, speed: normalizeAudioSpeedValue(config.audioSpeed), instructions: config.audioInstructions.trim() } },
+                signal: options?.signal,
+            });
+            return await audioPluginBlob(result, format);
+        } catch (error) {
+            throw new Error(readAxiosError(error, "音频生成失败"));
+        }
+    }
+    assertAudioConfig(requestConfig, model);
     const instructions = config.audioInstructions.trim();
 
     try {
@@ -44,6 +63,20 @@ export async function requestAudioGeneration(config: AiConfig, prompt: string, o
     }
 }
 
+async function audioPluginBlob(result: unknown, format: string): Promise<Blob> {
+    if (result instanceof Blob) return result.type.startsWith("audio/") ? result : new Blob([result], { type: audioMimeType(format) });
+    let source = "";
+    if (typeof result === "string") source = result;
+    else if (result && typeof result === "object") {
+        const record = result as Record<string, unknown>;
+        source = typeof record.b64_json === "string" ? record.b64_json : typeof record.data === "string" ? record.data : typeof record.url === "string" ? record.url : "";
+    }
+    if (!source) throw new Error("模型调用脚本没有返回音频");
+    const url = source.startsWith("data:") || /^https?:/i.test(source) ? source : `data:${audioMimeType(format)};base64,${source}`;
+    const blob = await (await fetch(url)).blob();
+    return blob.type.startsWith("audio/") ? blob : new Blob([blob], { type: audioMimeType(format) });
+}
+
 export async function storeGeneratedAudio(blob: Blob, format = "mp3"): Promise<UploadedFile> {
     const audio = blob.type.startsWith("audio/") ? blob : new Blob([blob], { type: audioMimeType(format) });
     return uploadMediaFile(audio, "audio");

web/src/services/api/image.ts

@@ -1,6 +1,7 @@
 import axios from "axios";
 
-import { buildApiUrl, resolveModelRequestConfig, type AiConfig, type ModelChannel } from "@/stores/use-config-store";
+import { buildApiUrl, resolveModelRequestConfig, resolveModelScript, type AiConfig, type ModelChannel } from "@/stores/use-config-store";
+import { normalizePluginImages, runModelPlugin } from "./model-plugin";
 import { nanoid } from "nanoid";
 import { dataUrlToFile } from "@/lib/image-utils";
 import { buildImageReferencePromptText } from "@/lib/image-reference-prompt";
@@ -656,6 +657,28 @@ function parseGeminiImagePayload(payload: GeminiPayload) {
 export async function requestGeneration(config: AiConfig, prompt: string, options?: RequestOptions) {
     const requestConfig = resolveModelRequestConfig(config, config.model || config.imageModel);
     const n = Math.max(1, Math.min(15, Math.floor(Math.abs(Number(config.count)) || 1)));
+    const script = resolveModelScript(config, config.model || config.imageModel);
+    if (script) {
+        const quality = normalizeQuality(config.quality);
+        const requestSize = resolveRequestSize(quality, config.size);
+        try {
+            const result = await runModelPlugin({
+                capability: "image",
+                script,
+                config: requestConfig,
+                input: {
+                    prompt: withSystemPrompt(requestConfig, prompt),
+                    references: [],
+                    params: { size: requestSize, quality, count: n },
+                    body: { model: requestConfig.model, n, ...(quality ? { quality } : {}), ...(requestSize ? { size: requestSize } : {}), response_format: "b64_json", output_format: IMAGE_OUTPUT_FORMAT },
+                },
+                signal: options?.signal,
+            });
+            return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl }));
+        } catch (error) {
+            throw new Error(readAxiosError(error, "请求失败"));
+        }
+    }
     if (requestConfig.apiFormat === "gemini") {
         try {
             return await requestGeminiImages(requestConfig, prompt, [], n, options);
@@ -693,6 +716,29 @@ export async function requestEdit(config: AiConfig, prompt: string, references:
     const requestConfig = resolveModelRequestConfig(config, config.model || config.imageModel);
     const n = Math.max(1, Math.min(15, Math.floor(Math.abs(Number(config.count)) || 1)));
     const requestPrompt = buildImageReferencePromptText(prompt, references);
+    const script = resolveModelScript(config, config.model || config.imageModel);
+    if (script) {
+        const quality = normalizeQuality(config.quality);
+        const requestSize = resolveRequestSize(quality, config.size);
+        const refs = await Promise.all(references.map((image) => imageToDataUrl(image)));
+        try {
+            const result = await runModelPlugin({
+                capability: "image",
+                script,
+                config: requestConfig,
+                input: {
+                    prompt: withSystemPrompt(requestConfig, requestPrompt),
+                    references: refs,
+                    params: { size: requestSize, quality, count: n },
+                    body: { model: requestConfig.model, n, ...(quality ? { quality } : {}), ...(requestSize ? { size: requestSize } : {}), response_format: "b64_json", output_format: IMAGE_OUTPUT_FORMAT },
+                },
+                signal: options?.signal,
+            });
+            return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl }));
+        } catch (error) {
+            throw new Error(readAxiosError(error, "请求失败"));
+        }
+    }
     if (requestConfig.apiFormat === "gemini") {
         if (mask) throw new Error("Gemini 调用格式暂不支持蒙版编辑");
         try {
@@ -730,6 +776,24 @@ export async function requestEdit(config: AiConfig, prompt: string, references:
 
 export async function requestImageQuestion(config: AiConfig, messages: AiTextMessage[], onDelta: (text: string) => void, options?: RequestOptions) {
     const requestConfig = resolveModelRequestConfig(config, config.model || config.textModel);
+    const script = resolveModelScript(config, config.model || config.textModel);
+    if (script) {
+        try {
+            const answer = await runModelPlugin<string>({
+                capability: "text",
+                script,
+                config: requestConfig,
+                input: { messages: withSystemMessage(requestConfig, messages), body: { model: requestConfig.model } },
+                signal: options?.signal,
+                onDelta,
+            });
+            const text = String(answer ?? "").trim() || "没有返回内容";
+            if (text === "没有返回内容") onDelta(text);
+            return text;
+        } catch (error) {
+            throw new Error(readAxiosError(error, "请求失败"));
+        }
+    }
     try {
         if (requestConfig.apiFormat === "gemini") {
             const answer = (await requestGeminiStreamingResponse(requestConfig, toGeminiBody(requestConfig, messages), onDelta, options)).content || "没有返回内容";

web/src/services/api/model-plugin.ts

@@ -0,0 +1,204 @@
+import axios from "axios";
+
+import { buildApiUrl, type AiConfig, type ModelCapability } from "@/stores/use-config-store";
+
+type RequestOptions = { signal?: AbortSignal };
+
+export type PluginHttpOptions = {
+    headers?: Record<string, string>;
+    params?: Record<string, unknown>;
+    responseType?: "json" | "blob" | "text" | "arraybuffer";
+};
+
+export type PluginHttp = {
+    url: (path: string) => string;
+    post: (path: string, body?: unknown, options?: PluginHttpOptions) => Promise<unknown>;
+    get: (path: string, options?: PluginHttpOptions) => Promise<unknown>;
+};
+
+export type PluginPollOptions = { intervalMs?: number; timeoutMs?: number };
+
+export type PluginConfigView = {
+    baseUrl: string;
+    apiKey: string;
+    model: string;
+    apiFormat: string;
+    systemPrompt: string;
+};
+
+export type RunPluginArgs = {
+    capability: ModelCapability;
+    script: string;
+    config: AiConfig;
+    input: Record<string, unknown>;
+    signal?: AbortSignal;
+    onDelta?: (text: string) => void;
+};
+
+function pluginHeaders(config: AiConfig, extra?: Record<string, string>, hasJsonBody = false): Record<string, string> {
+    const headers: Record<string, string> = {};
+    if (config.apiFormat === "gemini") headers["x-goog-api-key"] = config.apiKey;
+    else headers.Authorization = `Bearer ${config.apiKey}`;
+    if (hasJsonBody) headers["Content-Type"] = "application/json";
+    return { ...headers, ...extra };
+}
+
+function pluginUrl(config: AiConfig, path: string) {
+    if (/^https?:/i.test(path)) return path;
+    return buildApiUrl(config.baseUrl, path.startsWith("/") ? path : `/${path}`);
+}
+
+function createPluginHttp(config: AiConfig, options?: RequestOptions): PluginHttp {
+    const request = async (method: "get" | "post", path: string, body: unknown, opts?: PluginHttpOptions) => {
+        const isForm = typeof FormData !== "undefined" && body instanceof FormData;
+        const response = await axios.request({
+            method,
+            url: pluginUrl(config, path),
+            data: method === "post" ? body : undefined,
+            params: opts?.params,
+            headers: pluginHeaders(config, opts?.headers, method === "post" && !isForm && body !== undefined),
+            responseType: opts?.responseType || "json",
+            signal: options?.signal,
+        });
+        return response.data;
+    };
+    return {
+        url: (path) => pluginUrl(config, path),
+        post: (path, body, opts) => request("post", path, body, opts),
+        get: (path, opts) => request("get", path, undefined, opts),
+    };
+}
+
+function sleep(ms: number, signal?: AbortSignal) {
+    return new Promise<void>((resolve, reject) => {
+        if (signal?.aborted) {
+            reject(new DOMException("Aborted", "AbortError"));
+            return;
+        }
+        const timer = setTimeout(resolve, ms);
+        signal?.addEventListener(
+            "abort",
+            () => {
+                clearTimeout(timer);
+                reject(new DOMException("Aborted", "AbortError"));
+            },
+            { once: true },
+        );
+    });
+}
+
+function createPoll(signal?: AbortSignal) {
+    return async function poll<T, R>(request: () => Promise<T>, extract: (value: T) => R | null | undefined | false, options?: PluginPollOptions): Promise<R> {
+        const intervalMs = options?.intervalMs ?? 2500;
+        const timeoutMs = options?.timeoutMs ?? 300000;
+        const deadline = performance.now() + timeoutMs;
+        for (;;) {
+            if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+            const result = extract(await request());
+            if (result !== null && result !== undefined && result !== false) return result;
+            if (performance.now() >= deadline) throw new Error("插件轮询超时，请检查调用脚本或稍后重试");
+            await sleep(intervalMs, signal);
+        }
+    };
+}
+
+/**
+ * Run a user-authored model call script. The script body runs as an async function with these locals:
+ *   input   —— normalized request input for this capability (prompt / references / messages / params / body)
+ *   config  —— { baseUrl, apiKey, model, apiFormat, systemPrompt }
+ *   http    —— { url(path), post(path, body, opts), get(path, opts) } bound to the model's channel
+ *   poll    —— poll(request, extract, { intervalMs, timeoutMs }) resolves with the first truthy extract result
+ *   sleep   —— sleep(ms)
+ *   signal  —— AbortSignal for cancellation
+ *   onDelta —— (text) => void, push streaming text (text capability only)
+ * The script must `return` the result; each caller normalizes it to its capability's shape.
+ */
+export async function runModelPlugin<T = unknown>(args: RunPluginArgs): Promise<T> {
+    const configView: PluginConfigView = {
+        baseUrl: args.config.baseUrl,
+        apiKey: args.config.apiKey,
+        model: args.config.model,
+        apiFormat: args.config.apiFormat,
+        systemPrompt: args.config.systemPrompt,
+    };
+    const http = createPluginHttp(args.config, { signal: args.signal });
+    const poll = createPoll(args.signal);
+    const runner = new Function(
+        "input",
+        "config",
+        "http",
+        "poll",
+        "sleep",
+        "signal",
+        "onDelta",
+        `"use strict"; return (async () => {\n${args.script}\n})();`,
+    ) as (input: unknown, config: PluginConfigView, http: PluginHttp, poll: ReturnType<typeof createPoll>, sleep: (ms: number) => Promise<void>, signal: AbortSignal | undefined, onDelta?: (text: string) => void) => Promise<T>;
+    try {
+        return await runner(args.input, configView, http, poll, (ms: number) => sleep(ms, args.signal), args.signal, args.onDelta);
+    } catch (error) {
+        if (error instanceof DOMException && error.name === "AbortError") throw error;
+        if (axios.isCancel(error)) throw error;
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`模型调用脚本执行失败：${message}`);
+    }
+}
+
+export const PLUGIN_TEMPLATES: Record<ModelCapability, string> = {
+    image: `// 输入：input.prompt / input.references(dataURL[]) / input.body(默认请求体) / input.params
+// 返回：dataURL 或 URL 字符串，或它们的数组，或 [{ dataUrl }]
+const data = await http.post("/images/generations", {
+  ...input.body,
+  model: config.model,
+  prompt: input.prompt,
+});
+return (data.data || []).map((item) => item.b64_json ? \`data:image/png;base64,\${item.b64_json}\` : item.url);`,
+    video: `// 输入：input.prompt / input.references(dataURL[]) / input.params
+// 返回：{ url } 或 { blob } 或视频 URL 字符串
+const task = await http.post("/videos", {
+  model: config.model,
+  prompt: input.prompt,
+  seconds: input.params.seconds,
+});
+return await poll(
+  () => http.get(\`/videos/\${task.id}\`),
+  (state) => state.status === "completed" ? { url: state.video_url || state.url } : null,
+  { intervalMs: 2500, timeoutMs: 300000 },
+);`,
+    audio: `// 输入：input.prompt / input.params(voice/format/speed/instructions)
+// 返回：Blob，或 base64/dataURL 字符串
+return await http.post("/audio/speech", {
+  model: config.model,
+  input: input.prompt,
+  voice: input.params.voice,
+  response_format: input.params.format,
+  speed: Number(input.params.speed),
+}, { responseType: "blob" });`,
+    text: `// 输入：input.messages([{role,content}]) / input.body
+// 用 onDelta(text) 推送流式文本；返回最终完整文本
+const data = await http.post("/chat/completions", {
+  model: config.model,
+  messages: input.messages,
+});
+const text = data.choices?.[0]?.message?.content || "";
+onDelta(text);
+return text;`,
+};
+
+/** Normalize whatever an image script returns into the app's generated-image shape. */
+export function normalizePluginImages(result: unknown): string[] {
+    const items = Array.isArray(result) ? result : [result];
+    const urls = items
+        .map((item) => {
+            if (typeof item === "string") return item;
+            if (item && typeof item === "object") {
+                const record = item as Record<string, unknown>;
+                if (typeof record.dataUrl === "string") return record.dataUrl;
+                if (typeof record.url === "string") return record.url;
+                if (typeof record.b64_json === "string") return `data:image/png;base64,${record.b64_json}`;
+            }
+            return "";
+        })
+        .filter(Boolean);
+    if (!urls.length) throw new Error("模型调用脚本没有返回图片");
+    return urls;
+}

web/src/services/api/video.ts

@@ -1,10 +1,12 @@
 import axios from "axios";
+import { nanoid } from "nanoid";
 
 import { dataUrlToFile } from "@/lib/image-utils";
 import { getMediaBlob, uploadMediaFile, type UploadedFile } from "@/services/file-storage";
 import { imageToDataUrl } from "@/services/image-storage";
 import { boolConfig, buildSeedancePromptText, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceVideoReferenceError, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video";
-import { buildApiUrl, modelOptionName, resolveModelRequestConfig, type AiConfig } from "@/stores/use-config-store";
+import { buildApiUrl, modelOptionName, resolveModelRequestConfig, resolveModelScript, type AiConfig } from "@/stores/use-config-store";
+import { runModelPlugin } from "./model-plugin";
 import type { ReferenceImage } from "@/types/image";
 import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
 
@@ -23,9 +25,12 @@ type ApiEnvelope<T> = T | { code?: number | string; data?: T | null; msg?: strin
 type RequestOptions = { signal?: AbortSignal };
 
 export type VideoGenerationResult = { blob?: Blob; url?: string; mimeType?: string };
-export type VideoGenerationTask = { id: string; provider: "openai" | "seedance"; model: string };
+export type VideoGenerationTask = { id: string; provider: "openai" | "seedance" | "plugin"; model: string };
 export type VideoGenerationTaskState = { status: "pending" } | { status: "completed"; result: VideoGenerationResult } | { status: "failed"; error: string };
 
+/** Results for scripted (plugin) video models, which run their own create+poll in one shot at task creation. */
+const pluginVideoResults = new Map<string, VideoGenerationResult>();
+
 function aiApiUrl(config: AiConfig, path: string) {
     return buildApiUrl(config.baseUrl, path);
 }
@@ -54,6 +59,8 @@ export async function requestVideoGeneration(config: AiConfig, prompt: string, r
 export async function createVideoGenerationTask(config: AiConfig, prompt: string, references: ReferenceImage[] = [], videoReferences: ReferenceVideo[] = [], audioReferences: ReferenceAudio[] = [], options?: RequestOptions): Promise<VideoGenerationTask> {
     const selectedModel = (config.model || config.videoModel).trim();
     const requestConfig = resolveModelRequestConfig(config, selectedModel);
+    const script = resolveModelScript(config, selectedModel);
+    if (script) return createPluginVideoTask(requestConfig, selectedModel, script, prompt, references, options);
     assertVideoConfig(requestConfig, requestConfig.model);
     if (isSeedanceVideoConfig(requestConfig)) {
         return createSeedanceTask(requestConfig, selectedModel, prompt, references, videoReferences, audioReferences, options);
@@ -65,11 +72,56 @@ export async function createVideoGenerationTask(config: AiConfig, prompt: string
 }
 
 export async function pollVideoGenerationTask(config: AiConfig, task: VideoGenerationTask, options?: RequestOptions): Promise<VideoGenerationTaskState> {
+    if (task.provider === "plugin") {
+        const result = pluginVideoResults.get(task.id);
+        return result ? { status: "completed", result } : { status: "failed", error: "插件视频任务已失效，请重新生成" };
+    }
     const requestConfig = resolveModelRequestConfig(config, task.model);
     assertVideoConfig(requestConfig, requestConfig.model);
     return task.provider === "seedance" ? pollSeedanceTask(requestConfig, task, options) : pollOpenAIVideoTask(requestConfig, task, options);
 }
 
+async function createPluginVideoTask(config: AiConfig, model: string, script: string, prompt: string, references: ReferenceImage[], options?: RequestOptions): Promise<VideoGenerationTask> {
+    if (!config.baseUrl.trim()) throw new Error("请先配置 Base URL");
+    if (!config.apiKey.trim()) throw new Error("请先配置 API Key");
+    const refs = await Promise.all(references.map((image) => imageToDataUrl(image)));
+    const result = videoPluginResult(
+        await runModelPlugin({
+            capability: "video",
+            script,
+            config,
+            input: {
+                prompt,
+                references: refs,
+                params: {
+                    seconds: normalizeVideoSeconds(config.videoSeconds),
+                    size: normalizeVideoSize(config.size),
+                    resolution: normalizeVideoResolution(config.vquality),
+                    ratio: config.size,
+                    generateAudio: boolConfig(config.videoGenerateAudio, true),
+                    watermark: boolConfig(config.videoWatermark, false),
+                },
+            },
+            signal: options?.signal,
+        }),
+    );
+    const id = nanoid();
+    pluginVideoResults.set(id, result);
+    return { id, provider: "plugin", model };
+}
+
+function videoPluginResult(result: unknown): VideoGenerationResult {
+    if (result instanceof Blob) return { blob: result };
+    if (typeof result === "string") return { url: result, mimeType: "video/mp4" };
+    if (result && typeof result === "object") {
+        const record = result as Record<string, unknown>;
+        if (record.blob instanceof Blob) return { blob: record.blob };
+        const url = [record.url, record.video_url, record.result_url].find((value) => typeof value === "string" && value) as string | undefined;
+        if (url) return { url, mimeType: "video/mp4" };
+    }
+    throw new Error("模型调用脚本没有返回视频");
+}
+
 export async function storeGeneratedVideo(result: VideoGenerationResult): Promise<UploadedFile> {
     if (result.blob) return uploadMediaFile(result.blob, "video");
     if (result.url) {