diff --git a/web/package.json b/web/package.json index fbd4357..f0fec03 100644 --- a/web/package.json +++ b/web/package.json @@ -14,6 +14,7 @@ "dependencies": { "@ant-design/icons": "^6.1.1", "@ant-design/pro-components": "3.0.0-beta.3", + "@codemirror/lang-javascript": "^6.2.5", "@codemirror/lang-json": "^6.0.2", "@tanstack/react-query": "^5.100.9", "@uiw/react-codemirror": "^4.25.9", diff --git a/web/src/components/layout/model-script-editor.tsx b/web/src/components/layout/model-script-editor.tsx index 1aa586e..cc8e754 100644 --- a/web/src/components/layout/model-script-editor.tsx +++ b/web/src/components/layout/model-script-editor.tsx @@ -1,18 +1,16 @@ -import { Button, Input, Modal } from "antd"; +import { javascript } from "@codemirror/lang-javascript"; +import CodeMirror from "@uiw/react-codemirror"; +import { Button, Modal } from "antd"; import { useEffect, useState } from "react"; -import { PLUGIN_TEMPLATES } from "@/services/api/model-plugin"; +import { PLUGIN_RETURNS, PLUGIN_TEMPLATES, PLUGIN_VARIABLES } from "@/services/api/model-plugin"; import type { ModelCapability } from "@/stores/use-config-store"; const capabilityLabels: Record = { image: "生图", video: "视频", text: "文本", audio: "音频" }; -const variableHints = [ - "input：本次请求的归一化输入（prompt / references / messages / params / body）", - "config：{ baseUrl, apiKey, model, apiFormat, systemPrompt }", - "http：{ url(path), post(path, body, opts), get(path, opts) }，已绑定当前渠道", - "poll(request, extract, { intervalMs, timeoutMs })：轮询直到 extract 返回真值", - "sleep(ms)、signal：延时与取消信号；onDelta(text)：推送流式文本（文本模型）", -]; +function isDarkMode() { + return typeof document !== "undefined" && document.documentElement.classList.contains("dark"); +} export function ModelScriptEditor({ open, capability, modelName, value, onSave, onClose }: { open: boolean; capability: ModelCapability; modelName: string; value: string; onSave: (script: string) => void; onClose: () => void }) { const [draft, setDraft] = useState(value); @@ -20,48 +18,95 @@ export function ModelScriptEditor({ open, capability, modelName, value, onSave, if (open) setDraft(value); }, [open, value]); + const variables = PLUGIN_VARIABLES.filter((variable) => !variable.capabilities || variable.capabilities.includes(capability)); + return ( -
调用脚本 · {modelName}
-
当前能力：{capabilityLabels[capability]}。留空则使用系统默认调用方式。
+
+ {capabilityLabels[capability]} + {modelName ? ` - ${modelName}` : ""} +
+
脚本是一段异步函数体，直接使用下方变量，最后 return 结果；留空则使用系统默认调用。
} - width={720} + width={1080} centered onCancel={onClose} - styles={{ body: { maxHeight: "64vh", overflowY: "auto" } }} - footer={[ - , - , - , - , - ]} + styles={{ body: { padding: 0 } }} + footer={ +
+
+ {PLUGIN_TEMPLATES[capability].map((template) => ( + + ))} + +
+
+ + +
+
+ } > -
-
可用变量（异步函数体，需 return 结果）
- {variableHints.map((hint) => ( -
· {hint}
- ))} +
+ +
+ +
- setDraft(event.target.value)} autoSize={{ minRows: 12, maxRows: 24 }} spellCheck={false} placeholder="留空使用系统默认调用；点击“插入模板”查看示例。" style={{ fontFamily: "var(--font-mono, ui-monospace, SFMono-Regular, Menlo, monospace)", fontSize: 12 }} /> ); } diff --git a/web/src/services/api/audio.ts b/web/src/services/api/audio.ts index 611c242..a0ad8c8 100644 --- a/web/src/services/api/audio.ts +++ b/web/src/services/api/audio.ts @@ -32,7 +32,8 @@ export async function requestAudioGeneration(config: AiConfig, prompt: string, o capability: "audio", script, config: requestConfig, - input: { prompt, params: { voice: normalizeAudioVoiceValue(config.audioVoice), format, speed: normalizeAudioSpeedValue(config.audioSpeed), instructions: config.audioInstructions.trim() } }, + prompt, + params: { voice: normalizeAudioVoiceValue(config.audioVoice), format, speed: normalizeAudioSpeedValue(config.audioSpeed), instructions: config.audioInstructions.trim() }, signal: options?.signal, }); return await audioPluginBlob(result, format); diff --git a/web/src/services/api/image.ts b/web/src/services/api/image.ts index e072317..3ffa00d 100644 --- a/web/src/services/api/image.ts +++ b/web/src/services/api/image.ts @@ -666,12 +666,9 @@ export async function requestGeneration(config: AiConfig, prompt: string, option capability: "image", script, config: requestConfig, - input: { - prompt: withSystemPrompt(requestConfig, prompt), - references: [], - params: { size: requestSize, quality, count: n }, - body: { model: requestConfig.model, n, ...(quality ? { quality } : {}), ...(requestSize ? { size: requestSize } : {}), response_format: "b64_json", output_format: IMAGE_OUTPUT_FORMAT }, - }, + prompt: withSystemPrompt(requestConfig, prompt), + images: [], + params: { size: requestSize, quality, count: n }, signal: options?.signal, }); return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl })); @@ -726,12 +723,9 @@ export async function requestEdit(config: AiConfig, prompt: string, references: capability: "image", script, config: requestConfig, - input: { - prompt: withSystemPrompt(requestConfig, requestPrompt), - references: refs, - params: { size: requestSize, quality, count: n }, - body: { model: requestConfig.model, n, ...(quality ? { quality } : {}), ...(requestSize ? { size: requestSize } : {}), response_format: "b64_json", output_format: IMAGE_OUTPUT_FORMAT }, - }, + prompt: withSystemPrompt(requestConfig, requestPrompt), + images: refs, + params: { size: requestSize, quality, count: n }, signal: options?.signal, }); return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl })); @@ -783,7 +777,7 @@ export async function requestImageQuestion(config: AiConfig, messages: AiTextMes capability: "text", script, config: requestConfig, - input: { messages: withSystemMessage(requestConfig, messages), body: { model: requestConfig.model } }, + messages: withSystemMessage(requestConfig, messages), signal: options?.signal, onDelta, }); diff --git a/web/src/services/api/model-plugin.ts b/web/src/services/api/model-plugin.ts index 092e82f..409b1de 100644 --- a/web/src/services/api/model-plugin.ts +++ b/web/src/services/api/model-plugin.ts @@ -1,4 +1,4 @@ -import axios from "axios"; +import axios, { type AxiosRequestConfig } from "axios"; import { buildApiUrl, type AiConfig, type ModelCapability } from "@/stores/use-config-store"; @@ -18,27 +18,20 @@ export type PluginHttp = { export type PluginPollOptions = { intervalMs?: number; timeoutMs?: number }; -export type PluginConfigView = { - baseUrl: string; - apiKey: string; - model: string; - apiFormat: string; - systemPrompt: string; -}; - export type RunPluginArgs = { capability: ModelCapability; script: string; config: AiConfig; - input: Record; + prompt?: string; + images?: string[]; + messages?: unknown[]; + params?: Record; signal?: AbortSignal; onDelta?: (text: string) => void; }; -function pluginHeaders(config: AiConfig, extra?: Record, hasJsonBody = false): Record { +function pluginHeaders(extra?: Record, hasJsonBody = false): Record { const headers: Record = {}; - if (config.apiFormat === "gemini") headers["x-goog-api-key"] = config.apiKey; - else headers.Authorization = `Bearer ${config.apiKey}`; if (hasJsonBody) headers["Content-Type"] = "application/json"; return { ...headers, ...extra }; } @@ -49,14 +42,14 @@ function pluginUrl(config: AiConfig, path: string) { } function createPluginHttp(config: AiConfig, options?: RequestOptions): PluginHttp { - const request = async (method: "get" | "post", path: string, body: unknown, opts?: PluginHttpOptions) => { + const run = async (method: "get" | "post", path: string, body: unknown, opts?: PluginHttpOptions) => { const isForm = typeof FormData !== "undefined" && body instanceof FormData; const response = await axios.request({ method, url: pluginUrl(config, path), data: method === "post" ? body : undefined, params: opts?.params, - headers: pluginHeaders(config, opts?.headers, method === "post" && !isForm && body !== undefined), + headers: pluginHeaders({ Authorization: `Bearer ${config.apiKey}`, ...opts?.headers }, method === "post" && !isForm && body !== undefined), responseType: opts?.responseType || "json", signal: options?.signal, }); @@ -64,8 +57,16 @@ function createPluginHttp(config: AiConfig, options?: RequestOptions): PluginHtt }; return { url: (path) => pluginUrl(config, path), - post: (path, body, opts) => request("post", path, body, opts), - get: (path, opts) => request("get", path, undefined, opts), + post: (path, body, opts) => run("post", path, body, opts), + get: (path, opts) => run("get", path, undefined, opts), + }; +} + +/** Raw request with no automatic auth header — the script controls method, url, headers, body entirely. */ +function createPluginRequest(config: AiConfig, options?: RequestOptions) { + return async (requestConfig: AxiosRequestConfig & { url: string }) => { + const response = await axios.request({ ...requestConfig, url: pluginUrl(config, requestConfig.url), signal: options?.signal }); + return response.data; }; } @@ -103,38 +104,51 @@ function createPoll(signal?: AbortSignal) { } /** - * Run a user-authored model call script. The script body runs as an async function with these locals: - * input —— normalized request input for this capability (prompt / references / messages / params / body) - * config —— { baseUrl, apiKey, model, apiFormat, systemPrompt } - * http —— { url(path), post(path, body, opts), get(path, opts) } bound to the model's channel - * poll —— poll(request, extract, { intervalMs, timeoutMs }) resolves with the first truthy extract result - * sleep —— sleep(ms) - * signal —— AbortSignal for cancellation - * onDelta —— (text) => void, push streaming text (text capability only) + * Run a user-authored model call script as an async function body with flat locals (see PLUGIN_VARIABLES): + * prompt / images / messages / params —— 本次请求的输入 + * model / baseUrl / apiKey / systemPrompt —— 当前渠道信息 + * http / request / poll / sleep / signal / onDelta —— 调用辅助 * The script must `return` the result; each caller normalizes it to its capability's shape. */ export async function runModelPlugin(args: RunPluginArgs): Promise { - const configView: PluginConfigView = { - baseUrl: args.config.baseUrl, - apiKey: args.config.apiKey, - model: args.config.model, - apiFormat: args.config.apiFormat, - systemPrompt: args.config.systemPrompt, - }; - const http = createPluginHttp(args.config, { signal: args.signal }); + const { config } = args; + const http = createPluginHttp(config, { signal: args.signal }); + const request = createPluginRequest(config, { signal: args.signal }); const poll = createPoll(args.signal); const runner = new Function( - "input", - "config", + "prompt", + "images", + "messages", + "params", + "model", + "baseUrl", + "apiKey", + "systemPrompt", "http", + "request", "poll", "sleep", "signal", "onDelta", `"use strict"; return (async () => {\n${args.script}\n})();`, - ) as (input: unknown, config: PluginConfigView, http: PluginHttp, poll: ReturnType, sleep: (ms: number) => Promise, signal: AbortSignal | undefined, onDelta?: (text: string) => void) => Promise; + ) as (...fnArgs: unknown[]) => Promise; try { - return await runner(args.input, configView, http, poll, (ms: number) => sleep(ms, args.signal), args.signal, args.onDelta); + return await runner( + args.prompt || "", + args.images || [], + args.messages || [], + args.params || {}, + config.model, + config.baseUrl, + config.apiKey, + config.systemPrompt || "", + http, + request, + poll, + (ms: number) => sleep(ms, args.signal), + args.signal, + args.onDelta, + ); } catch (error) { if (error instanceof DOMException && error.name === "AbortError") throw error; if (axios.isCancel(error)) throw error; @@ -143,45 +157,201 @@ export async function runModelPlugin(args: RunPluginArgs): Promise< } } -export const PLUGIN_TEMPLATES: Record = { - image: `// 输入：input.prompt / input.references(dataURL[]) / input.body(默认请求体) / input.params -// 返回：dataURL 或 URL 字符串，或它们的数组，或 [{ dataUrl }] -const data = await http.post("/images/generations", { - ...input.body, - model: config.model, - prompt: input.prompt, +export type PluginVariable = { name: string; type: string; desc: string; capabilities?: ModelCapability[] }; + +/** Documentation surface shown in the script editor. */ +export const PLUGIN_VARIABLES: PluginVariable[] = [ + { name: "prompt", type: "string", desc: "用户输入的提示词（已拼接系统提示词）", capabilities: ["image", "video", "audio"] }, + { name: "images", type: "string[]", desc: "参考图，dataURL 数组（改图 / 图生视频时有值）", capabilities: ["image", "video"] }, + { name: "messages", type: "{ role, content }[]", desc: "对话消息数组，含系统消息", capabilities: ["text"] }, + { name: "params", type: "object", desc: "生成参数：生图 {size,quality,count}、视频 {seconds,size,resolution,ratio,generateAudio,watermark}、音频 {voice,format,speed,instructions}" }, + { name: "model", type: "string", desc: "模型名称（不含渠道前缀）" }, + { name: "baseUrl", type: "string", desc: "渠道接口地址（原样，未拼 /v1）" }, + { name: "apiKey", type: "string", desc: "渠道 API Key，请求头里自己带上" }, + { name: "systemPrompt", type: "string", desc: "系统提示词原文" }, + { name: "http", type: "object", desc: "便捷请求：http.post(path, body, {headers,params,responseType})、http.get(path, opts)、http.url(path)；默认带 Authorization: Bearer apiKey，可用 headers 覆盖；path 相对时按 baseUrl 拼 /v1" }, + { name: "request", type: "function", desc: "原始请求 request({ method, url, headers, params, data, responseType })，不加任何默认头，鉴权头自己写；url 相对时按 baseUrl 拼接（不加 /v1）" }, + { name: "poll", type: "function", desc: "轮询 poll(request, extract, {intervalMs,timeoutMs})，extract 返回真值即结束" }, + { name: "sleep", type: "function", desc: "sleep(ms) 延时" }, + { name: "signal", type: "AbortSignal", desc: "取消信号，可透传给 http/request" }, + { name: "onDelta", type: "function", desc: "onDelta(text) 推送流式文本（文本模型）", capabilities: ["text"] }, +]; + +export const PLUGIN_RETURNS: Record = { + image: "文生图（images 为空）和图生图（images 有参考图）接口不同，脚本需自行区分；返回图片 URL 或 dataURL 字符串，也可返回它们的数组，或 [{ dataUrl }] / [{ url }] / [{ b64_json }]", + video: "脚本内部完成轮询，返回 { url } 或 { blob } 或视频 URL 字符串", + audio: "返回 Blob，或 base64 / dataURL 字符串，或 { b64_json } / { data } / { url }", + text: "用 onDelta(text) 推送流式，最终 return 完整文本字符串", +}; + +export type PluginTemplate = { label: string; script: string }; + +export const PLUGIN_TEMPLATES: Record = { + image: [ + { + label: "OpenAI 规范", + script: `// 生图 / 改图：两者接口不同，用 images 是否为空来区分。 +// 可用：prompt、images(dataURL[])、params{size,quality,count}、model、baseUrl、apiKey +if (images.length === 0) { + // 文生图：/images/generations（JSON） + const data = await request({ + method: "post", + url: \`\${baseUrl}/v1/images/generations\`, + headers: { "Content-Type": "application/json", Authorization: \`Bearer \${apiKey}\` }, + data: { model, prompt, n: params.count, size: params.size, response_format: "b64_json" }, + }); + return (data.data || []).map((item) => item.b64_json ? \`data:image/png;base64,\${item.b64_json}\` : item.url); +} + +// 图生图：/images/edits（multipart/form-data，参考图作为文件上传） +const form = new FormData(); +form.set("model", model); +form.set("prompt", prompt); +form.set("n", String(params.count)); +form.set("response_format", "b64_json"); +for (const dataUrl of images) { + form.append("image", await (await fetch(dataUrl)).blob(), "ref.png"); +} +const edited = await request({ + method: "post", + url: \`\${baseUrl}/v1/images/edits\`, + headers: { Authorization: \`Bearer \${apiKey}\` }, // 不要手动设 Content-Type，交给浏览器带 boundary + data: form, }); -return (data.data || []).map((item) => item.b64_json ? \`data:image/png;base64,\${item.b64_json}\` : item.url);`, - video: `// 输入：input.prompt / input.references(dataURL[]) / input.params -// 返回：{ url } 或 { blob } 或视频 URL 字符串 -const task = await http.post("/videos", { - model: config.model, - prompt: input.prompt, - seconds: input.params.seconds, +return (edited.data || []).map((item) => item.b64_json ? \`data:image/png;base64,\${item.b64_json}\` : item.url);`, + }, + { + label: "Gemini 规范", + script: `// Gemini 文生图 / 图生图：都走 generateContent，参考图放进 parts 的 inline_data。 +// 可用：prompt、images(dataURL[])、model、baseUrl、apiKey +const parts = [{ text: prompt }]; +for (const dataUrl of images) { + const match = dataUrl.match(/^data:([^;]+);base64,(.*)$/); + if (match) parts.push({ inline_data: { mime_type: match[1], data: match[2] } }); +} +const data = await request({ + method: "post", + url: \`\${baseUrl}/v1beta/models/\${model}:generateContent\`, + headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey }, + data: { contents: [{ role: "user", parts }], generationConfig: { responseModalities: ["IMAGE"] } }, +}); +return (data.candidates || []) + .flatMap((c) => c.content?.parts || []) + .map((p) => p.inlineData || p.inline_data) + .filter(Boolean) + .map((img) => \`data:\${img.mimeType || img.mime_type || "image/png"};base64,\${img.data}\`);`, + }, + ], + video: [ + { + label: "OpenAI 规范", + script: `// 视频（脚本内部自行轮询）。可用：prompt、images(dataURL[])、params{seconds,size,resolution,ratio} +const headers = { "Content-Type": "application/json", Authorization: \`Bearer \${apiKey}\` }; +const task = await request({ + method: "post", + url: \`\${baseUrl}/v1/videos\`, + headers, + data: { model, prompt, seconds: params.seconds }, }); return await poll( - () => http.get(\`/videos/\${task.id}\`), + () => request({ method: "get", url: \`\${baseUrl}/v1/videos/\${task.id}\`, headers }), (state) => state.status === "completed" ? { url: state.video_url || state.url } : null, { intervalMs: 2500, timeoutMs: 300000 }, );`, - audio: `// 输入：input.prompt / input.params(voice/format/speed/instructions) -// 返回：Blob，或 base64/dataURL 字符串 -return await http.post("/audio/speech", { - model: config.model, - input: input.prompt, - voice: input.params.voice, - response_format: input.params.format, - speed: Number(input.params.speed), -}, { responseType: "blob" });`, - text: `// 输入：input.messages([{role,content}]) / input.body -// 用 onDelta(text) 推送流式文本；返回最终完整文本 -const data = await http.post("/chat/completions", { - model: config.model, - messages: input.messages, + }, + { + label: "Gemini 规范", + script: `// Gemini(Veo) 视频：predictLongRunning 提交，轮询 operation 拿视频 URI。 +// 可用：prompt、images(dataURL[])、params、model、baseUrl、apiKey +const headers = { "Content-Type": "application/json", "x-goog-api-key": apiKey }; +const instance = { prompt }; +const first = images[0] && images[0].match(/^data:([^;]+);base64,(.*)$/); +if (first) instance.image = { bytesBase64Encoded: first[2], mimeType: first[1] }; +const op = await request({ + method: "post", + url: \`\${baseUrl}/v1beta/models/\${model}:predictLongRunning\`, + headers, + data: { instances: [instance], parameters: { aspectRatio: params.ratio } }, }); -const text = data.choices?.[0]?.message?.content || ""; +return await poll( + () => request({ method: "get", url: \`\${baseUrl}/v1beta/\${op.name}\`, headers }), + (state) => { + if (!state.done) return null; + const uri = state.response?.generateVideoResponse?.generatedSamples?.[0]?.video?.uri; + if (!uri) throw new Error("Gemini 未返回视频 URI"); + return { url: uri.includes("key=") ? uri : \`\${uri}\${uri.includes("?") ? "&" : "?"}key=\${apiKey}\` }; + }, + { intervalMs: 5000, timeoutMs: 300000 }, +);`, + }, + ], + audio: [ + { + label: "OpenAI 规范", + script: `// 音频 TTS。可用：prompt、params{voice,format,speed,instructions}、model +return await request({ + method: "post", + url: \`\${baseUrl}/v1/audio/speech\`, + headers: { "Content-Type": "application/json", Authorization: \`Bearer \${apiKey}\` }, + responseType: "blob", + data: { model, input: prompt, voice: params.voice, response_format: params.format, speed: Number(params.speed) }, +});`, + }, + { + label: "Gemini 规范", + script: `// Gemini TTS：generateContent + AUDIO 模态，返回 base64 PCM（音频数据在 inlineData.data）。 +// 可用：prompt、params{voice}、model、baseUrl、apiKey +const data = await request({ + method: "post", + url: \`\${baseUrl}/v1beta/models/\${model}:generateContent\`, + headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey }, + data: { + contents: [{ role: "user", parts: [{ text: prompt }] }], + generationConfig: { + responseModalities: ["AUDIO"], + speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: params.voice } } }, + }, + }, +}); +const audio = data.candidates?.[0]?.content?.parts?.map((p) => p.inlineData || p.inline_data).find(Boolean); +if (!audio?.data) throw new Error("Gemini 未返回音频"); +return { data: audio.data };`, + }, + ], + text: [ + { + label: "OpenAI 规范", + script: `// 文本对话（OpenAI Responses 接口）。可用：messages([{role,content}])、systemPrompt、model +const data = await request({ + method: "post", + url: \`\${baseUrl}/v1/responses\`, + headers: { "Content-Type": "application/json", Authorization: \`Bearer \${apiKey}\` }, + data: { model, input: messages }, +}); +const text = data.output_text + || (data.output || []).flatMap((o) => o.content || []).map((c) => c.text || "").join("") + || ""; onDelta(text); return text;`, + }, + { + label: "Gemini 规范", + script: `// Gemini 文本：generateContent，system 消息放 systemInstruction。 +// 可用：messages([{role,content}])、systemPrompt、model、baseUrl、apiKey +const contents = messages + .filter((m) => m.role !== "system") + .map((m) => ({ role: m.role === "assistant" ? "model" : "user", parts: [{ text: m.content }] })); +const data = await request({ + method: "post", + url: \`\${baseUrl}/v1beta/models/\${model}:generateContent\`, + headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey }, + data: { contents, ...(systemPrompt ? { systemInstruction: { parts: [{ text: systemPrompt }] } } : {}) }, +}); +const text = data.candidates?.[0]?.content?.parts?.map((p) => p.text || "").join("") || ""; +onDelta(text); +return text;`, + }, + ], }; /** Normalize whatever an image script returns into the app's generated-image shape. */ diff --git a/web/src/services/api/video.ts b/web/src/services/api/video.ts index 3ce8e6d..cb7f830 100644 --- a/web/src/services/api/video.ts +++ b/web/src/services/api/video.ts @@ -90,17 +90,15 @@ async function createPluginVideoTask(config: AiConfig, model: string, script: st capability: "video", script, config, - input: { - prompt, - references: refs, - params: { - seconds: normalizeVideoSeconds(config.videoSeconds), - size: normalizeVideoSize(config.size), - resolution: normalizeVideoResolution(config.vquality), - ratio: config.size, - generateAudio: boolConfig(config.videoGenerateAudio, true), - watermark: boolConfig(config.videoWatermark, false), - }, + prompt, + images: refs, + params: { + seconds: normalizeVideoSeconds(config.videoSeconds), + size: normalizeVideoSize(config.size), + resolution: normalizeVideoResolution(config.vquality), + ratio: config.size, + generateAudio: boolConfig(config.videoGenerateAudio, true), + watermark: boolConfig(config.videoWatermark, false), }, signal: options?.signal, }),