mirror of
https://github.com/basketikun/infinite-canvas.git
synced 2026-07-24 06:54:06 +08:00
feat(channel): add customizable JS model-call scripts with split-panel editor
- Run user-authored async scripts per model with flat locals (prompt, images, messages, params, model, baseUrl, apiKey, http, request, poll, sleep, signal, onDelta); empty script falls back to system default. - Wire image/video/audio/text request entrypoints through the plugin runtime, keeping default handlers unchanged. - Split-panel script editor: variable/return docs on the left, CodeMirror JS editor on the right; title shows capability - model. - Provide OpenAI and Gemini template buttons per capability; text template uses the /responses API; image template branches text2img vs img2img. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,7 @@
|
||||
"dependencies": {
|
||||
"@ant-design/icons": "^6.1.1",
|
||||
"@ant-design/pro-components": "3.0.0-beta.3",
|
||||
"@codemirror/lang-javascript": "^6.2.5",
|
||||
"@codemirror/lang-json": "^6.0.2",
|
||||
"@tanstack/react-query": "^5.100.9",
|
||||
"@uiw/react-codemirror": "^4.25.9",
|
||||
|
||||
@@ -1,18 +1,16 @@
|
||||
import { Button, Input, Modal } from "antd";
|
||||
import { javascript } from "@codemirror/lang-javascript";
|
||||
import CodeMirror from "@uiw/react-codemirror";
|
||||
import { Button, Modal } from "antd";
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
import { PLUGIN_TEMPLATES } from "@/services/api/model-plugin";
|
||||
import { PLUGIN_RETURNS, PLUGIN_TEMPLATES, PLUGIN_VARIABLES } from "@/services/api/model-plugin";
|
||||
import type { ModelCapability } from "@/stores/use-config-store";
|
||||
|
||||
const capabilityLabels: Record<ModelCapability, string> = { image: "生图", video: "视频", text: "文本", audio: "音频" };
|
||||
|
||||
const variableHints = [
|
||||
"input:本次请求的归一化输入(prompt / references / messages / params / body)",
|
||||
"config:{ baseUrl, apiKey, model, apiFormat, systemPrompt }",
|
||||
"http:{ url(path), post(path, body, opts), get(path, opts) },已绑定当前渠道",
|
||||
"poll(request, extract, { intervalMs, timeoutMs }):轮询直到 extract 返回真值",
|
||||
"sleep(ms)、signal:延时与取消信号;onDelta(text):推送流式文本(文本模型)",
|
||||
];
|
||||
function isDarkMode() {
|
||||
return typeof document !== "undefined" && document.documentElement.classList.contains("dark");
|
||||
}
|
||||
|
||||
export function ModelScriptEditor({ open, capability, modelName, value, onSave, onClose }: { open: boolean; capability: ModelCapability; modelName: string; value: string; onSave: (script: string) => void; onClose: () => void }) {
|
||||
const [draft, setDraft] = useState(value);
|
||||
@@ -20,48 +18,95 @@ export function ModelScriptEditor({ open, capability, modelName, value, onSave,
|
||||
if (open) setDraft(value);
|
||||
}, [open, value]);
|
||||
|
||||
const variables = PLUGIN_VARIABLES.filter((variable) => !variable.capabilities || variable.capabilities.includes(capability));
|
||||
|
||||
return (
|
||||
<Modal
|
||||
open={open}
|
||||
title={
|
||||
<div>
|
||||
<div className="text-base font-semibold">调用脚本 · {modelName}</div>
|
||||
<div className="mt-1 text-xs font-normal text-stone-500">当前能力:{capabilityLabels[capability]}。留空则使用系统默认调用方式。</div>
|
||||
<div className="text-base font-semibold">
|
||||
{capabilityLabels[capability]}
|
||||
{modelName ? ` - ${modelName}` : ""}
|
||||
</div>
|
||||
<div className="mt-1 text-xs font-normal text-stone-500">脚本是一段异步函数体,直接使用下方变量,最后 return 结果;留空则使用系统默认调用。</div>
|
||||
</div>
|
||||
}
|
||||
width={720}
|
||||
width={1080}
|
||||
centered
|
||||
onCancel={onClose}
|
||||
styles={{ body: { maxHeight: "64vh", overflowY: "auto" } }}
|
||||
footer={[
|
||||
<Button key="template" onClick={() => setDraft(PLUGIN_TEMPLATES[capability])}>
|
||||
插入模板
|
||||
</Button>,
|
||||
<Button key="reset" danger onClick={() => setDraft("")}>
|
||||
恢复默认调用
|
||||
</Button>,
|
||||
<Button key="cancel" onClick={onClose}>
|
||||
取消
|
||||
</Button>,
|
||||
<Button
|
||||
key="save"
|
||||
type="primary"
|
||||
onClick={() => {
|
||||
onSave(draft.trim());
|
||||
onClose();
|
||||
}}
|
||||
>
|
||||
保存
|
||||
</Button>,
|
||||
]}
|
||||
styles={{ body: { padding: 0 } }}
|
||||
footer={
|
||||
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
{PLUGIN_TEMPLATES[capability].map((template) => (
|
||||
<Button key={template.label} size="small" onClick={() => setDraft(template.script)}>
|
||||
插入{template.label}模板
|
||||
</Button>
|
||||
))}
|
||||
<Button size="small" danger onClick={() => setDraft("")}>
|
||||
恢复默认调用
|
||||
</Button>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Button onClick={onClose}>取消</Button>
|
||||
<Button
|
||||
type="primary"
|
||||
onClick={() => {
|
||||
onSave(draft.trim());
|
||||
onClose();
|
||||
}}
|
||||
>
|
||||
保存
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
>
|
||||
<div className="mb-3 rounded-md border border-stone-200 bg-stone-50 p-3 text-xs leading-6 text-stone-600 dark:border-stone-800 dark:bg-stone-900/40 dark:text-stone-300">
|
||||
<div className="mb-1 font-semibold">可用变量(异步函数体,需 return 结果)</div>
|
||||
{variableHints.map((hint) => (
|
||||
<div key={hint}>· {hint}</div>
|
||||
))}
|
||||
<div className="flex h-[60vh] min-h-[420px] border-t border-stone-200 dark:border-stone-800">
|
||||
<aside className="flex w-[320px] shrink-0 flex-col overflow-y-auto border-r border-stone-200 bg-stone-50/80 dark:border-stone-800 dark:bg-stone-900/40">
|
||||
<div className="border-b border-stone-200/70 px-4 py-3 dark:border-stone-800/70">
|
||||
<div className="mb-1.5 text-[11px] font-semibold uppercase tracking-wide text-stone-400">返回要求</div>
|
||||
<div className="text-xs leading-6 text-stone-600 dark:text-stone-300">{PLUGIN_RETURNS[capability]}</div>
|
||||
</div>
|
||||
<div className="px-4 py-3">
|
||||
<div className="mb-2.5 flex items-center justify-between">
|
||||
<span className="text-[11px] font-semibold uppercase tracking-wide text-stone-400">可用变量</span>
|
||||
<span className="text-[10px] text-stone-400">点击插入</span>
|
||||
</div>
|
||||
<div className="space-y-1.5">
|
||||
{variables.map((variable) => (
|
||||
<button
|
||||
key={variable.name}
|
||||
type="button"
|
||||
onClick={() => setDraft((current) => (current ? `${current}\n${variable.name}` : variable.name))}
|
||||
className="group block w-full rounded-lg border border-transparent px-2.5 py-2 text-left transition-colors hover:border-stone-200 hover:bg-white dark:hover:border-stone-700 dark:hover:bg-stone-800/60"
|
||||
>
|
||||
<div className="flex flex-wrap items-baseline gap-1.5">
|
||||
<code className="rounded bg-stone-200/80 px-1.5 py-0.5 font-mono text-[11px] font-semibold text-stone-800 group-hover:bg-blue-100 group-hover:text-blue-700 dark:bg-stone-800 dark:text-stone-100 dark:group-hover:bg-blue-950 dark:group-hover:text-blue-300">
|
||||
{variable.name}
|
||||
</code>
|
||||
<span className="font-mono text-[10px] text-stone-400">{variable.type}</span>
|
||||
</div>
|
||||
<div className="mt-1 text-xs leading-5 text-stone-500 dark:text-stone-400">{variable.desc}</div>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</aside>
|
||||
<div className="min-w-0 flex-1 overflow-hidden bg-white dark:bg-stone-950">
|
||||
<CodeMirror
|
||||
value={draft}
|
||||
onChange={setDraft}
|
||||
height="100%"
|
||||
theme={isDarkMode() ? "dark" : "light"}
|
||||
extensions={[javascript()]}
|
||||
placeholder={"// 留空使用系统默认调用;点击右下角「插入模板」查看示例。"}
|
||||
style={{ height: "100%", fontSize: 13 }}
|
||||
className="h-full [&_.cm-editor]:h-full [&_.cm-gutters]:border-none [&_.cm-scroller]:overflow-auto"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<Input.TextArea value={draft} onChange={(event) => setDraft(event.target.value)} autoSize={{ minRows: 12, maxRows: 24 }} spellCheck={false} placeholder="留空使用系统默认调用;点击“插入模板”查看示例。" style={{ fontFamily: "var(--font-mono, ui-monospace, SFMono-Regular, Menlo, monospace)", fontSize: 12 }} />
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -32,7 +32,8 @@ export async function requestAudioGeneration(config: AiConfig, prompt: string, o
|
||||
capability: "audio",
|
||||
script,
|
||||
config: requestConfig,
|
||||
input: { prompt, params: { voice: normalizeAudioVoiceValue(config.audioVoice), format, speed: normalizeAudioSpeedValue(config.audioSpeed), instructions: config.audioInstructions.trim() } },
|
||||
prompt,
|
||||
params: { voice: normalizeAudioVoiceValue(config.audioVoice), format, speed: normalizeAudioSpeedValue(config.audioSpeed), instructions: config.audioInstructions.trim() },
|
||||
signal: options?.signal,
|
||||
});
|
||||
return await audioPluginBlob(result, format);
|
||||
|
||||
@@ -666,12 +666,9 @@ export async function requestGeneration(config: AiConfig, prompt: string, option
|
||||
capability: "image",
|
||||
script,
|
||||
config: requestConfig,
|
||||
input: {
|
||||
prompt: withSystemPrompt(requestConfig, prompt),
|
||||
references: [],
|
||||
params: { size: requestSize, quality, count: n },
|
||||
body: { model: requestConfig.model, n, ...(quality ? { quality } : {}), ...(requestSize ? { size: requestSize } : {}), response_format: "b64_json", output_format: IMAGE_OUTPUT_FORMAT },
|
||||
},
|
||||
prompt: withSystemPrompt(requestConfig, prompt),
|
||||
images: [],
|
||||
params: { size: requestSize, quality, count: n },
|
||||
signal: options?.signal,
|
||||
});
|
||||
return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl }));
|
||||
@@ -726,12 +723,9 @@ export async function requestEdit(config: AiConfig, prompt: string, references:
|
||||
capability: "image",
|
||||
script,
|
||||
config: requestConfig,
|
||||
input: {
|
||||
prompt: withSystemPrompt(requestConfig, requestPrompt),
|
||||
references: refs,
|
||||
params: { size: requestSize, quality, count: n },
|
||||
body: { model: requestConfig.model, n, ...(quality ? { quality } : {}), ...(requestSize ? { size: requestSize } : {}), response_format: "b64_json", output_format: IMAGE_OUTPUT_FORMAT },
|
||||
},
|
||||
prompt: withSystemPrompt(requestConfig, requestPrompt),
|
||||
images: refs,
|
||||
params: { size: requestSize, quality, count: n },
|
||||
signal: options?.signal,
|
||||
});
|
||||
return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl }));
|
||||
@@ -783,7 +777,7 @@ export async function requestImageQuestion(config: AiConfig, messages: AiTextMes
|
||||
capability: "text",
|
||||
script,
|
||||
config: requestConfig,
|
||||
input: { messages: withSystemMessage(requestConfig, messages), body: { model: requestConfig.model } },
|
||||
messages: withSystemMessage(requestConfig, messages),
|
||||
signal: options?.signal,
|
||||
onDelta,
|
||||
});
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import axios from "axios";
|
||||
import axios, { type AxiosRequestConfig } from "axios";
|
||||
|
||||
import { buildApiUrl, type AiConfig, type ModelCapability } from "@/stores/use-config-store";
|
||||
|
||||
@@ -18,27 +18,20 @@ export type PluginHttp = {
|
||||
|
||||
export type PluginPollOptions = { intervalMs?: number; timeoutMs?: number };
|
||||
|
||||
export type PluginConfigView = {
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
apiFormat: string;
|
||||
systemPrompt: string;
|
||||
};
|
||||
|
||||
export type RunPluginArgs = {
|
||||
capability: ModelCapability;
|
||||
script: string;
|
||||
config: AiConfig;
|
||||
input: Record<string, unknown>;
|
||||
prompt?: string;
|
||||
images?: string[];
|
||||
messages?: unknown[];
|
||||
params?: Record<string, unknown>;
|
||||
signal?: AbortSignal;
|
||||
onDelta?: (text: string) => void;
|
||||
};
|
||||
|
||||
function pluginHeaders(config: AiConfig, extra?: Record<string, string>, hasJsonBody = false): Record<string, string> {
|
||||
function pluginHeaders(extra?: Record<string, string>, hasJsonBody = false): Record<string, string> {
|
||||
const headers: Record<string, string> = {};
|
||||
if (config.apiFormat === "gemini") headers["x-goog-api-key"] = config.apiKey;
|
||||
else headers.Authorization = `Bearer ${config.apiKey}`;
|
||||
if (hasJsonBody) headers["Content-Type"] = "application/json";
|
||||
return { ...headers, ...extra };
|
||||
}
|
||||
@@ -49,14 +42,14 @@ function pluginUrl(config: AiConfig, path: string) {
|
||||
}
|
||||
|
||||
function createPluginHttp(config: AiConfig, options?: RequestOptions): PluginHttp {
|
||||
const request = async (method: "get" | "post", path: string, body: unknown, opts?: PluginHttpOptions) => {
|
||||
const run = async (method: "get" | "post", path: string, body: unknown, opts?: PluginHttpOptions) => {
|
||||
const isForm = typeof FormData !== "undefined" && body instanceof FormData;
|
||||
const response = await axios.request({
|
||||
method,
|
||||
url: pluginUrl(config, path),
|
||||
data: method === "post" ? body : undefined,
|
||||
params: opts?.params,
|
||||
headers: pluginHeaders(config, opts?.headers, method === "post" && !isForm && body !== undefined),
|
||||
headers: pluginHeaders({ Authorization: `Bearer ${config.apiKey}`, ...opts?.headers }, method === "post" && !isForm && body !== undefined),
|
||||
responseType: opts?.responseType || "json",
|
||||
signal: options?.signal,
|
||||
});
|
||||
@@ -64,8 +57,16 @@ function createPluginHttp(config: AiConfig, options?: RequestOptions): PluginHtt
|
||||
};
|
||||
return {
|
||||
url: (path) => pluginUrl(config, path),
|
||||
post: (path, body, opts) => request("post", path, body, opts),
|
||||
get: (path, opts) => request("get", path, undefined, opts),
|
||||
post: (path, body, opts) => run("post", path, body, opts),
|
||||
get: (path, opts) => run("get", path, undefined, opts),
|
||||
};
|
||||
}
|
||||
|
||||
/** Raw request with no automatic auth header — the script controls method, url, headers, body entirely. */
|
||||
function createPluginRequest(config: AiConfig, options?: RequestOptions) {
|
||||
return async (requestConfig: AxiosRequestConfig & { url: string }) => {
|
||||
const response = await axios.request({ ...requestConfig, url: pluginUrl(config, requestConfig.url), signal: options?.signal });
|
||||
return response.data;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -103,38 +104,51 @@ function createPoll(signal?: AbortSignal) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a user-authored model call script. The script body runs as an async function with these locals:
|
||||
* input —— normalized request input for this capability (prompt / references / messages / params / body)
|
||||
* config —— { baseUrl, apiKey, model, apiFormat, systemPrompt }
|
||||
* http —— { url(path), post(path, body, opts), get(path, opts) } bound to the model's channel
|
||||
* poll —— poll(request, extract, { intervalMs, timeoutMs }) resolves with the first truthy extract result
|
||||
* sleep —— sleep(ms)
|
||||
* signal —— AbortSignal for cancellation
|
||||
* onDelta —— (text) => void, push streaming text (text capability only)
|
||||
* Run a user-authored model call script as an async function body with flat locals (see PLUGIN_VARIABLES):
|
||||
* prompt / images / messages / params —— 本次请求的输入
|
||||
* model / baseUrl / apiKey / systemPrompt —— 当前渠道信息
|
||||
* http / request / poll / sleep / signal / onDelta —— 调用辅助
|
||||
* The script must `return` the result; each caller normalizes it to its capability's shape.
|
||||
*/
|
||||
export async function runModelPlugin<T = unknown>(args: RunPluginArgs): Promise<T> {
|
||||
const configView: PluginConfigView = {
|
||||
baseUrl: args.config.baseUrl,
|
||||
apiKey: args.config.apiKey,
|
||||
model: args.config.model,
|
||||
apiFormat: args.config.apiFormat,
|
||||
systemPrompt: args.config.systemPrompt,
|
||||
};
|
||||
const http = createPluginHttp(args.config, { signal: args.signal });
|
||||
const { config } = args;
|
||||
const http = createPluginHttp(config, { signal: args.signal });
|
||||
const request = createPluginRequest(config, { signal: args.signal });
|
||||
const poll = createPoll(args.signal);
|
||||
const runner = new Function(
|
||||
"input",
|
||||
"config",
|
||||
"prompt",
|
||||
"images",
|
||||
"messages",
|
||||
"params",
|
||||
"model",
|
||||
"baseUrl",
|
||||
"apiKey",
|
||||
"systemPrompt",
|
||||
"http",
|
||||
"request",
|
||||
"poll",
|
||||
"sleep",
|
||||
"signal",
|
||||
"onDelta",
|
||||
`"use strict"; return (async () => {\n${args.script}\n})();`,
|
||||
) as (input: unknown, config: PluginConfigView, http: PluginHttp, poll: ReturnType<typeof createPoll>, sleep: (ms: number) => Promise<void>, signal: AbortSignal | undefined, onDelta?: (text: string) => void) => Promise<T>;
|
||||
) as (...fnArgs: unknown[]) => Promise<T>;
|
||||
try {
|
||||
return await runner(args.input, configView, http, poll, (ms: number) => sleep(ms, args.signal), args.signal, args.onDelta);
|
||||
return await runner(
|
||||
args.prompt || "",
|
||||
args.images || [],
|
||||
args.messages || [],
|
||||
args.params || {},
|
||||
config.model,
|
||||
config.baseUrl,
|
||||
config.apiKey,
|
||||
config.systemPrompt || "",
|
||||
http,
|
||||
request,
|
||||
poll,
|
||||
(ms: number) => sleep(ms, args.signal),
|
||||
args.signal,
|
||||
args.onDelta,
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
||||
if (axios.isCancel(error)) throw error;
|
||||
@@ -143,45 +157,201 @@ export async function runModelPlugin<T = unknown>(args: RunPluginArgs): Promise<
|
||||
}
|
||||
}
|
||||
|
||||
export const PLUGIN_TEMPLATES: Record<ModelCapability, string> = {
|
||||
image: `// 输入:input.prompt / input.references(dataURL[]) / input.body(默认请求体) / input.params
|
||||
// 返回:dataURL 或 URL 字符串,或它们的数组,或 [{ dataUrl }]
|
||||
const data = await http.post("/images/generations", {
|
||||
...input.body,
|
||||
model: config.model,
|
||||
prompt: input.prompt,
|
||||
export type PluginVariable = { name: string; type: string; desc: string; capabilities?: ModelCapability[] };
|
||||
|
||||
/** Documentation surface shown in the script editor. */
|
||||
export const PLUGIN_VARIABLES: PluginVariable[] = [
|
||||
{ name: "prompt", type: "string", desc: "用户输入的提示词(已拼接系统提示词)", capabilities: ["image", "video", "audio"] },
|
||||
{ name: "images", type: "string[]", desc: "参考图,dataURL 数组(改图 / 图生视频时有值)", capabilities: ["image", "video"] },
|
||||
{ name: "messages", type: "{ role, content }[]", desc: "对话消息数组,含系统消息", capabilities: ["text"] },
|
||||
{ name: "params", type: "object", desc: "生成参数:生图 {size,quality,count}、视频 {seconds,size,resolution,ratio,generateAudio,watermark}、音频 {voice,format,speed,instructions}" },
|
||||
{ name: "model", type: "string", desc: "模型名称(不含渠道前缀)" },
|
||||
{ name: "baseUrl", type: "string", desc: "渠道接口地址(原样,未拼 /v1)" },
|
||||
{ name: "apiKey", type: "string", desc: "渠道 API Key,请求头里自己带上" },
|
||||
{ name: "systemPrompt", type: "string", desc: "系统提示词原文" },
|
||||
{ name: "http", type: "object", desc: "便捷请求:http.post(path, body, {headers,params,responseType})、http.get(path, opts)、http.url(path);默认带 Authorization: Bearer apiKey,可用 headers 覆盖;path 相对时按 baseUrl 拼 /v1" },
|
||||
{ name: "request", type: "function", desc: "原始请求 request({ method, url, headers, params, data, responseType }),不加任何默认头,鉴权头自己写;url 相对时按 baseUrl 拼接(不加 /v1)" },
|
||||
{ name: "poll", type: "function", desc: "轮询 poll(request, extract, {intervalMs,timeoutMs}),extract 返回真值即结束" },
|
||||
{ name: "sleep", type: "function", desc: "sleep(ms) 延时" },
|
||||
{ name: "signal", type: "AbortSignal", desc: "取消信号,可透传给 http/request" },
|
||||
{ name: "onDelta", type: "function", desc: "onDelta(text) 推送流式文本(文本模型)", capabilities: ["text"] },
|
||||
];
|
||||
|
||||
export const PLUGIN_RETURNS: Record<ModelCapability, string> = {
|
||||
image: "文生图(images 为空)和图生图(images 有参考图)接口不同,脚本需自行区分;返回图片 URL 或 dataURL 字符串,也可返回它们的数组,或 [{ dataUrl }] / [{ url }] / [{ b64_json }]",
|
||||
video: "脚本内部完成轮询,返回 { url } 或 { blob } 或视频 URL 字符串",
|
||||
audio: "返回 Blob,或 base64 / dataURL 字符串,或 { b64_json } / { data } / { url }",
|
||||
text: "用 onDelta(text) 推送流式,最终 return 完整文本字符串",
|
||||
};
|
||||
|
||||
export type PluginTemplate = { label: string; script: string };
|
||||
|
||||
export const PLUGIN_TEMPLATES: Record<ModelCapability, PluginTemplate[]> = {
|
||||
image: [
|
||||
{
|
||||
label: "OpenAI 规范",
|
||||
script: `// 生图 / 改图:两者接口不同,用 images 是否为空来区分。
|
||||
// 可用:prompt、images(dataURL[])、params{size,quality,count}、model、baseUrl、apiKey
|
||||
if (images.length === 0) {
|
||||
// 文生图:/images/generations(JSON)
|
||||
const data = await request({
|
||||
method: "post",
|
||||
url: \`\${baseUrl}/v1/images/generations\`,
|
||||
headers: { "Content-Type": "application/json", Authorization: \`Bearer \${apiKey}\` },
|
||||
data: { model, prompt, n: params.count, size: params.size, response_format: "b64_json" },
|
||||
});
|
||||
return (data.data || []).map((item) => item.b64_json ? \`data:image/png;base64,\${item.b64_json}\` : item.url);
|
||||
}
|
||||
|
||||
// 图生图:/images/edits(multipart/form-data,参考图作为文件上传)
|
||||
const form = new FormData();
|
||||
form.set("model", model);
|
||||
form.set("prompt", prompt);
|
||||
form.set("n", String(params.count));
|
||||
form.set("response_format", "b64_json");
|
||||
for (const dataUrl of images) {
|
||||
form.append("image", await (await fetch(dataUrl)).blob(), "ref.png");
|
||||
}
|
||||
const edited = await request({
|
||||
method: "post",
|
||||
url: \`\${baseUrl}/v1/images/edits\`,
|
||||
headers: { Authorization: \`Bearer \${apiKey}\` }, // 不要手动设 Content-Type,交给浏览器带 boundary
|
||||
data: form,
|
||||
});
|
||||
return (data.data || []).map((item) => item.b64_json ? \`data:image/png;base64,\${item.b64_json}\` : item.url);`,
|
||||
video: `// 输入:input.prompt / input.references(dataURL[]) / input.params
|
||||
// 返回:{ url } 或 { blob } 或视频 URL 字符串
|
||||
const task = await http.post("/videos", {
|
||||
model: config.model,
|
||||
prompt: input.prompt,
|
||||
seconds: input.params.seconds,
|
||||
return (edited.data || []).map((item) => item.b64_json ? \`data:image/png;base64,\${item.b64_json}\` : item.url);`,
|
||||
},
|
||||
{
|
||||
label: "Gemini 规范",
|
||||
script: `// Gemini 文生图 / 图生图:都走 generateContent,参考图放进 parts 的 inline_data。
|
||||
// 可用:prompt、images(dataURL[])、model、baseUrl、apiKey
|
||||
const parts = [{ text: prompt }];
|
||||
for (const dataUrl of images) {
|
||||
const match = dataUrl.match(/^data:([^;]+);base64,(.*)$/);
|
||||
if (match) parts.push({ inline_data: { mime_type: match[1], data: match[2] } });
|
||||
}
|
||||
const data = await request({
|
||||
method: "post",
|
||||
url: \`\${baseUrl}/v1beta/models/\${model}:generateContent\`,
|
||||
headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey },
|
||||
data: { contents: [{ role: "user", parts }], generationConfig: { responseModalities: ["IMAGE"] } },
|
||||
});
|
||||
return (data.candidates || [])
|
||||
.flatMap((c) => c.content?.parts || [])
|
||||
.map((p) => p.inlineData || p.inline_data)
|
||||
.filter(Boolean)
|
||||
.map((img) => \`data:\${img.mimeType || img.mime_type || "image/png"};base64,\${img.data}\`);`,
|
||||
},
|
||||
],
|
||||
video: [
|
||||
{
|
||||
label: "OpenAI 规范",
|
||||
script: `// 视频(脚本内部自行轮询)。可用:prompt、images(dataURL[])、params{seconds,size,resolution,ratio}
|
||||
const headers = { "Content-Type": "application/json", Authorization: \`Bearer \${apiKey}\` };
|
||||
const task = await request({
|
||||
method: "post",
|
||||
url: \`\${baseUrl}/v1/videos\`,
|
||||
headers,
|
||||
data: { model, prompt, seconds: params.seconds },
|
||||
});
|
||||
return await poll(
|
||||
() => http.get(\`/videos/\${task.id}\`),
|
||||
() => request({ method: "get", url: \`\${baseUrl}/v1/videos/\${task.id}\`, headers }),
|
||||
(state) => state.status === "completed" ? { url: state.video_url || state.url } : null,
|
||||
{ intervalMs: 2500, timeoutMs: 300000 },
|
||||
);`,
|
||||
audio: `// 输入:input.prompt / input.params(voice/format/speed/instructions)
|
||||
// 返回:Blob,或 base64/dataURL 字符串
|
||||
return await http.post("/audio/speech", {
|
||||
model: config.model,
|
||||
input: input.prompt,
|
||||
voice: input.params.voice,
|
||||
response_format: input.params.format,
|
||||
speed: Number(input.params.speed),
|
||||
}, { responseType: "blob" });`,
|
||||
text: `// 输入:input.messages([{role,content}]) / input.body
|
||||
// 用 onDelta(text) 推送流式文本;返回最终完整文本
|
||||
const data = await http.post("/chat/completions", {
|
||||
model: config.model,
|
||||
messages: input.messages,
|
||||
},
|
||||
{
|
||||
label: "Gemini 规范",
|
||||
script: `// Gemini(Veo) 视频:predictLongRunning 提交,轮询 operation 拿视频 URI。
|
||||
// 可用:prompt、images(dataURL[])、params、model、baseUrl、apiKey
|
||||
const headers = { "Content-Type": "application/json", "x-goog-api-key": apiKey };
|
||||
const instance = { prompt };
|
||||
const first = images[0] && images[0].match(/^data:([^;]+);base64,(.*)$/);
|
||||
if (first) instance.image = { bytesBase64Encoded: first[2], mimeType: first[1] };
|
||||
const op = await request({
|
||||
method: "post",
|
||||
url: \`\${baseUrl}/v1beta/models/\${model}:predictLongRunning\`,
|
||||
headers,
|
||||
data: { instances: [instance], parameters: { aspectRatio: params.ratio } },
|
||||
});
|
||||
const text = data.choices?.[0]?.message?.content || "";
|
||||
return await poll(
|
||||
() => request({ method: "get", url: \`\${baseUrl}/v1beta/\${op.name}\`, headers }),
|
||||
(state) => {
|
||||
if (!state.done) return null;
|
||||
const uri = state.response?.generateVideoResponse?.generatedSamples?.[0]?.video?.uri;
|
||||
if (!uri) throw new Error("Gemini 未返回视频 URI");
|
||||
return { url: uri.includes("key=") ? uri : \`\${uri}\${uri.includes("?") ? "&" : "?"}key=\${apiKey}\` };
|
||||
},
|
||||
{ intervalMs: 5000, timeoutMs: 300000 },
|
||||
);`,
|
||||
},
|
||||
],
|
||||
audio: [
|
||||
{
|
||||
label: "OpenAI 规范",
|
||||
script: `// 音频 TTS。可用:prompt、params{voice,format,speed,instructions}、model
|
||||
return await request({
|
||||
method: "post",
|
||||
url: \`\${baseUrl}/v1/audio/speech\`,
|
||||
headers: { "Content-Type": "application/json", Authorization: \`Bearer \${apiKey}\` },
|
||||
responseType: "blob",
|
||||
data: { model, input: prompt, voice: params.voice, response_format: params.format, speed: Number(params.speed) },
|
||||
});`,
|
||||
},
|
||||
{
|
||||
label: "Gemini 规范",
|
||||
script: `// Gemini TTS:generateContent + AUDIO 模态,返回 base64 PCM(音频数据在 inlineData.data)。
|
||||
// 可用:prompt、params{voice}、model、baseUrl、apiKey
|
||||
const data = await request({
|
||||
method: "post",
|
||||
url: \`\${baseUrl}/v1beta/models/\${model}:generateContent\`,
|
||||
headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey },
|
||||
data: {
|
||||
contents: [{ role: "user", parts: [{ text: prompt }] }],
|
||||
generationConfig: {
|
||||
responseModalities: ["AUDIO"],
|
||||
speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: params.voice } } },
|
||||
},
|
||||
},
|
||||
});
|
||||
const audio = data.candidates?.[0]?.content?.parts?.map((p) => p.inlineData || p.inline_data).find(Boolean);
|
||||
if (!audio?.data) throw new Error("Gemini 未返回音频");
|
||||
return { data: audio.data };`,
|
||||
},
|
||||
],
|
||||
text: [
|
||||
{
|
||||
label: "OpenAI 规范",
|
||||
script: `// 文本对话(OpenAI Responses 接口)。可用:messages([{role,content}])、systemPrompt、model
|
||||
const data = await request({
|
||||
method: "post",
|
||||
url: \`\${baseUrl}/v1/responses\`,
|
||||
headers: { "Content-Type": "application/json", Authorization: \`Bearer \${apiKey}\` },
|
||||
data: { model, input: messages },
|
||||
});
|
||||
const text = data.output_text
|
||||
|| (data.output || []).flatMap((o) => o.content || []).map((c) => c.text || "").join("")
|
||||
|| "";
|
||||
onDelta(text);
|
||||
return text;`,
|
||||
},
|
||||
{
|
||||
label: "Gemini 规范",
|
||||
script: `// Gemini 文本:generateContent,system 消息放 systemInstruction。
|
||||
// 可用:messages([{role,content}])、systemPrompt、model、baseUrl、apiKey
|
||||
const contents = messages
|
||||
.filter((m) => m.role !== "system")
|
||||
.map((m) => ({ role: m.role === "assistant" ? "model" : "user", parts: [{ text: m.content }] }));
|
||||
const data = await request({
|
||||
method: "post",
|
||||
url: \`\${baseUrl}/v1beta/models/\${model}:generateContent\`,
|
||||
headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey },
|
||||
data: { contents, ...(systemPrompt ? { systemInstruction: { parts: [{ text: systemPrompt }] } } : {}) },
|
||||
});
|
||||
const text = data.candidates?.[0]?.content?.parts?.map((p) => p.text || "").join("") || "";
|
||||
onDelta(text);
|
||||
return text;`,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
/** Normalize whatever an image script returns into the app's generated-image shape. */
|
||||
|
||||
@@ -90,17 +90,15 @@ async function createPluginVideoTask(config: AiConfig, model: string, script: st
|
||||
capability: "video",
|
||||
script,
|
||||
config,
|
||||
input: {
|
||||
prompt,
|
||||
references: refs,
|
||||
params: {
|
||||
seconds: normalizeVideoSeconds(config.videoSeconds),
|
||||
size: normalizeVideoSize(config.size),
|
||||
resolution: normalizeVideoResolution(config.vquality),
|
||||
ratio: config.size,
|
||||
generateAudio: boolConfig(config.videoGenerateAudio, true),
|
||||
watermark: boolConfig(config.videoWatermark, false),
|
||||
},
|
||||
prompt,
|
||||
images: refs,
|
||||
params: {
|
||||
seconds: normalizeVideoSeconds(config.videoSeconds),
|
||||
size: normalizeVideoSize(config.size),
|
||||
resolution: normalizeVideoResolution(config.vquality),
|
||||
ratio: config.size,
|
||||
generateAudio: boolConfig(config.videoGenerateAudio, true),
|
||||
watermark: boolConfig(config.videoWatermark, false),
|
||||
},
|
||||
signal: options?.signal,
|
||||
}),
|
||||
|
||||
Reference in New Issue
Block a user