From c57f7d61a7a7bf0b459cad0a09ee24f1e99ac464 Mon Sep 17 00:00:00 2001 From: HouYunFei <1844025705@qq.com> Date: Wed, 15 Jul 2026 10:13:32 +0800 Subject: [PATCH 1/7] feat(model): enhance model configuration and script handling for audio and video capabilities --- CHANGELOG.md | 4 +- .../canvas/canvas-config-node-panel.tsx | 4 +- .../canvas/canvas-node-prompt-panel.tsx | 4 +- .../components/layout/app-config-modal.tsx | 236 ++++-------------- .../layout/channel-editor-drawer.tsx | 134 ++++++++++ .../components/layout/model-script-editor.tsx | 67 +++++ web/src/components/model-picker.tsx | 2 +- web/src/lib/agent/agent-site-tools.ts | 6 +- web/src/services/api/audio.ts | 37 ++- web/src/services/api/image.ts | 66 ++++- web/src/services/api/model-plugin.ts | 204 +++++++++++++++ web/src/services/api/video.ts | 56 ++++- web/src/stores/use-config-store.ts | 136 +++++----- 13 files changed, 690 insertions(+), 266 deletions(-) create mode 100644 web/src/components/layout/channel-editor-drawer.tsx create mode 100644 web/src/components/layout/model-script-editor.tsx create mode 100644 web/src/services/api/model-plugin.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a8480fd..59003cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,13 @@ ## Unreleased ++ [新增] 支持自定义生图/视频接口调用方式以适配不同中转站。 + ## v0.7.0 - 2026-07-14 + [新增] Agent 对话消息改用 streamdown 流式渲染，提升Markdown 内容展示效果。 + [新增] Agent 新增画布、工作台、提示词库和素材等站点级工具。 -+ [新增] Agent 面板改为全站常驻右侧栏，开关时同步推动顶栏和页面内容，并保留独立入口按钮。 ++ [新增] Agent 面板改为全站常驻右侧栏，开关时同步推动顶栏和页面内容。 + [新增] Agent 新增 `site_navigate` 工具，支持页面跳转。 + [新增] Agent 对话运行中支持一键停止，中断当前 Codex turn。 + [新增] 画布节点支持统一维护名称字段，默认显示在节点上方，并可直接双击名称编辑。 diff --git a/web/src/components/canvas/canvas-config-node-panel.tsx b/web/src/components/canvas/canvas-config-node-panel.tsx index 8eb2283..4e265bf 100644 --- a/web/src/components/canvas/canvas-config-node-panel.tsx +++ b/web/src/components/canvas/canvas-config-node-panel.tsx @@ -146,9 +146,9 @@ function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: Can const defaultModel = mode === "image" ? globalConfig.imageModel : mode === "video" ? globalConfig.videoModel : mode === "audio" ? globalConfig.audioModel : globalConfig.textModel; const fallbackModel = mode === "image" ? defaultConfig.imageModel : mode === "video" ? defaultConfig.videoModel : mode === "audio" ? defaultConfig.audioModel : defaultConfig.textModel; const currentModel = node.metadata?.model; - const model = currentModel && modelMatchesCapability(currentModel, mode) + const model = currentModel && modelMatchesCapability(globalConfig, currentModel, mode) ? currentModel - : defaultModel && modelMatchesCapability(defaultModel, mode) + : defaultModel && modelMatchesCapability(globalConfig, defaultModel, mode) ? defaultModel : fallbackModel; return { diff --git a/web/src/components/canvas/canvas-node-prompt-panel.tsx b/web/src/components/canvas/canvas-node-prompt-panel.tsx index aff57e2..ad1ee7e 100644 --- a/web/src/components/canvas/canvas-node-prompt-panel.tsx +++ b/web/src/components/canvas/canvas-node-prompt-panel.tsx @@ -134,9 +134,9 @@ function buildNodeConfig(globalConfig: AiConfig, node: CanvasNodeData, mode: Can const defaultModel = mode === "image" ? globalConfig.imageModel : mode === "video" ? globalConfig.videoModel : mode === "audio" ? globalConfig.audioModel : globalConfig.textModel; const fallbackModel = mode === "image" ? defaultConfig.imageModel : mode === "video" ? defaultConfig.videoModel : mode === "audio" ? defaultConfig.audioModel : defaultConfig.textModel; const currentModel = node.metadata?.model; - const model = currentModel && modelMatchesCapability(currentModel, mode) + const model = currentModel && modelMatchesCapability(globalConfig, currentModel, mode) ? currentModel - : defaultModel && modelMatchesCapability(defaultModel, mode) + : defaultModel && modelMatchesCapability(globalConfig, defaultModel, mode) ? defaultModel : fallbackModel; return { diff --git a/web/src/components/layout/app-config-modal.tsx b/web/src/components/layout/app-config-modal.tsx index b52ab5f..45bcc3b 100644 --- a/web/src/components/layout/app-config-modal.tsx +++ b/web/src/components/layout/app-config-modal.tsx @@ -1,20 +1,18 @@ import { App, Button, Form, Input, Modal, Progress, Select, Tabs } from "antd"; -import { CircleAlert, Cloud, Plus, RefreshCw, Trash2, Wifi } from "lucide-react"; +import { Cloud, Pencil, Plus, RefreshCw, Trash2, Wifi } from "lucide-react"; import { useEffect, useState } from "react"; import { ModelPicker } from "@/components/model-picker"; -import { fetchChannelModels } from "@/services/api/image"; +import { ChannelEditorDrawer } from "@/components/layout/channel-editor-drawer"; import { syncAppDataToWebdav, type AppSyncDomainKey, type AppSyncProgressEvent } from "@/services/app-sync"; import { testWebdavConnection, WEBDAV_MANIFEST_FILE_NAME } from "@/services/webdav-sync"; import { audioFormatOptions, audioVoiceOptions, normalizeAudioSpeedValue } from "@/lib/audio-generation"; -import { createModelChannel, defaultBaseUrlForApiFormat, filterModelsByCapability, modelOptionLabel, modelOptionsFromChannels, normalizeModelOptionValue, useConfigStore, type AiConfig, type ApiCallFormat, type ConfigTabKey, type ModelCapability, type ModelChannel } from "@/stores/use-config-store"; +import { createModelChannel, modelOptionsFromChannels, normalizeModelOptionValue, selectableModelsByCapability, useConfigStore, type AiConfig, type ApiCallFormat, type ConfigTabKey, type ModelCapability, type ModelChannel } from "@/stores/use-config-store"; type ModelGroup = { capability: ModelCapability; modelKey: "imageModel" | "videoModel" | "textModel" | "audioModel"; - modelsKey: "imageModels" | "videoModels" | "textModels" | "audioModels"; defaultLabel: string; - optionsLabel: string; }; type WebdavDomainProgress = { @@ -26,15 +24,10 @@ type WebdavDomainProgress = { }; const modelGroups: ModelGroup[] = [ - { capability: "image", modelKey: "imageModel", modelsKey: "imageModels", defaultLabel: "默认生图模型", optionsLabel: "生图模型可选项" }, - { capability: "video", modelKey: "videoModel", modelsKey: "videoModels", defaultLabel: "默认视频模型", optionsLabel: "视频模型可选项" }, - { capability: "text", modelKey: "textModel", modelsKey: "textModels", defaultLabel: "默认文本模型", optionsLabel: "文本模型可选项" }, - { capability: "audio", modelKey: "audioModel", modelsKey: "audioModels", defaultLabel: "默认音频模型", optionsLabel: "音频模型可选项" }, -]; - -const apiFormatOptions: Array<{ label: string; value: ApiCallFormat }> = [ - { label: "OpenAI", value: "openai" }, - { label: "Gemini", value: "gemini" }, + { capability: "image", modelKey: "imageModel", defaultLabel: "默认生图模型" }, + { capability: "video", modelKey: "videoModel", defaultLabel: "默认视频模型" }, + { capability: "text", modelKey: "textModel", defaultLabel: "默认文本模型" }, + { capability: "audio", modelKey: "audioModel", defaultLabel: "默认音频模型" }, ]; const webdavDomainKeys: AppSyncDomainKey[] = ["canvas", "assets", "image-workbench", "video-workbench"]; @@ -58,7 +51,7 @@ function createWebdavDomainProgress(): Record(initialTab); - const [loadingChannelId, setLoadingChannelId] = useState(""); + const [editingChannelId, setEditingChannelId] = useState(""); const [testingWebdav, setTestingWebdav] = useState(false); const [syncingWebdav, setSyncingWebdav] = useState(false); const [webdavSyncStatus, setWebdavSyncStatus] = useState(""); @@ -70,8 +63,8 @@ export function AppConfigPanel({ showDoneButton = false, initialTab = "channels" const shouldPromptContinue = useConfigStore((state) => state.shouldPromptContinue); const setConfigDialogOpen = useConfigStore((state) => state.setConfigDialogOpen); const clearPromptContinue = useConfigStore((state) => state.clearPromptContinue); - const modelOptions = config.models.map((model) => ({ label: modelOptionLabel(config, model), value: model })); const webdavReady = Boolean(webdav.url.trim()); + const editingChannel = config.channels.find((channel) => channel.id === editingChannelId) || null; useEffect(() => setActiveTab(initialTab), [initialTab]); const saveConfig = (nextConfig: AiConfig) => { @@ -86,22 +79,12 @@ export function AppConfigPanel({ showDoneButton = false, initialTab = "channels" clearPromptContinue(); }; - const updateChannels = (channels: ModelChannel[]) => { - const nextConfig = withChannels(config, channels); - saveConfig(nextConfig); - }; - - const updateChannel = (id: string, patch: Partial) => { - updateChannels(config.channels.map((channel) => (channel.id === id ? { ...channel, ...patch, models: patch.models ? uniqueModels(patch.models) : channel.models } : channel))); - }; - - const updateChannelApiFormat = (channel: ModelChannel, apiFormat: ApiCallFormat) => { - const baseUrl = !channel.baseUrl.trim() || channel.baseUrl.trim() === defaultBaseUrlForApiFormat(channel.apiFormat) ? defaultBaseUrlForApiFormat(apiFormat) : channel.baseUrl; - updateChannel(channel.id, { apiFormat, baseUrl }); - }; + const updateChannels = (channels: ModelChannel[]) => saveConfig(withChannels(config, channels)); const addChannel = () => { - updateChannels([...config.channels, createModelChannel({ name: `渠道 ${config.channels.length + 1}` })]); + const channel = createModelChannel({ name: `渠道 ${config.channels.length + 1}` }); + updateChannels([...config.channels, channel]); + setEditingChannelId(channel.id); }; const deleteChannel = (id: string) => { @@ -112,46 +95,8 @@ export function AppConfigPanel({ showDoneButton = false, initialTab = "channels" updateChannels(config.channels.filter((channel) => channel.id !== id)); }; - const refreshChannelModels = async (channel: ModelChannel) => { - if (!channel.baseUrl.trim() || !channel.apiKey.trim()) { - message.error("请先填写该渠道的 Base URL 和 API Key"); - return; - } - setLoadingChannelId(channel.id); - try { - const models = await fetchChannelModels(channel); - updateChannels(config.channels.map((item) => (item.id === channel.id ? { ...item, models } : item))); - message.success(`${channel.name} 模型列表已更新`); - } catch (error) { - message.error(error instanceof Error ? error.message : "读取模型失败"); - } finally { - setLoadingChannelId(""); - } - }; - - const refreshAllModels = async () => { - const runnable = config.channels.filter((channel) => channel.baseUrl.trim() && channel.apiKey.trim()); - if (!runnable.length) { - message.error("请先填写至少一个渠道的 Base URL 和 API Key"); - return; - } - setLoadingChannelId("all"); - try { - const entries = await Promise.all(runnable.map(async (channel) => [channel.id, await fetchChannelModels(channel)] as const)); - const modelMap = new Map(entries); - updateChannels(config.channels.map((channel) => (modelMap.has(channel.id) ? { ...channel, models: modelMap.get(channel.id) || [] } : channel))); - message.success("模型列表已更新"); - } catch (error) { - message.error(error instanceof Error ? error.message : "读取模型失败"); - } finally { - setLoadingChannelId(""); - } - }; - - const updateCapabilityModels = (group: ModelGroup, models: string[]) => { - const next = uniqueModels(models.map((model) => normalizeModelOptionValue(model, config.channels)).filter(Boolean)); - updateConfig(group.modelsKey, next); - if (!next.includes(config[group.modelKey])) updateConfig(group.modelKey, next[0] || ""); + const saveChannel = (channel: ModelChannel) => { + updateChannels(config.channels.map((item) => (item.id === channel.id ? channel : item))); }; const testWebdav = async () => { @@ -215,107 +160,48 @@ export function AppConfigPanel({ showDoneButton = false, initialTab = "channels" key: "channels", label: "渠道", children: ( -
-
-
-
- - 重要： - 新增或拉取模型后，需要到“模型”Tab 选择可选项才会显示。 - -
-
-
- - -
+
+
+
每个渠道选择一个协议并拉取模型，为每个模型指定能力（生图/视频/文本/音频），并可自定义调用脚本。
+
-
+
{config.channels.map((channel) => ( -
-
-
-
{channel.name || "未命名渠道"}
-
- {apiFormatLabel(channel.apiFormat)} · 已保存 {channel.models.length} 个模型 -
-
-
- - + + + } + > +
+ + + +
+ +
+
+
渠道模型
+
已选 {draft.models.length} 个；为每个模型指定能力并可自定义调用脚本。
+
+ +
+ +
+ {draft.models.length ? ( + draft.models.map((model) => ( +
+ + {model.name} + +
+ setCapability(model.name, value as ModelCapability)} /> + +
+
+ )) + ) : ( +
点击「选择模型」拉取或手动增加模型。
+ )} +
+ + model.name)} onConfirm={applySelection} onClose={() => setSelectOpen(false)} /> + + scriptTarget && setScript(scriptTarget.name, script)} + onClose={() => setScriptTarget(null)} + /> + + ); +} diff --git a/web/src/components/layout/model-script-editor.tsx b/web/src/components/layout/model-script-editor.tsx new file mode 100644 index 0000000..1aa586e --- /dev/null +++ b/web/src/components/layout/model-script-editor.tsx @@ -0,0 +1,67 @@ +import { Button, Input, Modal } from "antd"; +import { useEffect, useState } from "react"; + +import { PLUGIN_TEMPLATES } from "@/services/api/model-plugin"; +import type { ModelCapability } from "@/stores/use-config-store"; + +const capabilityLabels: Record = { image: "生图", video: "视频", text: "文本", audio: "音频" }; + +const variableHints = [ + "input：本次请求的归一化输入（prompt / references / messages / params / body）", + "config：{ baseUrl, apiKey, model, apiFormat, systemPrompt }", + "http：{ url(path), post(path, body, opts), get(path, opts) }，已绑定当前渠道", + "poll(request, extract, { intervalMs, timeoutMs })：轮询直到 extract 返回真值", + "sleep(ms)、signal：延时与取消信号；onDelta(text)：推送流式文本（文本模型）", +]; + +export function ModelScriptEditor({ open, capability, modelName, value, onSave, onClose }: { open: boolean; capability: ModelCapability; modelName: string; value: string; onSave: (script: string) => void; onClose: () => void }) { + const [draft, setDraft] = useState(value); + useEffect(() => { + if (open) setDraft(value); + }, [open, value]); + + return ( + +
调用脚本 · {modelName}
+
当前能力：{capabilityLabels[capability]}。留空则使用系统默认调用方式。
+
+ } + width={720} + centered + onCancel={onClose} + styles={{ body: { maxHeight: "64vh", overflowY: "auto" } }} + footer={[ + , + , + , + , + ]} + > +
+
可用变量（异步函数体，需 return 结果）
+ {variableHints.map((hint) => ( +
· {hint}
+ ))} +
+ setDraft(event.target.value)} autoSize={{ minRows: 12, maxRows: 24 }} spellCheck={false} placeholder="留空使用系统默认调用；点击“插入模板”查看示例。" style={{ fontFamily: "var(--font-mono, ui-monospace, SFMono-Regular, Menlo, monospace)", fontSize: 12 }} /> + + ); +} diff --git a/web/src/components/model-picker.tsx b/web/src/components/model-picker.tsx index 1ebf373..b6e2372 100644 --- a/web/src/components/model-picker.tsx +++ b/web/src/components/model-picker.tsx @@ -83,7 +83,7 @@ export function ModelPicker({ config, value, onChange, capability, className, fu function emptyModelLabel(config: AiConfig, capability?: ModelCapability) { const label = capability === "image" ? "生图" : capability === "video" ? "视频" : capability === "text" ? "文本" : capability === "audio" ? "音频" : ""; - if (capability && config.models.length) return "请先在上方配置可选模型"; + if (capability && config.models.length) return `请先在渠道里为${label}指定模型`; return config.models.length ? `暂无匹配的${label}模型` : "请先到配置里添加渠道和模型"; } diff --git a/web/src/lib/agent/agent-site-tools.ts b/web/src/lib/agent/agent-site-tools.ts index 575a04c..6539629 100644 --- a/web/src/lib/agent/agent-site-tools.ts +++ b/web/src/lib/agent/agent-site-tools.ts @@ -6,7 +6,7 @@ import { imageAspectOptions, imageQualityOptions } from "@/components/image-sett import { videoResolutionOptions, videoSecondOptions, videoSizeOptions } from "@/components/video-settings-panel"; import { useCanvasStore } from "@/stores/canvas/use-canvas-store"; import { useAssetStore } from "@/stores/use-asset-store"; -import { modelOptionLabel, modelOptionName, normalizeModelOptionValue, useConfigStore } from "@/stores/use-config-store"; +import { modelOptionLabel, modelOptionName, normalizeModelOptionValue, selectableModelsByCapability, useConfigStore } from "@/stores/use-config-store"; import { useWorkbenchAgentStore } from "@/stores/use-workbench-agent-store"; // 在网页端执行 Agent 的「站点级」工具（画布列表、工作台生成、提示词搜索、素材增删查等）。 @@ -87,7 +87,7 @@ function getImageConfig() { const model = config.imageModel || config.model; return { current: { model, modelName: modelOptionName(model), quality: config.quality || "auto", size: config.size || "1:1", count: config.count || "1" }, - models: config.imageModels.map((value) => ({ value, label: modelOptionLabel(config, value) })), + models: selectableModelsByCapability(config, "image").map((value) => ({ value, label: modelOptionLabel(config, value) })), qualityOptions: imageQualityOptions, sizeOptions: imageAspectOptions, countRange: { min: 1, max: 15 }, @@ -135,7 +135,7 @@ function getVideoConfig() { generateAudio: config.videoGenerateAudio !== "false", watermark: config.videoWatermark === "true", }, - models: config.videoModels.map((value) => ({ value, label: modelOptionLabel(config, value) })), + models: selectableModelsByCapability(config, "video").map((value) => ({ value, label: modelOptionLabel(config, value) })), sizeOptions: videoSizeOptions, secondsOptions: videoSecondOptions, resolutionOptions: videoResolutionOptions, diff --git a/web/src/services/api/audio.ts b/web/src/services/api/audio.ts index d84a547..611c242 100644 --- a/web/src/services/api/audio.ts +++ b/web/src/services/api/audio.ts @@ -2,7 +2,8 @@ import axios from "axios"; import { audioMimeType, normalizeAudioFormatValue, normalizeAudioSpeedValue, normalizeAudioVoiceValue } from "@/lib/audio-generation"; import { uploadMediaFile, type UploadedFile } from "@/services/file-storage"; -import { buildApiUrl, resolveModelRequestConfig, type AiConfig } from "@/stores/use-config-store"; +import { buildApiUrl, resolveModelRequestConfig, resolveModelScript, type AiConfig } from "@/stores/use-config-store"; +import { runModelPlugin } from "./model-plugin"; type RequestOptions = { signal?: AbortSignal }; @@ -20,8 +21,26 @@ function aiHeaders(config: AiConfig) { export async function requestAudioGeneration(config: AiConfig, prompt: string, options?: RequestOptions): Promise { const requestConfig = resolveModelRequestConfig(config, config.model || config.audioModel); const model = requestConfig.model.trim(); - assertAudioConfig(requestConfig, model); const format = normalizeAudioFormatValue(config.audioFormat); + const script = resolveModelScript(config, config.model || config.audioModel); + if (script) { + if (!model) throw new Error("请先配置音频模型"); + if (!requestConfig.baseUrl.trim()) throw new Error("请先配置 Base URL"); + if (!requestConfig.apiKey.trim()) throw new Error("请先配置 API Key"); + try { + const result = await runModelPlugin({ + capability: "audio", + script, + config: requestConfig, + input: { prompt, params: { voice: normalizeAudioVoiceValue(config.audioVoice), format, speed: normalizeAudioSpeedValue(config.audioSpeed), instructions: config.audioInstructions.trim() } }, + signal: options?.signal, + }); + return await audioPluginBlob(result, format); + } catch (error) { + throw new Error(readAxiosError(error, "音频生成失败")); + } + } + assertAudioConfig(requestConfig, model); const instructions = config.audioInstructions.trim(); try { @@ -44,6 +63,20 @@ export async function requestAudioGeneration(config: AiConfig, prompt: string, o } } +async function audioPluginBlob(result: unknown, format: string): Promise { + if (result instanceof Blob) return result.type.startsWith("audio/") ? result : new Blob([result], { type: audioMimeType(format) }); + let source = ""; + if (typeof result === "string") source = result; + else if (result && typeof result === "object") { + const record = result as Record; + source = typeof record.b64_json === "string" ? record.b64_json : typeof record.data === "string" ? record.data : typeof record.url === "string" ? record.url : ""; + } + if (!source) throw new Error("模型调用脚本没有返回音频"); + const url = source.startsWith("data:") || /^https?:/i.test(source) ? source : `data:${audioMimeType(format)};base64,${source}`; + const blob = await (await fetch(url)).blob(); + return blob.type.startsWith("audio/") ? blob : new Blob([blob], { type: audioMimeType(format) }); +} + export async function storeGeneratedAudio(blob: Blob, format = "mp3"): Promise { const audio = blob.type.startsWith("audio/") ? blob : new Blob([blob], { type: audioMimeType(format) }); return uploadMediaFile(audio, "audio"); diff --git a/web/src/services/api/image.ts b/web/src/services/api/image.ts index 1959759..e072317 100644 --- a/web/src/services/api/image.ts +++ b/web/src/services/api/image.ts @@ -1,6 +1,7 @@ import axios from "axios"; -import { buildApiUrl, resolveModelRequestConfig, type AiConfig, type ModelChannel } from "@/stores/use-config-store"; +import { buildApiUrl, resolveModelRequestConfig, resolveModelScript, type AiConfig, type ModelChannel } from "@/stores/use-config-store"; +import { normalizePluginImages, runModelPlugin } from "./model-plugin"; import { nanoid } from "nanoid"; import { dataUrlToFile } from "@/lib/image-utils"; import { buildImageReferencePromptText } from "@/lib/image-reference-prompt"; @@ -656,6 +657,28 @@ function parseGeminiImagePayload(payload: GeminiPayload) { export async function requestGeneration(config: AiConfig, prompt: string, options?: RequestOptions) { const requestConfig = resolveModelRequestConfig(config, config.model || config.imageModel); const n = Math.max(1, Math.min(15, Math.floor(Math.abs(Number(config.count)) || 1))); + const script = resolveModelScript(config, config.model || config.imageModel); + if (script) { + const quality = normalizeQuality(config.quality); + const requestSize = resolveRequestSize(quality, config.size); + try { + const result = await runModelPlugin({ + capability: "image", + script, + config: requestConfig, + input: { + prompt: withSystemPrompt(requestConfig, prompt), + references: [], + params: { size: requestSize, quality, count: n }, + body: { model: requestConfig.model, n, ...(quality ? { quality } : {}), ...(requestSize ? { size: requestSize } : {}), response_format: "b64_json", output_format: IMAGE_OUTPUT_FORMAT }, + }, + signal: options?.signal, + }); + return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl })); + } catch (error) { + throw new Error(readAxiosError(error, "请求失败")); + } + } if (requestConfig.apiFormat === "gemini") { try { return await requestGeminiImages(requestConfig, prompt, [], n, options); @@ -693,6 +716,29 @@ export async function requestEdit(config: AiConfig, prompt: string, references: const requestConfig = resolveModelRequestConfig(config, config.model || config.imageModel); const n = Math.max(1, Math.min(15, Math.floor(Math.abs(Number(config.count)) || 1))); const requestPrompt = buildImageReferencePromptText(prompt, references); + const script = resolveModelScript(config, config.model || config.imageModel); + if (script) { + const quality = normalizeQuality(config.quality); + const requestSize = resolveRequestSize(quality, config.size); + const refs = await Promise.all(references.map((image) => imageToDataUrl(image))); + try { + const result = await runModelPlugin({ + capability: "image", + script, + config: requestConfig, + input: { + prompt: withSystemPrompt(requestConfig, requestPrompt), + references: refs, + params: { size: requestSize, quality, count: n }, + body: { model: requestConfig.model, n, ...(quality ? { quality } : {}), ...(requestSize ? { size: requestSize } : {}), response_format: "b64_json", output_format: IMAGE_OUTPUT_FORMAT }, + }, + signal: options?.signal, + }); + return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl })); + } catch (error) { + throw new Error(readAxiosError(error, "请求失败")); + } + } if (requestConfig.apiFormat === "gemini") { if (mask) throw new Error("Gemini 调用格式暂不支持蒙版编辑"); try { @@ -730,6 +776,24 @@ export async function requestEdit(config: AiConfig, prompt: string, references: export async function requestImageQuestion(config: AiConfig, messages: AiTextMessage[], onDelta: (text: string) => void, options?: RequestOptions) { const requestConfig = resolveModelRequestConfig(config, config.model || config.textModel); + const script = resolveModelScript(config, config.model || config.textModel); + if (script) { + try { + const answer = await runModelPlugin({ + capability: "text", + script, + config: requestConfig, + input: { messages: withSystemMessage(requestConfig, messages), body: { model: requestConfig.model } }, + signal: options?.signal, + onDelta, + }); + const text = String(answer ?? "").trim() || "没有返回内容"; + if (text === "没有返回内容") onDelta(text); + return text; + } catch (error) { + throw new Error(readAxiosError(error, "请求失败")); + } + } try { if (requestConfig.apiFormat === "gemini") { const answer = (await requestGeminiStreamingResponse(requestConfig, toGeminiBody(requestConfig, messages), onDelta, options)).content || "没有返回内容"; diff --git a/web/src/services/api/model-plugin.ts b/web/src/services/api/model-plugin.ts new file mode 100644 index 0000000..092e82f --- /dev/null +++ b/web/src/services/api/model-plugin.ts @@ -0,0 +1,204 @@ +import axios from "axios"; + +import { buildApiUrl, type AiConfig, type ModelCapability } from "@/stores/use-config-store"; + +type RequestOptions = { signal?: AbortSignal }; + +export type PluginHttpOptions = { + headers?: Record; + params?: Record; + responseType?: "json" | "blob" | "text" | "arraybuffer"; +}; + +export type PluginHttp = { + url: (path: string) => string; + post: (path: string, body?: unknown, options?: PluginHttpOptions) => Promise; + get: (path: string, options?: PluginHttpOptions) => Promise; +}; + +export type PluginPollOptions = { intervalMs?: number; timeoutMs?: number }; + +export type PluginConfigView = { + baseUrl: string; + apiKey: string; + model: string; + apiFormat: string; + systemPrompt: string; +}; + +export type RunPluginArgs = { + capability: ModelCapability; + script: string; + config: AiConfig; + input: Record; + signal?: AbortSignal; + onDelta?: (text: string) => void; +}; + +function pluginHeaders(config: AiConfig, extra?: Record, hasJsonBody = false): Record { + const headers: Record = {}; + if (config.apiFormat === "gemini") headers["x-goog-api-key"] = config.apiKey; + else headers.Authorization = `Bearer ${config.apiKey}`; + if (hasJsonBody) headers["Content-Type"] = "application/json"; + return { ...headers, ...extra }; +} + +function pluginUrl(config: AiConfig, path: string) { + if (/^https?:/i.test(path)) return path; + return buildApiUrl(config.baseUrl, path.startsWith("/") ? path : `/${path}`); +} + +function createPluginHttp(config: AiConfig, options?: RequestOptions): PluginHttp { + const request = async (method: "get" | "post", path: string, body: unknown, opts?: PluginHttpOptions) => { + const isForm = typeof FormData !== "undefined" && body instanceof FormData; + const response = await axios.request({ + method, + url: pluginUrl(config, path), + data: method === "post" ? body : undefined, + params: opts?.params, + headers: pluginHeaders(config, opts?.headers, method === "post" && !isForm && body !== undefined), + responseType: opts?.responseType || "json", + signal: options?.signal, + }); + return response.data; + }; + return { + url: (path) => pluginUrl(config, path), + post: (path, body, opts) => request("post", path, body, opts), + get: (path, opts) => request("get", path, undefined, opts), + }; +} + +function sleep(ms: number, signal?: AbortSignal) { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new DOMException("Aborted", "AbortError")); + return; + } + const timer = setTimeout(resolve, ms); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + reject(new DOMException("Aborted", "AbortError")); + }, + { once: true }, + ); + }); +} + +function createPoll(signal?: AbortSignal) { + return async function poll(request: () => Promise, extract: (value: T) => R | null | undefined | false, options?: PluginPollOptions): Promise { + const intervalMs = options?.intervalMs ?? 2500; + const timeoutMs = options?.timeoutMs ?? 300000; + const deadline = performance.now() + timeoutMs; + for (;;) { + if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); + const result = extract(await request()); + if (result !== null && result !== undefined && result !== false) return result; + if (performance.now() >= deadline) throw new Error("插件轮询超时，请检查调用脚本或稍后重试"); + await sleep(intervalMs, signal); + } + }; +} + +/** + * Run a user-authored model call script. The script body runs as an async function with these locals: + * input —— normalized request input for this capability (prompt / references / messages / params / body) + * config —— { baseUrl, apiKey, model, apiFormat, systemPrompt } + * http —— { url(path), post(path, body, opts), get(path, opts) } bound to the model's channel + * poll —— poll(request, extract, { intervalMs, timeoutMs }) resolves with the first truthy extract result + * sleep —— sleep(ms) + * signal —— AbortSignal for cancellation + * onDelta —— (text) => void, push streaming text (text capability only) + * The script must `return` the result; each caller normalizes it to its capability's shape. + */ +export async function runModelPlugin(args: RunPluginArgs): Promise { + const configView: PluginConfigView = { + baseUrl: args.config.baseUrl, + apiKey: args.config.apiKey, + model: args.config.model, + apiFormat: args.config.apiFormat, + systemPrompt: args.config.systemPrompt, + }; + const http = createPluginHttp(args.config, { signal: args.signal }); + const poll = createPoll(args.signal); + const runner = new Function( + "input", + "config", + "http", + "poll", + "sleep", + "signal", + "onDelta", + `"use strict"; return (async () => {\n${args.script}\n})();`, + ) as (input: unknown, config: PluginConfigView, http: PluginHttp, poll: ReturnType, sleep: (ms: number) => Promise, signal: AbortSignal | undefined, onDelta?: (text: string) => void) => Promise; + try { + return await runner(args.input, configView, http, poll, (ms: number) => sleep(ms, args.signal), args.signal, args.onDelta); + } catch (error) { + if (error instanceof DOMException && error.name === "AbortError") throw error; + if (axios.isCancel(error)) throw error; + const message = error instanceof Error ? error.message : String(error); + throw new Error(`模型调用脚本执行失败：${message}`); + } +} + +export const PLUGIN_TEMPLATES: Record = { + image: `// 输入：input.prompt / input.references(dataURL[]) / input.body(默认请求体) / input.params +// 返回：dataURL 或 URL 字符串，或它们的数组，或 [{ dataUrl }] +const data = await http.post("/images/generations", { + ...input.body, + model: config.model, + prompt: input.prompt, +}); +return (data.data || []).map((item) => item.b64_json ? \`data:image/png;base64,\${item.b64_json}\` : item.url);`, + video: `// 输入：input.prompt / input.references(dataURL[]) / input.params +// 返回：{ url } 或 { blob } 或视频 URL 字符串 +const task = await http.post("/videos", { + model: config.model, + prompt: input.prompt, + seconds: input.params.seconds, +}); +return await poll( + () => http.get(\`/videos/\${task.id}\`), + (state) => state.status === "completed" ? { url: state.video_url || state.url } : null, + { intervalMs: 2500, timeoutMs: 300000 }, +);`, + audio: `// 输入：input.prompt / input.params(voice/format/speed/instructions) +// 返回：Blob，或 base64/dataURL 字符串 +return await http.post("/audio/speech", { + model: config.model, + input: input.prompt, + voice: input.params.voice, + response_format: input.params.format, + speed: Number(input.params.speed), +}, { responseType: "blob" });`, + text: `// 输入：input.messages([{role,content}]) / input.body +// 用 onDelta(text) 推送流式文本；返回最终完整文本 +const data = await http.post("/chat/completions", { + model: config.model, + messages: input.messages, +}); +const text = data.choices?.[0]?.message?.content || ""; +onDelta(text); +return text;`, +}; + +/** Normalize whatever an image script returns into the app's generated-image shape. */ +export function normalizePluginImages(result: unknown): string[] { + const items = Array.isArray(result) ? result : [result]; + const urls = items + .map((item) => { + if (typeof item === "string") return item; + if (item && typeof item === "object") { + const record = item as Record; + if (typeof record.dataUrl === "string") return record.dataUrl; + if (typeof record.url === "string") return record.url; + if (typeof record.b64_json === "string") return `data:image/png;base64,${record.b64_json}`; + } + return ""; + }) + .filter(Boolean); + if (!urls.length) throw new Error("模型调用脚本没有返回图片"); + return urls; +} diff --git a/web/src/services/api/video.ts b/web/src/services/api/video.ts index d99bb78..3ce8e6d 100644 --- a/web/src/services/api/video.ts +++ b/web/src/services/api/video.ts @@ -1,10 +1,12 @@ import axios from "axios"; +import { nanoid } from "nanoid"; import { dataUrlToFile } from "@/lib/image-utils"; import { getMediaBlob, uploadMediaFile, type UploadedFile } from "@/services/file-storage"; import { imageToDataUrl } from "@/services/image-storage"; import { boolConfig, buildSeedancePromptText, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceVideoReferenceError, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video"; -import { buildApiUrl, modelOptionName, resolveModelRequestConfig, type AiConfig } from "@/stores/use-config-store"; +import { buildApiUrl, modelOptionName, resolveModelRequestConfig, resolveModelScript, type AiConfig } from "@/stores/use-config-store"; +import { runModelPlugin } from "./model-plugin"; import type { ReferenceImage } from "@/types/image"; import type { ReferenceAudio, ReferenceVideo } from "@/types/media"; @@ -23,9 +25,12 @@ type ApiEnvelope = T | { code?: number | string; data?: T | null; msg?: strin type RequestOptions = { signal?: AbortSignal }; export type VideoGenerationResult = { blob?: Blob; url?: string; mimeType?: string }; -export type VideoGenerationTask = { id: string; provider: "openai" | "seedance"; model: string }; +export type VideoGenerationTask = { id: string; provider: "openai" | "seedance" | "plugin"; model: string }; export type VideoGenerationTaskState = { status: "pending" } | { status: "completed"; result: VideoGenerationResult } | { status: "failed"; error: string }; +/** Results for scripted (plugin) video models, which run their own create+poll in one shot at task creation. */ +const pluginVideoResults = new Map(); + function aiApiUrl(config: AiConfig, path: string) { return buildApiUrl(config.baseUrl, path); } @@ -54,6 +59,8 @@ export async function requestVideoGeneration(config: AiConfig, prompt: string, r export async function createVideoGenerationTask(config: AiConfig, prompt: string, references: ReferenceImage[] = [], videoReferences: ReferenceVideo[] = [], audioReferences: ReferenceAudio[] = [], options?: RequestOptions): Promise { const selectedModel = (config.model || config.videoModel).trim(); const requestConfig = resolveModelRequestConfig(config, selectedModel); + const script = resolveModelScript(config, selectedModel); + if (script) return createPluginVideoTask(requestConfig, selectedModel, script, prompt, references, options); assertVideoConfig(requestConfig, requestConfig.model); if (isSeedanceVideoConfig(requestConfig)) { return createSeedanceTask(requestConfig, selectedModel, prompt, references, videoReferences, audioReferences, options); @@ -65,11 +72,56 @@ export async function createVideoGenerationTask(config: AiConfig, prompt: string } export async function pollVideoGenerationTask(config: AiConfig, task: VideoGenerationTask, options?: RequestOptions): Promise { + if (task.provider === "plugin") { + const result = pluginVideoResults.get(task.id); + return result ? { status: "completed", result } : { status: "failed", error: "插件视频任务已失效，请重新生成" }; + } const requestConfig = resolveModelRequestConfig(config, task.model); assertVideoConfig(requestConfig, requestConfig.model); return task.provider === "seedance" ? pollSeedanceTask(requestConfig, task, options) : pollOpenAIVideoTask(requestConfig, task, options); } +async function createPluginVideoTask(config: AiConfig, model: string, script: string, prompt: string, references: ReferenceImage[], options?: RequestOptions): Promise { + if (!config.baseUrl.trim()) throw new Error("请先配置 Base URL"); + if (!config.apiKey.trim()) throw new Error("请先配置 API Key"); + const refs = await Promise.all(references.map((image) => imageToDataUrl(image))); + const result = videoPluginResult( + await runModelPlugin({ + capability: "video", + script, + config, + input: { + prompt, + references: refs, + params: { + seconds: normalizeVideoSeconds(config.videoSeconds), + size: normalizeVideoSize(config.size), + resolution: normalizeVideoResolution(config.vquality), + ratio: config.size, + generateAudio: boolConfig(config.videoGenerateAudio, true), + watermark: boolConfig(config.videoWatermark, false), + }, + }, + signal: options?.signal, + }), + ); + const id = nanoid(); + pluginVideoResults.set(id, result); + return { id, provider: "plugin", model }; +} + +function videoPluginResult(result: unknown): VideoGenerationResult { + if (result instanceof Blob) return { blob: result }; + if (typeof result === "string") return { url: result, mimeType: "video/mp4" }; + if (result && typeof result === "object") { + const record = result as Record; + if (record.blob instanceof Blob) return { blob: record.blob }; + const url = [record.url, record.video_url, record.result_url].find((value) => typeof value === "string" && value) as string | undefined; + if (url) return { url, mimeType: "video/mp4" }; + } + throw new Error("模型调用脚本没有返回视频"); +} + export async function storeGeneratedVideo(result: VideoGenerationResult): Promise { if (result.blob) return uploadMediaFile(result.blob, "video"); if (result.url) { diff --git a/web/src/stores/use-config-store.ts b/web/src/stores/use-config-store.ts index b54cf37..de9d1f4 100644 --- a/web/src/stores/use-config-store.ts +++ b/web/src/stores/use-config-store.ts @@ -4,6 +4,13 @@ import { persist } from "zustand/middleware"; import { nanoid } from "nanoid"; export type ApiCallFormat = "openai" | "gemini"; +export type ModelCapability = "image" | "video" | "text" | "audio"; + +export type ChannelModel = { + name: string; + capability: ModelCapability; + script?: string; +}; export type ModelChannel = { id: string; @@ -11,7 +18,7 @@ export type ModelChannel = { baseUrl: string; apiKey: string; apiFormat: ApiCallFormat; - models: string[]; + models: ChannelModel[]; }; export type AiConfig = { @@ -35,10 +42,6 @@ export type AiConfig = { videoWatermark: string; systemPrompt: string; models: string[]; - imageModels: string[]; - videoModels: string[]; - textModels: string[]; - audioModels: string[]; quality: string; size: string; count: string; @@ -52,10 +55,9 @@ export type WebdavSyncConfig = { directory: string; lastSyncedAt: string; }; -export type ConfigTabKey = "channels" | "models" | "preferences" | "webdav"; +export type ConfigTabKey = "channels" | "preferences" | "webdav"; export const CONFIG_STORE_KEY = "infinite-canvas:ai_config_store"; -export type ModelCapability = "image" | "video" | "text" | "audio"; const CHANNEL_MODEL_SEPARATOR = "::"; const OPENAI_BASE_URL = "https://api.openai.com"; const GEMINI_BASE_URL = "https://generativelanguage.googleapis.com"; @@ -72,7 +74,12 @@ export const defaultConfig: AiConfig = { baseUrl: OPENAI_BASE_URL, apiKey: "", apiFormat: "openai", - models: ["gpt-image-2", "grok-imagine-video", "gpt-5.5", "gpt-4o-mini-tts"], + models: [ + { name: "gpt-image-2", capability: "image" }, + { name: "grok-imagine-video", capability: "video" }, + { name: "gpt-5.5", capability: "text" }, + { name: "gpt-4o-mini-tts", capability: "audio" }, + ], }, ], model: "default::gpt-image-2", @@ -90,10 +97,6 @@ export const defaultConfig: AiConfig = { videoWatermark: "false", systemPrompt: "", models: ["default::gpt-image-2", "default::grok-imagine-video", "default::gpt-5.5", "default::gpt-4o-mini-tts"], - imageModels: ["default::gpt-image-2"], - videoModels: ["default::grok-imagine-video"], - textModels: ["default::gpt-5.5"], - audioModels: ["default::gpt-4o-mini-tts"], quality: "auto", size: "1:1", count: "1", @@ -122,44 +125,44 @@ type ConfigStore = { clearPromptContinue: () => void; }; -function isVideoModelName(model: string) { - const value = modelOptionName(model).toLowerCase(); - return value.includes("seedance") || value.includes("video") || value.includes("sora") || value.includes("veo") || value.includes("kling") || value.includes("wan") || value.includes("hailuo"); +const VIDEO_KEYWORDS = ["seedance", "video", "sora", "veo", "kling", "wan", "hailuo"]; +const AUDIO_KEYWORDS = ["audio", "tts", "speech", "voice", "music", "sound"]; +const IMAGE_KEYWORDS = ["seedream", "gpt-image", "image", "dall-e", "dalle", "imagen", "flux", "sdxl", "stable-diffusion", "midjourney"]; + +/** Best-effort default capability for a freshly fetched model name; user can override in the channel editor. */ +export function guessCapability(name: string): ModelCapability { + const value = name.toLowerCase(); + if (VIDEO_KEYWORDS.some((keyword) => value.includes(keyword))) return "video"; + if (AUDIO_KEYWORDS.some((keyword) => value.includes(keyword))) return "audio"; + if (IMAGE_KEYWORDS.some((keyword) => value.includes(keyword))) return "image"; + return "text"; } -function isImageModelName(model: string) { - const value = modelOptionName(model).toLowerCase(); - return !isVideoModelName(model) && !isAudioModelName(model) && (value.includes("seedream") || value.includes("gpt-image") || value.includes("image") || value.includes("dall-e") || value.includes("dalle") || value.includes("imagen") || value.includes("flux") || value.includes("sdxl") || value.includes("stable-diffusion") || value.includes("midjourney")); +function findChannelModel(config: AiConfig, value: string): { channel: ModelChannel; model: ChannelModel } | null { + const decoded = decodeChannelModel(value); + const name = decoded?.model || value; + const channel = decoded ? config.channels.find((item) => item.id === decoded.channelId) : config.channels.find((item) => item.models.some((model) => model.name === name)); + const model = channel?.models.find((item) => item.name === name); + return channel && model ? { channel, model } : null; } -function isAudioModelName(model: string) { - const value = modelOptionName(model).toLowerCase(); - return value.includes("audio") || value.includes("tts") || value.includes("speech") || value.includes("voice") || value.includes("music") || value.includes("sound"); +export function modelCapabilityOf(config: AiConfig, value: string): ModelCapability | undefined { + return findChannelModel(config, value)?.model.capability; } -function isTextModelName(model: string) { - return !isImageModelName(model) && !isVideoModelName(model) && !isAudioModelName(model); -} - -export function modelMatchesCapability(model: string, capability?: ModelCapability) { +export function modelMatchesCapability(config: AiConfig, value: string, capability?: ModelCapability) { if (!capability) return true; - if (capability === "image") return isImageModelName(model); - if (capability === "video") return isVideoModelName(model); - if (capability === "audio") return isAudioModelName(model); - return isTextModelName(model); -} - -export function filterModelsByCapability(models: string[], capability?: ModelCapability) { - return capability ? models.filter((model) => modelMatchesCapability(model, capability)) : models; + return modelCapabilityOf(config, value) === capability; } export function selectableModelsByCapability(config: AiConfig, capability?: ModelCapability) { if (!capability) return config.models; - return config[modelListKey(capability)]; + return config.channels.flatMap((channel) => channel.models.filter((model) => model.capability === capability).map((model) => encodeChannelModel(channel.id, model.name))); } -function modelListKey(capability: ModelCapability) { - return `${capability}Models` as "imageModels" | "videoModels" | "textModels" | "audioModels"; +/** The user script (if any) attached to a model; empty string means use the system default call. */ +export function resolveModelScript(config: AiConfig, value: string) { + return findChannelModel(config, value)?.model.script?.trim() || ""; } function isAiConfigReady(config: AiConfig, model: string) { @@ -215,7 +218,7 @@ export const useConfigStore = create()( channels, models, imageModel: normalizeModelOptionValue(config.imageModel || config.model, channels), - videoModel: normalizeModelOptionValue(config.videoModel || "grok-imagine-video", channels), + videoModel: normalizeModelOptionValue(config.videoModel, channels), textModel: normalizeModelOptionValue(config.textModel || config.model, channels), audioModel: normalizeModelOptionValue(config.audioModel || defaultConfig.audioModel, channels), audioVoice: config.audioVoice || defaultConfig.audioVoice, @@ -227,10 +230,6 @@ export const useConfigStore = create()( videoGenerateAudio: config.videoGenerateAudio || "true", videoWatermark: config.videoWatermark || "false", canvasImageCount: config.canvasImageCount || "3", - imageModels: Array.isArray(persistedConfig.imageModels) ? normalizeModelList(config.imageModels, channels) : filterModelsByCapability(models, "image"), - videoModels: Array.isArray(persistedConfig.videoModels) ? normalizeModelList(config.videoModels, channels) : filterModelsByCapability(models, "video"), - textModels: Array.isArray(persistedConfig.textModels) ? normalizeModelList(config.textModels, channels) : filterModelsByCapability(models, "text"), - audioModels: Array.isArray(persistedConfig.audioModels) ? normalizeModelList(config.audioModels, channels) : filterModelsByCapability(models, "audio"), }, }; }, @@ -238,18 +237,26 @@ export const useConfigStore = create()( ), ); -function normalizeModelList(models: string[], channels: ModelChannel[]) { - const allModelOptions = channels.flatMap((channel) => channel.models.map((model) => encodeChannelModel(channel.id, model))); - return Array.from(new Set((models || []).map((model) => model.trim()).filter(Boolean))) - .map((model) => normalizeModelOptionValue(model, channels)) - .filter((model) => !allModelOptions.length || allModelOptions.includes(model) || !isChannelModelValue(model)); -} - export function useEffectiveConfig() { const config = useConfigStore((state) => state.config); return useMemo(() => ({ ...config, channelMode: "local" as const }), [config]); } +/** Normalize a mixed list of raw model names or model objects into deduped ChannelModel entries. */ +export function normalizeChannelModels(models: Array | undefined): ChannelModel[] { + const seen = new Set(); + const result: ChannelModel[] = []; + for (const item of models || []) { + const name = (typeof item === "string" ? item : item?.name || "").trim(); + if (!name || seen.has(name)) continue; + seen.add(name); + const capability = typeof item === "string" ? guessCapability(name) : item.capability || guessCapability(name); + const script = typeof item === "string" ? undefined : item.script?.trim() || undefined; + result.push({ name, capability, script }); + } + return result; +} + export function createModelChannel(channel?: Partial): ModelChannel { const apiFormat = normalizeApiFormat(channel?.apiFormat); return { @@ -258,7 +265,7 @@ export function createModelChannel(channel?: Partial): ModelChanne baseUrl: channel?.baseUrl?.trim() || defaultBaseUrlForApiFormat(apiFormat), apiKey: channel?.apiKey || "", apiFormat, - models: uniqueRawModels(channel?.models || []), + models: normalizeChannelModels(channel?.models), }; } @@ -288,7 +295,7 @@ export function modelOptionLabel(config: AiConfig, value: string) { } export function modelOptionsFromChannels(channels: ModelChannel[]) { - return uniqueModelOptions(channels.flatMap((channel) => channel.models.map((model) => encodeChannelModel(channel.id, model)))); + return uniqueModelOptions(channels.flatMap((channel) => channel.models.map((model) => encodeChannelModel(channel.id, model.name)))); } export function normalizeModelOptionValue(value: string | undefined, channels: ModelChannel[]) { @@ -297,17 +304,17 @@ export function normalizeModelOptionValue(value: string | undefined, channels: M const decoded = decodeChannelModel(model); if (decoded) { const channel = channels.find((item) => item.id === decoded.channelId); - return channel && channel.models.includes(decoded.model) ? model : ""; + return channel && channel.models.some((item) => item.name === decoded.model) ? model : ""; } - const channel = channels.find((item) => item.models.includes(model)) || channels[0]; - return channel && channel.models.includes(model) ? encodeChannelModel(channel.id, model) : model; + const channel = channels.find((item) => item.models.some((entry) => entry.name === model)) || channels[0]; + return channel && channel.models.some((item) => item.name === model) ? encodeChannelModel(channel.id, model) : model; } export function resolveModelChannel(config: AiConfig, value: string) { const decoded = decodeChannelModel(value); const model = decoded?.model || value; - const matched = decoded ? config.channels.find((channel) => channel.id === decoded.channelId) : config.channels.find((channel) => channel.models.includes(model)); - return matched || config.channels[0] || createModelChannel({ id: "default", name: "默认渠道", baseUrl: config.baseUrl, apiKey: config.apiKey, apiFormat: config.apiFormat, models: config.models.map(modelOptionName) }); + const matched = decoded ? config.channels.find((channel) => channel.id === decoded.channelId) : config.channels.find((channel) => channel.models.some((item) => item.name === model)); + return matched || config.channels[0] || createModelChannel({ id: "default", name: "默认渠道", baseUrl: config.baseUrl, apiKey: config.apiKey, apiFormat: config.apiFormat, models: config.models.map(modelOptionName).map((name) => ({ name, capability: guessCapability(name) })) }); } export function resolveModelRequestConfig(config: AiConfig, value: string) { @@ -328,7 +335,7 @@ function normalizeChannels(config: AiConfig) { ...channel, id: channel.id || (index === 0 ? "default" : `channel-${index + 1}`), name: channel.name || (index === 0 ? "默认渠道" : `渠道 ${index + 1}`), - models: uniqueRawModels(channel.models || []), + models: normalizeChannelModels(channel.models), }), ); if (!channels.length) { @@ -339,18 +346,11 @@ function normalizeChannels(config: AiConfig) { baseUrl: config.baseUrl || defaultConfig.baseUrl, apiKey: config.apiKey || "", apiFormat: config.apiFormat || defaultConfig.apiFormat, - models: uniqueRawModels([ - ...(config.models || []), - config.model, - config.imageModel, - config.videoModel, - config.textModel, - config.audioModel, - ]), + models: normalizeChannelModels([config.model, config.imageModel, config.videoModel, config.textModel, config.audioModel].map(modelOptionName)), }), ); } - return channels.map((channel) => ({ ...channel, models: uniqueRawModels(channel.models) })); + return channels; } export function defaultBaseUrlForApiFormat(apiFormat: ApiCallFormat) { @@ -361,10 +361,6 @@ function normalizeApiFormat(apiFormat: unknown): ApiCallFormat { return apiFormat === "gemini" ? "gemini" : "openai"; } -function uniqueRawModels(models: string[]) { - return Array.from(new Set((models || []).map((model) => modelOptionName(model).trim()).filter(Boolean))); -} - function uniqueModelOptions(models: string[]) { return Array.from(new Set((models || []).map((model) => model.trim()).filter(Boolean))); } From 3963c6eebf5911343a2fe053149f7e4d247c8a5e Mon Sep 17 00:00:00 2001 From: HouYunFei <1844025705@qq.com> Date: Wed, 15 Jul 2026 10:13:37 +0800 Subject: [PATCH 2/7] feat(model): enhance model configuration and script handling for audio and video capabilities --- .../components/layout/model-select-modal.tsx | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 web/src/components/layout/model-select-modal.tsx diff --git a/web/src/components/layout/model-select-modal.tsx b/web/src/components/layout/model-select-modal.tsx new file mode 100644 index 0000000..80df491 --- /dev/null +++ b/web/src/components/layout/model-select-modal.tsx @@ -0,0 +1,153 @@ +import { App, Button, Checkbox, Input, Modal, Tabs } from "antd"; +import { RefreshCw, Search } from "lucide-react"; +import { useEffect, useMemo, useState } from "react"; + +import { fetchChannelModels } from "@/services/api/image"; +import type { ModelChannel } from "@/stores/use-config-store"; + +// 选择渠道模型弹窗：拉取上游模型列表或手动增加，勾选后才会进入渠道模型列表。 +export function ModelSelectModal({ open, channel, selectedNames, onConfirm, onClose }: { open: boolean; channel: ModelChannel | null; selectedNames: string[]; onConfirm: (names: string[]) => void; onClose: () => void }) { + const { message } = App.useApp(); + const [existing, setExisting] = useState([]); + const [fetched, setFetched] = useState([]); + const [selected, setSelected] = useState>(new Set()); + const [activeTab, setActiveTab] = useState("new"); + const [search, setSearch] = useState(""); + const [manual, setManual] = useState(""); + const [loading, setLoading] = useState(false); + + useEffect(() => { + if (!open) return; + setExisting(selectedNames); + setFetched([]); + setSelected(new Set(selectedNames)); + setActiveTab(selectedNames.length ? "existing" : "new"); + setSearch(""); + setManual(""); + }, [open, selectedNames]); + + const currentList = activeTab === "new" ? fetched : existing; + const visibleList = useMemo(() => { + const keyword = search.trim().toLowerCase(); + return keyword ? currentList.filter((name) => name.toLowerCase().includes(keyword)) : currentList; + }, [currentList, search]); + const visibleSelectedCount = visibleList.filter((name) => selected.has(name)).length; + + const toggle = (name: string, checked: boolean) => + setSelected((current) => { + const next = new Set(current); + if (checked) next.add(name); + else next.delete(name); + return next; + }); + + const selectVisible = (checked: boolean) => + setSelected((current) => { + const next = new Set(current); + visibleList.forEach((name) => (checked ? next.add(name) : next.delete(name))); + return next; + }); + + const addManual = () => { + const name = manual.trim(); + if (!name) return; + if (!fetched.includes(name) && !existing.includes(name)) setFetched((current) => [name, ...current]); + setSelected((current) => new Set(current).add(name)); + setManual(""); + setActiveTab("new"); + }; + + const fetchModels = async () => { + if (!channel) return; + if (!channel.baseUrl.trim() || !channel.apiKey.trim()) { + message.error("请先填写接口地址和 API Key"); + return; + } + setLoading(true); + try { + const models = await fetchChannelModels(channel); + setFetched(models); + setActiveTab("new"); + message.success(`已拉取 ${models.length} 个模型`); + } catch (error) { + message.error(error instanceof Error ? error.message : "拉取模型失败"); + } finally { + setLoading(false); + } + }; + + const confirm = () => { + const ordered = [...existing, ...fetched].filter((name, index, list) => list.indexOf(name) === index).filter((name) => selected.has(name)); + onConfirm(ordered); + onClose(); + }; + + return ( + + 选择渠道模型 已选择 {selected.size} / {new Set([...existing, ...fetched]).size} + + } + styles={{ body: { maxHeight: "62vh", overflowY: "auto" } }} + footer={[ + , + , + ]} + > +
+ setSearch(event.target.value)} placeholder="搜索模型" prefix={} allowClear /> + setManual(event.target.value)} onPressEnter={addManual} placeholder="输入模型名称" /> + + +
+
如果上游不提供 OpenAI /models 模型列表接口，请在这里手动增加模型名称。
+ + + +
+ 当前列表已选择 {visibleSelectedCount} / {visibleList.length} +
+ + +
+
+ + {visibleList.length ? ( +
+ {visibleList.map((name) => ( + toggle(name, event.target.checked)}> + + {name} + + + ))} +
+ ) : ( +
{activeTab === "new" ? "点击「拉取模型列表」获取上游模型，或手动增加模型名称。" : "暂无已选择的模型。"}
+ )} + + ); +} From f0db29d8e325996a848684b15673e882412e943e Mon Sep 17 00:00:00 2001 From: HouYunFei <1844025705@qq.com> Date: Wed, 15 Jul 2026 11:24:34 +0800 Subject: [PATCH 3/7] feat(channel): add customizable JS model-call scripts with split-panel editor - Run user-authored async scripts per model with flat locals (prompt, images, messages, params, model, baseUrl, apiKey, http, request, poll, sleep, signal, onDelta); empty script falls back to system default. - Wire image/video/audio/text request entrypoints through the plugin runtime, keeping default handlers unchanged. - Split-panel script editor: variable/return docs on the left, CodeMirror JS editor on the right; title shows capability - model. - Provide OpenAI and Gemini template buttons per capability; text template uses the /responses API; image template branches text2img vs img2img. Co-Authored-By: Claude Opus 4.8 (1M context) --- web/package.json | 1 + .../components/layout/model-script-editor.tsx | 125 ++++--- web/src/services/api/audio.ts | 3 +- web/src/services/api/image.ts | 20 +- web/src/services/api/model-plugin.ts | 304 ++++++++++++++---- web/src/services/api/video.ts | 20 +- 6 files changed, 341 insertions(+), 132 deletions(-) diff --git a/web/package.json b/web/package.json index fbd4357..f0fec03 100644 --- a/web/package.json +++ b/web/package.json @@ -14,6 +14,7 @@ "dependencies": { "@ant-design/icons": "^6.1.1", "@ant-design/pro-components": "3.0.0-beta.3", + "@codemirror/lang-javascript": "^6.2.5", "@codemirror/lang-json": "^6.0.2", "@tanstack/react-query": "^5.100.9", "@uiw/react-codemirror": "^4.25.9", diff --git a/web/src/components/layout/model-script-editor.tsx b/web/src/components/layout/model-script-editor.tsx index 1aa586e..cc8e754 100644 --- a/web/src/components/layout/model-script-editor.tsx +++ b/web/src/components/layout/model-script-editor.tsx @@ -1,18 +1,16 @@ -import { Button, Input, Modal } from "antd"; +import { javascript } from "@codemirror/lang-javascript"; +import CodeMirror from "@uiw/react-codemirror"; +import { Button, Modal } from "antd"; import { useEffect, useState } from "react"; -import { PLUGIN_TEMPLATES } from "@/services/api/model-plugin"; +import { PLUGIN_RETURNS, PLUGIN_TEMPLATES, PLUGIN_VARIABLES } from "@/services/api/model-plugin"; import type { ModelCapability } from "@/stores/use-config-store"; const capabilityLabels: Record = { image: "生图", video: "视频", text: "文本", audio: "音频" }; -const variableHints = [ - "input：本次请求的归一化输入（prompt / references / messages / params / body）", - "config：{ baseUrl, apiKey, model, apiFormat, systemPrompt }", - "http：{ url(path), post(path, body, opts), get(path, opts) }，已绑定当前渠道", - "poll(request, extract, { intervalMs, timeoutMs })：轮询直到 extract 返回真值", - "sleep(ms)、signal：延时与取消信号；onDelta(text)：推送流式文本（文本模型）", -]; +function isDarkMode() { + return typeof document !== "undefined" && document.documentElement.classList.contains("dark"); +} export function ModelScriptEditor({ open, capability, modelName, value, onSave, onClose }: { open: boolean; capability: ModelCapability; modelName: string; value: string; onSave: (script: string) => void; onClose: () => void }) { const [draft, setDraft] = useState(value); @@ -20,48 +18,95 @@ export function ModelScriptEditor({ open, capability, modelName, value, onSave, if (open) setDraft(value); }, [open, value]); + const variables = PLUGIN_VARIABLES.filter((variable) => !variable.capabilities || variable.capabilities.includes(capability)); + return ( -
调用脚本 · {modelName}
-
当前能力：{capabilityLabels[capability]}。留空则使用系统默认调用方式。
+
+ {capabilityLabels[capability]} + {modelName ? ` - ${modelName}` : ""} +
+
脚本是一段异步函数体，直接使用下方变量，最后 return 结果；留空则使用系统默认调用。
} - width={720} + width={1080} centered onCancel={onClose} - styles={{ body: { maxHeight: "64vh", overflowY: "auto" } }} - footer={[ - , - , - , - , - ]} + styles={{ body: { padding: 0 } }} + footer={ +
+
+ {PLUGIN_TEMPLATES[capability].map((template) => ( + + ))} + +
+
+ + +
+
+ } > -
-
可用变量（异步函数体，需 return 结果）
- {variableHints.map((hint) => ( -
· {hint}
- ))} +
+ +
+ +
- setDraft(event.target.value)} autoSize={{ minRows: 12, maxRows: 24 }} spellCheck={false} placeholder="留空使用系统默认调用；点击“插入模板”查看示例。" style={{ fontFamily: "var(--font-mono, ui-monospace, SFMono-Regular, Menlo, monospace)", fontSize: 12 }} /> ); } diff --git a/web/src/services/api/audio.ts b/web/src/services/api/audio.ts index 611c242..a0ad8c8 100644 --- a/web/src/services/api/audio.ts +++ b/web/src/services/api/audio.ts @@ -32,7 +32,8 @@ export async function requestAudioGeneration(config: AiConfig, prompt: string, o capability: "audio", script, config: requestConfig, - input: { prompt, params: { voice: normalizeAudioVoiceValue(config.audioVoice), format, speed: normalizeAudioSpeedValue(config.audioSpeed), instructions: config.audioInstructions.trim() } }, + prompt, + params: { voice: normalizeAudioVoiceValue(config.audioVoice), format, speed: normalizeAudioSpeedValue(config.audioSpeed), instructions: config.audioInstructions.trim() }, signal: options?.signal, }); return await audioPluginBlob(result, format); diff --git a/web/src/services/api/image.ts b/web/src/services/api/image.ts index e072317..3ffa00d 100644 --- a/web/src/services/api/image.ts +++ b/web/src/services/api/image.ts @@ -666,12 +666,9 @@ export async function requestGeneration(config: AiConfig, prompt: string, option capability: "image", script, config: requestConfig, - input: { - prompt: withSystemPrompt(requestConfig, prompt), - references: [], - params: { size: requestSize, quality, count: n }, - body: { model: requestConfig.model, n, ...(quality ? { quality } : {}), ...(requestSize ? { size: requestSize } : {}), response_format: "b64_json", output_format: IMAGE_OUTPUT_FORMAT }, - }, + prompt: withSystemPrompt(requestConfig, prompt), + images: [], + params: { size: requestSize, quality, count: n }, signal: options?.signal, }); return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl })); @@ -726,12 +723,9 @@ export async function requestEdit(config: AiConfig, prompt: string, references: capability: "image", script, config: requestConfig, - input: { - prompt: withSystemPrompt(requestConfig, requestPrompt), - references: refs, - params: { size: requestSize, quality, count: n }, - body: { model: requestConfig.model, n, ...(quality ? { quality } : {}), ...(requestSize ? { size: requestSize } : {}), response_format: "b64_json", output_format: IMAGE_OUTPUT_FORMAT }, - }, + prompt: withSystemPrompt(requestConfig, requestPrompt), + images: refs, + params: { size: requestSize, quality, count: n }, signal: options?.signal, }); return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl })); @@ -783,7 +777,7 @@ export async function requestImageQuestion(config: AiConfig, messages: AiTextMes capability: "text", script, config: requestConfig, - input: { messages: withSystemMessage(requestConfig, messages), body: { model: requestConfig.model } }, + messages: withSystemMessage(requestConfig, messages), signal: options?.signal, onDelta, }); diff --git a/web/src/services/api/model-plugin.ts b/web/src/services/api/model-plugin.ts index 092e82f..409b1de 100644 --- a/web/src/services/api/model-plugin.ts +++ b/web/src/services/api/model-plugin.ts @@ -1,4 +1,4 @@ -import axios from "axios"; +import axios, { type AxiosRequestConfig } from "axios"; import { buildApiUrl, type AiConfig, type ModelCapability } from "@/stores/use-config-store"; @@ -18,27 +18,20 @@ export type PluginHttp = { export type PluginPollOptions = { intervalMs?: number; timeoutMs?: number }; -export type PluginConfigView = { - baseUrl: string; - apiKey: string; - model: string; - apiFormat: string; - systemPrompt: string; -}; - export type RunPluginArgs = { capability: ModelCapability; script: string; config: AiConfig; - input: Record; + prompt?: string; + images?: string[]; + messages?: unknown[]; + params?: Record; signal?: AbortSignal; onDelta?: (text: string) => void; }; -function pluginHeaders(config: AiConfig, extra?: Record, hasJsonBody = false): Record { +function pluginHeaders(extra?: Record, hasJsonBody = false): Record { const headers: Record = {}; - if (config.apiFormat === "gemini") headers["x-goog-api-key"] = config.apiKey; - else headers.Authorization = `Bearer ${config.apiKey}`; if (hasJsonBody) headers["Content-Type"] = "application/json"; return { ...headers, ...extra }; } @@ -49,14 +42,14 @@ function pluginUrl(config: AiConfig, path: string) { } function createPluginHttp(config: AiConfig, options?: RequestOptions): PluginHttp { - const request = async (method: "get" | "post", path: string, body: unknown, opts?: PluginHttpOptions) => { + const run = async (method: "get" | "post", path: string, body: unknown, opts?: PluginHttpOptions) => { const isForm = typeof FormData !== "undefined" && body instanceof FormData; const response = await axios.request({ method, url: pluginUrl(config, path), data: method === "post" ? body : undefined, params: opts?.params, - headers: pluginHeaders(config, opts?.headers, method === "post" && !isForm && body !== undefined), + headers: pluginHeaders({ Authorization: `Bearer ${config.apiKey}`, ...opts?.headers }, method === "post" && !isForm && body !== undefined), responseType: opts?.responseType || "json", signal: options?.signal, }); @@ -64,8 +57,16 @@ function createPluginHttp(config: AiConfig, options?: RequestOptions): PluginHtt }; return { url: (path) => pluginUrl(config, path), - post: (path, body, opts) => request("post", path, body, opts), - get: (path, opts) => request("get", path, undefined, opts), + post: (path, body, opts) => run("post", path, body, opts), + get: (path, opts) => run("get", path, undefined, opts), + }; +} + +/** Raw request with no automatic auth header — the script controls method, url, headers, body entirely. */ +function createPluginRequest(config: AiConfig, options?: RequestOptions) { + return async (requestConfig: AxiosRequestConfig & { url: string }) => { + const response = await axios.request({ ...requestConfig, url: pluginUrl(config, requestConfig.url), signal: options?.signal }); + return response.data; }; } @@ -103,38 +104,51 @@ function createPoll(signal?: AbortSignal) { } /** - * Run a user-authored model call script. The script body runs as an async function with these locals: - * input —— normalized request input for this capability (prompt / references / messages / params / body) - * config —— { baseUrl, apiKey, model, apiFormat, systemPrompt } - * http —— { url(path), post(path, body, opts), get(path, opts) } bound to the model's channel - * poll —— poll(request, extract, { intervalMs, timeoutMs }) resolves with the first truthy extract result - * sleep —— sleep(ms) - * signal —— AbortSignal for cancellation - * onDelta —— (text) => void, push streaming text (text capability only) + * Run a user-authored model call script as an async function body with flat locals (see PLUGIN_VARIABLES): + * prompt / images / messages / params —— 本次请求的输入 + * model / baseUrl / apiKey / systemPrompt —— 当前渠道信息 + * http / request / poll / sleep / signal / onDelta —— 调用辅助 * The script must `return` the result; each caller normalizes it to its capability's shape. */ export async function runModelPlugin(args: RunPluginArgs): Promise { - const configView: PluginConfigView = { - baseUrl: args.config.baseUrl, - apiKey: args.config.apiKey, - model: args.config.model, - apiFormat: args.config.apiFormat, - systemPrompt: args.config.systemPrompt, - }; - const http = createPluginHttp(args.config, { signal: args.signal }); + const { config } = args; + const http = createPluginHttp(config, { signal: args.signal }); + const request = createPluginRequest(config, { signal: args.signal }); const poll = createPoll(args.signal); const runner = new Function( - "input", - "config", + "prompt", + "images", + "messages", + "params", + "model", + "baseUrl", + "apiKey", + "systemPrompt", "http", + "request", "poll", "sleep", "signal", "onDelta", `"use strict"; return (async () => {\n${args.script}\n})();`, - ) as (input: unknown, config: PluginConfigView, http: PluginHttp, poll: ReturnType, sleep: (ms: number) => Promise, signal: AbortSignal | undefined, onDelta?: (text: string) => void) => Promise; + ) as (...fnArgs: unknown[]) => Promise; try { - return await runner(args.input, configView, http, poll, (ms: number) => sleep(ms, args.signal), args.signal, args.onDelta); + return await runner( + args.prompt || "", + args.images || [], + args.messages || [], + args.params || {}, + config.model, + config.baseUrl, + config.apiKey, + config.systemPrompt || "", + http, + request, + poll, + (ms: number) => sleep(ms, args.signal), + args.signal, + args.onDelta, + ); } catch (error) { if (error instanceof DOMException && error.name === "AbortError") throw error; if (axios.isCancel(error)) throw error; @@ -143,45 +157,201 @@ export async function runModelPlugin(args: RunPluginArgs): Promise< } } -export const PLUGIN_TEMPLATES: Record = { - image: `// 输入：input.prompt / input.references(dataURL[]) / input.body(默认请求体) / input.params -// 返回：dataURL 或 URL 字符串，或它们的数组，或 [{ dataUrl }] -const data = await http.post("/images/generations", { - ...input.body, - model: config.model, - prompt: input.prompt, +export type PluginVariable = { name: string; type: string; desc: string; capabilities?: ModelCapability[] }; + +/** Documentation surface shown in the script editor. */ +export const PLUGIN_VARIABLES: PluginVariable[] = [ + { name: "prompt", type: "string", desc: "用户输入的提示词（已拼接系统提示词）", capabilities: ["image", "video", "audio"] }, + { name: "images", type: "string[]", desc: "参考图，dataURL 数组（改图 / 图生视频时有值）", capabilities: ["image", "video"] }, + { name: "messages", type: "{ role, content }[]", desc: "对话消息数组，含系统消息", capabilities: ["text"] }, + { name: "params", type: "object", desc: "生成参数：生图 {size,quality,count}、视频 {seconds,size,resolution,ratio,generateAudio,watermark}、音频 {voice,format,speed,instructions}" }, + { name: "model", type: "string", desc: "模型名称（不含渠道前缀）" }, + { name: "baseUrl", type: "string", desc: "渠道接口地址（原样，未拼 /v1）" }, + { name: "apiKey", type: "string", desc: "渠道 API Key，请求头里自己带上" }, + { name: "systemPrompt", type: "string", desc: "系统提示词原文" }, + { name: "http", type: "object", desc: "便捷请求：http.post(path, body, {headers,params,responseType})、http.get(path, opts)、http.url(path)；默认带 Authorization: Bearer apiKey，可用 headers 覆盖；path 相对时按 baseUrl 拼 /v1" }, + { name: "request", type: "function", desc: "原始请求 request({ method, url, headers, params, data, responseType })，不加任何默认头，鉴权头自己写；url 相对时按 baseUrl 拼接（不加 /v1）" }, + { name: "poll", type: "function", desc: "轮询 poll(request, extract, {intervalMs,timeoutMs})，extract 返回真值即结束" }, + { name: "sleep", type: "function", desc: "sleep(ms) 延时" }, + { name: "signal", type: "AbortSignal", desc: "取消信号，可透传给 http/request" }, + { name: "onDelta", type: "function", desc: "onDelta(text) 推送流式文本（文本模型）", capabilities: ["text"] }, +]; + +export const PLUGIN_RETURNS: Record = { + image: "文生图（images 为空）和图生图（images 有参考图）接口不同，脚本需自行区分；返回图片 URL 或 dataURL 字符串，也可返回它们的数组，或 [{ dataUrl }] / [{ url }] / [{ b64_json }]", + video: "脚本内部完成轮询，返回 { url } 或 { blob } 或视频 URL 字符串", + audio: "返回 Blob，或 base64 / dataURL 字符串，或 { b64_json } / { data } / { url }", + text: "用 onDelta(text) 推送流式，最终 return 完整文本字符串", +}; + +export type PluginTemplate = { label: string; script: string }; + +export const PLUGIN_TEMPLATES: Record = { + image: [ + { + label: "OpenAI 规范", + script: `// 生图 / 改图：两者接口不同，用 images 是否为空来区分。 +// 可用：prompt、images(dataURL[])、params{size,quality,count}、model、baseUrl、apiKey +if (images.length === 0) { + // 文生图：/images/generations（JSON） + const data = await request({ + method: "post", + url: \`\${baseUrl}/v1/images/generations\`, + headers: { "Content-Type": "application/json", Authorization: \`Bearer \${apiKey}\` }, + data: { model, prompt, n: params.count, size: params.size, response_format: "b64_json" }, + }); + return (data.data || []).map((item) => item.b64_json ? \`data:image/png;base64,\${item.b64_json}\` : item.url); +} + +// 图生图：/images/edits（multipart/form-data，参考图作为文件上传） +const form = new FormData(); +form.set("model", model); +form.set("prompt", prompt); +form.set("n", String(params.count)); +form.set("response_format", "b64_json"); +for (const dataUrl of images) { + form.append("image", await (await fetch(dataUrl)).blob(), "ref.png"); +} +const edited = await request({ + method: "post", + url: \`\${baseUrl}/v1/images/edits\`, + headers: { Authorization: \`Bearer \${apiKey}\` }, // 不要手动设 Content-Type，交给浏览器带 boundary + data: form, }); -return (data.data || []).map((item) => item.b64_json ? \`data:image/png;base64,\${item.b64_json}\` : item.url);`, - video: `// 输入：input.prompt / input.references(dataURL[]) / input.params -// 返回：{ url } 或 { blob } 或视频 URL 字符串 -const task = await http.post("/videos", { - model: config.model, - prompt: input.prompt, - seconds: input.params.seconds, +return (edited.data || []).map((item) => item.b64_json ? \`data:image/png;base64,\${item.b64_json}\` : item.url);`, + }, + { + label: "Gemini 规范", + script: `// Gemini 文生图 / 图生图：都走 generateContent，参考图放进 parts 的 inline_data。 +// 可用：prompt、images(dataURL[])、model、baseUrl、apiKey +const parts = [{ text: prompt }]; +for (const dataUrl of images) { + const match = dataUrl.match(/^data:([^;]+);base64,(.*)$/); + if (match) parts.push({ inline_data: { mime_type: match[1], data: match[2] } }); +} +const data = await request({ + method: "post", + url: \`\${baseUrl}/v1beta/models/\${model}:generateContent\`, + headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey }, + data: { contents: [{ role: "user", parts }], generationConfig: { responseModalities: ["IMAGE"] } }, +}); +return (data.candidates || []) + .flatMap((c) => c.content?.parts || []) + .map((p) => p.inlineData || p.inline_data) + .filter(Boolean) + .map((img) => \`data:\${img.mimeType || img.mime_type || "image/png"};base64,\${img.data}\`);`, + }, + ], + video: [ + { + label: "OpenAI 规范", + script: `// 视频（脚本内部自行轮询）。可用：prompt、images(dataURL[])、params{seconds,size,resolution,ratio} +const headers = { "Content-Type": "application/json", Authorization: \`Bearer \${apiKey}\` }; +const task = await request({ + method: "post", + url: \`\${baseUrl}/v1/videos\`, + headers, + data: { model, prompt, seconds: params.seconds }, }); return await poll( - () => http.get(\`/videos/\${task.id}\`), + () => request({ method: "get", url: \`\${baseUrl}/v1/videos/\${task.id}\`, headers }), (state) => state.status === "completed" ? { url: state.video_url || state.url } : null, { intervalMs: 2500, timeoutMs: 300000 }, );`, - audio: `// 输入：input.prompt / input.params(voice/format/speed/instructions) -// 返回：Blob，或 base64/dataURL 字符串 -return await http.post("/audio/speech", { - model: config.model, - input: input.prompt, - voice: input.params.voice, - response_format: input.params.format, - speed: Number(input.params.speed), -}, { responseType: "blob" });`, - text: `// 输入：input.messages([{role,content}]) / input.body -// 用 onDelta(text) 推送流式文本；返回最终完整文本 -const data = await http.post("/chat/completions", { - model: config.model, - messages: input.messages, + }, + { + label: "Gemini 规范", + script: `// Gemini(Veo) 视频：predictLongRunning 提交，轮询 operation 拿视频 URI。 +// 可用：prompt、images(dataURL[])、params、model、baseUrl、apiKey +const headers = { "Content-Type": "application/json", "x-goog-api-key": apiKey }; +const instance = { prompt }; +const first = images[0] && images[0].match(/^data:([^;]+);base64,(.*)$/); +if (first) instance.image = { bytesBase64Encoded: first[2], mimeType: first[1] }; +const op = await request({ + method: "post", + url: \`\${baseUrl}/v1beta/models/\${model}:predictLongRunning\`, + headers, + data: { instances: [instance], parameters: { aspectRatio: params.ratio } }, }); -const text = data.choices?.[0]?.message?.content || ""; +return await poll( + () => request({ method: "get", url: \`\${baseUrl}/v1beta/\${op.name}\`, headers }), + (state) => { + if (!state.done) return null; + const uri = state.response?.generateVideoResponse?.generatedSamples?.[0]?.video?.uri; + if (!uri) throw new Error("Gemini 未返回视频 URI"); + return { url: uri.includes("key=") ? uri : \`\${uri}\${uri.includes("?") ? "&" : "?"}key=\${apiKey}\` }; + }, + { intervalMs: 5000, timeoutMs: 300000 }, +);`, + }, + ], + audio: [ + { + label: "OpenAI 规范", + script: `// 音频 TTS。可用：prompt、params{voice,format,speed,instructions}、model +return await request({ + method: "post", + url: \`\${baseUrl}/v1/audio/speech\`, + headers: { "Content-Type": "application/json", Authorization: \`Bearer \${apiKey}\` }, + responseType: "blob", + data: { model, input: prompt, voice: params.voice, response_format: params.format, speed: Number(params.speed) }, +});`, + }, + { + label: "Gemini 规范", + script: `// Gemini TTS：generateContent + AUDIO 模态，返回 base64 PCM（音频数据在 inlineData.data）。 +// 可用：prompt、params{voice}、model、baseUrl、apiKey +const data = await request({ + method: "post", + url: \`\${baseUrl}/v1beta/models/\${model}:generateContent\`, + headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey }, + data: { + contents: [{ role: "user", parts: [{ text: prompt }] }], + generationConfig: { + responseModalities: ["AUDIO"], + speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: params.voice } } }, + }, + }, +}); +const audio = data.candidates?.[0]?.content?.parts?.map((p) => p.inlineData || p.inline_data).find(Boolean); +if (!audio?.data) throw new Error("Gemini 未返回音频"); +return { data: audio.data };`, + }, + ], + text: [ + { + label: "OpenAI 规范", + script: `// 文本对话（OpenAI Responses 接口）。可用：messages([{role,content}])、systemPrompt、model +const data = await request({ + method: "post", + url: \`\${baseUrl}/v1/responses\`, + headers: { "Content-Type": "application/json", Authorization: \`Bearer \${apiKey}\` }, + data: { model, input: messages }, +}); +const text = data.output_text + || (data.output || []).flatMap((o) => o.content || []).map((c) => c.text || "").join("") + || ""; onDelta(text); return text;`, + }, + { + label: "Gemini 规范", + script: `// Gemini 文本：generateContent，system 消息放 systemInstruction。 +// 可用：messages([{role,content}])、systemPrompt、model、baseUrl、apiKey +const contents = messages + .filter((m) => m.role !== "system") + .map((m) => ({ role: m.role === "assistant" ? "model" : "user", parts: [{ text: m.content }] })); +const data = await request({ + method: "post", + url: \`\${baseUrl}/v1beta/models/\${model}:generateContent\`, + headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey }, + data: { contents, ...(systemPrompt ? { systemInstruction: { parts: [{ text: systemPrompt }] } } : {}) }, +}); +const text = data.candidates?.[0]?.content?.parts?.map((p) => p.text || "").join("") || ""; +onDelta(text); +return text;`, + }, + ], }; /** Normalize whatever an image script returns into the app's generated-image shape. */ diff --git a/web/src/services/api/video.ts b/web/src/services/api/video.ts index 3ce8e6d..cb7f830 100644 --- a/web/src/services/api/video.ts +++ b/web/src/services/api/video.ts @@ -90,17 +90,15 @@ async function createPluginVideoTask(config: AiConfig, model: string, script: st capability: "video", script, config, - input: { - prompt, - references: refs, - params: { - seconds: normalizeVideoSeconds(config.videoSeconds), - size: normalizeVideoSize(config.size), - resolution: normalizeVideoResolution(config.vquality), - ratio: config.size, - generateAudio: boolConfig(config.videoGenerateAudio, true), - watermark: boolConfig(config.videoWatermark, false), - }, + prompt, + images: refs, + params: { + seconds: normalizeVideoSeconds(config.videoSeconds), + size: normalizeVideoSize(config.size), + resolution: normalizeVideoResolution(config.vquality), + ratio: config.size, + generateAudio: boolConfig(config.videoGenerateAudio, true), + watermark: boolConfig(config.videoWatermark, false), }, signal: options?.signal, }), From eef4d967878d73a3d39d23013a61de962dd7c397 Mon Sep 17 00:00:00 2001 From: HouYunFei <1844025705@qq.com> Date: Wed, 15 Jul 2026 11:47:25 +0800 Subject: [PATCH 4/7] feat(plugin): implement canvas node plugin system with dynamic registration and remote installation --- .gitignore | 5 +- CHANGELOG.md | 1 + SECURITY.md | 16 + docs/content/docs/progress/pending-test.mdx | 8 +- plan-plugins-node.md | 155 +++++ plugins/canvas/README.md | 126 ++++ plugins/canvas/html/README.md | 17 + plugins/canvas/html/build.mjs | 50 ++ plugins/canvas/html/package-lock.json | 499 ++++++++++++++ plugins/canvas/html/package.json | 14 + plugins/canvas/html/src/index.jsx | 49 ++ plugins/canvas/markdown/README.md | 23 + plugins/canvas/markdown/build.mjs | 50 ++ plugins/canvas/markdown/package-lock.json | 499 ++++++++++++++ plugins/canvas/markdown/package.json | 14 + plugins/canvas/markdown/src/index.jsx | 64 ++ plugins/canvas/markdown/src/styles.css | 59 ++ plugins/canvas/panorama/README.md | 17 + plugins/canvas/panorama/build.mjs | 50 ++ plugins/canvas/panorama/package-lock.json | 499 ++++++++++++++ plugins/canvas/panorama/package.json | 14 + plugins/canvas/panorama/src/index.jsx | 126 ++++ plugins/canvas/sticky-note/README.md | 17 + plugins/canvas/sticky-note/build.mjs | 50 ++ plugins/canvas/sticky-note/package-lock.json | 499 ++++++++++++++ plugins/canvas/sticky-note/package.json | 14 + plugins/canvas/sticky-note/src/index.jsx | 64 ++ plugins/canvas/svg/README.md | 17 + plugins/canvas/svg/build.mjs | 50 ++ plugins/canvas/svg/package-lock.json | 499 ++++++++++++++ plugins/canvas/svg/package.json | 14 + plugins/canvas/svg/src/index.jsx | 51 ++ web/bun.lock | 618 +++++++++++++++++- web/src/components/canvas/canvas-mini-map.tsx | 5 +- .../canvas/canvas-node-hover-toolbar.tsx | 5 +- web/src/components/canvas/canvas-node.tsx | 42 +- .../canvas/canvas-plugin-manager-modal.tsx | 105 +++ .../components/canvas/nodes/builtin-nodes.tsx | 37 ++ web/src/constant/canvas.ts | 8 +- web/src/lib/canvas/canvas-agent-ops.ts | 10 +- web/src/lib/canvas/canvas-event-bus.ts | 47 ++ .../lib/canvas/canvas-resource-references.ts | 12 +- web/src/lib/canvas/node-registry.ts | 59 ++ web/src/lib/canvas/plugin-loader.ts | 121 ++++ web/src/lib/canvas/plugin-node-context.ts | 26 + web/src/lib/canvas/plugin-runtime.ts | 42 ++ web/src/pages/canvas/project.tsx | 81 ++- web/src/stores/canvas/use-plugin-store.ts | 43 ++ web/src/types/canvas-plugin.ts | 102 +++ web/src/types/canvas.ts | 7 +- web/src/vite-env.d.ts | 5 + 51 files changed, 4933 insertions(+), 72 deletions(-) create mode 100644 plan-plugins-node.md create mode 100644 plugins/canvas/README.md create mode 100644 plugins/canvas/html/README.md create mode 100644 plugins/canvas/html/build.mjs create mode 100644 plugins/canvas/html/package-lock.json create mode 100644 plugins/canvas/html/package.json create mode 100644 plugins/canvas/html/src/index.jsx create mode 100644 plugins/canvas/markdown/README.md create mode 100644 plugins/canvas/markdown/build.mjs create mode 100644 plugins/canvas/markdown/package-lock.json create mode 100644 plugins/canvas/markdown/package.json create mode 100644 plugins/canvas/markdown/src/index.jsx create mode 100644 plugins/canvas/markdown/src/styles.css create mode 100644 plugins/canvas/panorama/README.md create mode 100644 plugins/canvas/panorama/build.mjs create mode 100644 plugins/canvas/panorama/package-lock.json create mode 100644 plugins/canvas/panorama/package.json create mode 100644 plugins/canvas/panorama/src/index.jsx create mode 100644 plugins/canvas/sticky-note/README.md create mode 100644 plugins/canvas/sticky-note/build.mjs create mode 100644 plugins/canvas/sticky-note/package-lock.json create mode 100644 plugins/canvas/sticky-note/package.json create mode 100644 plugins/canvas/sticky-note/src/index.jsx create mode 100644 plugins/canvas/svg/README.md create mode 100644 plugins/canvas/svg/build.mjs create mode 100644 plugins/canvas/svg/package-lock.json create mode 100644 plugins/canvas/svg/package.json create mode 100644 plugins/canvas/svg/src/index.jsx create mode 100644 web/src/components/canvas/canvas-plugin-manager-modal.tsx create mode 100644 web/src/components/canvas/nodes/builtin-nodes.tsx create mode 100644 web/src/lib/canvas/canvas-event-bus.ts create mode 100644 web/src/lib/canvas/node-registry.ts create mode 100644 web/src/lib/canvas/plugin-loader.ts create mode 100644 web/src/lib/canvas/plugin-node-context.ts create mode 100644 web/src/lib/canvas/plugin-runtime.ts create mode 100644 web/src/stores/canvas/use-plugin-store.ts create mode 100644 web/src/types/canvas-plugin.ts diff --git a/.gitignore b/.gitignore index 4bb71a1..69d22e8 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,7 @@ data/ *.tsbuildinfo .DS_Store .idea -web/dist \ No newline at end of file +web/dist +plugins/canvas/*/dist +plugins/canvas/*/node_modules +web/public/plugins \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 59003cf..4ba266d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased ++ [新增] 画布节点插件系统:节点类型改为统一注册表,支持通过 URL 动态安装/启用/更新/卸载远程节点插件;内置节点保持文本、图片、视频、音频、生成配置、组六种,并在 `plugins/canvas/` 提供 Markdown、SVG、HTML、3D 全景、便利贴等独立示例插件(各自独立目录与构建)。 + [新增] 支持自定义生图/视频接口调用方式以适配不同中转站。 ## v0.7.0 - 2026-07-14 diff --git a/SECURITY.md b/SECURITY.md index 3a12c7a..d65c860 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -31,6 +31,22 @@ Please include: ## Scope +### Canvas node plugins + +The canvas supports third-party node plugins loaded from a remote URL. By +design, an installed plugin's code runs directly inside the web app with full +access to the page, including locally stored data such as AI API keys. This is +an intentional trade-off for extensibility, and the installer shows a warning +before installing. Therefore: + +- Only install plugins from sources you trust. +- Reports that a *malicious plugin* can access page data or API keys are **out + of scope** — that is the documented behavior of the trust model. +- Reports **in scope** include: the app loading/executing plugin code without + the install confirmation, a plugin escaping its declared node type to break + core app integrity in ways not implied by "runs in the page", or the plugin + source cache being writable by an unrelated origin. + Examples of in-scope reports: - Cross-site scripting or token exfiltration in the web app. diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index edd676e..fb1b2dd 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -5,7 +5,13 @@ description: 当前版本已实现但仍需人工验证的变更项 # 待测试 -- 画布节点创建：鼠标左键双击画布空白区域会在点击位置打开节点选择菜单，可创建文本、图片、视频、音频、生成配置和组节点；双击已有节点或连线不应触发菜单。 +- 画布节点插件系统:内置节点(图片/文本/视频/音频/配置/组)改为统一注册表管理,创建菜单由注册表动态生成;需验证六种内置节点的创建、渲染、缩放、连线、小地图颜色、作为生成输入等行为与之前一致。内置节点仅这六种,其余均为插件。 +- 画布节点插件系统:左上菜单新增「节点插件」,可通过 URL 安装远程插件、启用/禁用、更新、卸载;安装时有安全警告说明插件代码在页面内执行、可访问本地数据。可用 `/plugins/sticky-note.js` 安装便利贴节点验证:换色、编辑、「衍生文本节点」(演示 applyOps 增节点+连线)。 +- 画布节点插件系统:示例插件 Markdown(编辑/渲染)、HTML(沙箱 iframe 渲染,支持 {{input}} 注入上游文本)、SVG(渲染/编辑,可取上游文本 SVG 源码)、3D 全景(three.js CDN 动态加载,可从上游图片节点取全景图并拖拽查看);均为远程插件,需分别安装 `/plugins/markdown.js`、`/plugins/html.js`、`/plugins/svg.js`、`/plugins/panorama.js` 后验证创建、渲染、编辑及与上游节点的交互。 +- 画布节点插件系统:卸载或禁用插件后,画布上遗留的该类型节点显示「缺少插件」占位且数据保留,重新安装/启用后恢复渲染;需验证刷新后已启用插件自动加载。 +- 画布节点插件系统:每个插件为 `plugins/canvas//` 独立目录(各自 package.json + esbuild + src + README,互不耦合),`npm run build` 产物落到 `dist/.js` 并同步到 `web/public/plugins/`;需验证各插件可独立构建与安装。 +- 画布节点插件本地开发:每个插件支持 `npm run dev`(watch 自动构建并同步到 `web/public/plugins/`),配合 `web/.env.local` 的 `VITE_DEV_PLUGINS`(逗号分隔 URL)每次刷新重新拉取、无需反复安装;需验证改源码刷新页面即生效。 +- 画布节点创建:鼠标左键双击画布空白区域会在点击位置打开节点选择菜单,可创建文本、图片、视频、音频、生成配置和组节点;双击已有节点或连线不应触发菜单。 - 本地 Agent 连接说明：新增提醒说明只有安装 Codex 插件或手动添加 MCP 后才会增加 Codex token 消耗，直接运行 `npx -y @basketikun/canvas-agent` 不会安装 MCP；需验证文案位置清晰。 - 本地 Agent 连接说明：画布面板和配置页 Codex Tab 改为区分“Codex 插件启动画布”和“直接运行 Agent 后网页连接”两种方式，需验证文案和布局清晰。 - 全站 Agent：Agent 面板从画布页抽离为全站常驻面板，挂在全局布局右侧，开/关只挤压左侧内容不遮挡；顶栏和画布工具条各有一个开关按钮，需验证在首页、生图、视频、素材、提示词库、画布等页面都能打开面板并保持 SSE 连接不断开。 diff --git a/plan-plugins-node.md b/plan-plugins-node.md new file mode 100644 index 0000000..ba9d73b --- /dev/null +++ b/plan-plugins-node.md @@ -0,0 +1,155 @@ +# 画布节点插件系统设计方案 + +> 目标:让画布支持自定义节点(SVG、3D 全景、导演台、Markdown、HTML 渲染等),插件可从远程 URL 动态安装,插件节点可与画布及其他节点交互。 + +## 一、现状梳理(设计依据) + +- **渲染架构**:纯 DOM + CSS transform(`web/src/components/canvas/infinite-canvas.tsx:188-195`),每个节点是绝对定位 div,插件节点渲染真 React 组件毫无障碍。 +- **节点模型**:`CanvasNodeData.type` 是封闭枚举 `CanvasNodeType`(`web/src/types/canvas.ts:12-19`),metadata 是扁平可选字段袋——对扩展友好。 +- **已有雏形**:`web/src/components/canvas/canvas-node.tsx:420-427` 的 `nodeContentRenderers` 已经是 `type → 渲染器` 映射,注册表化就是把它开放。 +- **关键分散点**(所有按 type 分支的地方,都是要收敛到注册表的接口): + - 默认尺寸/初始 metadata:`web/src/constant/canvas.ts` NODE_SPECS + - 内容渲染:`canvas-node.tsx` NodeContent + - 等比缩放判断:`canvas-node.tsx:262` + - 双击行为:`canvas-node.tsx:340-354` + - hover 工具栏按钮:`web/src/components/canvas/canvas-node-hover-toolbar.tsx:106-152` + - 创建菜单 ×2:`web/src/pages/canvas/project.tsx:172-236`(NodeCreateMenu / ConnectionCreateMenu) + - 小地图颜色:`web/src/components/canvas/canvas-mini-map.tsx:116` + - 节点信息弹窗类型名:`canvas-node-hover-toolbar.tsx:257` + - 上游输入采集(节点作为生成输入):`web/src/lib/canvas/canvas-resource-references.ts:93-96`、`web/src/components/canvas/canvas-node-generation.ts` + - Agent 操作校验:`web/src/lib/canvas/canvas-agent-ops.ts:45`(add_node 只认枚举) +- **持久化**:zustand persist → localforage,节点是纯 JSON,`type` 改成 string 后旧结构天然兼容(项目未上线,无需迁移逻辑)。 +- **一个重要复用点**:`canvas-agent-ops.ts` 的 `applyCanvasAgentOps` 已经是一套完整的"画布操作指令集"(增删节点/连线/选择/视口/触发生成),**插件上下文直接复用这套 op API**,插件就拥有了和 AI Agent 同级的画布操作能力,零新增协议。 + +## 二、总体设计 + +已确认的决策: + +- **信任/运行模型**:ESM 直连加载,插件代码在页面内全权执行,安装时弹确认警告(不做 iframe 沙箱运行时;但 HTML 节点渲染内容本身用 sandbox iframe 隔离)。 +- **内置节点也迁到同一注册表**,统一架构。 +- **示例插件**:Markdown / HTML / SVG / 3D 全景 四个节点。 + +``` +┌─ 插件管理 UI (安装/启用/禁用/删除, URL 安装) +│ │ +├─ use-plugin-store (zustand+localforage: {id,url,version,enabled,source缓存}) +│ │ +├─ plugin-loader (fetch源码 → blob URL import → 校验 manifest → 注册) +│ │ +├─ node-registry (Map + 版本计数器触发UI更新) +│ ↑ 内置6种节点 + first-party插件(markdown/html/svg/panorama) + 远程插件 +│ │ +└─ canvas-node.tsx / 创建菜单 / 工具栏 / 小地图 / 生成输入采集 → 全部查注册表 +``` + +### 1. 核心类型 `web/src/types/canvas-plugin.ts` + +```ts +export type CanvasNodeDefinition = { + type: string; // 内置 "image";插件 ":" + title: string; // 菜单名/默认标题 + icon: ReactNode; + description?: string; + defaultSize: { width: number; height: number }; + defaultMetadata?: CanvasNodeMetadata; + minimapColor?: string; + showInCreateMenu?: boolean; // 默认 true + hasSourceHandle?: boolean; // 右侧连接点,Config 为 false + keepAspectRatio?: (node: CanvasNodeData) => boolean; + // 作为上游输入被消费时输出什么(接入生成/引用体系) + resource?: (node: CanvasNodeData) => { kind: "text" | "image" | "video" | "audio"; text?: string; url?: string } | null; + // 渲染 + Content: ComponentType<{ node: CanvasNodeData; ctx: CanvasNodeContext }>; + Panel?: ComponentType<{ node: CanvasNodeData; ctx: CanvasNodeContext; onClose: () => void }>; // 节点下方面板 + toolbar?: (node: CanvasNodeData, ctx: CanvasNodeContext) => Array<{ id; title; label; icon; onClick; danger? }>; + onDoubleClick?: (node: CanvasNodeData, ctx: CanvasNodeContext) => boolean; // true=已处理 +}; + +export type CanvasPlugin = { + id: string; // 唯一,kebab-case + name: string; + version: string; + minAppVersion?: string; + nodes: CanvasNodeDefinition[]; + setup?: (app: CanvasPluginApp) => void | (() => void); // 可选初始化/清理 +}; +``` + +### 2. 节点上下文 `CanvasNodeContext`(节点交互的核心接口) + +由 project.tsx 构建、经 CanvasNode 注入每个节点渲染器: + +```ts +export type CanvasNodeContext = { + // 自身数据 + updateMetadata(patch: CanvasNodeMetadata): void; + updateNode(patch: Partial>): void; + // 图访问与操作(复用 CanvasAgentOp 指令集 → 插件可增删节点/连线/触发生成) + getNode(id: string): CanvasNodeData | null; + getUpstream(): CanvasNodeData[]; // 输入节点(含 resource 解析结果) + getDownstream(): CanvasNodeData[]; + applyOps(ops: CanvasAgentOp[]): void; + // 节点间/插件间通信 + on(event: string, handler: (payload: unknown) => void): () => void; + emit(event: string, payload?: unknown): void; // 轻量事件总线,内置事件: node:updated / connection:added ... + // 环境 + theme: CanvasTheme; // 当前主题 token,保证插件 UI 跟主题 + scale: number; // 当前缩放 + storage: { get; set; remove }; // localforage 命名空间 "plugin:" +}; +``` + +### 3. 插件运行时与远程加载 + +- `web/src/lib/canvas/plugin-runtime.ts`:启动时挂 `window.InfiniteCanvas = { React, ReactDOM, version }`。远程插件构建时把 `react` external 到该全局(提供 esbuild 示例配置),避免双 React 实例;插件自己的重依赖(如 three)可从 esm.sh 动态 import。 +- `web/src/lib/canvas/plugin-loader.ts`: + - `installPlugin(url)`:fetch 源码 → `import(blobUrl)` 校验 default export → 存入 store(**缓存源码**,离线可用、版本固定)→ 注册。 + - `loadInstalledPlugins()`:应用启动时从 store 恢复所有 enabled 插件。 + - `unloadPlugin(id)`:从注册表移除其节点类型;画布上遗留节点渲染成"缺少插件"占位节点(数据完整保留,重装即恢复)。 +- `web/src/stores/canvas/use-plugin-store.ts`:插件记录持久化。 +- 安装时 Modal 明确警告:插件代码将在页面内完整执行、可访问本地数据(含 API Key),仅安装可信来源。 + +### 4. 注册表 `web/src/lib/canvas/node-registry.ts` + +模块级 Map + zustand 计数器(注册/卸载时 +1,创建菜单等 UI 响应更新)。`getNodeDefinition(type)`、`listNodeDefinitions()`、`registerNodeDefinitions(defs)`、`unregisterByPlugin(id)`。未知 type 统一走 `MissingPluginContent` 兜底。 + +## 三、内置节点迁移 + +1. `types/canvas.ts`:`CanvasNodeData.type` 改为 `string`;`CanvasNodeType` 保留为内置类型字符串常量(所有现有 `CanvasNodeType.Image` 判断照常编译)。 +2. 把 `canvas-node.tsx` 中 6 个内容渲染器抽到 `web/src/components/canvas/nodes/`(text/image/video/audio/config/group 各一文件),连同默认尺寸、图标、minimap 颜色、resource 函数一起写成 `CanvasNodeDefinition`,`registerBuiltinNodes()` 启动注册。`canvas-node.tsx` 只留外壳(边框/标题/缩放把手/连接点)。 +3. `constant/canvas.ts` 的 `getNodeSpec`/`NODE_DEFAULT_SIZE` 改为读注册表(保留函数签名,减少调用方改动)。 +4. 通用分支改查注册表:创建菜单 ×2、小地图颜色、信息弹窗类型名、等比缩放、双击行为、hover 工具栏(内置逻辑保留,追加 `definition.toolbar` 项)。 +5. `canvas-resource-references.ts` / `canvas-node-generation.ts` 的输入采集改走 `definition.resource()` → **插件节点(如 Markdown)可以直接作为生成输入连给图片/配置节点**。 +6. `canvas-agent-ops.ts` add_node 校验放开为"注册表中存在的 type" → AI Agent 自动获得操作插件节点的能力。 +7. project.tsx 中图片生成/批量等**媒体专属业务分支保持不动**(那是生成管线固有逻辑,不属于节点渲染扩展面)。 + +## 四、示例插件 + +First-party 插件(`web/src/plugins/nodes/`,与远程插件同一套 API,验证设计): + +| 插件 | 实现 | 交互演示 | +|---|---|---| +| **Markdown** | 已有 `streamdown` 依赖渲染;双击编辑源码 | `resource` 输出 text,可连入生成 | +| **HTML** | sandbox iframe `srcdoc`(`allow-scripts`,默认隔离) | 上游文本节点内容可注入渲染 | +| **SVG** | 直接渲染 SVG 源码,面板编辑 | 可从上游文本节点取 SVG 代码 | +| **3D 全景** | 新增 `three` 依赖(动态 import 按需分包),等距柱状全景查看器 | 从上游图片节点取全景图,演示 `getUpstream()` | + +远程插件示例:`plugins/examples/hello-node/`——独立 ESM 源码 + esbuild 构建脚本 + README(如何构建、托管、经 URL 安装),打通远程链路验证。 + +## 五、插件管理 UI + +`web/src/components/canvas/canvas-plugin-manager-modal.tsx`:URL 安装(含安全警告)、列表(名称/版本/来源/节点数)、启用开关、更新(重新拉取)、删除。入口挂在画布左上 Dropdown 菜单。遵循画布主题 token。 + +## 六、实施顺序 + +1. **注册表 + 类型放开 + 内置迁移**(改动最大,先落地保证现状不回归) +2. **NodeContext + 事件总线**(注入渲染链路) +3. **插件运行时 + loader + store + 管理 UI + 缺失插件占位** +4. **四个示例插件 + 远程示例** +5. **agent ops 放开校验;更新 CHANGELOG(Unreleased)、todo/pending-test 文档;SECURITY.md 补插件安全说明** + +## 风险与边界 + +- **安全**:ESM 直连=插件可读本地 API Key,靠安装警告+可信来源约束(已确认此取舍)。HTML 节点内容本身仍走 sandbox iframe 隔离。 +- **React 单例**:远程插件必须 external react 用全局运行时,SDK 文档写清楚,loader 校验失败给出明确报错。 +- **project.tsx 不做整体重构**:只替换类型分支为注册表查询,生成管线逻辑原样保留,控制回归面。 diff --git a/plugins/canvas/README.md b/plugins/canvas/README.md new file mode 100644 index 0000000..fe2d39a --- /dev/null +++ b/plugins/canvas/README.md @@ -0,0 +1,126 @@ +# Infinite Canvas 画布节点插件 + +给画布扩展自定义节点。每个插件是一个**独立目录**,自带 `package.json` / `build.mjs` / `src/index.jsx` / `dist/`,互不耦合,可单独构建、发布、升级。 + +内置节点只有文本、图片、视频、音频、生成配置、组六种;其余节点(Markdown、SVG、HTML、3D 全景、便利贴……)都是插件。 + +## 目录约定 + +``` +plugins/canvas/ + markdown/ # 每个插件一个独立目录 + package.json + build.mjs # esbuild 构建,产物名取目录名 → dist/markdown.js + src/index.jsx # 插件源码(默认导出工厂函数) + README.md + svg/ html/ panorama/ sticky-note/ ... +``` + +## 构建 / 发布 / 升级 + +```bash +cd plugins/canvas/ +npm install +npm run build # → dist/.js,并同步到 web/public/plugins/.js +npm run dev # watch,改动自动构建并同步 +``` + +把 `dist/.js` 托管到任意静态地址(CDN、GitHub Raw、对象存储),用户在画布「节点插件」管理器填该 URL 安装。升级时重新构建覆盖同一 URL,用户点「更新」即可。 + +## 本地开发 + +`npm run dev` 起 watch,在 `web/.env.local` 声明开发插件(逗号分隔多个): + +```env +VITE_DEV_PLUGINS=/plugins/markdown.js,/plugins/svg.js +``` + +再起画布 `web`(`npm run dev`)。`VITE_DEV_PLUGINS` 里的插件每次刷新页面都会**重新拉取**(不缓存、不落库),流程即:改 `src/index.jsx` → watch 自动构建 → 刷新画布看到最新效果,无需反复安装。 + +## 插件文件构成 + +每个插件最终打包成**单个 `.js` 文件**(加载器 `fetch 单文件 → import`),但源码可以随意拆分: + +- **多个 JS/JSX**:在 `src/index.jsx` 里 `import` 其它模块,esbuild `bundle` 会合并进一个产物。 +- **CSS**:写独立 `.css` 文件,`import css from "./styles.css"` 拿到字符串(esbuild 用 `text` loader),放到插件的 `css` 字段即可——启用时自动注入 `