From 7893bd72dddb7bf66a11b33817d67740a6116ba2 Mon Sep 17 00:00:00 2001 From: HouYunFei <1844025705@qq.com> Date: Tue, 16 Jun 2026 15:54:20 +0800 Subject: [PATCH] feat(agent): refactor tool response handling and enhance online agent functionality --- CLA.md | 91 ++++ README.md | 20 +- SECURITY.md | 64 +++ docs/content/docs/business/cla.mdx | 22 + docs/content/docs/business/meta.json | 1 + docs/content/docs/overview/features.mdx | 2 +- docs/content/docs/progress/pending-test.mdx | 7 +- docs/content/docs/support/meta.json | 3 +- docs/content/docs/support/security.mdx | 35 ++ docs/index.md | 5 +- .../(user)/canvas/[id]/canvas-client-page.tsx | 6 +- .../components/canvas-assistant-panel.tsx | 460 +++++++++++++----- .../components/canvas-node-generation.ts | 4 +- web/src/services/api/image.ts | 195 +++++--- 14 files changed, 698 insertions(+), 217 deletions(-) create mode 100644 CLA.md create mode 100644 SECURITY.md create mode 100644 docs/content/docs/business/cla.mdx create mode 100644 docs/content/docs/support/security.mdx diff --git a/CLA.md b/CLA.md new file mode 100644 index 0000000..978d561 --- /dev/null +++ b/CLA.md @@ -0,0 +1,91 @@ +# Contributor License Agreement + +This Contributor License Agreement ("Agreement") applies to any code, +documentation, design asset, issue text, pull request, patch, or other +material ("Contribution") that you submit to the infinite-canvas project +("Project"). + +By submitting a Contribution, you agree to the terms below. + +## 1. Copyright License + +You keep ownership of your Contributions. + +You grant the Project maintainers, including the repository owner, a +perpetual, worldwide, non-exclusive, royalty-free, irrevocable license to use, +reproduce, modify, prepare derivative works of, publicly display, publicly +perform, sublicense, distribute, commercialize, and otherwise exploit your +Contributions as part of the Project and related distributions. + +This license allows the maintainers to publish your Contributions under the +Project's open-source license, currently GNU Affero General Public License +v3.0, and to use them in commercial, proprietary, privately licensed, +source-available, or other versions, services, support offerings, or +derivatives related to the Project. + +## 2. Relicensing + +You grant the Project maintainers, including the repository owner, the +perpetual, worldwide, irrevocable right to relicense the Project, in whole or +in part, under any license terms, whether existing now or created in the +future. + +You agree that your Contributions may be incorporated into and distributed as +part of versions of the Project released under open-source, source-available, +proprietary, commercial, private, dual-license, or other licensing models, +without requiring additional permission, notice, approval, or compensation. + +This right includes distribution of the Project as software, hosted services, +Software-as-a-Service (SaaS), managed services, commercial offerings, OEM +products, embedded products, and any other form of distribution or use. + +## 3. Patent License + +If your Contribution is covered by patents that you own or control, you grant +the Project maintainers and recipients of the Project a perpetual, worldwide, +non-exclusive, royalty-free, irrevocable patent license to make, use, sell, +offer for sale, import, and otherwise run, modify, and distribute your +Contribution as part of the Project. + +## 4. Right to Contribute + +You represent that: + +* You have the legal right to submit the Contribution. +* The Contribution is your original work, or you have permission to submit it + under this Agreement. +* Any third-party material included in the Contribution is clearly identified + and compatible with the Project's license and this Agreement. +* If you submit a Contribution on behalf of an employer or organization, you + are authorized to do so. + +## 5. No Obligation + +The maintainers are not required to accept, review, merge, publish, or keep any +Contribution. The maintainers may modify, reject, revert, or remove +Contributions at their sole discretion. + +## 6. No Warranty + +You provide your Contributions "as is", without warranties or conditions of any +kind, express or implied, including warranties of merchantability, fitness for +a particular purpose, title, and non-infringement. + +## 7. How This Agreement Is Accepted + +Submitting a pull request, patch, commit, issue attachment, design asset, +documentation update, or any other Contribution to this repository constitutes +acceptance of this Agreement. + +Maintainers may request explicit confirmation by asking contributors to comment: + +```text +I have read and agree to CLA.md. +``` + +or by requiring acceptance through an automated CLA workflow. + +## 8. Contact + +For questions about this Agreement, contact the maintainers through the +repository or email `1844025705@qq.com`. diff --git a/README.md b/README.md index 01edadf..a6c632e 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ - 本地 Agent：通过本机 Canvas Agent 连接 Codex / Claude Code，让 Agent 通过 MCP 操作当前画布。 - 提示词库：Next.js route 抓取多个 GitHub 开源项目，并缓存在运行实例内存中。 -完整功能说明见 [docs/features.md](docs2/features.md)。 +完整功能说明见 [功能介绍](docs/content/docs/overview/features.mdx)。 如果你在为担心没有合适的生图API来发愁，可以查看该免费生图项目：[chatgpt2api](https://github.com/basketikun/chatgpt2api) @@ -88,15 +88,23 @@ https://canvas.best?apiKey={key}&baseUrl={address} 5 image + + image + image + ## 文档 -- [功能介绍](docs2/features.md) -- [部署说明](docs2/deployment.md) -- [画布节点操作手册](docs2/canvas-node-manual.md) -- [画布快捷键](docs2/canvas-shortcuts.md) -- [待办事项](docs2/todo.md) +- [快速开始](docs/content/docs/overview/quick-start.mdx) +- [功能介绍](docs/content/docs/overview/features.mdx) +- [Render 部署](docs/content/docs/overview/render.mdx) +- [Docker 部署](docs/content/docs/overview/docker.mdx) +- [画布节点操作手册](docs/content/docs/canvas/canvas-node-manual.mdx) +- [画布快捷键](docs/content/docs/canvas/canvas-shortcuts.mdx) +- [贡献者协议](CLA.md) +- [漏洞提交](SECURITY.md) +- [待办事项](docs/content/docs/progress/todo.mdx) - [本地 Canvas Agent](canvas-agent/README.md) ## 赞助支持 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..3a12c7a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,64 @@ +# Security Policy + +## Supported Versions + +infinite-canvas is in active development. Security fixes are accepted for the +`main` branch and the latest tagged release. Older versions may be handled on a +best-effort basis. + +## Reporting a Vulnerability + +Please do not open a public issue with exploit details, credentials, private +API keys, proof-of-concept code, or screenshots that reveal sensitive data. + +Preferred reporting channels: + +1. Use GitHub private vulnerability reporting or a GitHub Security Advisory for + this repository, if available. +2. If a private GitHub report is not available, email `1844025705@qq.com` with + the subject `[infinite-canvas security]`. +3. If neither private channel is available, open a public issue that asks for a + private contact channel and does not include technical exploit details. + +Please include: + +- Affected version, commit, branch, or deployment mode. +- Clear reproduction steps. +- Impact and attack scenario. +- Any relevant logs, screenshots, or proof of concept, with secrets removed. +- Whether the issue affects local-only usage, hosted deployments, browser + storage, WebDAV sync, AI provider configuration, or API proxy behavior. + +## Scope + +Examples of in-scope reports: + +- Cross-site scripting or token exfiltration in the web app. +- Exposure of locally stored API keys or synced canvas data caused by project + code. +- Unsafe file handling, import/export behavior, or WebDAV proxy behavior. +- Authentication, authorization, or access-control flaws in project-managed + features. +- Supply-chain issues that are exploitable through this repository's shipped + code or default configuration. + +Examples that are usually out of scope: + +- Vulnerabilities in third-party AI providers, model APIs, hosting platforms, + or browser extensions outside this repository. +- Compromise of a user's own API key outside the app. +- Denial-of-service reports that require unrealistic traffic volume or physical + access to the user's device. +- Missing security headers without a demonstrated exploit path. +- Social engineering, phishing, spam, or account recovery requests. +- Dependency reports without a practical impact on this project. + +## Disclosure + +The maintainers aim to acknowledge valid reports within 7 days and coordinate a +fix before public disclosure. Response and fix timelines are best effort for +this community project. + +Please allow time for investigation and remediation before publishing details. +Credit will be given on request unless you prefer to remain anonymous. + diff --git a/docs/content/docs/business/cla.mdx b/docs/content/docs/business/cla.mdx new file mode 100644 index 0000000..6feec9c --- /dev/null +++ b/docs/content/docs/business/cla.mdx @@ -0,0 +1,22 @@ +--- +title: 贡献者协议 +description: 贡献代码、文档或素材前需要了解的 CLA +--- + +# 贡献者协议 + +Contributor License Agreement，见 [CLA.md](https://github.com/basketikun/infinite-canvas/blob/main/CLA.md)。 + +向本仓库提交 Pull Request、补丁、文档、素材或其他贡献，即表示你同意该协议。维护者也可能要求你在 PR 中明确回复： + +```text +I have read and agree to CLA.md. +``` + +## 贡献前请确认 + +- 你有权提交相关代码、文档或素材。 +- 第三方内容已明确标注来源，并且授权方式与本项目兼容。 +- 提交内容可以随项目以 AGPL-3.0 以及项目相关商业授权方式发布。 +- 安全漏洞不要通过公开 Issue 直接披露细节，请按[漏洞提交](/docs/support/security)流程处理。 + diff --git a/docs/content/docs/business/meta.json b/docs/content/docs/business/meta.json index e58651f..2d2736f 100644 --- a/docs/content/docs/business/meta.json +++ b/docs/content/docs/business/meta.json @@ -4,6 +4,7 @@ "defaultOpen": true, "pages": [ "license", + "cla", "business" ] } diff --git a/docs/content/docs/overview/features.mdx b/docs/content/docs/overview/features.mdx index e5fcfdc..29cb409 100644 --- a/docs/content/docs/overview/features.mdx +++ b/docs/content/docs/overview/features.mdx @@ -63,7 +63,7 @@ OpenAI 兼容图像和文本能力继续复用现有接口： - `/v1/images/generations`：文生图。 - `/v1/images/edits`：图生图/参考图编辑。 -- `/v1/chat/completions`：文本问答和带图问答。 +- `/v1/responses`：文本问答、带图问答和在线 Agent 工具调用。 - `/v1/models`：读取模型列表；火山方舟 Agent Plan 专属 `/api/plan/v3` 当前未提供 OpenAI `/models` 模型列表接口，需要手动填写模型名。 视频能力支持两类接口： diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index 90598f2..ad3ceda 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -5,9 +5,4 @@ description: 当前版本已实现但仍需人工验证的变更项 # 待测试 -- 右上角 `Agent` 按钮在面板已打开时再次点击会走与右侧收起图标一致的收起动画，不再立即卸载面板；需要验证打开、右上角再次点击收起、右侧收起图标收起和收起过程中再次打开都正常。 -- 右侧 `Agent` 面板默认打开宽度已调大；需要验证首次打开空间更宽，手动拖拽调整宽度仍正常。 -- 在线 `Agent` 输入框底部的文本模型控件改为无背景无阴影的轻量下拉，只展示图标、模型名和渠道，并移除了输入区上方分割线；需要验证能正常切换文本模型。 -- 在线 `Agent` 询问“画布里有什么内容”等只读问题时不再进入工具确认流程；需要验证只展示普通回答，不出现“等待确认”工具卡片。 -- 在线 `Agent` 连接文本节点和生成配置节点时会把 `Note`、`生成配置` 等别名归一化为真实节点 id，模型漏传连接端点时也会回退到当前画布的文本节点与生成配置节点；需要验证不会再因标题当 id 或空端点而失败。 -- 在线 `Agent` 日志页改为类似本机 Agent 的可复制诊断日志视图；需要验证排查日志、原始 JSON、复制、最近错误和清空操作正常。 +- 在线 `Agent` 工具卡片详情不再保存和展示完整上下文消息，只展示当前工具调用和执行结果；需要验证详情内容体积不会随对话历史膨胀。 diff --git a/docs/content/docs/support/meta.json b/docs/content/docs/support/meta.json index 4370ec4..79d6e75 100644 --- a/docs/content/docs/support/meta.json +++ b/docs/content/docs/support/meta.json @@ -1,8 +1,9 @@ { - "title": "赞助支持", + "title": "支持与安全", "root": true, "defaultOpen": true, "pages": [ + "security", "donate", "sponsor" ] diff --git a/docs/content/docs/support/security.mdx b/docs/content/docs/support/security.mdx new file mode 100644 index 0000000..40d9fbf --- /dev/null +++ b/docs/content/docs/support/security.mdx @@ -0,0 +1,35 @@ +--- +title: 漏洞提交 +description: 安全漏洞提交和负责任披露说明 +--- + +# 漏洞提交 + +安全策略英文原文见 [SECURITY.md](https://github.com/basketikun/infinite-canvas/blob/main/SECURITY.md)。 + +请不要在公开 Issue 中直接发布漏洞细节、可利用代码、私密 API Key、截图中的敏感信息或真实用户数据。 + +## 提交流程 + +优先使用 GitHub 的私密漏洞报告或 Security Advisory。如果仓库没有开启对应入口，请发送邮件至 [1844025705@qq.com](mailto:1844025705@qq.com)，邮件标题建议使用： + +```text +[infinite-canvas security] +``` + +如果暂时无法使用私密渠道，可以先开一个公开 Issue 请求维护者提供私密联系方式，但不要在 Issue 中写入漏洞细节。 + +## 建议提供的信息 + +- 受影响的版本、提交、分支或部署方式。 +- 清晰的复现步骤。 +- 漏洞影响、攻击场景和触发条件。 +- 已脱敏的日志、截图或 PoC。 +- 是否涉及浏览器本地存储、API Key、WebDAV 同步、AI 渠道配置或代理接口。 + +## 范围说明 + +项目会优先处理由本仓库代码或默认配置导致的 XSS、敏感信息泄露、文件处理、导入导出、WebDAV 代理、权限控制和供应链风险。 + +第三方模型服务、托管平台、浏览器插件、用户自行泄露的 API Key、没有实际利用路径的安全头建议、纯社工钓鱼或账号找回问题通常不属于本项目漏洞范围。 + diff --git a/docs/index.md b/docs/index.md index b878567..43a1f25 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,11 +21,12 @@ ## 商务合作 - [开源协议](/docs/business/license) +- [贡献者协议](/docs/business/cla) - [商务合作](/docs/business/business) -## 赞助支持 +## 支持与安全 -- [文档](/docs/support/docs) +- [漏洞提交](/docs/support/security) - [打赏支持](/docs/support/donate) - [广告赞助](/docs/support/sponsor) diff --git a/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx b/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx index 11fb699..400a6b1 100644 --- a/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx +++ b/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx @@ -33,7 +33,7 @@ import { CanvasNodeCropDialog, type CanvasImageCropRect } from "../components/ca import { CanvasNodeMaskEditDialog, type CanvasImageMaskEditPayload } from "../components/canvas-node-mask-edit-dialog"; import { CanvasNodeSplitDialog, type CanvasImageSplitParams } from "../components/canvas-node-split-dialog"; import { CanvasNodeUpscaleDialog, type CanvasImageUpscaleParams } from "../components/canvas-node-upscale-dialog"; -import { buildNodeChatMessages, buildNodeGenerationContext, buildNodeGenerationInputs, hydrateNodeGenerationContext, type NodeGenerationInput } from "../components/canvas-node-generation"; +import { buildNodeGenerationContext, buildNodeGenerationInputs, buildNodeResponseMessages, hydrateNodeGenerationContext, type NodeGenerationInput } from "../components/canvas-node-generation"; import { CanvasNodeHoverToolbar, CanvasNodeInfoModal } from "../components/canvas-node-hover-toolbar"; import { InfiniteCanvas } from "../components/infinite-canvas"; import { Minimap } from "../components/canvas-mini-map"; @@ -2118,7 +2118,7 @@ function InfiniteCanvasPage() { const answers = await Promise.all( (childIds.length ? childIds : [nodeId]).map((targetNodeId) => { let localStreamed = ""; - return requestImageQuestion(generationConfig, buildNodeChatMessages({ ...generationContext, prompt: effectivePrompt }), (text) => { + return requestImageQuestion(generationConfig, buildNodeResponseMessages({ ...generationContext, prompt: effectivePrompt }), (text) => { localStreamed = text; streamed = text; if (isConfigNode) return; @@ -2199,7 +2199,7 @@ function InfiniteCanvasPage() { if (node.type === CanvasNodeType.Text) { if (!context) return; let streamed = ""; - const answer = await requestImageQuestion(generationConfig, buildNodeChatMessages({ ...context, prompt }), (text) => { + const answer = await requestImageQuestion(generationConfig, buildNodeResponseMessages({ ...context, prompt }), (text) => { streamed = text; setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, type: CanvasNodeType.Text, metadata: { ...item.metadata, content: text, status: NODE_STATUS_LOADING } } : item))); }); diff --git a/web/src/app/(user)/canvas/components/canvas-assistant-panel.tsx b/web/src/app/(user)/canvas/components/canvas-assistant-panel.tsx index 447102f..1bf7713 100644 --- a/web/src/app/(user)/canvas/components/canvas-assistant-panel.tsx +++ b/web/src/app/(user)/canvas/components/canvas-assistant-panel.tsx @@ -9,7 +9,7 @@ import { motion } from "motion/react"; import { modelOptionName, resolveModelChannel, selectableModelsByCapability, useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store"; import { canvasThemes } from "@/lib/canvas-theme"; import { nanoid } from "nanoid"; -import { requestImageQuestion, type ChatCompletionMessage } from "@/services/api/image"; +import { requestToolResponse, type ResponseFunctionTool, type ResponseInputMessage, type ResponseToolCall } from "@/services/api/image"; import { imageToDataUrl } from "@/services/image-storage"; import { useAssetStore } from "@/stores/use-asset-store"; import { useThemeStore } from "@/stores/use-theme-store"; @@ -22,17 +22,115 @@ import { AgentChatComposer, AgentChatMessage, AgentModeSwitch, AgentPanelTabs, A import { CanvasLocalAgentPanel } from "./canvas-local-agent-panel"; import { CanvasNodeType, type CanvasAssistantMessage, type CanvasAssistantReference, type CanvasAssistantSession, type CanvasNodeData } from "../types"; import { useCanvasAgentStore } from "../stores/use-canvas-agent-store"; -import { summarizeCanvasAgentOps, type CanvasAgentOp, type CanvasAgentSnapshot } from "../utils/canvas-agent-ops"; +import { type CanvasAgentOp, type CanvasAgentSnapshot } from "../utils/canvas-agent-ops"; export const CANVAS_AGENT_PANEL_MOTION_MS = 500; const PANEL_MOTION_SECONDS = CANVAS_AGENT_PANEL_MOTION_MS / 1000; const ONLINE_AGENT_MAX_STEPS = 4; const ONLINE_AGENT_PROMPT = - '你是 Infinite Canvas 网页内置在线画布助手。你只能返回 JSON，不要 Markdown，不要解释。格式：{"reply":"给用户看的中文说明","ops":[...]}。用户只是询问、查看、总结、描述或分析当前画布内容时，必须返回 ops:[]，reply 直接回答画布现状，不要出现“等待确认”。只有需要修改画布时才返回 ops，且 reply 只能说明“准备执行/等待确认”，不能说“已完成/已删除/已连接/已调整”，因为工具操作需要用户确认后才会执行。返回 ops 时必须使用当前画布 JSON 里的真实 node.id / connection.id，不要使用节点标题、类型名、Note、生成配置等别名当 id。工具执行结果返回后，你要判断用户任务是否完成；完成时返回 ops:[]，未完成时返回下一步 ops。ops 可用类型：add_node、update_node、delete_node、delete_connections、connect_nodes、set_viewport、select_nodes、run_generation。add_node 支持 nodeType: text/image/config/video/audio，position:{x,y}，metadata。delete_node 必须带 id/ids，或用 nodeType:"config" 删除全部生成配置节点。delete_connections 可用 all:true 删除全部连线。文本内容放 metadata.content。用户要求生图、生成文字、视频或音频时，不要直接生成最终内容，要创建提示词文本节点、config 节点、connect_nodes，并追加 run_generation 触发画布已有生成工具；config 节点 metadata 至少包含 generationMode、composerContent、prompt、status:"idle"，composerContent/prompt 用 @[node:id] 引用提示词节点或参考节点。只输出能直接 JSON.parse 的对象。'; + "你是 Infinite Canvas 网页内置在线画布助手。当前画布 JSON 会随用户消息提供。首轮必须调用工具：只读问题调用 canvas_get_state，需要修改画布时调用对应写工具。不要输出 JSON ops，不要编造执行结果。工具参数必须完全符合 schema，并且必须使用当前画布 JSON 中真实存在的 id；不要把 title、type、Note、生成配置等名称当 id。缺少必要 id 或用户意图不明确时直接说明需要用户明确选择或说明，不要猜测。生图、生成文本、视频或音频时，如果使用节点作为输入，先调用 canvas_connect_nodes 连接输入节点到生成配置节点，再调用 canvas_configure_generation，最后调用 canvas_run_generation。工具返回结果后，再根据真实结果回答用户。"; +const ONLINE_AGENT_TOOLS: ResponseFunctionTool[] = [ + { + type: "function", + function: { + name: "canvas_get_state", + description: "读取当前画布状态。只读问题或需要确认节点 id 时使用。", + parameters: { type: "object", properties: {}, required: [], additionalProperties: false }, + strict: true, + }, + }, + { + type: "function", + function: { + name: "canvas_connect_nodes", + description: "连接两个已存在节点。必须传真实 fromNodeId 和 toNodeId。", + parameters: { + type: "object", + properties: { + fromNodeId: { type: "string" }, + toNodeId: { type: "string" }, + }, + required: ["fromNodeId", "toNodeId"], + additionalProperties: false, + }, + strict: true, + }, + }, + { + type: "function", + function: { + name: "canvas_configure_generation", + description: "配置一个生成配置节点的生成模式和提示词引用。必须传真实 configNodeId；promptNodeIds 必须已连接到该生成配置节点。", + parameters: { + type: "object", + properties: { + configNodeId: { type: "string" }, + mode: { type: "string", enum: ["text", "image", "video", "audio"] }, + promptNodeIds: { type: "array", items: { type: "string" } }, + prompt: { type: "string" }, + }, + required: ["configNodeId", "mode", "promptNodeIds", "prompt"], + additionalProperties: false, + }, + strict: true, + }, + }, + { + type: "function", + function: { + name: "canvas_run_generation", + description: "触发一个生成配置节点执行生成。必须传真实 configNodeId。", + parameters: { + type: "object", + properties: { + configNodeId: { type: "string" }, + mode: { type: "string", enum: ["text", "image", "video", "audio"] }, + }, + required: ["configNodeId", "mode"], + additionalProperties: false, + }, + strict: true, + }, + }, + { + type: "function", + function: { + name: "canvas_delete_nodes", + description: "删除指定节点。必须传真实 nodeIds。", + parameters: { + type: "object", + properties: { nodeIds: { type: "array", items: { type: "string" }, minItems: 1 } }, + required: ["nodeIds"], + additionalProperties: false, + }, + strict: true, + }, + }, + { + type: "function", + function: { + name: "canvas_delete_connections", + description: "删除指定连线，或删除全部连线。", + parameters: { + type: "object", + properties: { + connectionIds: { type: "array", items: { type: "string" } }, + all: { type: "boolean" }, + }, + required: ["connectionIds", "all"], + additionalProperties: false, + }, + strict: true, + }, + }, +]; type OnlineAgentTab = "setup" | "chat" | "history" | "log"; type OnlineAgentLog = { id: string; time: string; title: string; data?: unknown }; type OnlineAgentLogContext = { model: string; running: boolean; confirmTools: boolean; messages: number; nodes: number; connections: number }; -type OnlineLoopContext = { step: number; previous?: unknown }; +type OnlineLoopContext = { step: number }; +type OnlineToolResult = { ok: true; message: string; data?: unknown } | { ok: false; message: string }; +type OnlineExecutedToolCall = { toolCallId: string; name: string; result: OnlineToolResult }; +type PendingOnlineToolContext = { messages: ResponseInputMessage[]; toolCalls: ResponseToolCall[]; assistantId: string; step: number }; type CanvasAssistantPanelProps = { nodes: CanvasNodeData[]; @@ -73,6 +171,7 @@ export function CanvasAssistantPanel({ nodes, selectedNodeIds, snapshot, session const [localSessions, setLocalSessions] = useState(() => (sessions.length ? sessions : [createSession()])); const [localActiveSessionId, setLocalActiveSessionId] = useState(activeSessionId); const snapshotRef = useRef(snapshot); + const pendingToolContextRef = useRef(new Map()); useEffect(() => { if (!sessions.length) return; @@ -184,77 +283,204 @@ export function CanvasAssistantPanel({ nodes, selectedNodeIds, snapshot, session const runOnlineAgentStep = async (sessionId: string, assistantId: string, history: CanvasAssistantMessage[], userMessage: CanvasAssistantMessage, loop: OnlineLoopContext) => { const requestConfig = { ...effectiveConfig, model: effectiveConfig.textModel || effectiveConfig.model }; - let continued = false; try { setIsRunning(true); - addOnlineLog(`Agent Loop ${loop.step} 开始`, loop.previous); - const answer = await requestImageQuestion({ ...requestConfig, systemPrompt: "" }, await buildAgentMessages(snapshotRef.current, history, userMessage, loop), (streamText) => { - const reply = partialAgentReply(streamText); - if (reply) upsertMessage(sessionId, { id: assistantId, role: "assistant", text: pendingReply(reply) }); - }); - addOnlineLog("模型原始回复", answer); - const result = parseAgentResult(answer); - const readOnly = isReadOnlyCanvasQuestion(userMessage.text); - const reply = readOnly ? cleanReadOnlyReply(result.reply, snapshotRef.current) : result.reply; - addOnlineLog("解析结果", result); - const ops = readOnly ? [] : normalizeOnlineOps(result.ops, userMessage.text, snapshotRef.current); - addOnlineLog("归一化操作", ops); - if (ops.length && sameOps(ops, objectDetail(loop.previous).ops)) { - addOnlineLog(`Agent Loop ${loop.step} 停止`, { reason: "same_ops", ops }); - upsertMessage(sessionId, { id: assistantId, role: "assistant", text: "画布状态已更新，后续操作与上一轮重复，已停止继续执行。" }); - return; - } - if (ops.length) { - upsertMessage(sessionId, { id: assistantId, role: "assistant", text: pendingReply(reply) }); - const toolMessage: CanvasAssistantMessage = { id: nanoid(), role: "tool", title: confirmTools ? "确认工具调用" : "画布操作执行中", text: summarizeCanvasAgentOps(ops) || "画布操作", detail: { name: "canvas_apply_ops", ops, intent: userMessage.text, assistantId, step: loop.step, status: confirmTools ? "pending" : "running" } }; - appendMessage(sessionId, toolMessage); - addOnlineLog(confirmTools ? "等待用户确认" : "自动执行工具", { step: loop.step, ops }); - if (!confirmTools) continued = executeOnlineTool(sessionId, toolMessage.id, ops, { assistantId, userMessage, history, step: loop.step }); + const messages = await buildToolAgentMessages(snapshotRef.current, history, userMessage); + addOnlineLog(`Agent Tool Loop ${loop.step} 开始`, { toolChoice: "required" }); + const result = await requestToolResponse({ ...requestConfig, systemPrompt: "" }, messages, ONLINE_AGENT_TOOLS, "required"); + addOnlineLog("模型工具回复", result); + if (result.toolCalls.length) { + const writableCalls = result.toolCalls.filter(isWritableToolCall); + if (confirmTools && writableCalls.length) { + upsertMessage(sessionId, { id: assistantId, role: "assistant", text: result.content || "准备执行工具，等待确认。" }); + const toolMessageId = nanoid(); + pendingToolContextRef.current.set(toolMessageId, { messages, toolCalls: result.toolCalls, assistantId, step: loop.step }); + const toolMessage: CanvasAssistantMessage = { id: toolMessageId, role: "tool", title: "确认工具调用", text: summarizeToolCalls(result.toolCalls), detail: { status: "pending", step: loop.step, toolCalls: result.toolCalls } }; + appendMessage(sessionId, toolMessage); + addOnlineLog("等待用户确认", result.toolCalls); + return; + } + await continueOnlineToolLoop(sessionId, assistantId, messages, result, loop.step); } else { - addOnlineLog(`Agent Loop ${loop.step} 结束`, { reply, reason: readOnly ? "readonly" : "no_ops" }); - upsertMessage(sessionId, { id: assistantId, role: "assistant", text: reply }); + if (!result.content.trim()) throw new Error("模型没有返回工具调用，画布操作未执行。"); + upsertMessage(sessionId, { id: assistantId, role: "assistant", text: result.content || "没有返回内容。" }); + addOnlineLog(`Agent Tool Loop ${loop.step} 结束`, { reply: result.content }); } } catch (error) { addOnlineLog("请求失败", error instanceof Error ? error.message : error); appendMessage(sessionId, { id: nanoid(), role: "error", title: "操作失败", text: error instanceof Error ? error.message : "操作失败" }); } finally { - if (!continued) setIsRunning(false); + setIsRunning(false); } }; - const executeOnlineTool = (sessionId: string, messageId: string, ops: CanvasAgentOp[], loop?: { assistantId?: string; userMessage?: CanvasAssistantMessage; history?: CanvasAssistantMessage[]; step?: number }) => { + const continueOnlineToolLoop = async (sessionId: string, assistantId: string, messages: ResponseInputMessage[], result: { content: string; toolCalls: ResponseToolCall[] }, step: number) => { + const toolResults = executeOnlineToolCalls(result.toolCalls); + addOnlineLog("工具执行结果", toolResults); + await continueOnlineToolLoopAfterResults(sessionId, assistantId, messages, result.toolCalls, toolResults, step); + }; + + const continueOnlineToolLoopAfterResults = async (sessionId: string, assistantId: string, messages: ResponseInputMessage[], toolCalls: ResponseToolCall[], toolResults: OnlineExecutedToolCall[], step: number) => { + const nextMessages: ResponseInputMessage[] = [ + ...messages, + ...toolCalls.map(toolCallToResponseInput), + ...toolResults.map((item) => ({ role: "tool" as const, tool_call_id: item.toolCallId, content: JSON.stringify(item.result) })), + ]; + if (step >= ONLINE_AGENT_MAX_STEPS) { + upsertMessage(sessionId, { id: assistantId, role: "assistant", text: toolResults.map((item) => toolResultText(item.result)).join("\n") || "工具已执行。" }); + addOnlineLog("Agent Tool Loop 达到步数上限", { maxSteps: ONLINE_AGENT_MAX_STEPS }); + return; + } + const requestConfig = { ...effectiveConfig, model: effectiveConfig.textModel || effectiveConfig.model }; + const next = await requestToolResponse({ ...requestConfig, systemPrompt: "" }, nextMessages, ONLINE_AGENT_TOOLS); + addOnlineLog(`Agent Tool Loop ${step + 1} 回复`, next); + if (next.toolCalls.length) { + const writableCalls = next.toolCalls.filter(isWritableToolCall); + if (confirmTools && writableCalls.length) { + upsertMessage(sessionId, { id: assistantId, role: "assistant", text: next.content || "准备执行工具，等待确认。" }); + const toolMessageId = nanoid(); + pendingToolContextRef.current.set(toolMessageId, { messages: nextMessages, toolCalls: next.toolCalls, assistantId, step: step + 1 }); + appendMessage(sessionId, { id: toolMessageId, role: "tool", title: "确认工具调用", text: summarizeToolCalls(next.toolCalls), detail: { status: "pending", step: step + 1, toolCalls: next.toolCalls } }); + addOnlineLog("等待用户确认", next.toolCalls); + return; + } + await continueOnlineToolLoop(sessionId, assistantId, nextMessages, next, step + 1); + return; + } + upsertMessage(sessionId, { id: assistantId, role: "assistant", text: next.content || toolResults.map((item) => toolResultText(item.result)).join("\n") || "工具已执行。" }); + }; + + const executeOps = (ops: CanvasAgentOp[]) => { const beforeSnapshot = snapshotRef.current; const before = snapshotSignature(beforeSnapshot); const next = onApplyOps(ops); snapshotRef.current = next; - const ranGeneration = ops.some((op) => op.type === "run_generation" && op.nodeId && beforeSnapshot.nodes.some((node) => node.id === op.nodeId)); + const ranGeneration = ops.some((op) => op.type === "run_generation" && Boolean(op.nodeId)); const changed = before !== snapshotSignature(next) || ranGeneration; const noopReason = changed ? "" : explainNoop(ops, beforeSnapshot); - addOnlineLog(changed ? "执行成功" : "执行未生效", { ops, ranGeneration, noopReason, before: JSON.parse(before), after: JSON.parse(snapshotSignature(next)) }); - upsertMessage(sessionId, { id: messageId, role: "tool", title: changed ? "画布操作完成" : "画布操作未生效", text: changed ? summarizeCanvasAgentOps(ops) || "画布操作" : noopReason, detail: { name: "canvas_apply_ops", ops, status: changed ? "completed" : "noop", noopReason } }); - if (changed && loop?.assistantId && loop.userMessage) { - const step = loop.step || 1; - if (step < ONLINE_AGENT_MAX_STEPS) { - void runOnlineAgentStep(sessionId, nanoid(), loop.history || [], loop.userMessage, { step: step + 1, previous: { changed, ops, snapshot: compactSnapshot(next) } }); - return true; - } - else addOnlineLog("Agent Loop 达到步数上限", { maxSteps: ONLINE_AGENT_MAX_STEPS }); - } - return false; + return { changed, ops, ranGeneration, noopReason, before: JSON.parse(before), after: JSON.parse(snapshotSignature(next)) }; }; - const approveOnlineTool = (messageId: string) => { + const executeOnlineTool = (name: string, args: Record): OnlineToolResult => { + const current = snapshotRef.current; + const requireNode = (id: unknown, expectedType?: CanvasNodeType) => { + if (typeof id !== "string" || !id) throw new Error("缺少真实节点 id"); + const node = current.nodes.find((item) => item.id === id); + if (!node) throw new Error(`节点不存在：${id}`); + if (expectedType && node.type !== expectedType) throw new Error(`节点类型不匹配：${id}`); + return node; + }; + try { + if (name === "canvas_get_state") return { ok: true, message: describeCanvasSnapshot(current), data: compactSnapshot(current) }; + if (name === "canvas_connect_nodes") { + const fromNode = requireNode(args.fromNodeId); + const toNode = requireNode(args.toNodeId); + if (fromNode.id === toNode.id) throw new Error("不能连接同一个节点"); + const existed = current.connections.some((conn) => conn.fromNodeId === fromNode.id && conn.toNodeId === toNode.id); + const result = executeOps([{ type: "connect_nodes", fromNodeId: fromNode.id, toNodeId: toNode.id }]); + return { ok: true, message: result.changed ? "已连接节点。" : existed ? "节点已存在连线。" : result.noopReason, data: result }; + } + if (name === "canvas_configure_generation") { + const configNode = requireNode(args.configNodeId, CanvasNodeType.Config); + const mode = requireGenerationMode(args.mode); + const promptNodeIds = requireStringArray(args.promptNodeIds, "promptNodeIds").map((id) => requireNode(id).id); + promptNodeIds.forEach((id) => { + if (!snapshotRef.current.connections.some((conn) => conn.fromNodeId === id && conn.toNodeId === configNode.id)) throw new Error(`提示词节点未连接到生成配置节点：${id}`); + }); + if (typeof args.prompt !== "string") throw new Error("prompt 必须是字符串"); + const prompt = args.prompt.trim(); + if (!prompt && !promptNodeIds.length) throw new Error("缺少提示词文本或提示词节点 id"); + const composerContent = promptNodeIds.map((id) => `@[node:${id}]`).concat(prompt ? [prompt] : []).join("\n"); + const result = executeOps([{ type: "update_node", id: configNode.id, metadata: { generationMode: mode, composerContent, prompt: composerContent, status: "idle" } }]); + return { ok: true, message: result.changed ? "已配置生成节点。" : "生成节点配置已是目标值。", data: result }; + } + if (name === "canvas_run_generation") { + const configNode = requireNode(args.configNodeId, CanvasNodeType.Config); + const mode = requireGenerationMode(args.mode); + const result = executeOps([{ type: "run_generation", nodeId: configNode.id, mode }]); + return { ok: result.changed, message: result.changed ? "已触发生成。" : result.noopReason, data: result }; + } + if (name === "canvas_delete_nodes") { + const nodeIds = requireStringArray(args.nodeIds, "nodeIds"); + if (!nodeIds.length) throw new Error("缺少要删除的节点 id"); + nodeIds.forEach((id) => requireNode(id)); + const result = executeOps([{ type: "delete_node", ids: nodeIds }]); + return { ok: result.changed, message: result.changed ? "已删除节点。" : result.noopReason, data: result }; + } + if (name === "canvas_delete_connections") { + if (typeof args.all !== "boolean") throw new Error("all 必须是布尔值"); + const all = args.all; + const connectionIds = requireStringArray(args.connectionIds, "connectionIds"); + if (!all && !connectionIds.length) throw new Error("缺少要删除的连线 id"); + if (!all) connectionIds.forEach((id) => { + if (!current.connections.some((conn) => conn.id === id)) throw new Error(`连线不存在：${id}`); + }); + const result = executeOps([{ type: "delete_connections", all, ids: connectionIds }]); + return { ok: result.changed, message: result.changed ? "已删除连线。" : result.noopReason, data: result }; + } + return { ok: false, message: `不支持的工具：${name}` }; + } catch (error) { + return { ok: false, message: error instanceof Error ? error.message : "工具执行失败" }; + } + }; + + const executeOnlineToolCall = (toolCall: ResponseToolCall): OnlineExecutedToolCall => { + try { + const result = executeOnlineTool(toolCall.function.name, parseToolArguments(toolCall.function.arguments)); + return { toolCallId: toolCall.id, name: toolCall.function.name, result }; + } catch (error) { + return { toolCallId: toolCall.id, name: toolCall.function.name, result: { ok: false, message: error instanceof Error ? error.message : "工具参数错误" } }; + } + }; + + const executeOnlineToolCalls = (toolCalls: ResponseToolCall[]) => { + const results: OnlineExecutedToolCall[] = []; + let stopped = false; + toolCalls.forEach((toolCall) => { + if (stopped) { + results.push({ toolCallId: toolCall.id, name: toolCall.function.name, result: { ok: false, message: "前一个工具调用失败，未继续执行。" } }); + return; + } + const result = executeOnlineToolCall(toolCall); + results.push(result); + if (!result.result.ok) stopped = true; + }); + return results; + }; + + const approveOnlineTool = async (messageId: string) => { const message = safeSessions.flatMap((session) => session.messages).find((item) => item.id === messageId); const detail = objectDetail(message?.detail); - const ops = normalizeOnlineOps(toolOps(detail), String(detail.intent || ""), snapshotRef.current); + const pendingContext = pendingToolContextRef.current.get(messageId); + const toolCalls = pendingContext?.toolCalls || toolCallsFromDetail(detail); + const previousMessages = pendingContext?.messages || []; const session = safeSessions.find((session) => session.messages.some((item) => item.id === messageId)); - addOnlineLog("批准工具", { messageId, ops }); - if (session && ops.length) executeOnlineTool(session.id, messageId, ops, { assistantId: String(detail.assistantId || ""), userMessage: { id: "", role: "user", text: String(detail.intent || "") }, history: messages, step: Number(detail.step) || 1 }); + addOnlineLog("批准工具", { messageId, toolCalls }); + const assistantId = pendingContext?.assistantId || ""; + if (!session) return; + if (!toolCalls.length || !previousMessages.length || !assistantId) { + upsertMessage(session.id, { id: messageId, role: "tool", title: "工具执行失败", text: "工具上下文不完整，无法执行。", detail: { ...detail, status: "failed" } }); + return; + } + try { + setIsRunning(true); + const results = executeOnlineToolCalls(toolCalls); + addOnlineLog("工具执行结果", results); + upsertMessage(session.id, { id: messageId, role: "tool", title: "工具执行完成", text: results.map((item) => toolResultText(item.result)).join("\n"), detail: { ...detail, results, status: "completed" } }); + pendingToolContextRef.current.delete(messageId); + await continueOnlineToolLoopAfterResults(session.id, assistantId, previousMessages, toolCalls, results, pendingContext?.step || Number(detail.step) || 1); + } catch (error) { + addOnlineLog("工具续跑失败", error instanceof Error ? error.message : error); + appendMessage(session.id, { id: nanoid(), role: "error", title: "操作失败", text: error instanceof Error ? error.message : "操作失败" }); + } finally { + setIsRunning(false); + } }; const rejectOnlineTool = (messageId: string) => { const session = safeSessions.find((session) => session.messages.some((item) => item.id === messageId)); addOnlineLog("拒绝工具", { messageId }); + pendingToolContextRef.current.delete(messageId); if (session) upsertMessage(session.id, { id: messageId, role: "tool", title: "已拒绝执行", text: "工具调用已取消", detail: { ...objectDetail(session.messages.find((item) => item.id === messageId)?.detail), status: "rejected" } }); }; @@ -708,15 +934,6 @@ function objectDetail(value: unknown) { return value && typeof value === "object" ? (value as Record) : {}; } -function toolOps(value: unknown) { - const ops = objectDetail(value).ops; - return Array.isArray(ops) ? (ops as CanvasAgentOp[]) : []; -} - -function sameOps(a: CanvasAgentOp[], b: unknown) { - return Array.isArray(b) && JSON.stringify(a) === JSON.stringify(b); -} - function stringifyLog(value: unknown) { return typeof value === "string" ? value : JSON.stringify(value, null, 2); } @@ -740,28 +957,6 @@ function formatOnlineLogJson(logs: OnlineAgentLog[], context: OnlineAgentLogCont return JSON.stringify({ context, logs: logs.map(({ time, title, data }) => ({ time, title, data })) }, null, 2); } -function pendingReply(text: string) { - return text - .replace(/已(?:经)?完成/g, "准备执行") - .replace(/已(?:经)?删除/g, "准备删除") - .replace(/已(?:经)?连接/g, "准备连接") - .replace(/已(?:经)?调整/g, "准备调整") - .replace(/已(?:经)?整理/g, "准备整理") - .replace(/已(?:经)?移动/g, "准备移动") - .replace(/已(?:经)?创建/g, "准备创建") - .replace(/已(?:经)?帮你/g, "准备帮你") - .replace(/已(?:经)?将/g, "准备将"); -} - -function isReadOnlyCanvasQuestion(text: string) { - return /(?:有什么|有哪些|看一下|查看|说一下|总结|概括|描述|介绍|分析|当前|现在).*(?:画布|内容|节点|连线)|(?:画布|内容|节点|连线).*(?:有什么|有哪些|是什么|多少|情况|状态)/.test(text); -} - -function cleanReadOnlyReply(reply: string, snapshot: CanvasAgentSnapshot) { - const text = reply.replace(/^等待确认[:：]\s*/, "").trim(); - return text || describeCanvasSnapshot(snapshot); -} - function describeCanvasSnapshot(snapshot: CanvasAgentSnapshot) { const counts = snapshot.nodes.reduce>((acc, node) => { acc[node.type] = (acc[node.type] || 0) + 1; @@ -770,44 +965,61 @@ function describeCanvasSnapshot(snapshot: CanvasAgentSnapshot) { return `当前画布有 ${snapshot.nodes.length} 个节点、${snapshot.connections.length} 条连线。文本 ${counts[CanvasNodeType.Text] || 0} 个，图片 ${counts[CanvasNodeType.Image] || 0} 个，生成配置 ${counts[CanvasNodeType.Config] || 0} 个，视频 ${counts[CanvasNodeType.Video] || 0} 个，音频 ${counts[CanvasNodeType.Audio] || 0} 个。`; } -function normalizeOnlineOps(ops: CanvasAgentOp[], intent: string, snapshot: CanvasAgentSnapshot) { - if (/删|删除|移除|清空/.test(intent) && /连线|连接线|线条|边/.test(intent)) return snapshot.connections.length ? [...ops.filter((op) => op.type !== "connect_nodes"), { type: "delete_connections", all: true }] : ops; - if (/删|删除|移除/.test(intent) && /生成配置|配置节点|config/i.test(intent)) { - const ids = snapshot.nodes.filter((node) => node.type === CanvasNodeType.Config).map((node) => node.id); - return ids.length ? (ops.some((op) => op.type === "delete_node") ? ops.map((op) => (op.type === "delete_node" && !op.id && !op.ids?.length ? { ...op, ids } : op)) : [...ops, { type: "delete_node", ids }]) : ops; +function parseToolArguments(value: string) { + try { + const parsed = JSON.parse(value || "{}"); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("工具参数必须是 JSON 对象"); + return parsed as Record; + } catch { + throw new Error("工具参数不是合法 JSON 对象"); } - if (/连线|连接|串联/.test(intent)) { - const connectOps = ops.filter((op): op is Extract => op.type === "connect_nodes"); - const resolved = connectOps.map((op) => ({ ...op, fromNodeId: resolveCanvasNodeId(op.fromNodeId, snapshot, intent), toNodeId: resolveCanvasNodeId(op.toNodeId, snapshot, intent) })).filter((op) => op.fromNodeId && op.toNodeId && op.fromNodeId !== op.toNodeId); - if (resolved.length) return [...ops.filter((op) => op.type !== "connect_nodes"), ...resolved]; - const fallback = fallbackConnectOps(snapshot); - return fallback.length ? [...ops.filter((op) => op.type !== "connect_nodes"), ...fallback] : ops; - } - return ops; } -function fallbackConnectOps(snapshot: CanvasAgentSnapshot) { - const textNode = snapshot.nodes.find((node) => node.type === CanvasNodeType.Text); - const configNode = snapshot.nodes.find((node) => node.type === CanvasNodeType.Config); - if (textNode && configNode) return [{ type: "connect_nodes" as const, fromNodeId: textNode.id, toNodeId: configNode.id }]; - const nodes = snapshot.nodes.filter((node) => node.type !== CanvasNodeType.Config).sort((a, b) => a.position.x - b.position.x || a.position.y - b.position.y); - return nodes.slice(1).map((node, index) => ({ type: "connect_nodes" as const, fromNodeId: nodes[index].id, toNodeId: node.id })); +function isWritableToolCall(call: ResponseToolCall) { + return call.function.name !== "canvas_get_state"; } -function resolveCanvasNodeId(value: unknown, snapshot: CanvasAgentSnapshot, intent: string) { - const raw = typeof value === "string" ? value : ""; - if (!raw) return ""; - if (snapshot.nodes.some((node) => node.id === raw)) return raw; - const text = normalizeNodeToken(raw); - const titleMatch = snapshot.nodes.find((node) => normalizeNodeToken(node.title) === text); - if (titleMatch) return titleMatch.id; - if (/note|文本|文字|提示词|prompt/i.test(raw) || /文本节点|note/i.test(intent)) return snapshot.nodes.find((node) => node.type === CanvasNodeType.Text)?.id || raw; - if (/config|生成配置|配置节点|配置/i.test(raw) || /生成配置|配置节点|config/i.test(intent)) return snapshot.nodes.find((node) => node.type === CanvasNodeType.Config)?.id || raw; - return raw; +function toolCallsFromDetail(detail: Record): ResponseToolCall[] { + return Array.isArray(detail.toolCalls) ? (detail.toolCalls.filter(isResponseToolCall) as ResponseToolCall[]) : []; } -function normalizeNodeToken(value: unknown) { - return (typeof value === "string" ? value : "").trim().toLowerCase().replace(/\s+/g, ""); +function isResponseToolCall(value: unknown): value is ResponseToolCall { + const item = objectDetail(value); + const fn = objectDetail(item.function); + return typeof item.id === "string" && item.type === "function" && typeof fn.name === "string" && typeof fn.arguments === "string"; +} + +function toolCallToResponseInput(call: ResponseToolCall): ResponseInputMessage { + return { type: "function_call", call_id: call.id, name: call.function.name, arguments: call.function.arguments }; +} + +function summarizeToolCalls(calls: ResponseToolCall[]) { + return calls.map((call) => toolCallLabel(call.function.name)).join("，") || "工具调用"; +} + +function toolCallLabel(name: string) { + if (name === "canvas_get_state") return "读取画布"; + if (name === "canvas_connect_nodes") return "连接节点"; + if (name === "canvas_configure_generation") return "配置生成"; + if (name === "canvas_run_generation") return "触发生成"; + if (name === "canvas_delete_nodes") return "删除节点"; + if (name === "canvas_delete_connections") return "删除连线"; + return name; +} + +function toolResultText(result: OnlineToolResult) { + return result.message; +} + +function requireGenerationMode(value: unknown): "text" | "image" | "video" | "audio" { + if (value === "text" || value === "image" || value === "video" || value === "audio") return value; + throw new Error("生成模式必须是 text、image、video 或 audio"); +} + +function requireStringArray(value: unknown, field: string): string[] { + if (!Array.isArray(value)) throw new Error(`${field} 必须是字符串数组`); + if (!value.every((item) => typeof item === "string" && Boolean(item))) throw new Error(`${field} 必须只包含非空字符串`); + return value as string[]; } function snapshotSignature(snapshot: CanvasAgentSnapshot) { @@ -835,7 +1047,7 @@ function explainNoop(ops: CanvasAgentOp[], snapshot: CanvasAgentSnapshot) { if (generationOps.length && generationOps.every((op) => !nodeIds.has(op.nodeId))) return "没有找到要触发生成的节点。"; if (ops.every((op) => op.type === "set_viewport")) return "视图已经是目标状态。"; if (selectOps.length && selectOps.every((op) => JSON.stringify(op.ids || []) === JSON.stringify(snapshot.selectedNodeIds))) return "选区已经是目标状态。"; - return "工具已执行，但画布状态没有变化；请在日志 tab 查看归一化操作和执行前后状态。"; + return "工具已执行，但画布状态没有变化；请在日志 tab 查看工具参数和执行前后状态。"; } function nodeToReference(node: CanvasNodeData): CanvasAssistantReference | null { @@ -857,17 +1069,19 @@ function buildAssistantReferences(nodes: CanvasNodeData[], selectedNodeIds: Set< .filter((item): item is CanvasAssistantReference => Boolean(item)); } -async function buildAgentMessages(snapshot: CanvasAgentSnapshot, history: CanvasAssistantMessage[], userMessage: CanvasAssistantMessage, loop?: OnlineLoopContext): Promise { +async function buildToolAgentMessages(snapshot: CanvasAgentSnapshot, history: CanvasAssistantMessage[], userMessage: CanvasAssistantMessage): Promise { const refs = userMessage.references || []; - const loopText = loop?.previous ? `\n\n上一轮工具执行结果：${JSON.stringify(loop.previous)}\n请判断用户任务是否已经完成。完成则返回 {"reply":"完成说明","ops":[]}；未完成则只返回下一步 ops。` : ""; return [ { role: "system", content: ONLINE_AGENT_PROMPT }, - ...history.slice(-8).map((message): ChatCompletionMessage => ({ role: message.role === "user" ? "user" : message.role === "system" ? "system" : "assistant", content: message.text })), + ...history + .filter((message) => message.role === "user" || message.role === "assistant" || message.role === "system") + .slice(-8) + .map((message): ResponseInputMessage => ({ role: message.role, content: message.text })), { role: "user", content: [ ...refs.flatMap((item) => (item.text ? [{ type: "text" as const, text: `选中节点 ${item.title}：${item.text}` }] : [])), - { type: "text", text: `当前画布：${JSON.stringify(compactSnapshot(snapshot))}\n\n用户需求：${userMessage.text}${loopText}` }, + { type: "text", text: `当前画布：${JSON.stringify(compactSnapshot(snapshot))}\n\n用户需求：${userMessage.text}` }, ...(await Promise.all(refs.filter((item) => item.dataUrl).map(async (item) => ({ type: "image_url" as const, image_url: { url: await imageToDataUrl(item) } })))), ], }, @@ -903,22 +1117,6 @@ function compactMetadata(metadata: CanvasNodeData["metadata"]) { }; } -function parseAgentResult(text: string): { reply: string; ops: CanvasAgentOp[] } { - const payload = JSON.parse(text.trim().replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, "")) as { reply?: unknown; ops?: unknown }; - const ops = Array.isArray(payload.ops) ? (payload.ops.filter((op) => op && typeof op === "object" && typeof (op as CanvasAgentOp).type === "string") as CanvasAgentOp[]) : []; - return { reply: String(payload.reply || (ops.length ? "已完成画布操作" : "没有需要执行的画布操作")), ops }; -} - -function partialAgentReply(text: string) { - const match = text.match(/"reply"\s*:\s*"((?:\\.|[^"\\])*)/); - if (!match) return ""; - try { - return JSON.parse(`"${match[1].replace(/\\?$/, "")}"`); - } catch { - return match[1]; - } -} - function createSession(): CanvasAssistantSession { const now = new Date().toISOString(); return { id: nanoid(), title: "新对话", messages: [], createdAt: now, updatedAt: now }; diff --git a/web/src/app/(user)/canvas/components/canvas-node-generation.ts b/web/src/app/(user)/canvas/components/canvas-node-generation.ts index 9397b1b..3fd8ecf 100644 --- a/web/src/app/(user)/canvas/components/canvas-node-generation.ts +++ b/web/src/app/(user)/canvas/components/canvas-node-generation.ts @@ -1,4 +1,4 @@ -import type { ChatCompletionMessage } from "@/services/api/image"; +import type { AiTextMessage } from "@/services/api/image"; import { imageReferenceLabel } from "@/lib/image-reference-prompt"; import { seedanceReferenceLabel } from "@/lib/seedance-video"; import type { ReferenceImage } from "@/types/image"; @@ -127,7 +127,7 @@ export function buildNodeGenerationInputs(nodeId: string, nodes: CanvasNodeData[ }); } -export function buildNodeChatMessages(context: NodeGenerationContext): ChatCompletionMessage[] { +export function buildNodeResponseMessages(context: NodeGenerationContext): AiTextMessage[] { if (!context.referenceImages.length) { return [{ role: "user", content: context.prompt }]; } diff --git a/web/src/services/api/image.ts b/web/src/services/api/image.ts index a04813d..6a0d364 100644 --- a/web/src/services/api/image.ts +++ b/web/src/services/api/image.ts @@ -7,11 +7,63 @@ import { buildImageReferencePromptText } from "@/lib/image-reference-prompt"; import { imageToDataUrl } from "@/services/image-storage"; import type { ReferenceImage } from "@/types/image"; -export type ChatCompletionMessage = { +export type AiTextMessage = { role: "system" | "user" | "assistant"; content: string | Array<{ type: "text"; text: string } | { type: "image_url"; image_url: { url: string } }>; }; +export type ResponseToolCall = { + id: string; + type: "function"; + function: { name: string; arguments: string }; +}; + +export type ResponseInputMessage = + | AiTextMessage + | { type: "function_call"; call_id: string; name: string; arguments: string } + | { role: "tool"; tool_call_id: string; content: string }; + +export type ResponseFunctionTool = { + type: "function"; + function: { + name: string; + description?: string; + parameters: Record; + strict?: boolean; + }; +}; + +export type ToolResponseResult = { + content: string; + toolCalls: ResponseToolCall[]; +}; + +type ToolChoice = "auto" | "required" | { type: "function"; name: string }; +type ResponseMessageContent = AiTextMessage["content"] | string; +type ResponseInputContent = { type: "input_text"; text: string } | { type: "input_image"; image_url: string }; +type ResponseInputItem = + | { role: "system" | "user" | "assistant"; content: string | ResponseInputContent[] } + | { type: "function_call"; call_id: string; name: string; arguments: string } + | { type: "function_call_output"; call_id: string; output: string }; +type ResponseApiToolDefinition = { + type: "function"; + name: string; + description?: string; + parameters: Record; + strict?: boolean; +}; +type ResponseApiOutputItem = + | { type?: "message"; content?: Array<{ type?: string; text?: string }> } + | { type?: "function_call"; id?: string; call_id?: string; name?: string; arguments?: string }; +type ResponseApiPayload = { + id?: string; + output?: ResponseApiOutputItem[]; + output_text?: string; + error?: { message?: string }; + code?: number; + msg?: string; +}; + type ImageApiResponse = { data?: Array>; error?: { message?: string }; @@ -148,20 +200,6 @@ function readStatusError(status: number | undefined, fallback: string) { return status ? `${fallback}：${status}` : fallback; } -function parseStreamChunk(chunk: string, onDelta: (value: string) => void) { - let deltaText = ""; - for (const eventBlock of chunk.split("\n\n")) { - const data = eventBlock - .split("\n") - .find((line) => line.startsWith("data: ")) - ?.slice(6); - if (!data || data === "[DONE]") continue; - const delta = (JSON.parse(data) as { choices?: Array<{ delta?: { content?: string } }> }).choices?.[0]?.delta?.content || ""; - deltaText += delta; - } - if (deltaText) onDelta(deltaText); -} - function withSystemPrompt(config: AiConfig, prompt: string) { const systemPrompt = config.systemPrompt.trim(); return systemPrompt ? `${systemPrompt}\n\n${prompt}` : prompt; @@ -178,11 +216,53 @@ function aiHeaders(config: AiConfig, contentType?: string) { }; } -function withSystemMessage(config: AiConfig, messages: ChatCompletionMessage[]) { +function withSystemMessage(config: AiConfig, messages: T[]): ResponseInputMessage[] { const systemPrompt = config.systemPrompt.trim(); return systemPrompt ? [{ role: "system" as const, content: systemPrompt }, ...messages] : messages; } +function toResponseInput(messages: ResponseInputMessage[]): ResponseInputItem[] { + return messages.flatMap((message): ResponseInputItem[] => { + if ("type" in message) return [message]; + if (message.role === "tool") return [{ type: "function_call_output", call_id: message.tool_call_id, output: message.content }]; + return [{ role: message.role, content: toResponseContent(message.content || "") }]; + }); +} + +function toResponseContent(content: ResponseMessageContent): string | ResponseInputContent[] { + if (!Array.isArray(content)) return String(content || ""); + return content.map((item) => (item.type === "text" ? { type: "input_text" as const, text: item.text } : { type: "input_image" as const, image_url: item.image_url.url })); +} + +function toResponseTool(tool: ResponseFunctionTool): ResponseApiToolDefinition { + return { + type: "function", + name: tool.function.name, + description: tool.function.description, + parameters: tool.function.parameters, + strict: tool.function.strict, + }; +} + +function parseToolResponse(payload: ResponseApiPayload): ToolResponseResult { + const output = payload.output || []; + const content = + payload.output_text || + output + .flatMap((item) => (item.type === "message" ? item.content || [] : [])) + .map((item) => item.text || "") + .join(""); + const toolCalls = output + .filter((item): item is Extract => item.type === "function_call") + .map((item) => ({ + id: item.call_id || item.id || "", + type: "function" as const, + function: { name: item.name || "", arguments: item.arguments || "{}" }, + })) + .filter((item) => item.id && item.function.name); + return { content, toolCalls }; +} + export async function requestGeneration(config: AiConfig, prompt: string) { const requestConfig = resolveModelRequestConfig(config, config.model || config.imageModel); const n = Math.max(1, Math.min(15, Math.floor(Math.abs(Number(config.count)) || 1))); @@ -242,66 +322,51 @@ export async function requestEdit(config: AiConfig, prompt: string, references: } } -export async function requestImageQuestion(config: AiConfig, messages: ChatCompletionMessage[], onDelta: (text: string) => void) { +export async function requestImageQuestion(config: AiConfig, messages: AiTextMessage[], onDelta: (text: string) => void) { const requestConfig = resolveModelRequestConfig(config, config.model || config.textModel); - let buffer = ""; - let answer = ""; - let processedLength = 0; - try { - const response = await axios.post( - aiApiUrl(requestConfig, "/chat/completions"), + const response = await axios.post( + aiApiUrl(requestConfig, "/responses"), { model: requestConfig.model, - messages: withSystemMessage(requestConfig, messages), - stream: true, + input: toResponseInput(withSystemMessage(requestConfig, messages)), }, { - headers: { - ...aiHeaders(requestConfig, "application/json"), - } as Record, - responseType: "text", - onDownloadProgress: (event) => { - const responseText = String(event.event?.target?.responseText || ""); - const nextText = responseText.slice(processedLength); - processedLength = responseText.length; - buffer += nextText; - const chunks = buffer.split("\n\n"); - buffer = chunks.pop() || ""; - for (const chunk of chunks) { - parseStreamChunk(chunk, (delta) => { - answer += delta; - onDelta(answer); - }); - } - }, + headers: aiHeaders(requestConfig, "application/json"), }, ); - if (typeof response.data === "object" && response.data && "code" in response.data && (response.data as { code?: number; msg?: string }).code !== 0) { - throw new Error((response.data as { msg?: string }).msg || "请求失败"); - } - if (typeof response.data === "string") { - let apiError = ""; - try { - const payload = JSON.parse(response.data) as { code?: number; msg?: string }; - if (typeof payload.code === "number" && payload.code !== 0) { - apiError = payload.msg || "请求失败"; - } - } catch { - // ignore plain text stream content - } - if (apiError) throw new Error(apiError); - } - if (buffer) { - parseStreamChunk(buffer, (delta) => { - answer += delta; - onDelta(answer); - }); - } + if (typeof response.data.code === "number" && response.data.code !== 0) throw new Error(response.data.msg || "请求失败"); + if (response.data.error?.message) throw new Error(response.data.error.message); + const answer = parseToolResponse(response.data).content || "没有返回内容"; + onDelta(answer); + return answer; + } catch (error) { + throw new Error(readAxiosError(error, "请求失败")); + } +} + +export async function requestToolResponse(config: AiConfig, messages: ResponseInputMessage[], tools: ResponseFunctionTool[], toolChoice: ToolChoice = "auto"): Promise { + const requestConfig = resolveModelRequestConfig(config, config.model || config.textModel); + try { + const response = await axios.post( + aiApiUrl(requestConfig, "/responses"), + { + model: requestConfig.model, + input: toResponseInput(withSystemMessage(requestConfig, messages)), + tools: tools.map(toResponseTool), + tool_choice: toolChoice, + parallel_tool_calls: false, + }, + { + headers: aiHeaders(requestConfig, "application/json"), + }, + ); + if (typeof response.data.code === "number" && response.data.code !== 0) throw new Error(response.data.msg || "请求失败"); + if (response.data.error?.message) throw new Error(response.data.error.message); + return parseToolResponse(response.data); } catch (error) { throw new Error(readAxiosError(error, "请求失败")); } - return answer || "没有返回内容"; } export async function fetchImageModels(config: Pick) {