diff --git a/CHANGELOG.md b/CHANGELOG.md index 5143415..aa1d0a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ + [修复] 本地 Agent 请求与发起标签页强绑定，校验工具结果归属，并在页面关闭后回退到最近聚焦页面。 + [修复] 多标签页共享 Codex 会话改由 Agent 广播同步，按线程过滤消息和状态，并禁止运行中切换会话。 + [修复] 本地 Agent 统一广播 Codex 运行状态，新连接标签页会同步禁用发送，并明确区分工具完成与本轮完成。 ++ [修复] 本地 Agent 上传的图片附件可创建为画布图片节点并连接生成流程，不再出现参考图空节点。 + [新增] 提示词来源增加自定义脚本抓取功能、支持新增来源。 ## v0.9.0 - 2026-07-17 diff --git a/canvas-agent/src/agents.ts b/canvas-agent/src/agents.ts index e215bf8..fb23c5b 100644 --- a/canvas-agent/src/agents.ts +++ b/canvas-agent/src/agents.ts @@ -482,6 +482,7 @@ function toolName(name: string) { if (name === "canvas_get_state") return "读取画布"; if (name === "canvas_get_selection") return "读取选区"; if (name === "canvas_export_snapshot") return "导出快照"; + if (name === "canvas_create_attachment_nodes") return "添加附件图片"; if (name === "canvas_create_text_node") return "创建文本"; if (name === "canvas_create_image_prompt_flow") return "创建生图流程"; if (name === "canvas_create_generation_flow") return "创建生成流程"; diff --git a/canvas-agent/src/canvas-session.test.ts b/canvas-agent/src/canvas-session.test.ts index 20780ae..a6742f1 100644 --- a/canvas-agent/src/canvas-session.test.ts +++ b/canvas-agent/src/canvas-session.test.ts @@ -43,6 +43,53 @@ test("画布写操作只发送给当前激活网页", async (t) => { assert.deepEqual(await result, { ok: true }); }); +test("当前 turn 的图片附件可在发起标签页画布创建图片节点", async (t) => { + const session = new CanvasSession(); + const first = connect(session, "first"); + t.after(() => first.close()); + const dataUrl = "data:image/png;base64,aW1hZ2U="; + session.setTurnAttachments("first", [{ id: "attachment-1", name: "商品.png", type: "image/png", size: 5, width: 1200, height: 600, dataUrl }]); + session.bindClient("first"); + + const result = session.callTool("canvas_create_attachment_nodes", { attachmentIds: ["attachment-1"], x: 100, y: 200 }); + const call = first.event("tool_call"); + const input = field(call, "input") as Record; + const nodes = input.nodes as Array>; + assert.equal(field(call, "name"), "canvas_create_attachment_nodes"); + assert.equal(nodes.length, 1); + assert.equal(nodes[0].attachmentId, "attachment-1"); + assert.equal(nodes[0].title, "商品.png"); + assert.deepEqual(nodes[0].position, { x: 100, y: 200 }); + assert.equal(nodes[0].width, 640); + assert.equal(nodes[0].height, 320); + assert.equal("dataUrl" in nodes[0], false); + assert.equal(session.getTurnAttachment("first", "attachment-1").dataUrl, dataUrl); + + session.resolveResult("first", { requestId: String(field(call, "requestId")), result: { ok: true } }); + const created = (await result) as { nodes: Array<{ id: string; attachmentId: string; title: string }> }; + assert.equal(created.nodes[0].id, nodes[0].id); + assert.equal(created.nodes[0].attachmentId, "attachment-1"); + session.clearTurnAttachments("first"); + assert.throws(() => session.getTurnAttachment("first", "attachment-1"), /找不到/); +}); + +test("图片附件只允许发起 turn 的标签页读取和落入画布", async (t) => { + const session = new CanvasSession(); + const first = connect(session, "first"); + const second = connect(session, "second"); + t.after(() => { + first.close(); + second.close(); + }); + session.setTurnAttachments("first", [{ id: "attachment-1", name: "商品.png", type: "image/png", dataUrl: "data:image/png;base64,aW1hZ2U=" }]); + session.bindClient("second"); + + await assert.rejects(session.callTool("canvas_create_attachment_nodes", { attachmentIds: ["attachment-1"] }), /发起标签页/); + assert.throws(() => session.getTurnAttachment("second", "attachment-1"), /发起标签页/); + assert.equal(first.event("tool_call"), undefined); + assert.equal(second.event("tool_call"), undefined); +}); + test("tool result is accepted only from the request client", async (t) => { const session = new CanvasSession(); const first = connect(session, "first"); diff --git a/canvas-agent/src/canvas-session.ts b/canvas-agent/src/canvas-session.ts index aadc5f8..6e47e04 100644 --- a/canvas-agent/src/canvas-session.ts +++ b/canvas-agent/src/canvas-session.ts @@ -3,9 +3,10 @@ import type { ServerResponse } from "node:http"; import { type ToolName } from "./schemas.js"; import { compactCanvasState, compactNode, isToolName, nextCanvasX, parseToolInput } from "./tools.js"; -import type { CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js"; +import type { AgentAttachment, CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js"; type PendingRequest = { clientId: string; resolve: (value: unknown) => void; reject: (error: Error) => void }; +type TurnAttachment = { clientId: string; id: string; name: string; type: string; size: number; width: number; height: number; dataUrl: string }; export type CodexState = { busy: boolean; threadId: string; turnId: string }; const SITE_TOOLS = new Set([ @@ -26,6 +27,7 @@ export class CanvasSession { private clientFocusOrder = new Map(); private pending = new Map(); private canvasStates = new Map(); + private turnAttachments = new Map(); private activeClientId = ""; private boundClientId = ""; private focusSequence = 0; @@ -103,6 +105,39 @@ export class CanvasSession { if (this.boundClientId === clientId) this.boundClientId = ""; } + setTurnAttachments(clientId: string, attachments: AgentAttachment[]) { + this.turnAttachments.clear(); + return attachments.flatMap((item, index) => { + if (!item.dataUrl?.startsWith("data:image/")) return []; + const id = item.id?.trim() || `attachment-${crypto.randomUUID()}`; + const attachment: TurnAttachment = { + clientId, + id, + name: item.name?.trim() || `图片 ${index + 1}`, + type: item.type?.startsWith("image/") ? item.type : item.dataUrl.match(/^data:([^;]+)/)?.[1] || "image/png", + size: positiveNumber(item.size, 0), + width: positiveNumber(item.width, 1024), + height: positiveNumber(item.height, 1024), + dataUrl: item.dataUrl, + }; + this.turnAttachments.set(id, attachment); + return [{ id, name: attachment.name, type: attachment.type, size: attachment.size, width: attachment.width, height: attachment.height }]; + }); + } + + clearTurnAttachments(clientId?: string) { + this.turnAttachments.forEach((item, id) => { + if (!clientId || item.clientId === clientId) this.turnAttachments.delete(id); + }); + } + + getTurnAttachment(clientId: string, attachmentId: string) { + const attachment = this.turnAttachments.get(attachmentId); + if (!attachment) throw new Error(`找不到本轮图片附件：${attachmentId}`); + if (attachment.clientId !== clientId) throw new Error("图片附件不属于当前 turn 的发起标签页"); + return attachment; + } + resolveResult(clientId: string, body: { requestId?: string; error?: string; result?: unknown }) { const item = body.requestId ? this.pending.get(body.requestId) : null; if (!item || !body.requestId || item.clientId !== clientId) return false; @@ -134,6 +169,7 @@ export class CanvasSession { const ids = new Set(this.canvasState?.selectedNodeIds || []); return { nodes: (this.canvasState?.nodes || []).filter((node) => ids.has(node.id)).map(compactNode) }; } + if (tool === "canvas_create_attachment_nodes") return await this.createAttachmentNodes(input as { attachmentIds: string[]; x?: number; y?: number; gap?: number; direction?: "row" | "column" }); if (tool === "canvas_create_node") { const data = input as { nodeType: CanvasNodeType; title?: string; x?: number; y?: number; width?: number; height?: number; metadata?: Record }; input = { ops: [{ type: "add_node", nodeType: data.nodeType, title: data.title, position: { x: data.x ?? nextCanvasX(this.canvasState), y: data.y ?? 0 }, width: data.width, height: data.height, metadata: data.metadata }] }; @@ -228,6 +264,32 @@ export class CanvasSession { return await this.requestCanvasTool(tool, input); } + private async createAttachmentNodes(input: { attachmentIds: string[]; x?: number; y?: number; gap?: number; direction?: "row" | "column" }) { + const clientId = this.targetClientId; + if (!this.clients.has(clientId)) throw new Error("当前没有已连接画布"); + const attachments = input.attachmentIds.map((id) => this.getTurnAttachment(clientId, id)); + const x = Number(input.x ?? nextCanvasX(this.canvasState)); + const y = Number(input.y ?? 0); + const gap = Number(input.gap ?? 40); + const direction = input.direction || "row"; + let offset = 0; + const nodes = attachments.map((attachment) => { + const size = fitAttachmentNodeSize(attachment.width, attachment.height); + const node = { + id: `image-${crypto.randomUUID()}`, + attachmentId: attachment.id, + title: attachment.name, + position: { x: direction === "row" ? x + offset : x, y: direction === "column" ? y + offset : y }, + width: size.width, + height: size.height, + }; + offset += (direction === "row" ? size.width : size.height) + gap; + return node; + }); + await this.requestCanvasTool("canvas_create_attachment_nodes", { nodes }); + return { nodes: nodes.map(({ id, attachmentId, title }) => ({ id, attachmentId, title })) }; + } + private async requestCanvasTool(name: ToolName, input: Record) { const requestId = crypto.randomUUID(); const clientId = this.targetClientId; @@ -326,3 +388,13 @@ function findNode(state: CanvasSnapshot | null, id: string): CanvasNode | undefi function cleanRecord(value: Record) { return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined && item !== "")); } + +function positiveNumber(value: unknown, fallback: number) { + const number = Number(value); + return Number.isFinite(number) && number > 0 ? number : fallback; +} + +function fitAttachmentNodeSize(width: number, height: number) { + const scale = Math.min(1, 640 / width, 640 / height); + return { width: width * scale, height: height * scale }; +} diff --git a/canvas-agent/src/config.ts b/canvas-agent/src/config.ts index 9c43170..633df39 100644 --- a/canvas-agent/src/config.ts +++ b/canvas-agent/src/config.ts @@ -7,7 +7,7 @@ export const DEFAULT_PORT = 17371; export const CONFIG_DIR = path.join(os.homedir(), ".infinite-canvas"); export const CONFIG_FILE = path.join(CONFIG_DIR, "canvas-agent.json"); export const VERSION = readPackageVersion(); -export const AGENT_PROMPT = "你正在帮助用户操作 Infinite Canvas 网站。切换网站页面用 site_navigate，可跳 / (首页)、/canvas (我的画布)、/canvas/:id (指定画布)、/image、/video、/prompts、/assets、/config。需要改动画布时优先使用已配置的 infinite-canvas MCP 工具：先 canvas_get_state 读取当前画布，再根据任务使用 canvas_create_text_node、canvas_generate_text、canvas_generate_image、canvas_generate_video、canvas_generate_audio、canvas_create_generation_flow、canvas_create_config_node、canvas_run_generation、canvas_update_node、canvas_connect_nodes 等通用工具；复杂批量改动再用 canvas_apply_ops，删除连线可用 delete_connections。若当前不在画布页，画布工具会报错，需先用 site_navigate 打开画布。想了解或打开用户已有画布，用 canvas_list_projects 获取画布清单和 id，再用 site_navigate 跳 /canvas/:id 打开。生图工作台可用 workbench_image_get_config 看可选项、workbench_image_generate 填提示词并生成；视频创作台对应 workbench_video_get_config 与 workbench_video_generate；用 prompts_search 分页搜索提示词库；用 assets_list 查看「我的素材」、assets_add 新增文本或图片素材。需要生成内容时直接调用对应生成工具，不要绑定特定业务场景。不要模拟鼠标点击，不要要求用户手动复制 JSON。"; +export const AGENT_PROMPT = "你正在帮助用户操作 Infinite Canvas 网站。切换网站页面用 site_navigate，可跳 / (首页)、/canvas (我的画布)、/canvas/:id (指定画布)、/image、/video、/prompts、/assets、/config。需要改动画布时优先使用已配置的 infinite-canvas MCP 工具：先 canvas_get_state 读取当前画布，再根据任务使用 canvas_create_text_node、canvas_generate_text、canvas_generate_image、canvas_generate_video、canvas_generate_audio、canvas_create_generation_flow、canvas_create_config_node、canvas_run_generation、canvas_update_node、canvas_connect_nodes 等通用工具；复杂批量改动再用 canvas_apply_ops，删除连线可用 delete_connections。本轮若有用户上传的图片附件，会同时给出 attachmentId；用户要求把附件放入画布或作为生成参考图时，必须先用 canvas_create_attachment_nodes 创建真实图片节点，再把返回的节点 ID 传给 canvas_create_generation_flow.referenceNodeIds，不要创建空图片占位节点。若当前不在画布页，画布工具会报错，需先用 site_navigate 打开画布。想了解或打开用户已有画布，用 canvas_list_projects 获取画布清单和 id，再用 site_navigate 跳 /canvas/:id 打开。生图工作台可用 workbench_image_get_config 看可选项、workbench_image_generate 填提示词并生成；视频创作台对应 workbench_video_get_config 与 workbench_video_generate；用 prompts_search 分页搜索提示词库；用 assets_list 查看「我的素材」、assets_add 新增文本或图片素材。需要生成内容时直接调用对应生成工具，不要绑定特定业务场景。不要模拟鼠标点击，不要要求用户手动复制 JSON。"; export type SiteWorkspaceConfig = { workspacePath: string; activeThreadId?: string; pinnedThreadIds?: string[] }; export type CanvasAgentConfig = { url: string; token: string; origins?: string[]; workspace?: SiteWorkspaceConfig }; diff --git a/canvas-agent/src/http-server.ts b/canvas-agent/src/http-server.ts index f2d8c9d..40bc768 100644 --- a/canvas-agent/src/http-server.ts +++ b/canvas-agent/src/http-server.ts @@ -50,6 +50,13 @@ export function startHttpServer() { const ok = session.resolveResult(String(req.query.clientId || ""), req.body); res.status(ok ? 200 : 409).json({ ok }); }); + app.get("/agent/attachments/:attachmentId", route(async (req, res) => { + const attachment = session.getTurnAttachment(String(req.query.clientId || ""), routeParam(req.params.attachmentId)); + const data = attachment.dataUrl.split(",", 2)[1]; + if (!data) throw new Error("图片附件内容无效"); + res.setHeader("Cache-Control", "no-store"); + res.type(attachment.type).send(Buffer.from(data, "base64")); + })); app.post("/api/tools", route(async (req, res) => res.json({ ok: true, result: await session.callTool(req.body?.name, req.body?.input || {}) }))); app.get("/agent/codex/workspace", (_req, res) => { const workspace = ensureSiteWorkspace(config); @@ -113,6 +120,7 @@ export function startHttpServer() { await verifyCodexThreadWorkspace(emit, threadId, workspace.workspacePath); setActiveThread(threadId); } + const attachmentRefs = session.setTurnAttachments(clientId, attachments); const chatMessage = { sourceClientId: clientId, message: { id: String(req.body?.messageId || Date.now()), role: "user", text: String(req.body?.messageText || prompt || `发送了 ${attachments.length} 张图片`) }, @@ -122,7 +130,7 @@ export function startHttpServer() { const data = payload && typeof payload === "object" && !Array.isArray(payload) ? payload as Record : { value: payload }; session.emitThread(type, threadId, data); }; - void runCodexTurn(withAgentPrompt(prompt), turnEmit, attachments, { + void runCodexTurn(withAgentPrompt(withAttachmentContext(prompt, attachmentRefs)), turnEmit, attachments, { threadId, cwd: workspace.workspacePath, appEmit: emit, @@ -143,6 +151,7 @@ export function startHttpServer() { session.setCodexState({ busy: true, threadId, turnId }); }, onFinish: () => { + session.clearTurnAttachments(clientId); if (clientId) session.releaseClient(clientId); session.setCodexState({ busy: false, threadId, turnId }); }, @@ -206,3 +215,9 @@ function validToken(req: Request, url: URL, token: string) { const header = req.headers["x-canvas-agent-token"]; return url.searchParams.get("token") === token || header === token || (Array.isArray(header) && header.includes(token)); } + +function withAttachmentContext(prompt: string, attachments: Array<{ id: string; name: string }>) { + if (!attachments.length) return prompt; + const list = attachments.map((item, index) => `${index + 1}. attachmentId=${item.id}, name=${JSON.stringify(item.name)}`).join("\n"); + return `${prompt}\n\n本轮可用图片附件（顺序与图片输入一致）：\n${list}\n需要把附件放入画布或作为生成参考图时，先调用 canvas_create_attachment_nodes，再使用返回的画布节点 ID 创建生成流程。`; +} diff --git a/canvas-agent/src/schemas.ts b/canvas-agent/src/schemas.ts index 479098b..b155598 100644 --- a/canvas-agent/src/schemas.ts +++ b/canvas-agent/src/schemas.ts @@ -14,6 +14,7 @@ export const toolNames = [ "canvas_export_snapshot", "canvas_apply_ops", "canvas_create_node", + "canvas_create_attachment_nodes", "canvas_create_text_node", "canvas_create_text_nodes", "canvas_create_config_node", @@ -94,6 +95,7 @@ export const toolInputSchemas = { canvas_export_snapshot: z.object({}).passthrough(), canvas_apply_ops: z.object({ ops: z.array(canvasOpSchema) }), canvas_create_node: z.object({ nodeType: nodeTypeSchema, title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), metadata: recordSchema.optional() }), + canvas_create_attachment_nodes: z.object({ attachmentIds: z.array(z.string()).min(1), x: z.number().optional(), y: z.number().optional(), gap: z.number().optional(), direction: z.enum(["row", "column"]).optional() }), canvas_create_text_node: z.object({ text: z.string().optional(), x: z.number().optional(), y: z.number().optional(), title: z.string().optional(), width: z.number().optional(), height: z.number().optional() }), canvas_create_text_nodes: z.object({ items: z.array(textNodeSchema).min(1), x: z.number().optional(), y: z.number().optional(), gap: z.number().optional(), direction: z.enum(["row", "column"]).optional() }), canvas_create_config_node: z.object({ prompt: z.string().optional(), mode: generationModeSchema.optional(), title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), autoRun: z.boolean().optional() }).merge(generationOptionsSchema), @@ -130,6 +132,7 @@ export const toolDescriptions: Record = { canvas_export_snapshot: "导出当前画布快照，用于理解布局。", canvas_apply_ops: "批量操作当前网页画布。ops 支持 add_node、update_node、delete_node、delete_connections、connect_nodes、set_viewport、select_nodes、run_generation。", canvas_create_node: "创建任意类型节点：text、image、config、video、audio。适合创建占位图、媒体占位、配置节点或自定义 metadata 节点。", + canvas_create_attachment_nodes: "把当前对话中用户上传的图片附件创建成真实画布图片节点。attachmentIds 使用本轮附件清单中的 ID；返回的节点 ID 可传给 canvas_create_generation_flow.referenceNodeIds 作为生成参考图。", canvas_create_text_node: "在当前画布创建单个文本节点。", canvas_create_text_nodes: "批量创建文本节点，适合生成标题、段落、脚本、说明等内容块。", canvas_create_config_node: "创建生成配置节点，可指定 text/image/video/audio 模式和生成参数，可选择立即触发生成。", diff --git a/canvas-agent/src/types.ts b/canvas-agent/src/types.ts index d513918..6c6f458 100644 --- a/canvas-agent/src/types.ts +++ b/canvas-agent/src/types.ts @@ -5,4 +5,4 @@ export type CanvasNode = { id: string; type: CanvasNodeType; title?: string; pos export type CanvasConnection = { id: string; fromNodeId: string; toNodeId: string }; export type CanvasSnapshot = { projectId?: string; title?: string; nodes?: CanvasNode[]; connections?: CanvasConnection[]; selectedNodeIds?: string[]; viewport?: Viewport; clientId?: string }; export type AgentEmit = (type: string, payload: unknown) => void; -export type AgentAttachment = { name?: string; type?: string; dataUrl?: string }; +export type AgentAttachment = { id?: string; name?: string; type?: string; size?: number; width?: number; height?: number; dataUrl?: string }; diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index 809e130..9e6abe8 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -9,3 +9,4 @@ description: 当前版本已实现但仍需人工验证的变更项 - 本地 Agent 多标签页隔离：同时打开两个不同画布并连接同一个 Agent，分别聚焦标签页后通过 MCP 读取和修改画布，操作应只落在当前聚焦页面；网页面板发起的整个 Codex turn 应固定操作发起页面，即使中途聚焦另一标签页也不能切换目标；关闭当前页面后应回退到最近聚焦且仍连接的页面，其他页面回传同一请求结果应被拒绝。 - 本地 Agent 多标签页会话同步：所有标签页共享同一个站点级 Codex 活跃线程；任一页面发送消息、新建、恢复或删除会话后，其他页面应同步活跃线程和聊天记录；Agent 输出仅显示在事件所属线程，运行中不能新建、恢复、删除或再次发送任务。 - 本地 Agent 运行状态同步：在一个标签页运行较长 Codex 任务，等待某张工具卡显示「工具完成」后再打开或刷新第二个标签页；第二个标签页应立即显示 Codex 正在运行并禁用发送，整轮结束后两个标签页同时恢复；工具卡只显示「工具完成」，整轮结束由「本轮完成」表示。 +- 本地 Agent 图片附件落画布：在右侧 Agent 上传参考图并要求基于商品信息创建生图流程，附件应创建为保持原比例的真实图片节点，分析提示词应创建为文本节点，二者都应连接到生成配置节点；刷新页面后参考图仍可显示并参与生成。任务中途切换到其他标签页时，附件只能写入发起任务的标签页；若发起标签页关闭，附件读取应失败且不能落入其他画布。 diff --git a/web/src/components/canvas/canvas-local-agent-panel.tsx b/web/src/components/canvas/canvas-local-agent-panel.tsx index 26bc9f4..4476b96 100644 --- a/web/src/components/canvas/canvas-local-agent-panel.tsx +++ b/web/src/components/canvas/canvas-local-agent-panel.tsx @@ -5,7 +5,11 @@ import copyToClipboard from "copy-to-clipboard"; import { Copy, FolderOpen, History, KeyRound, Link2, LoaderCircle, PlugZap, Plus, RefreshCw, Square, Terminal, Trash2 } from "lucide-react"; import { canvasThemes } from "@/lib/canvas-theme"; +import { imageMetadata } from "@/lib/canvas/canvas-node-factory"; +import { fitNodeSize } from "@/lib/canvas/canvas-node-size"; +import { readImageMeta } from "@/lib/image-utils"; import { randomId } from "@/lib/utils"; +import { uploadImage } from "@/services/image-storage"; import { useThemeStore } from "@/stores/use-theme-store"; import { useUserStore } from "@/stores/use-user-store"; import { useShallow } from "zustand/react/shallow"; @@ -276,7 +280,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb messageId, clientId: clientIdRef.current, threadId: useAgentStore.getState().activeThreadId || undefined, - attachments: files.map(({ name, type, dataUrl }) => ({ name, type, dataUrl })), + attachments: files.map(({ id, name, type, size, width, height, dataUrl }) => ({ id, name, type, size, width, height, dataUrl })), }), }); if (data.threadId) setAgentState({ activeThreadId: data.threadId }); @@ -316,9 +320,10 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb const next = await Promise.all( images.slice(0, Math.max(0, MAX_ATTACHMENTS - prev.length)).map(async (file) => { const dataUrl = await readDataUrl(file); + const meta = await readImageMeta(dataUrl); const url = URL.createObjectURL(file); attachmentUrlsRef.current.add(url); - return { id: createId(), name: file.name, type: file.type, size: file.size, url, dataUrl }; + return { id: createId(), name: file.name, type: file.type, size: file.size, width: meta.width, height: meta.height, url, dataUrl }; }), ); const merged = [...prev, ...next]; @@ -346,7 +351,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb }; const handleToolCall = async (endpoint: string, token: string, payload: AgentPendingToolCall) => { - if (confirmToolsRef.current && payload.name === "canvas_apply_ops") { + if (confirmToolsRef.current && isCanvasWriteTool(payload.name)) { if (pendingToolRef.current) { await postToolResult(endpoint, token, clientIdRef.current, { requestId: payload.requestId, error: "仍有待确认的画布工具调用" }); return; @@ -378,6 +383,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb const input: { ops?: CanvasAgentOp[]; path?: string } = payload.input || {}; addEventLog(toolName(payload.name), payload, payload); let result: unknown; + let appliedOps = input.ops || []; if (payload.name === "site_navigate") { const path = input.path || "/"; navigate(path); @@ -385,8 +391,14 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb } else if (payload.name === "canvas_apply_ops") { const context = canvasContextRef.current; if (!context) throw new Error("当前不在画布页，请先用 site_navigate 打开画布"); - result = context.applyOps(input.ops || []); + result = context.applyOps(appliedOps); void postState(endpoint, token, clientIdRef.current, result as CanvasAgentSnapshot); + } else if (payload.name === "canvas_create_attachment_nodes") { + const context = canvasContextRef.current; + if (!context) throw new Error("当前不在画布页，请先用 site_navigate 打开画布"); + appliedOps = await attachmentNodeOps(endpoint, token, clientIdRef.current, payload.input?.nodes); + result = context.applyOps(appliedOps); + await postState(endpoint, token, clientIdRef.current, result as CanvasAgentSnapshot); } else { const snapshot = canvasContextRef.current?.snapshot; if (!snapshot) throw new Error("当前不在画布页，请先用 site_navigate 打开画布"); @@ -397,7 +409,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb addMessage({ role: "tool", title: `${toolName(payload.name)}完成`, - text: payload.name === "canvas_apply_ops" ? summarizeCanvasAgentOps(input.ops || []) || "画布操作" : payload.name === "site_navigate" ? `已跳转到 ${input.path || "/"}` : "已完成", + text: appliedOps.length ? summarizeCanvasAgentOps(appliedOps) || "画布操作" : payload.name === "site_navigate" ? `已跳转到 ${input.path || "/"}` : "已完成", detail: { requestId: payload.requestId, name: payload.name, input, result }, }); } catch (error) { @@ -1113,6 +1125,7 @@ function toolName(name: string) { if (name === "canvas_get_selection") return "读取选区"; if (name === "canvas_export_snapshot") return "导出快照"; if (name === "canvas_create_node") return "创建节点"; + if (name === "canvas_create_attachment_nodes") return "添加附件图片"; if (name === "canvas_create_text_node") return "创建文本"; if (name === "canvas_create_text_nodes") return "批量创建文本"; if (name === "canvas_create_config_node") return "创建生成配置"; @@ -1220,9 +1233,7 @@ function mergeAgentText(prev: string, next: string) { } function promptWithAttachments(text: string, attachments: AgentAttachment[]) { - if (!attachments.length) return text; - const names = attachments.map((item) => item.name).join("、"); - return [text, `用户上传了 ${attachments.length} 张图片附件：${names}。`].filter(Boolean).join("\n\n"); + return text || (attachments.length ? "请处理上传的图片附件。" : ""); } function attachmentPayloadBytes(attachments: AgentAttachment[]) { @@ -1233,6 +1244,41 @@ function formatBytes(bytes: number) { return bytes > 1024 * 1024 ? `${(bytes / 1024 / 1024).toFixed(1)}MB` : `${Math.ceil(bytes / 1024)}KB`; } +function isCanvasWriteTool(name: string) { + return name === "canvas_apply_ops" || name === "canvas_create_attachment_nodes"; +} + +async function attachmentNodeOps(endpoint: string, token: string, clientId: string, value: unknown): Promise { + const nodes = Array.isArray(value) ? value : []; + if (!nodes.length) throw new Error("没有可添加的图片附件"); + return await Promise.all( + nodes.map(async (value) => { + const item = value as { id?: unknown; attachmentId?: unknown; title?: unknown; position?: unknown }; + const id = String(item.id || ""); + const attachmentId = String(item.attachmentId || ""); + if (!id || !attachmentId) throw new Error("图片附件节点参数无效"); + const res = await fetch(`${endpoint}/agent/attachments/${encodeURIComponent(attachmentId)}?token=${encodeURIComponent(token)}&clientId=${encodeURIComponent(clientId)}`); + if (!res.ok) { + const body = (await res.json().catch(() => null)) as { error?: string } | null; + throw new Error(body?.error || "读取图片附件失败"); + } + const image = await uploadImage(await res.blob()); + const size = fitNodeSize(image.width, image.height); + const position = item.position && typeof item.position === "object" ? (item.position as { x?: unknown; y?: unknown }) : {}; + return { + type: "add_node" as const, + id, + nodeType: "image" as const, + title: String(item.title || "参考图"), + position: { x: Number(position.x) || 0, y: Number(position.y) || 0 }, + width: size.width, + height: size.height, + metadata: imageMetadata(image), + }; + }), + ); +} + async function fetchAgentJson(endpoint: string, token: string, path: string, init?: RequestInit) { const url = `${endpoint}${path}${path.includes("?") ? "&" : "?"}token=${encodeURIComponent(token)}`; const res = await fetch(url, init); diff --git a/web/src/stores/use-agent-store.ts b/web/src/stores/use-agent-store.ts index 993cf96..20871f5 100644 --- a/web/src/stores/use-agent-store.ts +++ b/web/src/stores/use-agent-store.ts @@ -3,7 +3,7 @@ import { create } from "zustand"; import type { CanvasAgentOp, CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops"; export type AgentChatRole = "user" | "assistant" | "system" | "tool" | "error"; -export type AgentAttachment = { id: string; name: string; type: string; size: number; url: string; dataUrl: string }; +export type AgentAttachment = { id: string; name: string; type: string; size: number; width: number; height: number; url: string; dataUrl: string }; export type AgentChatItem = { id: string; role: AgentChatRole; title?: string; text: string; meta?: string; detail?: unknown; attachments?: AgentAttachment[]; streamId?: string }; export type AgentEventLog = { id: string; time: string; title: string; text: string; raw?: unknown }; export type AgentPendingToolCall = { requestId: string; name: string; input?: { ops?: CanvasAgentOp[]; path?: string } & Record };