From 1b289f6a7721fe1f2016145954fefefeb57b0bd6 Mon Sep 17 00:00:00 2001 From: HouYunFei <1844025705@qq.com> Date: Tue, 16 Jun 2026 16:52:52 +0800 Subject: [PATCH] feat(agent): align online Agent tools with local Agent capabilities and enhance node creation and generation configuration --- docs/content/docs/progress/pending-test.mdx | 2 + .../components/canvas-assistant-panel.tsx | 486 ++++++++++++------ 2 files changed, 330 insertions(+), 158 deletions(-) diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index e656fbc..8ff7c39 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -8,3 +8,5 @@ description: 当前版本已实现但仍需人工验证的变更项 - 在线 `Agent` 工具卡片详情不再保存和展示完整上下文消息，只展示当前工具调用和执行结果；需要验证详情内容体积不会随对话历史膨胀。 - 文本生成和在线 `Agent` 的 Responses API 请求已改为流式输出；需要验证生成文本、工具确认前后的对话回复会实时刷新。 - 在线 `Agent` 自动执行工具时会显示已完成工具卡片，并且从生成配置节点触发生成时会使用配置节点的提示词；需要验证关闭工具确认后连接节点并生图时请求不再发送空 `prompt`。 +- 网站端在线 `Agent` 工具已对齐本地 Agent 的通用画布工具，支持创建节点、创建生成流程、直接触发文本/图片/视频/音频生成和 `canvas_apply_ops`；需要验证只有文本节点时也能通过 `canvas_generate_image` 创建生图流程并执行。 +- 网站端在线 `Agent` 新建生成配置节点时会继承当前页面的默认生成设置，包括生图模型、尺寸和数量；需要验证 AI 创建生图流程时使用的模型和手动添加生成配置节点一致。 diff --git a/web/src/app/(user)/canvas/components/canvas-assistant-panel.tsx b/web/src/app/(user)/canvas/components/canvas-assistant-panel.tsx index 818e035..f82fdd2 100644 --- a/web/src/app/(user)/canvas/components/canvas-assistant-panel.tsx +++ b/web/src/app/(user)/canvas/components/canvas-assistant-panel.tsx @@ -20,109 +20,100 @@ import { Select, SelectContent, SelectItem, SelectTrigger } from "@/components/u import { CanvasPromptLibrary } from "./canvas-prompt-library"; import { AgentChatComposer, AgentChatMessage, AgentModeSwitch, AgentPanelTabs, AgentWorkingMessage, type CanvasAgentChatMessage, type CanvasAgentMode } from "./canvas-agent-chat-ui"; import { CanvasLocalAgentPanel } from "./canvas-local-agent-panel"; +import { NODE_DEFAULT_SIZE } from "../constants"; import { CanvasNodeType, type CanvasAssistantMessage, type CanvasAssistantReference, type CanvasAssistantSession, type CanvasNodeData } from "../types"; import { useCanvasAgentStore } from "../stores/use-canvas-agent-store"; -import { type CanvasAgentOp, type CanvasAgentSnapshot } from "../utils/canvas-agent-ops"; +import { summarizeCanvasAgentOps, type CanvasAgentOp, type CanvasAgentSnapshot } from "../utils/canvas-agent-ops"; export const CANVAS_AGENT_PANEL_MOTION_MS = 500; const PANEL_MOTION_SECONDS = CANVAS_AGENT_PANEL_MOTION_MS / 1000; const ONLINE_AGENT_MAX_STEPS = 4; const ONLINE_AGENT_PROMPT = - "你是 Infinite Canvas 网页内置在线画布助手。当前画布 JSON 会随用户消息提供。首轮必须调用工具：只读问题调用 canvas_get_state，需要修改画布时调用对应写工具。不要输出 JSON ops，不要编造执行结果。工具参数必须完全符合 schema，并且必须使用当前画布 JSON 中真实存在的 id；不要把 title、type、Note、生成配置等名称当 id。缺少必要 id 或用户意图不明确时直接说明需要用户明确选择或说明，不要猜测。生图、生成文本、视频或音频时，如果使用节点作为输入，先调用 canvas_connect_nodes 连接输入节点到生成配置节点，再调用 canvas_configure_generation，最后调用 canvas_run_generation。工具返回结果后，再根据真实结果回答用户。"; + "你是 Infinite Canvas 网页内置在线画布助手。当前画布 JSON 会随用户消息提供。首轮必须调用工具：只读问题调用 canvas_get_state，需要改动画布时调用和本地 Agent 一致的 infinite-canvas 工具。需要生成内容时直接调用 canvas_generate_text、canvas_generate_image、canvas_generate_video、canvas_generate_audio 或 canvas_create_generation_flow；需要精确批量操作时调用 canvas_apply_ops。不要输出 JSON ops，不要编造执行结果。工具参数涉及已有节点时必须使用当前画布 JSON 中真实存在的 id；缺少必要 id 或用户意图不明确时直接说明需要用户明确选择或说明，不要猜测。工具返回结果后，再根据真实结果回答用户。"; +const JSON_RECORD_SCHEMA = { type: "object", additionalProperties: true }; +const POSITION_SCHEMA = { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false }; +const VIEWPORT_SCHEMA = { type: "object", properties: { x: { type: "number" }, y: { type: "number" }, k: { type: "number" } }, required: ["x", "y", "k"], additionalProperties: false }; +const NODE_TYPE_SCHEMA = { type: "string", enum: ["image", "text", "config", "video", "audio"] }; +const GENERATION_MODE_SCHEMA = { type: "string", enum: ["text", "image", "video", "audio"] }; +const GENERATION_OPTION_PROPERTIES = { + model: { type: "string" }, + size: { type: "string" }, + quality: { type: "string" }, + count: { type: "number" }, + seconds: { type: "string" }, + vquality: { type: "string" }, + generateAudio: { type: "string" }, + watermark: { type: "string" }, + audioVoice: { type: "string" }, + audioFormat: { type: "string" }, + audioSpeed: { type: "string" }, + audioInstructions: { type: "string" }, +}; +const CANVAS_OP_SCHEMA = { + type: "object", + properties: { + type: { type: "string", enum: ["add_node", "update_node", "delete_node", "delete_connections", "connect_nodes", "set_viewport", "select_nodes", "run_generation"] }, + id: { type: "string" }, + ids: { type: "array", items: { type: "string" } }, + nodeType: NODE_TYPE_SCHEMA, + title: { type: "string" }, + x: { type: "number" }, + y: { type: "number" }, + width: { type: "number" }, + height: { type: "number" }, + position: POSITION_SCHEMA, + metadata: JSON_RECORD_SCHEMA, + patch: JSON_RECORD_SCHEMA, + all: { type: "boolean" }, + fromNodeId: { type: "string" }, + toNodeId: { type: "string" }, + viewport: VIEWPORT_SCHEMA, + nodeId: { type: "string" }, + mode: GENERATION_MODE_SCHEMA, + prompt: { type: "string" }, + }, + required: ["type"], + additionalProperties: false, +}; +const ONLINE_READ_TOOLS = new Set(["canvas_get_state", "canvas_get_selection", "canvas_export_snapshot"]); + +function toolDefinition(name: string, description: string, properties: Record, required: string[] = [], strict = false): ResponseFunctionTool { + return { type: "function", function: { name, description, parameters: { type: "object", properties, required, additionalProperties: false }, strict } }; +} + +function generationToolDefinition(name: string, description: string, mode?: "text" | "image" | "video" | "audio") { + return toolDefinition( + name, + description, + { prompt: { type: "string" }, title: { type: "string" }, x: { type: "number" }, y: { type: "number" }, referenceNodeIds: { type: "array", items: { type: "string" } }, ...(mode ? {} : { mode: GENERATION_MODE_SCHEMA }), autoRun: { type: "boolean" }, ...GENERATION_OPTION_PROPERTIES }, + ["prompt"], + ); +} + const ONLINE_AGENT_TOOLS: ResponseFunctionTool[] = [ - { - type: "function", - function: { - name: "canvas_get_state", - description: "读取当前画布状态。只读问题或需要确认节点 id 时使用。", - parameters: { type: "object", properties: {}, required: [], additionalProperties: false }, - strict: true, - }, - }, - { - type: "function", - function: { - name: "canvas_connect_nodes", - description: "连接两个已存在节点。必须传真实 fromNodeId 和 toNodeId。", - parameters: { - type: "object", - properties: { - fromNodeId: { type: "string" }, - toNodeId: { type: "string" }, - }, - required: ["fromNodeId", "toNodeId"], - additionalProperties: false, - }, - strict: true, - }, - }, - { - type: "function", - function: { - name: "canvas_configure_generation", - description: "配置一个生成配置节点的生成模式和提示词引用。必须传真实 configNodeId；promptNodeIds 必须已连接到该生成配置节点。", - parameters: { - type: "object", - properties: { - configNodeId: { type: "string" }, - mode: { type: "string", enum: ["text", "image", "video", "audio"] }, - promptNodeIds: { type: "array", items: { type: "string" } }, - prompt: { type: "string" }, - }, - required: ["configNodeId", "mode", "promptNodeIds", "prompt"], - additionalProperties: false, - }, - strict: true, - }, - }, - { - type: "function", - function: { - name: "canvas_run_generation", - description: "触发一个生成配置节点执行生成。必须传真实 configNodeId。", - parameters: { - type: "object", - properties: { - configNodeId: { type: "string" }, - mode: { type: "string", enum: ["text", "image", "video", "audio"] }, - }, - required: ["configNodeId", "mode"], - additionalProperties: false, - }, - strict: true, - }, - }, - { - type: "function", - function: { - name: "canvas_delete_nodes", - description: "删除指定节点。必须传真实 nodeIds。", - parameters: { - type: "object", - properties: { nodeIds: { type: "array", items: { type: "string" }, minItems: 1 } }, - required: ["nodeIds"], - additionalProperties: false, - }, - strict: true, - }, - }, - { - type: "function", - function: { - name: "canvas_delete_connections", - description: "删除指定连线，或删除全部连线。", - parameters: { - type: "object", - properties: { - connectionIds: { type: "array", items: { type: "string" } }, - all: { type: "boolean" }, - }, - required: ["connectionIds", "all"], - additionalProperties: false, - }, - strict: true, - }, - }, + toolDefinition("canvas_get_state", "读取当前网页画布的节点、连线、选区和视口。", {}), + toolDefinition("canvas_get_selection", "读取当前网页画布选中的节点。", {}), + toolDefinition("canvas_export_snapshot", "导出当前画布快照，用于理解布局。", {}), + toolDefinition("canvas_apply_ops", "批量操作当前网页画布。ops 支持 add_node、update_node、delete_node、delete_connections、connect_nodes、set_viewport、select_nodes、run_generation。", { ops: { type: "array", items: CANVAS_OP_SCHEMA } }, ["ops"], false), + toolDefinition("canvas_create_node", "创建任意类型节点：text、image、config、video、audio。适合创建占位图、媒体占位、配置节点或自定义 metadata 节点。", { nodeType: NODE_TYPE_SCHEMA, title: { type: "string" }, x: { type: "number" }, y: { type: "number" }, width: { type: "number" }, height: { type: "number" }, metadata: JSON_RECORD_SCHEMA }, ["nodeType"]), + toolDefinition("canvas_create_text_node", "在当前画布创建单个文本节点。", { text: { type: "string" }, x: { type: "number" }, y: { type: "number" }, title: { type: "string" }, width: { type: "number" }, height: { type: "number" } }), + toolDefinition("canvas_create_text_nodes", "批量创建文本节点，适合生成标题、段落、脚本、说明等内容块。", { items: { type: "array", minItems: 1, items: { type: "object", properties: { text: { type: "string" }, title: { type: "string" }, x: { type: "number" }, y: { type: "number" }, width: { type: "number" }, height: { type: "number" } }, required: ["text"], additionalProperties: false } }, x: { type: "number" }, y: { type: "number" }, gap: { type: "number" }, direction: { type: "string", enum: ["row", "column"] } }, ["items"]), + toolDefinition("canvas_create_config_node", "创建生成配置节点，可指定 text/image/video/audio 模式和生成参数，可选择立即触发生成。", { prompt: { type: "string" }, mode: GENERATION_MODE_SCHEMA, title: { type: "string" }, x: { type: "number" }, y: { type: "number" }, width: { type: "number" }, height: { type: "number" }, autoRun: { type: "boolean" }, ...GENERATION_OPTION_PROPERTIES }), + toolDefinition("canvas_create_image_prompt_flow", "创建提示词文本节点和图片生成配置节点，并自动连线，可选择立即触发生图。", { prompt: { type: "string" }, x: { type: "number" }, y: { type: "number" }, autoRun: { type: "boolean" }, ...GENERATION_OPTION_PROPERTIES }, ["prompt"]), + generationToolDefinition("canvas_create_generation_flow", "创建通用生成流程：提示词文本节点、生成配置节点、参考节点连线，可用于文案、生图、视频或音频。"), + generationToolDefinition("canvas_generate_text", "创建通用文本生成流程并立即触发生成。", "text"), + generationToolDefinition("canvas_generate_image", "创建通用图片生成流程并立即触发生成。", "image"), + generationToolDefinition("canvas_generate_video", "创建通用视频生成流程并立即触发生成。", "video"), + generationToolDefinition("canvas_generate_audio", "创建通用音频生成流程并立即触发生成。", "audio"), + toolDefinition("canvas_update_node", "更新节点基础字段或 metadata。", { id: { type: "string" }, patch: JSON_RECORD_SCHEMA, metadata: JSON_RECORD_SCHEMA }, ["id"]), + toolDefinition("canvas_update_node_text", "更新文本节点内容和标题。", { id: { type: "string" }, text: { type: "string" }, title: { type: "string" } }, ["id", "text"]), + toolDefinition("canvas_move_nodes", "移动一个或多个节点，支持绝对坐标或 dx/dy 偏移。", { items: { type: "array", minItems: 1, items: { type: "object", properties: { id: { type: "string" }, x: { type: "number" }, y: { type: "number" }, dx: { type: "number" }, dy: { type: "number" } }, required: ["id"], additionalProperties: false } } }, ["items"]), + toolDefinition("canvas_resize_node", "调整节点尺寸。", { id: { type: "string" }, width: { type: "number" }, height: { type: "number" }, freeResize: { type: "boolean" } }, ["id", "width", "height"]), + toolDefinition("canvas_delete_nodes", "删除指定节点及相关连线。", { ids: { type: "array", items: { type: "string" }, minItems: 1 } }, ["ids"]), + toolDefinition("canvas_connect_nodes", "批量连接节点。", { connections: { type: "array", minItems: 1, items: { type: "object", properties: { fromNodeId: { type: "string" }, toNodeId: { type: "string" } }, required: ["fromNodeId", "toNodeId"], additionalProperties: false } } }, ["connections"]), + toolDefinition("canvas_select_nodes", "设置当前选中节点。", { ids: { type: "array", items: { type: "string" } } }, ["ids"]), + toolDefinition("canvas_set_viewport", "调整画布视口。", { viewport: VIEWPORT_SCHEMA }, ["viewport"]), + toolDefinition("canvas_run_generation", "触发指定节点生成，通常用于配置节点或文本/图片/视频/音频节点。", { nodeId: { type: "string" }, mode: GENERATION_MODE_SCHEMA, prompt: { type: "string" } }, ["nodeId"]), ]; type OnlineAgentTab = "setup" | "chat" | "history" | "log"; type OnlineAgentLog = { id: string; time: string; title: string; data?: unknown }; @@ -378,62 +369,16 @@ export function CanvasAssistantPanel({ nodes, selectedNodeIds, snapshot, session const executeOnlineTool = (name: string, args: Record): OnlineToolResult => { const current = snapshotRef.current; - const requireNode = (id: unknown, expectedType?: CanvasNodeType) => { - if (typeof id !== "string" || !id) throw new Error("缺少真实节点 id"); - const node = current.nodes.find((item) => item.id === id); - if (!node) throw new Error(`节点不存在：${id}`); - if (expectedType && node.type !== expectedType) throw new Error(`节点类型不匹配：${id}`); - return node; - }; try { if (name === "canvas_get_state") return { ok: true, message: describeCanvasSnapshot(current), data: compactSnapshot(current) }; - if (name === "canvas_connect_nodes") { - const fromNode = requireNode(args.fromNodeId); - const toNode = requireNode(args.toNodeId); - if (fromNode.id === toNode.id) throw new Error("不能连接同一个节点"); - const existed = current.connections.some((conn) => conn.fromNodeId === fromNode.id && conn.toNodeId === toNode.id); - const result = executeOps([{ type: "connect_nodes", fromNodeId: fromNode.id, toNodeId: toNode.id }]); - return { ok: true, message: result.changed ? "已连接节点。" : existed ? "节点已存在连线。" : result.noopReason, data: result }; + if (name === "canvas_export_snapshot") return { ok: true, message: describeCanvasSnapshot(current), data: compactSnapshot(current) }; + if (name === "canvas_get_selection") { + const ids = new Set(current.selectedNodeIds || []); + return { ok: true, message: `当前选中 ${ids.size} 个节点。`, data: { nodes: compactSnapshot({ ...current, nodes: current.nodes.filter((node) => ids.has(node.id)) }).nodes } }; } - if (name === "canvas_configure_generation") { - const configNode = requireNode(args.configNodeId, CanvasNodeType.Config); - const mode = requireGenerationMode(args.mode); - const promptNodeIds = requireStringArray(args.promptNodeIds, "promptNodeIds").map((id) => requireNode(id).id); - promptNodeIds.forEach((id) => { - if (!snapshotRef.current.connections.some((conn) => conn.fromNodeId === id && conn.toNodeId === configNode.id)) throw new Error(`提示词节点未连接到生成配置节点：${id}`); - }); - if (typeof args.prompt !== "string") throw new Error("prompt 必须是字符串"); - const prompt = args.prompt.trim(); - if (!prompt && !promptNodeIds.length) throw new Error("缺少提示词文本或提示词节点 id"); - const composerContent = promptNodeIds.map((id) => `@[node:${id}]`).concat(prompt ? [prompt] : []).join("\n"); - const result = executeOps([{ type: "update_node", id: configNode.id, metadata: { generationMode: mode, composerContent, prompt: composerContent, status: "idle" } }]); - return { ok: true, message: result.changed ? "已配置生成节点。" : "生成节点配置已是目标值。", data: result }; - } - if (name === "canvas_run_generation") { - const configNode = requireNode(args.configNodeId, CanvasNodeType.Config); - const mode = requireGenerationMode(args.mode); - const result = executeOps([{ type: "run_generation", nodeId: configNode.id, mode }]); - return { ok: result.changed, message: result.changed ? "已触发生成。" : result.noopReason, data: result }; - } - if (name === "canvas_delete_nodes") { - const nodeIds = requireStringArray(args.nodeIds, "nodeIds"); - if (!nodeIds.length) throw new Error("缺少要删除的节点 id"); - nodeIds.forEach((id) => requireNode(id)); - const result = executeOps([{ type: "delete_node", ids: nodeIds }]); - return { ok: result.changed, message: result.changed ? "已删除节点。" : result.noopReason, data: result }; - } - if (name === "canvas_delete_connections") { - if (typeof args.all !== "boolean") throw new Error("all 必须是布尔值"); - const all = args.all; - const connectionIds = requireStringArray(args.connectionIds, "connectionIds"); - if (!all && !connectionIds.length) throw new Error("缺少要删除的连线 id"); - if (!all) connectionIds.forEach((id) => { - if (!current.connections.some((conn) => conn.id === id)) throw new Error(`连线不存在：${id}`); - }); - const result = executeOps([{ type: "delete_connections", all, ids: connectionIds }]); - return { ok: result.changed, message: result.changed ? "已删除连线。" : result.noopReason, data: result }; - } - return { ok: false, message: `不支持的工具：${name}` }; + const ops = onlineToolToOps(name, args, current, effectiveConfig); + const result = executeOps(ops); + return { ok: result.changed, message: result.changed ? summarizeCanvasAgentOps(ops) || "画布操作已执行。" : result.noopReason, data: result }; } catch (error) { return { ok: false, message: error instanceof Error ? error.message : "工具执行失败" }; } @@ -990,8 +935,114 @@ function parseToolArguments(value: string) { } } +function onlineToolToOps(name: string, input: Record, snapshot: CanvasAgentSnapshot, config: AiConfig): CanvasAgentOp[] { + if (name === "canvas_apply_ops") return requireOps(input.ops); + if (name === "canvas_create_node") { + const nodeType = requireNodeType(input.nodeType); + const x = numberOr(input.x, nextCanvasX(snapshot)); + const y = numberOr(input.y, 0); + if (nodeType === CanvasNodeType.Config) return [configNodeOp(stringOptional(input.id) || `config-${nanoid()}`, { ...recordOptional(input.metadata), ...input }, x, y, config)]; + return [{ type: "add_node", nodeType, title: stringOptional(input.title), position: { x, y }, width: numberOptional(input.width), height: numberOptional(input.height), metadata: recordOptional(input.metadata) as CanvasNodeData["metadata"] }]; + } + if (name === "canvas_create_text_node") return [textNodeOp(input, numberOr(input.x, nextCanvasX(snapshot)), numberOr(input.y, 0))]; + if (name === "canvas_create_text_nodes") { + const items = requireRecordArray(input.items, "items"); + const x = numberOr(input.x, nextCanvasX(snapshot)); + const y = numberOr(input.y, 0); + const gap = numberOr(input.gap, 40); + const direction = input.direction === "row" ? "row" : "column"; + return items.map((item, index) => textNodeOp({ ...item, text: requireString(item.text, "text") }, numberOr(item.x, direction === "row" ? x + index * (NODE_DEFAULT_SIZE[CanvasNodeType.Text].width + gap) : x), numberOr(item.y, direction === "row" ? y : y + index * (NODE_DEFAULT_SIZE[CanvasNodeType.Text].height + gap)))); + } + if (name === "canvas_create_image_prompt_flow") return generationFlowOps({ ...input, mode: "image" }, snapshot, config); + if (name === "canvas_create_config_node") { + const configId = `config-${nanoid()}`; + const mode = generationMode(input.mode); + return [configNodeOp(configId, input, numberOr(input.x, nextCanvasX(snapshot)), numberOr(input.y, 0), config), ...(input.autoRun ? [runGenerationOp(configId, mode, stringOptional(input.prompt))] : [])]; + } + if (name === "canvas_create_generation_flow") return generationFlowOps(input, snapshot, config); + if (name === "canvas_generate_text") return generationFlowOps({ ...input, mode: "text", autoRun: true }, snapshot, config); + if (name === "canvas_generate_image") return generationFlowOps({ ...input, mode: "image", autoRun: true }, snapshot, config); + if (name === "canvas_generate_video") return generationFlowOps({ ...input, mode: "video", autoRun: true }, snapshot, config); + if (name === "canvas_generate_audio") return generationFlowOps({ ...input, mode: "audio", autoRun: true }, snapshot, config); + if (name === "canvas_update_node") return [{ type: "update_node", id: requireString(input.id, "id"), patch: recordOptional(input.patch) as Partial | undefined, metadata: recordOptional(input.metadata) as CanvasNodeData["metadata"] }]; + if (name === "canvas_update_node_text") return [{ type: "update_node", id: requireString(input.id, "id"), patch: stringOptional(input.title) ? { title: stringOptional(input.title) } : undefined, metadata: { content: requireString(input.text, "text"), status: "success" } }]; + if (name === "canvas_move_nodes") { + return requireRecordArray(input.items, "items").map((item) => { + const id = requireString(item.id, "id"); + const current = snapshot.nodes.find((node) => node.id === id); + return { type: "update_node", id, patch: { position: { x: numberOr(item.x, (current?.position.x || 0) + numberOr(item.dx, 0)), y: numberOr(item.y, (current?.position.y || 0) + numberOr(item.dy, 0)) } } }; + }); + } + if (name === "canvas_resize_node") return [{ type: "update_node", id: requireString(input.id, "id"), patch: { width: requireNumber(input.width, "width"), height: requireNumber(input.height, "height") }, metadata: typeof input.freeResize === "boolean" ? { freeResize: input.freeResize } : undefined }]; + if (name === "canvas_delete_nodes") return [{ type: "delete_node", ids: requireStringArray(input.ids, "ids") }]; + if (name === "canvas_connect_nodes") return requireRecordArray(input.connections, "connections").map((connection) => ({ type: "connect_nodes", fromNodeId: requireString(connection.fromNodeId, "fromNodeId"), toNodeId: requireString(connection.toNodeId, "toNodeId") })); + if (name === "canvas_select_nodes") return [{ type: "select_nodes", ids: requireStringArray(input.ids, "ids") }]; + if (name === "canvas_set_viewport") return [{ type: "set_viewport", viewport: requireViewport(input.viewport) }]; + if (name === "canvas_run_generation") return [runGenerationOp(requireString(input.nodeId, "nodeId"), generationMode(input.mode), stringOptional(input.prompt))]; + throw new Error(`不支持的工具：${name}`); +} + +function generationFlowOps(input: Record, snapshot: CanvasAgentSnapshot, config: AiConfig): CanvasAgentOp[] { + const mode = generationMode(input.mode); + const prompt = requireString(input.prompt, "prompt"); + const x = numberOr(input.x, nextCanvasX(snapshot)); + const y = numberOr(input.y, 0); + const textId = `text-${nanoid()}`; + const configId = `config-${nanoid()}`; + const referenceNodeIds = Array.isArray(input.referenceNodeIds) ? input.referenceNodeIds.filter((id): id is string => typeof id === "string") : []; + const tokens = [`@[node:${textId}]`, ...referenceNodeIds.map((id) => `@[node:${id}]`)]; + return [ + textNodeOp({ id: textId, text: prompt, title: stringOptional(input.title) || "提示词" }, x, y), + configNodeOp(configId, { ...input, prompt: tokens.join("\n") }, x + NODE_DEFAULT_SIZE[CanvasNodeType.Text].width + 80, y, config), + { type: "connect_nodes", fromNodeId: textId, toNodeId: configId }, + ...referenceNodeIds.map((fromNodeId) => ({ type: "connect_nodes" as const, fromNodeId, toNodeId: configId })), + { type: "select_nodes", ids: [configId] }, + ...(input.autoRun ? [runGenerationOp(configId, mode, tokens.join("\n"))] : []), + ]; +} + +function textNodeOp(input: Record, x: number, y: number): CanvasAgentOp { + return { type: "add_node", id: stringOptional(input.id), nodeType: CanvasNodeType.Text, title: stringOptional(input.title), position: { x, y }, width: numberOptional(input.width), height: numberOptional(input.height), metadata: { content: stringOptional(input.text), status: "success", fontSize: 14 } }; +} + +function configNodeOp(id: string, input: Record, x: number, y: number, config: AiConfig): CanvasAgentOp { + const mode = generationMode(input.mode); + const prompt = stringOptional(input.prompt); + return { + type: "add_node", + id, + nodeType: CanvasNodeType.Config, + title: stringOptional(input.title) || generationTitle(mode), + position: { x, y }, + width: numberOptional(input.width), + height: numberOptional(input.height), + metadata: cleanRecord({ + generationMode: mode, + composerContent: prompt, + prompt, + status: "idle", + model: stringOptional(input.model) || defaultGenerationModel(config, mode), + size: stringOptional(input.size) || config.size, + quality: stringOptional(input.quality) || config.quality, + count: numberOptional(input.count) ?? generationCount(mode === "image" ? config.canvasImageCount || config.count : config.count), + seconds: stringOptional(input.seconds) || config.videoSeconds, + vquality: stringOptional(input.vquality) || config.vquality, + generateAudio: stringOptional(input.generateAudio) || config.videoGenerateAudio, + watermark: stringOptional(input.watermark) || config.videoWatermark, + audioVoice: stringOptional(input.audioVoice) || config.audioVoice, + audioFormat: stringOptional(input.audioFormat) || config.audioFormat, + audioSpeed: stringOptional(input.audioSpeed) || config.audioSpeed, + audioInstructions: stringOptional(input.audioInstructions) || config.audioInstructions, + }) as CanvasNodeData["metadata"], + }; +} + +function runGenerationOp(nodeId: string, mode: "text" | "image" | "video" | "audio", prompt?: string): CanvasAgentOp { + return { type: "run_generation", nodeId, mode, prompt }; +} + function isWritableToolCall(call: ResponseToolCall) { - return call.function.name !== "canvas_get_state"; + return !ONLINE_READ_TOOLS.has(call.function.name); } function toolCallsFromDetail(detail: Record): ResponseToolCall[] { @@ -1013,12 +1064,29 @@ function summarizeToolCalls(calls: ResponseToolCall[]) { } function toolCallLabel(name: string) { + if (name === "canvas_apply_ops") return "画布操作"; if (name === "canvas_get_state") return "读取画布"; - if (name === "canvas_connect_nodes") return "连接节点"; - if (name === "canvas_configure_generation") return "配置生成"; - if (name === "canvas_run_generation") return "触发生成"; + if (name === "canvas_get_selection") return "读取选区"; + if (name === "canvas_export_snapshot") return "导出快照"; + if (name === "canvas_create_node") return "创建节点"; + if (name === "canvas_create_text_node") return "创建文本"; + if (name === "canvas_create_text_nodes") return "批量创建文本"; + if (name === "canvas_create_config_node") return "创建生成配置"; + if (name === "canvas_create_image_prompt_flow") return "创建生图流程"; + if (name === "canvas_create_generation_flow") return "创建生成流程"; + if (name === "canvas_generate_text") return "生成文本"; + if (name === "canvas_generate_image") return "生成图片"; + if (name === "canvas_generate_video") return "生成视频"; + if (name === "canvas_generate_audio") return "生成音频"; + if (name === "canvas_update_node") return "更新节点"; + if (name === "canvas_update_node_text") return "更新文本"; + if (name === "canvas_move_nodes") return "移动节点"; + if (name === "canvas_resize_node") return "调整节点尺寸"; if (name === "canvas_delete_nodes") return "删除节点"; - if (name === "canvas_delete_connections") return "删除连线"; + if (name === "canvas_connect_nodes") return "连接节点"; + if (name === "canvas_select_nodes") return "选择节点"; + if (name === "canvas_set_viewport") return "调整视口"; + if (name === "canvas_run_generation") return "触发生成"; return name; } @@ -1026,17 +1094,119 @@ function toolResultText(result: OnlineToolResult) { return result.message; } -function requireGenerationMode(value: unknown): "text" | "image" | "video" | "audio" { - if (value === "text" || value === "image" || value === "video" || value === "audio") return value; - throw new Error("生成模式必须是 text、image、video 或 audio"); -} - function requireStringArray(value: unknown, field: string): string[] { if (!Array.isArray(value)) throw new Error(`${field} 必须是字符串数组`); if (!value.every((item) => typeof item === "string" && Boolean(item))) throw new Error(`${field} 必须只包含非空字符串`); return value as string[]; } +function requireOps(value: unknown): CanvasAgentOp[] { + if (!Array.isArray(value)) throw new Error("ops 必须是数组"); + return value.map(toCanvasAgentOp); +} + +function toCanvasAgentOp(value: unknown): CanvasAgentOp { + const item = objectDetail(value); + const type = item.type; + if (type === "add_node") { + return { + type, + id: stringOptional(item.id), + nodeType: item.nodeType ? requireNodeType(item.nodeType) : undefined, + title: stringOptional(item.title), + position: recordOptional(item.position) ? { x: requireNumber(objectDetail(item.position).x, "position.x"), y: requireNumber(objectDetail(item.position).y, "position.y") } : undefined, + x: numberOptional(item.x), + y: numberOptional(item.y), + width: numberOptional(item.width), + height: numberOptional(item.height), + metadata: recordOptional(item.metadata) as CanvasNodeData["metadata"], + }; + } + if (type === "update_node") return { type, id: requireString(item.id, "id"), patch: recordOptional(item.patch) as Partial | undefined, metadata: recordOptional(item.metadata) as CanvasNodeData["metadata"] }; + if (type === "delete_node") return { type, id: stringOptional(item.id), ids: Array.isArray(item.ids) ? requireStringArray(item.ids, "ids") : undefined }; + if (type === "delete_connections") return { type, id: stringOptional(item.id), ids: Array.isArray(item.ids) ? requireStringArray(item.ids, "ids") : undefined, all: typeof item.all === "boolean" ? item.all : undefined }; + if (type === "connect_nodes") return { type, id: stringOptional(item.id), fromNodeId: requireString(item.fromNodeId, "fromNodeId"), toNodeId: requireString(item.toNodeId, "toNodeId") }; + if (type === "set_viewport") return { type, viewport: requireViewport(item.viewport) }; + if (type === "select_nodes") return { type, ids: requireStringArray(item.ids, "ids") }; + if (type === "run_generation") return { type, nodeId: requireString(item.nodeId, "nodeId"), mode: generationMode(item.mode), prompt: stringOptional(item.prompt) }; + throw new Error("不支持的画布操作类型"); +} + +function requireRecordArray(value: unknown, field: string): Record[] { + if (!Array.isArray(value)) throw new Error(`${field} 必须是数组`); + return value.map((item) => { + const record = objectDetail(item); + if (!Object.keys(record).length) throw new Error(`${field} 必须只包含对象`); + return record; + }); +} + +function requireString(value: unknown, field: string) { + if (typeof value !== "string" || !value) throw new Error(`${field} 必须是非空字符串`); + return value; +} + +function requireNumber(value: unknown, field: string) { + if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`${field} 必须是数字`); + return value; +} + +function requireNodeType(value: unknown): CanvasNodeType { + if (Object.values(CanvasNodeType).includes(value as CanvasNodeType)) return value as CanvasNodeType; + throw new Error("节点类型必须是 text、image、config、video 或 audio"); +} + +function requireViewport(value: unknown) { + const item = objectDetail(value); + return { x: requireNumber(item.x, "viewport.x"), y: requireNumber(item.y, "viewport.y"), k: requireNumber(item.k, "viewport.k") }; +} + +function recordOptional(value: unknown) { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : undefined; +} + +function stringOptional(value: unknown) { + return typeof value === "string" ? value : ""; +} + +function numberOptional(value: unknown) { + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +function numberOr(value: unknown, fallback: number) { + return numberOptional(value) ?? fallback; +} + +function nextCanvasX(snapshot: CanvasAgentSnapshot) { + return snapshot.nodes.length ? Math.max(...snapshot.nodes.map((node) => node.position.x + node.width)) + 80 : 0; +} + +function generationMode(value: unknown): "text" | "image" | "video" | "audio" { + return value === "text" || value === "video" || value === "audio" ? value : "image"; +} + +function generationTitle(mode: "text" | "image" | "video" | "audio") { + if (mode === "text") return "文本生成"; + if (mode === "video") return "视频生成"; + if (mode === "audio") return "音频生成"; + return "图片生成"; +} + +function defaultGenerationModel(config: AiConfig, mode: "text" | "image" | "video" | "audio") { + if (mode === "image") return config.imageModel || config.model; + if (mode === "video") return config.videoModel || config.model; + if (mode === "audio") return config.audioModel || config.model; + return config.textModel || config.model; +} + +function generationCount(value: string) { + return Math.max(1, Math.min(15, Math.floor(Math.abs(Number(value)) || 1))); +} + +function cleanRecord(value: Record) { + return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined && item !== "")); +} + function snapshotSignature(snapshot: CanvasAgentSnapshot) { return JSON.stringify({ nodes: snapshot.nodes, connections: snapshot.connections, selectedNodeIds: snapshot.selectedNodeIds, viewport: snapshot.viewport }); }