From 2afe2f38985149fab495fb88bc10bb1e5e50a6b2 Mon Sep 17 00:00:00 2001 From: HouYunFei <1844025705@qq.com> Date: Wed, 29 Jul 2026 17:19:20 +0800 Subject: [PATCH] feat(agent): implement local image handling and enhance generated image integration in canvas --- CHANGELOG.md | 1 + canvas-agent/agent-instructions.md | 3 + canvas-agent/src/config.ts | 3 +- canvas-agent/src/server/http.ts | 10 +++- docs/content/docs/progress/pending-test.mdx | 1 + .../components/agent/local-agent-panel.tsx | 58 ++++++++++++++++++- 6 files changed, 71 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b793a85..6685443 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased ++ [修复] 画布 Agent 默认通过画布节点生成内容，仅在用户明确指定时使用 Codex 内置生图并将结果展示到对话和当前画布。 + [优化] Agent 面板标题、功能标签与新对话操作合并为单行顶部栏，减少面板空间占用。 + [新增] 画布 Agent 支持请求批准、自动审查和完全访问三档 Codex 权限，并可在对话中审批文件、命令与网络访问。 + [修复] Agent 回复中的本地绝对文件路径改为在系统文件管理器中定位，不再被浏览器错误拼接为 localhost 地址。 diff --git a/canvas-agent/agent-instructions.md b/canvas-agent/agent-instructions.md index d50a10f..c2f1496 100644 --- a/canvas-agent/agent-instructions.md +++ b/canvas-agent/agent-instructions.md @@ -7,4 +7,7 @@ - 修改当前画布时根据任务使用已配置的 infinite-canvas MCP 工具；复杂批量改动使用 `canvas_apply_ops`。 - 用户要求把上传附件放入画布或作为生成参考图时，必须先用 `canvas_create_attachment_nodes` 创建真实图片节点，再把节点 ID 传给生成流程，不要创建空图片占位节点。 - 生图与视频工作台分别使用 `workbench_image_*`、`workbench_video_*` 工具；提示词和素材分别使用 `prompts_search`、`assets_*` 工具。 +- 用户要求生成图片、视频、音频或文本时，默认调用对应的 `canvas_generate_image`、`canvas_generate_video`、`canvas_generate_audio`、`canvas_generate_text`，通过当前画布的生成节点完成任务。 +- 只有用户明确要求使用“Codex 内置生图”“ImageGen 技能”或意思明确相同的能力时，才使用 Codex 自带的 `imagegen`；不要因为用户只说“生成图片”就自行改用内置生图。内置生图完成后，其结果会由 Canvas Agent 自动展示到对话并插入当前画布，无需再创建空节点或重复生成。 +- 只有用户明确说要在生图/视频工作台生成时，才使用 `workbench_image_*`、`workbench_video_*`。生成任务提交后应说明已经在画布或工作台开始生成，不要在实际没有结果时声称“已生成”。 - 需要生成内容时直接调用对应生成工具，不要绑定特定业务场景，不要模拟鼠标点击，不要要求用户手动复制 JSON。 diff --git a/canvas-agent/src/config.ts b/canvas-agent/src/config.ts index 532b02d..e294431 100644 --- a/canvas-agent/src/config.ts +++ b/canvas-agent/src/config.ts @@ -61,7 +61,8 @@ function initializeWorkspace(workspacePath: string) { if (initializedWorkspaces.has(workspacePath)) return; fs.mkdirSync(workspacePath, { recursive: true }); const instructionsFile = path.join(workspacePath, "AGENTS.md"); - if (!fs.existsSync(instructionsFile)) fs.writeFileSync(instructionsFile, AGENT_PROMPT); + const current = fs.existsSync(instructionsFile) ? fs.readFileSync(instructionsFile, "utf8") : ""; + if (!current || current.startsWith("# Infinite Canvas Agent")) fs.writeFileSync(instructionsFile, AGENT_PROMPT); initializedWorkspaces.add(workspacePath); } diff --git a/canvas-agent/src/server/http.ts b/canvas-agent/src/server/http.ts index 0b9cdbe..617c48b 100644 --- a/canvas-agent/src/server/http.ts +++ b/canvas-agent/src/server/http.ts @@ -1,5 +1,5 @@ import { spawn } from "node:child_process"; -import { stat } from "node:fs/promises"; +import { readFile, stat } from "node:fs/promises"; import path from "node:path"; import express, { type NextFunction, type Request, type Response } from "express"; @@ -82,6 +82,14 @@ export function startHttpServer() { await revealLocalFile(filePath, file.isDirectory()); res.json({ ok: true }); })); + app.post("/agent/local-image", route(async (req, res) => { + const filePath = String(req.body?.path || ""); + if (!path.isAbsolute(filePath) || !/\.(?:avif|gif|jpe?g|png|webp)$/i.test(filePath)) return res.status(400).json({ ok: false, error: "图片路径无效" }); + const file = await stat(filePath); + if (!file.isFile()) return res.status(400).json({ ok: false, error: "图片文件无效" }); + res.setHeader("Cache-Control", "no-store"); + res.type(path.extname(filePath)).send(await readFile(filePath)); + })); app.post("/api/tools", route(async (req, res) => res.json({ ok: true, result: await session.callTool(req.body?.name, req.body?.input || {}) }))); app.get("/agent/codex/workspace", (_req, res) => { const workspace = ensureSiteWorkspace(config); diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index 515a644..e0b8bfe 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -5,6 +5,7 @@ description: 当前版本已实现但仍需人工验证的变更项 # 待测试 +- Agent 画布生图：重启 Canvas Agent 后，其自动生成的工作区 `AGENTS.md` 应同步为最新版指令（用户自行编写的其他 `AGENTS.md` 不应被覆盖）；输入“生成一张图片”时应调用 `canvas_generate_image`，通过当前画布节点创建并运行生成流程；只有明确输入“使用 Codex 内置生图/ImageGen 技能生成”时才允许调用内置能力，其结果应同时显示在对话中并作为保持原比例的图片节点添加到当前画布；Agent 不应在没有生成结果时提前声称“已生成”。 - Agent 顶部栏：标题只显示垂直居中的「Agent」，连接、对话、历史、日志、新对话和收起操作应位于同一行；面板较窄时标签仍可横向滚动且操作不应错位。 - Agent Markdown 样式：右侧 Agent 回复中的代码块应为横向占满消息区域、无语言标题和无双层边框的紧凑代码条，单行内容不应保留纵向大面积空白，复制操作仅在悬停时弱化显示；行内代码、链接在浅色和深色主题下应清晰；点击外部链接后应显示中文紧凑确认弹窗，长路径能够正常换行且复制、继续打开和关闭操作可用；点击 `/Users/`、`/home/` 等本地绝对文件路径时应改为提示在系统文件管理器中定位，支持复制路径，且浏览器地址不应跳转为 localhost 文件路径。 - Agent 工具确认模式：右侧面板标题栏不应再显示「工具确认」开关；对话输入框左下方应显示确认模式选择，默认选中「自动确认」，画布写入工具应直接执行；切换为「手动确认」后，画布写入工具应在对话中展示等待确认卡片，并支持批准或拒绝。 diff --git a/web/src/components/agent/local-agent-panel.tsx b/web/src/components/agent/local-agent-panel.tsx index 3954b8c..53d90b3 100644 --- a/web/src/components/agent/local-agent-panel.tsx +++ b/web/src/components/agent/local-agent-panel.tsx @@ -13,7 +13,7 @@ import { deleteAgentThreadMessages, readAgentUserMessages, saveAgentUserMessage import { useThemeStore } from "@/stores/use-theme-store"; import { useShallow } from "zustand/react/shallow"; import { useAgentStore, type AgentCanvasContext, type AgentChatItem, type AgentPendingApproval, type AgentPendingToolCall, type AgentPermissionMode, type AgentThreadSummary } from "@/stores/use-agent-store"; -import {summarizeCanvasAgentOps, type CanvasAgentOp, CanvasAgentSnapshot} from "@/lib/canvas/canvas-agent-ops"; +import { summarizeCanvasAgentOps, type CanvasAgentOp, type CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops"; import { isSiteTool, runSiteTool } from "@/lib/agent/agent-site-tools"; import { activateAgentClient, discoverAgentConfig, fetchAgentJson, postCodexApproval, postState, postToolResult } from "./agent-api"; import { AgentChatTimeline, AgentTaskProgress, AgentUsageBar } from "./agent-chat"; @@ -727,7 +727,7 @@ export function LocalAgentPanel({ embedded, headless, autoConnect }: { embedded? addEventLog("处理失败", error.text, value); }; - const handleAgentEvent = (event: AgentEventPayload) => { + const handleAgentEvent = async (event: AgentEventPayload) => { if (event.type === "usage.updated") setAgentState({ tokenUsage: eventUsage(event) }); const log = formatAgentEventLog(event); if (log) addEventLog(log.title, log.text); @@ -754,6 +754,31 @@ export function LocalAgentPanel({ embedded, headless, autoConnect }: { embedded? return; } } + if (event.type === "item.completed" && event.item?.type === "dynamic_tool_call" && event.item.id) { + const generated = await importGeneratedImages(endpoint, token, event.item); + if (generated.length) { + const context = canvasContextRef.current; + if (context) { + const right = Math.max(0, ...context.snapshot.nodes.map((node) => node.position.x + node.width)) + 80; + const ops = generated.map((image, index) => { + const size = fitNodeSize(image.upload.width, image.upload.height); + return { + type: "add_node", + id: `image-${createId()}`, + nodeType: "image", + title: image.name, + position: { x: right + index * 40, y: index * 40 }, + ...size, + metadata: imageMetadata(image.upload), + }; + }); + const result = context.applyOps(ops); + void postState(endpoint, token, clientIdRef.current, result); + } + addMessage({ id: `generated-${event.item.id}`, role: "assistant", text: context ? "已将生成图片添加到当前画布。" : "图片已生成。", attachments: generated.map((image) => image.attachment) }); + return; + } + } const activity = formatAgentActivity(event); if (activity && event.item?.id) { upsertActivityMessage({ ...activity, id: event.item.id }); @@ -927,7 +952,34 @@ function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); } -function readDataUrl(file: File) { +async function importGeneratedImages(endpoint: string, token: string, item: AgentEventItem) { + const sources = Array.from(generatedImageSources(item)); + return await Promise.all( + sources.map(async (source, index) => { + const response = source.startsWith("data:image/") + ? await fetch(source) + : await fetch(`${endpoint}/agent/local-image?token=${encodeURIComponent(token)}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ path: source }) }); + if (!response.ok) throw new Error("读取 Codex 生成图片失败"); + const blob = await response.blob(); + const upload = await uploadImage(blob); + const dataUrl = await readDataUrl(blob); + const name = source.startsWith("/") ? source.split("/").at(-1) || `生成图片 ${index + 1}` : `生成图片 ${index + 1}`; + return { upload, name, attachment: { id: createId(), name, type: blob.type || upload.mimeType, size: blob.size, width: upload.width, height: upload.height, url: upload.url, dataUrl } }; + }), + ); +} + +function generatedImageSources(value: unknown, result = new Set()) { + if (typeof value === "string") { + if (value.startsWith("data:image/") || (/^\/.+\.(?:avif|gif|jpe?g|png|webp)$/i.test(value) && !value.includes("\n"))) result.add(value); + return result; + } + if (Array.isArray(value)) value.forEach((item) => generatedImageSources(item, result)); + else if (value && typeof value === "object") Object.values(value).forEach((item) => generatedImageSources(item, result)); + return result; +} + +function readDataUrl(file: Blob) { return new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = () => resolve(String(reader.result || ""));