From 062e4569aa6b0f3cba47cf92288d7557daf21490 Mon Sep 17 00:00:00 2001 From: yu <2799341137@qq.com> Date: Fri, 17 Jul 2026 18:30:10 +0800 Subject: [PATCH 1/3] feat(agent): add task status and client-scoped operations --- CHANGELOG.md | 2 + canvas-agent/package.json | 1 + canvas-agent/src/canvas-session.test.ts | 170 ++++++++++++++++++ canvas-agent/src/canvas-session.ts | 53 ++++-- canvas-agent/src/http-server.ts | 8 +- canvas-agent/src/schemas.ts | 7 +- canvas-agent/tsconfig.json | 3 +- docs/content/docs/progress/pending-test.mdx | 3 + .../canvas/canvas-local-agent-panel.tsx | 26 ++- web/src/lib/agent/agent-site-tools.ts | 68 ++++++- web/src/pages/image/index.tsx | 25 ++- web/src/pages/video/index.tsx | 40 +++-- web/src/stores/use-workbench-agent-store.ts | 40 ++++- 13 files changed, 407 insertions(+), 39 deletions(-) create mode 100644 canvas-agent/src/canvas-session.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index de7e54d..60a478e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased ++ [新增] Agent 新增统一生成任务状态查询，支持查看画布、生图工作台和视频工作台任务进度。 ++ [修复] 修复多标签页同时连接本地 Agent 时 MCP 可能操作非当前页面的问题。 + [新增] 提示词来源增加自定义脚本抓取功能、支持新增来源。 ## v0.9.0 - 2026-07-17 diff --git a/canvas-agent/package.json b/canvas-agent/package.json index 3c6287a..69ebc60 100644 --- a/canvas-agent/package.json +++ b/canvas-agent/package.json @@ -13,6 +13,7 @@ ], "scripts": { "dev": "tsx src/index.ts", + "test": "tsx --test src/canvas-session.test.ts", "build": "tsc -p tsconfig.json", "start": "node dist/index.js", "prepack": "npm run build" diff --git a/canvas-agent/src/canvas-session.test.ts b/canvas-agent/src/canvas-session.test.ts new file mode 100644 index 0000000..4188772 --- /dev/null +++ b/canvas-agent/src/canvas-session.test.ts @@ -0,0 +1,170 @@ +import { EventEmitter } from "node:events"; +import type { ServerResponse } from "node:http"; +import assert from "node:assert/strict"; +import test from "node:test"; + +import { CanvasSession } from "./canvas-session.js"; + +test("MCP 读取当前激活网页的画布", async (t) => { + const session = new CanvasSession(); + const first = connect(session, "first"); + const second = connect(session, "second"); + t.after(() => { + first.close(); + second.close(); + }); + session.updateState(snapshot("canvas-first"), "first"); + session.updateState(snapshot("canvas-second"), "second"); + + session.activateClient("first"); + assert.equal(field(await session.callTool("canvas_get_state", {}), "projectId"), "canvas-first"); + + session.activateClient("second"); + assert.equal(field(await session.callTool("canvas_get_state", {}), "projectId"), "canvas-second"); +}); + +test("画布写操作只发送给当前激活网页", async (t) => { + const session = new CanvasSession(); + const first = connect(session, "first"); + const second = connect(session, "second"); + t.after(() => { + first.close(); + second.close(); + }); + session.updateState(snapshot("canvas-first"), "first"); + session.updateState(snapshot("canvas-second"), "second"); + session.activateClient("second"); + + const result = session.callTool("canvas_create_text_node", { text: "只写入第二个画布" }); + const call = second.event("tool_call"); + assert.equal(first.event("tool_call"), undefined); + assert.equal(field(call, "name"), "canvas_apply_ops"); + session.resolveResult("second", { requestId: String(field(call, "requestId")), result: { ok: true } }); + assert.deepEqual(await result, { ok: true }); +}); + +test("tool result is accepted only from the request client", async (t) => { + const session = new CanvasSession(); + const first = connect(session, "first"); + const second = connect(session, "second"); + t.after(() => { + first.close(); + second.close(); + }); + session.activateClient("first"); + + const result = session.callTool("canvas_create_text_node", { text: "first only" }); + const call = first.event("tool_call"); + const requestId = String(field(call, "requestId")); + + assert.equal(session.resolveResult("second", { requestId, result: { client: "second" } }), false); + assert.equal(session.resolveResult("first", { requestId, result: { client: "first" } }), true); + assert.deepEqual(await result, { client: "first" }); +}); + +test("生成状态查询由当前激活网页返回", async (t) => { + const session = new CanvasSession(); + const first = connect(session, "first"); + const second = connect(session, "second"); + t.after(() => { + first.close(); + second.close(); + }); + session.activateClient("second"); + + const result = session.callTool("generation_get_status", { scope: "all" }); + const call = second.event("tool_call"); + assert.equal(first.event("tool_call"), undefined); + assert.equal(field(call, "name"), "generation_get_status"); + session.resolveResult("second", { requestId: String(field(call, "requestId")), result: { total: 1, tasks: [{ id: "image-1", status: "running" }] } }); + assert.deepEqual(await result, { total: 1, tasks: [{ id: "image-1", status: "running" }] }); +}); + +test("活动网页关闭后回退到仍连接的画布", async (t) => { + const session = new CanvasSession(); + const first = connect(session, "first"); + const second = connect(session, "second"); + t.after(() => { + first.close(); + second.close(); + }); + session.updateState(snapshot("canvas-first"), "first"); + session.updateState(snapshot("canvas-second"), "second"); + session.activateClient("second"); + second.close(); + + assert.equal(field(await session.callTool("canvas_get_state", {}), "projectId"), "canvas-first"); +}); + +test("closing the active client falls back to the most recently focused client", async (t) => { + const session = new CanvasSession(); + const first = connect(session, "first"); + const second = connect(session, "second"); + const third = connect(session, "third"); + t.after(() => { + first.close(); + second.close(); + third.close(); + }); + session.updateState(snapshot("canvas-first"), "first"); + session.updateState(snapshot("canvas-second"), "second"); + session.updateState(snapshot("canvas-third"), "third"); + session.activateClient("third"); + session.activateClient("second"); + second.close(); + + assert.equal(field(await session.callTool("canvas_get_state", {}), "projectId"), "canvas-third"); +}); + +test("closing a client rejects its pending tool requests", async () => { + const session = new CanvasSession(); + const first = connect(session, "first"); + const result = session.callTool("canvas_create_text_node", { text: "pending" }); + const call = first.event("tool_call"); + const requestId = String(field(call, "requestId")); + first.close(); + + const outcome = await Promise.race([ + result.then(() => "resolved", (error) => error instanceof Error ? error.message : String(error)), + new Promise((resolve) => setTimeout(() => resolve("pending"), 20)), + ]); + if (outcome === "pending") session.resolveResult("first", { requestId, result: null }); + assert.match(outcome, /断开/); +}); + +function connect(session: CanvasSession, clientId: string) { + const response = new FakeSseResponse(); + session.openEvents(new URL(`http://127.0.0.1/events?clientId=${clientId}`), response as unknown as ServerResponse); + return response; +} + +function snapshot(projectId: string) { + return { projectId, title: projectId, nodes: [], connections: [], selectedNodeIds: [], viewport: { x: 0, y: 0, k: 1 } }; +} + +function field(value: unknown, key: string) { + return value && typeof value === "object" ? (value as Record)[key] : undefined; +} + +class FakeSseResponse extends EventEmitter { + private chunks: string[] = []; + + writeHead() { + return this; + } + + write(chunk: string) { + this.chunks.push(chunk); + return true; + } + + event(type: string) { + const chunk = this.chunks.find((item) => item.startsWith(`event: ${type}\n`)); + const data = chunk?.split("\n").find((line) => line.startsWith("data: "))?.slice(6); + return data ? (JSON.parse(data) as unknown) : undefined; + } + + close() { + this.emit("close"); + } +} diff --git a/canvas-agent/src/canvas-session.ts b/canvas-agent/src/canvas-session.ts index 6bc480c..1efaeea 100644 --- a/canvas-agent/src/canvas-session.ts +++ b/canvas-agent/src/canvas-session.ts @@ -5,7 +5,7 @@ import { type ToolName } from "./schemas.js"; import { compactCanvasState, compactNode, isToolName, nextCanvasX, parseToolInput } from "./tools.js"; import type { CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js"; -type PendingRequest = { resolve: (value: unknown) => void; reject: (error: Error) => void }; +type PendingRequest = { clientId: string; resolve: (value: unknown) => void; reject: (error: Error) => void }; const SITE_TOOLS = new Set([ "site_navigate", @@ -17,12 +17,20 @@ const SITE_TOOLS = new Set([ "prompts_search", "assets_list", "assets_add", + "generation_get_status", ]); export class CanvasSession { private clients = new Map(); + private clientFocusOrder = new Map(); private pending = new Map(); - private canvasState: CanvasSnapshot | null = null; + private canvasStates = new Map(); + private activeClientId = ""; + private focusSequence = 0; + + private get canvasState() { + return this.canvasStates.get(this.activeClientId) || null; + } health() { return { ok: true, hasCanvas: Boolean(this.canvasState), clients: this.clients.size }; @@ -32,25 +40,49 @@ export class CanvasSession { const clientId = url.searchParams.get("clientId") || crypto.randomUUID(); const statusOnly = url.searchParams.get("role") === "status"; res.writeHead(200, { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", Connection: "keep-alive" }); - if (!statusOnly) this.clients.set(clientId, res); + if (!statusOnly) { + this.clients.set(clientId, res); + if (!this.clientFocusOrder.has(clientId)) this.clientFocusOrder.set(clientId, 0); + if (!this.activeClientId) { + this.activeClientId = clientId; + this.clientFocusOrder.set(clientId, ++this.focusSequence); + } + } sendEvent(res, "hello", { ok: true, clientId }); const timer = setInterval(() => sendEvent(res, "ping", { time: Date.now() }), 15000); res.on("close", () => { clearInterval(timer); - if (!statusOnly) this.clients.delete(clientId); - if (this.canvasState?.clientId === clientId) this.canvasState = null; + if (statusOnly || this.clients.get(clientId) !== res) return; + this.clients.delete(clientId); + this.clientFocusOrder.delete(clientId); + this.canvasStates.delete(clientId); + this.pending.forEach((item, requestId) => { + if (item.clientId !== clientId) return; + this.pending.delete(requestId); + item.reject(new Error("请求页面已断开")); + }); + if (this.activeClientId === clientId) this.activeClientId = [...this.clients.keys()].sort((a, b) => (this.clientFocusOrder.get(b) || 0) - (this.clientFocusOrder.get(a) || 0))[0] || ""; }); } updateState(body: unknown, clientId?: string) { - this.canvasState = { ...((body && typeof body === "object" && !Array.isArray(body) ? body : {}) as Record), clientId } as CanvasSnapshot; + const targetClientId = clientId || this.activeClientId; + if (!targetClientId) return; + this.canvasStates.set(targetClientId, { ...((body && typeof body === "object" && !Array.isArray(body) ? body : {}) as Record), clientId: targetClientId } as CanvasSnapshot); } - resolveResult(body: { requestId?: string; error?: string; result?: unknown }) { + activateClient(clientId: string) { + if (!this.clients.has(clientId)) throw new Error("当前网页未连接"); + this.activeClientId = clientId; + this.clientFocusOrder.set(clientId, ++this.focusSequence); + } + + resolveResult(clientId: string, body: { requestId?: string; error?: string; result?: unknown }) { const item = body.requestId ? this.pending.get(body.requestId) : null; - if (!item || !body.requestId) return; + if (!item || !body.requestId || item.clientId !== clientId) return false; this.pending.delete(body.requestId); body.error ? item.reject(new Error(body.error)) : item.resolve(body.result); + return true; } emitAll(type: string, payload: unknown) { @@ -168,7 +200,8 @@ export class CanvasSession { private async requestCanvasTool(name: ToolName, input: Record) { const requestId = crypto.randomUUID(); - const client = this.clients.get(this.canvasState?.clientId || "") || this.clients.values().next().value; + const clientId = this.activeClientId; + const client = this.clients.get(clientId); if (!client) throw new Error("当前没有已连接画布"); sendEvent(client, "tool_call", { requestId, name, input }); return await new Promise((resolve, reject) => { @@ -176,7 +209,7 @@ export class CanvasSession { this.pending.delete(requestId); reject(new Error("画布操作超时")); }, 30000); - this.pending.set(requestId, { resolve: (value) => (clearTimeout(timer), resolve(value)), reject: (error) => (clearTimeout(timer), reject(error)) }); + this.pending.set(requestId, { clientId, resolve: (value) => (clearTimeout(timer), resolve(value)), reject: (error) => (clearTimeout(timer), reject(error)) }); }); } } diff --git a/canvas-agent/src/http-server.ts b/canvas-agent/src/http-server.ts index eeab8cd..d886236 100644 --- a/canvas-agent/src/http-server.ts +++ b/canvas-agent/src/http-server.ts @@ -33,10 +33,14 @@ export function startHttpServer() { session.updateState(req.body, String(req.query.clientId || "") || undefined); res.json({ ok: true }); }); - app.post("/canvas/result", (req, res) => { - session.resolveResult(req.body); + app.post("/canvas/activate", (req, res) => { + session.activateClient(String(req.query.clientId || "")); res.json({ ok: true }); }); + app.post("/canvas/result", (req, res) => { + const ok = session.resolveResult(String(req.query.clientId || ""), req.body); + res.status(ok ? 200 : 409).json({ ok }); + }); app.post("/api/tools", route(async (req, res) => res.json({ ok: true, result: await session.callTool(req.body?.name, req.body?.input || {}) }))); app.get("/agent/codex/workspace", (_req, res) => { const workspace = ensureSiteWorkspace(config); diff --git a/canvas-agent/src/schemas.ts b/canvas-agent/src/schemas.ts index a07af63..479098b 100644 --- a/canvas-agent/src/schemas.ts +++ b/canvas-agent/src/schemas.ts @@ -32,6 +32,7 @@ export const toolNames = [ "canvas_select_nodes", "canvas_set_viewport", "canvas_run_generation", + "generation_get_status", "workbench_image_get_config", "workbench_image_generate", "workbench_video_get_config", @@ -111,6 +112,7 @@ export const toolInputSchemas = { canvas_select_nodes: z.object({ ids: z.array(z.string()) }), canvas_set_viewport: z.object({ viewport: viewportSchema }), canvas_run_generation: z.object({ nodeId: z.string(), mode: generationModeSchema.optional(), prompt: z.string().optional() }), + generation_get_status: z.object({ scope: z.enum(["all", "canvas", "image", "video"]).optional(), taskId: z.string().optional(), nodeIds: z.array(z.string()).optional(), limit: z.number().optional() }), workbench_image_get_config: z.object({}).passthrough(), workbench_image_generate: z.object({ prompt: z.string(), model: z.string().optional(), quality: z.string().optional(), size: z.string().optional(), count: z.number().optional(), run: z.boolean().optional() }), workbench_video_get_config: z.object({}).passthrough(), @@ -146,10 +148,11 @@ export const toolDescriptions: Record = { canvas_select_nodes: "设置当前选中节点。", canvas_set_viewport: "调整画布视口。", canvas_run_generation: "触发指定节点生成，通常用于配置节点或文本/图片/视频/音频节点。", + generation_get_status: "查询当前活动网页的生成任务状态。默认返回画布、生图工作台和视频工作台最近任务；可用 scope 过滤来源，用 taskId 查询工作台任务，用 nodeIds 查询画布节点。", workbench_image_get_config: "读取生图工作台的当前参数和可选项（可用模型、质量、尺寸/宽高比、张数范围），在调用 workbench_image_generate 前先了解可选值。", - workbench_image_generate: "在生图工作台填入提示词并按需设置 model、quality、size（如 1:1 或 1024x1024）、count，run 默认 true 会自动点击生成按钮。会自动跳转到生图工作台。生成为异步过程，工具返回代表已提交，结果请在工作台查看。", + workbench_image_generate: "在生图工作台填入提示词并按需设置 model、quality、size（如 1:1 或 1024x1024）、count，run 默认 true 会自动点击生成按钮。会自动跳转到生图工作台。生成为异步过程，提交后返回 taskId，可用 generation_get_status 查询状态。", workbench_video_get_config: "读取视频创作台的当前参数和可选项（可用模型、尺寸/比例、时长、清晰度/分辨率、是否生成声音与水印）。", - workbench_video_generate: "在视频创作台填入提示词并按需设置 model、size、seconds、resolution、generateAudio、watermark，run 默认 true 会自动点击生成按钮。会自动跳转到视频创作台。生成为异步过程，工具返回代表已提交。", + workbench_video_generate: "在视频创作台填入提示词并按需设置 model、size、seconds、resolution、generateAudio、watermark，run 默认 true 会自动点击生成按钮。会自动跳转到视频创作台。生成为异步过程，提交后返回 taskId，可用 generation_get_status 查询状态。", prompts_search: "搜索提示词库（第三方提示词合集），支持 keyword、category、tags 过滤和 page/pageSize 分页，返回标题、提示词、分类、标签、封面等。", assets_list: "列出用户「我的素材」，支持 kind（text/image/video）过滤、keyword 搜索和 page/pageSize 分页。为控制体积不返回图片/视频原始 data，仅返回封面与元信息。", assets_add: "向「我的素材」新增素材。kind=text 时用 content 传文本内容；kind=image 时用 imageUrl 传图片地址或 dataURL。可附带 title、tags、source、note。", diff --git a/canvas-agent/tsconfig.json b/canvas-agent/tsconfig.json index 543a8e2..d312216 100644 --- a/canvas-agent/tsconfig.json +++ b/canvas-agent/tsconfig.json @@ -10,5 +10,6 @@ "rootDir": "src", "skipLibCheck": true }, - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"] } diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index a781a33..875286e 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -4,3 +4,6 @@ description: 当前版本已实现但仍需人工验证的变更项 --- # 待测试 + +- 全站 Agent：新增 `generation_get_status` 工具，画布生成节点可按 `nodeIds` 查询，生图和视频工作台提交后会返回 `taskId` 并可查询排队、运行、成功或失败状态；需验证查询只由当前活动标签页返回。 +- 本地 Agent 多标签页隔离：同时打开两个不同画布并连接同一个 Agent，分别聚焦标签页后通过 MCP 读取和修改画布，操作应只落在当前聚焦页面；关闭当前页面后应自动回退到仍连接的页面。 diff --git a/web/src/components/canvas/canvas-local-agent-panel.tsx b/web/src/components/canvas/canvas-local-agent-panel.tsx index fb20de8..eca3544 100644 --- a/web/src/components/canvas/canvas-local-agent-panel.tsx +++ b/web/src/components/canvas/canvas-local-agent-panel.tsx @@ -151,6 +151,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb setAgentState({ connected: true, activity: "已连接", connectError: "", silentConnect: false, messages: useAgentStore.getState().messages.filter((item) => !isConnectionErrorMessage(item)) }); if (!headless) message.success("本地 Agent 已连接"); void postState(endpoint, token, clientId, canvasContextRef.current?.snapshot || null); + if (document.visibilityState === "visible" && document.hasFocus()) void activateAgentClient(endpoint, token, clientId); }); source.addEventListener("tool_call", (event) => { const data = parseEventData(event); @@ -200,6 +201,19 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb if (connected) void loadThreads(); }, [connected, loadThreads]); + useEffect(() => { + if (!connected) return; + const activate = () => void activateAgentClient(endpoint, token, clientIdRef.current); + const activateVisible = () => { + if (document.visibilityState === "visible") activate(); + }; + window.addEventListener("focus", activate); + document.addEventListener("visibilitychange", activateVisible); + return () => { + window.removeEventListener("focus", activate); + document.removeEventListener("visibilitychange", activateVisible); + }; + }, [connected, endpoint, token]); const sendPrompt = async () => { const text = prompt.trim(); const files = attachments; @@ -304,7 +318,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb try { setAgentState({ activity: SITE_TOOL_LABELS[payload.name], waiting: true }); addEventLog(toolName(payload.name), payload, payload); - const result = await runSiteTool(payload.name, payload.input || {}, navigate); + const result = await runSiteTool(payload.name, payload.input || {}, navigate, { canvasSnapshot: canvasContextRef.current?.snapshot || null }); await postToolResult(endpoint, token, clientIdRef.current, { requestId: payload.requestId, result }); setAgentState({ activity: "工具完成", waiting: true }); addEventLog(`${toolName(payload.name)}完成`, result, result); @@ -955,6 +969,12 @@ async function postState(endpoint: string, token: string, clientId: string, snap } catch {} } +async function activateAgentClient(endpoint: string, token: string, clientId: string) { + try { + await fetch(`${endpoint}/canvas/activate?token=${encodeURIComponent(token)}&clientId=${encodeURIComponent(clientId)}`, { method: "POST" }); + } catch {} +} + async function postToolResult(endpoint: string, token: string, clientId: string, body: { requestId: string; result?: unknown; error?: string }) { await fetch(`${endpoint}/canvas/result?token=${encodeURIComponent(token)}&clientId=${encodeURIComponent(clientId)}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); } @@ -1093,6 +1113,10 @@ function siteToolSummary(name: string, result: unknown) { if (name === "prompts_search") return `找到 ${numberField(data, "total")} 条提示词`; if (name === "assets_list") return `共 ${numberField(data, "total")} 个资产`; if (name === "assets_add") return "已加入我的资产"; + if (name === "generation_get_status") { + const summary = data.summary && typeof data.summary === "object" ? (data.summary as Record) : {}; + return `共 ${numberField(data, "total")} 个任务，排队 ${numberField(summary, "queued")}，运行中 ${numberField(summary, "running")}，成功 ${numberField(summary, "succeeded")}，失败 ${numberField(summary, "failed")}`; + } if (name === "workbench_image_generate" || name === "workbench_video_generate") return typeof data.note === "string" ? data.note : "已在工作台执行"; if (name === "workbench_image_get_config" || name === "workbench_video_get_config") return "已读取工作台配置"; return "已完成"; diff --git a/web/src/lib/agent/agent-site-tools.ts b/web/src/lib/agent/agent-site-tools.ts index 1c5b49b..e031351 100644 --- a/web/src/lib/agent/agent-site-tools.ts +++ b/web/src/lib/agent/agent-site-tools.ts @@ -4,6 +4,7 @@ import { fetchPrompts } from "@/services/api/prompts"; import { uploadImage } from "@/services/image-storage"; import { imageAspectOptions, imageQualityOptions } from "@/components/image-settings-panel"; import { videoResolutionOptions, videoSecondOptions, videoSizeOptions } from "@/components/video-settings-panel"; +import type { CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops"; import { useCanvasStore } from "@/stores/canvas/use-canvas-store"; import { useAssetStore } from "@/stores/use-asset-store"; import { modelOptionLabel, modelOptionName, normalizeModelOptionValue, selectableModelsByCapability, useConfigStore } from "@/stores/use-config-store"; @@ -14,6 +15,7 @@ import { useWorkbenchAgentStore } from "@/stores/use-workbench-agent-store"; export const SITE_TOOL_NAMES = [ "canvas_list_projects", + "generation_get_status", "workbench_image_get_config", "workbench_image_generate", "workbench_video_get_config", @@ -31,6 +33,7 @@ export function isSiteTool(name: string): name is SiteToolName { export const SITE_TOOL_LABELS: Record = { canvas_list_projects: "画布列表", + generation_get_status: "生成任务状态", workbench_image_get_config: "生图配置", workbench_image_generate: "生图工作台生成", workbench_video_get_config: "视频配置", @@ -41,11 +44,16 @@ export const SITE_TOOL_LABELS: Record = { }; type SiteToolInput = Record; +type SiteToolContext = { canvasSnapshot?: CanvasAgentSnapshot | null }; +type GenerationStatus = "idle" | "queued" | "running" | "succeeded" | "failed"; +type GenerationStatusItem = { id: string; source: "canvas" | "image" | "video"; status: GenerationStatus; kind?: string; title?: string; prompt?: string; projectId?: string; createdAt?: string; updatedAt?: string; successCount?: number; failCount?: number; error?: string }; -export async function runSiteTool(name: SiteToolName, input: SiteToolInput, navigate: NavigateFunction): Promise { +export async function runSiteTool(name: SiteToolName, input: SiteToolInput, navigate: NavigateFunction, context: SiteToolContext = {}): Promise { switch (name) { case "canvas_list_projects": return listCanvasProjects(input); + case "generation_get_status": + return getGenerationStatus(input, context.canvasSnapshot); case "workbench_image_get_config": return getImageConfig(); case "workbench_image_generate": @@ -65,6 +73,56 @@ export async function runSiteTool(name: SiteToolName, input: SiteToolInput, navi } } +function getGenerationStatus(input: SiteToolInput, canvasSnapshot?: CanvasAgentSnapshot | null) { + const scope = input.scope === "canvas" || input.scope === "image" || input.scope === "video" ? input.scope : "all"; + const taskId = typeof input.taskId === "string" ? input.taskId : ""; + const nodeIds = new Set(Array.isArray(input.nodeIds) ? input.nodeIds.filter((id): id is string => typeof id === "string") : []); + const limit = Math.max(1, Math.min(100, Math.floor(Number(input.limit)) || 20)); + const tasks: GenerationStatusItem[] = []; + const includeCanvas = (scope === "all" || scope === "canvas") && (!taskId || nodeIds.size > 0); + const includeWorkbench = !nodeIds.size || Boolean(taskId); + + if (includeCanvas && canvasSnapshot) { + canvasSnapshot.nodes.forEach((node) => { + const status = normalizeCanvasGenerationStatus(node.metadata?.status); + if (!status || (nodeIds.size && !nodeIds.has(node.id))) return; + const metadata = node.metadata || {}; + if (!nodeIds.size && node.type !== "config" && status !== "running" && status !== "failed" && !metadata.generationMode && !metadata.generationType && !metadata.model) return; + tasks.push({ id: node.id, source: "canvas", status, kind: metadata.generationMode || node.type, title: node.title, prompt: compactPrompt(metadata.prompt || metadata.composerContent), projectId: canvasSnapshot.projectId, error: metadata.errorDetails }); + }); + } + + if (includeWorkbench) { + useWorkbenchAgentStore.getState().tasks.forEach((task) => { + if ((scope === "image" || scope === "video") && task.kind !== scope) return; + if (scope === "canvas" || (taskId && task.id !== taskId)) return; + tasks.push({ ...task, source: task.kind, prompt: compactPrompt(task.prompt) }); + }); + } + + tasks.sort((a, b) => generationStatusOrder(a.status) - generationStatusOrder(b.status) || (b.updatedAt || "").localeCompare(a.updatedAt || "")); + const summary: Record = { idle: 0, queued: 0, running: 0, succeeded: 0, failed: 0 }; + tasks.forEach((task) => (summary[task.status] += 1)); + return { total: tasks.length, summary, tasks: tasks.slice(0, limit) }; +} + +function generationStatusOrder(status: GenerationStatus) { + return status === "running" ? 0 : status === "queued" ? 1 : 2; +} + +function normalizeCanvasGenerationStatus(status: unknown): GenerationStatus | null { + if (status === "idle") return "idle"; + if (status === "loading") return "running"; + if (status === "success") return "succeeded"; + if (status === "error") return "failed"; + return null; +} + +function compactPrompt(prompt: unknown) { + const value = typeof prompt === "string" ? prompt.trim() : ""; + return value ? `${value.slice(0, 200)}${value.length > 200 ? "..." : ""}` : undefined; +} + function listCanvasProjects(input: SiteToolInput) { const { projects, hydrated } = useCanvasStore.getState(); if (!hydrated) throw new Error("画布还在加载中，请稍后重试"); @@ -118,8 +176,8 @@ function runImageWorkbench(input: SiteToolInput, navigate: NavigateFunction) { const prompt = typeof input.prompt === "string" ? input.prompt : undefined; const run = input.run !== false; navigate("/image"); - useWorkbenchAgentStore.getState().dispatchImage({ prompt, run }); - return { ok: true, navigated: "/image", prompt, run, applied, note: run ? "已跳转生图工作台并触发生成，结果请稍后在工作台查看" : "已跳转生图工作台并填入参数，未触发生成" }; + const taskId = useWorkbenchAgentStore.getState().dispatchImage({ prompt, run }); + return { ok: true, navigated: "/image", prompt, run, taskId, applied, note: run ? "已跳转生图工作台并触发生成，可用 generation_get_status 查询任务" : "已跳转生图工作台并填入参数，未触发生成" }; } function getVideoConfig() { @@ -173,8 +231,8 @@ function runVideoWorkbench(input: SiteToolInput, navigate: NavigateFunction) { const prompt = typeof input.prompt === "string" ? input.prompt : undefined; const run = input.run !== false; navigate("/video"); - useWorkbenchAgentStore.getState().dispatchVideo({ prompt, run }); - return { ok: true, navigated: "/video", prompt, run, applied, note: run ? "已跳转视频创作台并触发生成，结果请稍后在工作台查看" : "已跳转视频创作台并填入参数，未触发生成" }; + const taskId = useWorkbenchAgentStore.getState().dispatchVideo({ prompt, run }); + return { ok: true, navigated: "/video", prompt, run, taskId, applied, note: run ? "已跳转视频创作台并触发生成，可用 generation_get_status 查询任务" : "已跳转视频创作台并填入参数，未触发生成" }; } async function searchPrompts(input: SiteToolInput) { diff --git a/web/src/pages/image/index.tsx b/web/src/pages/image/index.tsx index b95ef3f..7d3b642 100644 --- a/web/src/pages/image/index.tsx +++ b/web/src/pages/image/index.tsx @@ -92,7 +92,9 @@ export default function ImagePage() { const [autoRunToken, setAutoRunToken] = useState(0); const imageCommand = useWorkbenchAgentStore((state) => state.imageCommand); const clearImageCommand = useWorkbenchAgentStore((state) => state.clearImageCommand); + const updateAgentTask = useWorkbenchAgentStore((state) => state.updateTask); const processedCommandRef = useRef(0); + const agentTaskIdRef = useRef(undefined); const model = effectiveConfig.imageModel || effectiveConfig.model; const canGenerate = Boolean(prompt.trim()); @@ -141,22 +143,30 @@ export default function ImagePage() { }; const generate = async () => { + const agentTaskId = agentTaskIdRef.current; + agentTaskIdRef.current = undefined; const text = prompt.trim(); if (!text) { message.error("请输入生图提示词"); + if (agentTaskId) updateAgentTask(agentTaskId, { status: "failed", error: "请输入生图提示词" }); return; } if (!isAiConfigReady(effectiveConfig, model)) { message.warning("请先完成配置"); openConfigDialog(true); + if (agentTaskId) updateAgentTask(agentTaskId, { status: "failed", error: "生图配置不完整" }); return; } const snapshot = buildRequestSnapshot(); - if (!snapshot) return; + if (!snapshot) { + if (agentTaskId) updateAgentTask(agentTaskId, { status: "failed", error: "生图参数无效" }); + return; + } setElapsedMs(0); setRunning(true); + if (agentTaskId) updateAgentTask(agentTaskId, { status: "running", error: undefined }); setPreviewLog(null); setResults(Array.from({ length: generationCount }, () => ({ id: nanoid(), status: "pending" }))); const batchStartedAt = performance.now(); @@ -169,6 +179,8 @@ export default function ImagePage() { const successCount = successImages.length; const failCount = generationCount - successCount; const failed = result.find((item): item is PromiseRejectedResult => item.status === "rejected"); + const error = failed?.reason instanceof Error ? failed.reason.message : failCount ? "生成失败" : undefined; + if (agentTaskId) updateAgentTask(agentTaskId, { status: successCount ? "succeeded" : "failed", successCount, failCount, error: successCount ? undefined : error }); try { const logImages = await Promise.all( @@ -202,8 +214,15 @@ export default function ImagePage() { processedCommandRef.current = imageCommand.nonce; clearImageCommand(); if (typeof imageCommand.prompt === "string") setPrompt(imageCommand.prompt); - if (imageCommand.run && !running) setAutoRunToken((value) => value + 1); - }, [imageCommand, clearImageCommand, running]); + if (imageCommand.run && running) { + if (imageCommand.taskId) updateAgentTask(imageCommand.taskId, { status: "failed", error: "生图工作台已有任务正在运行" }); + return; + } + if (imageCommand.run) { + agentTaskIdRef.current = imageCommand.taskId; + setAutoRunToken((value) => value + 1); + } + }, [imageCommand, clearImageCommand, running, updateAgentTask]); useEffect(() => { if (!autoRunToken) return; diff --git a/web/src/pages/video/index.tsx b/web/src/pages/video/index.tsx index f88caa9..f154100 100644 --- a/web/src/pages/video/index.tsx +++ b/web/src/pages/video/index.tsx @@ -97,7 +97,9 @@ export default function VideoPage() { const [autoRunToken, setAutoRunToken] = useState(0); const videoCommand = useWorkbenchAgentStore((state) => state.videoCommand); const clearVideoCommand = useWorkbenchAgentStore((state) => state.clearVideoCommand); + const updateAgentTask = useWorkbenchAgentStore((state) => state.updateTask); const processedCommandRef = useRef(0); + const agentTaskIdRef = useRef(undefined); const model = effectiveConfig.videoModel || effectiveConfig.model; const canGenerate = Boolean(prompt.trim()); @@ -170,10 +172,16 @@ export default function VideoPage() { } }; const generate = async () => { + const agentTaskId = agentTaskIdRef.current; + agentTaskIdRef.current = undefined; const snapshot = buildRequestSnapshot(); - if (!snapshot) return; + if (!snapshot) { + if (agentTaskId) updateAgentTask(agentTaskId, { status: "failed", error: "视频生成参数无效" }); + return; + } setElapsedMs(0); setRunning(true); + if (agentTaskId) updateAgentTask(agentTaskId, { status: "running", error: undefined }); setPreviewLog(null); setResults([{ id: nanoid(), status: "pending" }]); const batchStartedAt = performance.now(); @@ -181,11 +189,12 @@ export default function VideoPage() { try { const task = await createVideoGenerationTask(snapshot.config, snapshot.text, snapshot.references, snapshot.videoReferences, snapshot.audioReferences); const log = buildLog({ prompt: snapshot.text, model, config: snapshot.config, references: snapshot.references, videoReferences: snapshot.videoReferences, audioReferences: snapshot.audioReferences, durationMs: 0, status: "生成中", task }); - await saveLog(log); - void pollGenerationLog(log, snapshot.config); + await saveLog(log, false); + void pollGenerationLog(log, snapshot.config, agentTaskId); } catch (error) { const errorMessage = error instanceof Error ? error.message : "生成失败"; setResults([{ id: nanoid(), status: "failed", error: errorMessage }]); + if (agentTaskId) updateAgentTask(agentTaskId, { status: "failed", successCount: 0, failCount: 1, error: errorMessage }); await saveLog(buildLog({ prompt: snapshot.text, model, config: snapshot.config, references: snapshot.references, videoReferences: snapshot.videoReferences, audioReferences: snapshot.audioReferences, durationMs: performance.now() - batchStartedAt, status: "失败", error: errorMessage })); message.error(errorMessage); setRunning(false); @@ -198,8 +207,15 @@ export default function VideoPage() { processedCommandRef.current = videoCommand.nonce; clearVideoCommand(); if (typeof videoCommand.prompt === "string") setPrompt(videoCommand.prompt); - if (videoCommand.run && !running) setAutoRunToken((value) => value + 1); - }, [videoCommand, clearVideoCommand, running]); + if (videoCommand.run && running) { + if (videoCommand.taskId) updateAgentTask(videoCommand.taskId, { status: "failed", error: "视频工作台已有任务正在运行" }); + return; + } + if (videoCommand.run) { + agentTaskIdRef.current = videoCommand.taskId; + setAutoRunToken((value) => value + 1); + } + }, [videoCommand, clearVideoCommand, running, updateAgentTask]); useEffect(() => { if (!autoRunToken) return; @@ -276,7 +292,7 @@ export default function VideoPage() { .filter((log) => selectedLogIds.includes(log.id)) .map((log) => log.video?.storageKey) .filter((key): key is string => Boolean(key)); - void Promise.all([deleteStoredMedia(mediaKeys), ...selectedLogIds.map((id) => logStore.removeItem(id))]).then(refreshLogs); + void Promise.all([deleteStoredMedia(mediaKeys), ...selectedLogIds.map((id) => logStore.removeItem(id))]).then(() => refreshLogs()); if (previewLog && selectedLogIds.includes(previewLog.id)) { setPreviewLog(null); setResults([]); @@ -285,15 +301,15 @@ export default function VideoPage() { setDeleteConfirmOpen(false); }; - const saveLog = async (log: GenerationLog) => { + const saveLog = async (log: GenerationLog, resumePending = true) => { await logStore.setItem(log.id, serializeLog(log)); - await refreshLogs(); + await refreshLogs(resumePending); }; - const refreshLogs = async () => { + const refreshLogs = async (resumePending = true) => { const nextLogs = await readStoredLogs(); setLogs(nextLogs); - resumePendingLogs(nextLogs); + if (resumePending) resumePendingLogs(nextLogs); return nextLogs; }; @@ -303,7 +319,7 @@ export default function VideoPage() { } }; - const pollGenerationLog = async (log: GenerationLog, configOverride?: AiConfig) => { + const pollGenerationLog = async (log: GenerationLog, configOverride?: AiConfig, agentTaskId?: string) => { if (!log.task || activeLogIdsRef.current.has(log.id)) return; activeLogIdsRef.current.add(log.id); setRunning(true); @@ -326,6 +342,7 @@ export default function VideoPage() { mimeType: stored.mimeType, }; setResults([{ id: nextVideo.id, status: "success", video: nextVideo }]); + if (agentTaskId) updateAgentTask(agentTaskId, { status: "succeeded", successCount: 1, failCount: 0, error: undefined }); await saveLog({ ...log, status: "成功", durationMs: nextVideo.durationMs, video: nextVideo, error: undefined }); message.success("视频已生成"); return; @@ -337,6 +354,7 @@ export default function VideoPage() { } catch (error) { const errorMessage = error instanceof Error ? error.message : "生成失败"; setResults([{ id: log.id, status: "failed", error: errorMessage }]); + if (agentTaskId) updateAgentTask(agentTaskId, { status: "failed", successCount: 0, failCount: 1, error: errorMessage }); await saveLog({ ...log, status: "失败", durationMs: Date.now() - log.createdAt, error: errorMessage }); message.error(errorMessage); } finally { diff --git a/web/src/stores/use-workbench-agent-store.ts b/web/src/stores/use-workbench-agent-store.ts index 02d4c43..cb30422 100644 --- a/web/src/stores/use-workbench-agent-store.ts +++ b/web/src/stores/use-workbench-agent-store.ts @@ -6,15 +6,30 @@ import { create } from "zustand"; export type WorkbenchCommand = { nonce: number; + taskId?: string; prompt?: string; run: boolean; }; +export type WorkbenchGenerationTask = { + id: string; + kind: "image" | "video"; + status: "queued" | "running" | "succeeded" | "failed"; + prompt?: string; + createdAt: string; + updatedAt: string; + successCount?: number; + failCount?: number; + error?: string; +}; + type WorkbenchAgentStore = { imageCommand: WorkbenchCommand | null; videoCommand: WorkbenchCommand | null; - dispatchImage: (command: Omit) => void; - dispatchVideo: (command: Omit) => void; + tasks: WorkbenchGenerationTask[]; + dispatchImage: (command: Omit) => string | undefined; + dispatchVideo: (command: Omit) => string | undefined; + updateTask: (id: string, patch: Partial>) => void; clearImageCommand: () => void; clearVideoCommand: () => void; }; @@ -25,8 +40,25 @@ const nextNonce = () => (nonce += 1); export const useWorkbenchAgentStore = create((set) => ({ imageCommand: null, videoCommand: null, - dispatchImage: (command) => set({ imageCommand: { ...command, nonce: nextNonce() } }), - dispatchVideo: (command) => set({ videoCommand: { ...command, nonce: nextNonce() } }), + tasks: [], + dispatchImage: (command) => { + const commandNonce = nextNonce(); + const task = command.run ? createTask("image", commandNonce, command.prompt) : undefined; + set((state) => ({ imageCommand: { ...command, nonce: commandNonce, taskId: task?.id }, tasks: task ? [task, ...state.tasks].slice(0, 30) : state.tasks })); + return task?.id; + }, + dispatchVideo: (command) => { + const commandNonce = nextNonce(); + const task = command.run ? createTask("video", commandNonce, command.prompt) : undefined; + set((state) => ({ videoCommand: { ...command, nonce: commandNonce, taskId: task?.id }, tasks: task ? [task, ...state.tasks].slice(0, 30) : state.tasks })); + return task?.id; + }, + updateTask: (id, patch) => set((state) => ({ tasks: state.tasks.map((task) => (task.id === id ? { ...task, ...patch, updatedAt: new Date().toISOString() } : task)) })), clearImageCommand: () => set({ imageCommand: null }), clearVideoCommand: () => set({ videoCommand: null }), })); + +function createTask(kind: "image" | "video", commandNonce: number, prompt?: string): WorkbenchGenerationTask { + const now = new Date().toISOString(); + return { id: `${kind}-${commandNonce}`, kind, status: "queued", prompt, createdAt: now, updatedAt: now }; +} From 5e1fd7a825ebcb89c0351ce0aec05076d1b7aa99 Mon Sep 17 00:00:00 2001 From: yu <2799341137@qq.com> Date: Fri, 17 Jul 2026 18:30:10 +0800 Subject: [PATCH 2/3] fix(agent): synchronize shared Codex sessions --- CHANGELOG.md | 4 +- canvas-agent/src/agents.ts | 82 ++++++--- canvas-agent/src/canvas-session.test.ts | 72 ++++++++ canvas-agent/src/canvas-session.ts | 38 +++- canvas-agent/src/http-server.ts | 101 +++++++++-- docs/content/docs/progress/pending-test.mdx | 4 +- .../canvas/canvas-agent-chat-ui.tsx | 2 +- .../canvas/canvas-local-agent-panel.tsx | 171 ++++++++++-------- 8 files changed, 347 insertions(+), 127 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60a478e..5143415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,9 @@ ## Unreleased + [新增] Agent 新增统一生成任务状态查询，支持查看画布、生图工作台和视频工作台任务进度。 -+ [修复] 修复多标签页同时连接本地 Agent 时 MCP 可能操作非当前页面的问题。 ++ [修复] 本地 Agent 请求与发起标签页强绑定，校验工具结果归属，并在页面关闭后回退到最近聚焦页面。 ++ [修复] 多标签页共享 Codex 会话改由 Agent 广播同步，按线程过滤消息和状态，并禁止运行中切换会话。 ++ [修复] 本地 Agent 统一广播 Codex 运行状态，新连接标签页会同步禁用发送，并明确区分工具完成与本轮完成。 + [新增] 提示词来源增加自定义脚本抓取功能、支持新增来源。 ## v0.9.0 - 2026-07-17 diff --git a/canvas-agent/src/agents.ts b/canvas-agent/src/agents.ts index 209bbfa..e215bf8 100644 --- a/canvas-agent/src/agents.ts +++ b/canvas-agent/src/agents.ts @@ -11,11 +11,12 @@ import type { AgentAttachment, AgentEmit } from "./types.js"; type Json = Record; type AgentEvent = Json & { type: string; usage?: unknown }; type PendingRequest = { resolve: (value: unknown) => void; reject: (error: Error) => void }; -type CodexRunOptions = { threadId?: string; cwd?: string }; +type CodexRunOptions = { threadId?: string; cwd?: string; appEmit?: AgentEmit; onStart?: () => void; onThread?: (threadId: string) => void; onTurn?: (turnId: string) => void; onFinish?: () => void }; type AgentHistoryMessage = { id: string; role: "user" | "assistant" | "tool" | "error"; title?: string; text: string; detail?: unknown; streamId?: string }; let codexQueue: Promise = Promise.resolve(); let codexApp: CodexAppClient | null = null; +let codexAppStart: Promise | null = null; let codexThreadId = ""; const canvasAgentMcp = canvasAgentMcpCommand(); const require = createRequire(import.meta.url); @@ -30,52 +31,56 @@ export async function runCodexTurn(prompt: string, emit: AgentEmit, attachments: await codexQueue; } -export function interruptCodexTurn() { - if (!codexApp) return false; +export function interruptCodexTurn(threadId?: string) { + if (!codexApp || (threadId && threadId !== codexThreadId)) return false; return codexApp.interruptCurrentTurn(); } async function runCodexTurnNow(prompt: string, emit: AgentEmit, attachments: AgentAttachment[], options: CodexRunOptions) { let files: string[] = []; try { + options.onStart?.(); files = await writeAttachmentFiles(attachments); - codexApp ||= await CodexAppClient.start(emit); - let threadId = await ensureCodexThread(codexApp, options, emit); + const app = await getCodexApp(options.appEmit || emit); + let threadId = await ensureCodexThread(app, options, emit); + options.onThread?.(threadId); try { - await codexApp.startTurn(threadId, prompt, files); + await app.startTurn(threadId, prompt, files, options.onTurn); } catch (error) { if (!isRecoverableThreadError(error)) throw error; emit("agent_log", { text: `Codex thread unavailable, starting a new thread: ${errorMessage(error)}` }); codexThreadId = ""; - threadId = await ensureCodexThread(codexApp, { cwd: options.cwd }, emit); - await codexApp.startTurn(threadId, prompt, files); + threadId = await ensureCodexThread(app, { cwd: options.cwd }, emit); + options.onThread?.(threadId); + await app.startTurn(threadId, prompt, files, options.onTurn); } } catch (error) { emit("agent_error", { message: errorMessage(error) }); } finally { + options.onFinish?.(); await Promise.all(files.map((file) => fs.unlink(file).catch(() => undefined))); } } export async function startCodexThread(emit: AgentEmit, cwd?: string) { - codexApp ||= await CodexAppClient.start(emit); - const thread = await codexApp.startThread(cwd); + const app = await getCodexApp(emit); + const thread = await app.startThread(cwd); codexThreadId = String(field(thread, "id") || ""); return thread; } export async function resumeCodexThread(emit: AgentEmit, threadId: string, cwd?: string) { - codexApp ||= await CodexAppClient.start(emit); + const app = await getCodexApp(emit); await loadCodexThread(emit, threadId, cwd, false); - const thread = await codexApp.resumeThread(threadId, cwd); + const thread = await app.resumeThread(threadId, cwd); assertThreadWorkspace(thread, cwd); codexThreadId = String(field(thread, "id") || threadId); return { thread, messages: threadMessages(thread) }; } export async function listCodexThreads(emit: AgentEmit, options: { cwd: string; searchTerm?: string; limit?: number }) { - codexApp ||= await CodexAppClient.start(emit); - const result = await codexApp.listThreads({ + const app = await getCodexApp(emit); + const result = await app.listThreads({ limit: options.limit || 40, sortKey: "updated_at", sortDirection: "desc", @@ -97,9 +102,9 @@ export async function verifyCodexThreadWorkspace(emit: AgentEmit, threadId: stri } export async function archiveCodexThread(emit: AgentEmit, threadId: string, cwd?: string) { - codexApp ||= await CodexAppClient.start(emit); + const app = await getCodexApp(emit); await loadCodexThread(emit, threadId, cwd, false); - await codexApp.archiveThread(threadId); + await app.archiveThread(threadId); } export function runClaudeTurn(prompt: string, emit: AgentEmit) { @@ -131,7 +136,7 @@ async function ensureCodexThread(app: CodexAppClient, options: CodexRunOptions, return codexThreadId; } -function isRecoverableThreadError(error: unknown) { +export function isRecoverableThreadError(error: unknown) { return /thread not loaded|no rollout found/i.test(errorMessage(error)); } @@ -192,10 +197,11 @@ class CodexAppClient { return this.request("thread/archive", { threadId }); } - async startTurn(threadId: string, prompt: string, images: string[]) { + async startTurn(threadId: string, prompt: string, images: string[], onTurn?: (turnId: string) => void) { const result = await this.request("turn/start", { threadId, input: codexInput(prompt, images), approvalPolicy: "never" }); const turnId = String(field(field(result, "turn"), "id") || ""); if (!turnId) throw new Error("Codex app-server 没有返回 turn id"); + onTurn?.(turnId); const completed = this.completedTurns.get(turnId); if (this.completedTurns.has(turnId)) { this.completedTurns.delete(turnId); @@ -267,9 +273,9 @@ class CodexAppClient { } else if (turnId) { this.completedTurns.set(turnId, error ? new Error(String(field(error, "message") || "Codex turn failed")) : null); } - this.emit("agent_event", { agent: "codex", type: "stream.summary", delta_count: this.deltaCount }); + this.emit("agent_event", { agent: "codex", type: "stream.summary", delta_count: this.deltaCount, ...codexEventScope(params) }); this.deltaCount = 0; - this.emit("agent_done", { agent: "codex", usage: event.usage }); + this.emit("agent_done", { agent: "codex", usage: event.usage, ...codexEventScope(params) }); } } @@ -278,7 +284,7 @@ class CodexAppClient { const text = `${this.textByItem.get(id) || ""}${String(field(params, "delta") || "")}`; this.deltaCount += 1; this.textByItem.set(id, text); - this.emit("agent_event", { agent: "codex", type: "item.updated", item: { id, type: "agent_message", text } }); + this.emit("agent_event", { agent: "codex", type: "item.updated", item: { id, type: "agent_message", text }, ...codexEventScope(params) }); } private answerServerRequest(message: Json) { @@ -321,23 +327,41 @@ function codexInput(prompt: string, images: string[]) { } function normalizeCodexNotification(method: string, params: Json): AgentEvent | null { - if (method === "thread/started") return { type: "thread.started", thread_id: field(field(params, "thread"), "id") }; - if (method === "turn/started") return { type: "turn.started" }; - if (method === "turn/completed") return { type: "turn.completed", usage: null }; - if (method === "item/started") return { type: "item.started", item: normalizeItem(field(params, "item")) }; - if (method === "item/completed") return { type: "item.completed", item: normalizeItem(field(params, "item")) }; - if (method === "error") return { type: "error", message: field(params, "message") }; + const scope = codexEventScope(params); + if (method === "thread/started") return { type: "thread.started", ...scope }; + if (method === "turn/started") return { type: "turn.started", ...scope }; + if (method === "turn/completed") return { type: "turn.completed", usage: null, ...scope }; + if (method === "item/started") return { type: "item.started", item: normalizeItem(field(params, "item")), ...scope }; + if (method === "item/completed") return { type: "item.completed", item: normalizeItem(field(params, "item")), ...scope }; + if (method === "error") return { type: "error", message: field(params, "message"), ...scope }; return null; } +function codexEventScope(params: Json) { + const threadId = String(field(params, "threadId") || field(field(params, "thread"), "id") || ""); + const turnId = String(field(params, "turnId") || field(field(params, "turn"), "id") || ""); + return { ...(threadId ? { thread_id: threadId } : {}), ...(turnId ? { turn_id: turnId } : {}) }; +} + async function loadCodexThread(emit: AgentEmit, threadId: string, cwd: string | undefined, includeTurns: boolean) { - codexApp ||= await CodexAppClient.start(emit); - const result = await codexApp.readThread(threadId, includeTurns); + const app = await getCodexApp(emit); + const result = await app.readThread(threadId, includeTurns); const thread = field(result, "thread") || {}; assertThreadWorkspace(thread, cwd); return thread; } +async function getCodexApp(emit: AgentEmit) { + if (codexApp) return codexApp; + codexAppStart ||= CodexAppClient.start(emit); + try { + codexApp = await codexAppStart; + return codexApp; + } finally { + codexAppStart = null; + } +} + function assertThreadWorkspace(thread: unknown, cwd?: string) { if (!cwd || threadInWorkspace(thread, cwd)) return; throw new Error("该 Codex 会话不属于当前画布工作空间"); diff --git a/canvas-agent/src/canvas-session.test.ts b/canvas-agent/src/canvas-session.test.ts index 4188772..20780ae 100644 --- a/canvas-agent/src/canvas-session.test.ts +++ b/canvas-agent/src/canvas-session.test.ts @@ -132,6 +132,78 @@ test("closing a client rejects its pending tool requests", async () => { assert.match(outcome, /断开/); }); +test("shared thread events are broadcast with the active thread id", (t) => { + const session = new CanvasSession(); + const first = connect(session, "first"); + const second = connect(session, "second"); + t.after(() => { + first.close(); + second.close(); + }); + + session.emitThread("workspace_changed", "thread-2", { activeThreadId: "thread-2" }); + + assert.deepEqual(first.event("workspace_changed"), { activeThreadId: "thread-2", threadId: "thread-2" }); + assert.deepEqual(second.event("workspace_changed"), { activeThreadId: "thread-2", threadId: "thread-2" }); +}); + +test("new clients receive the current Codex state and later updates", (t) => { + const session = new CanvasSession(); + session.setCodexState({ busy: true, threadId: "thread-2", turnId: "turn-1" }); + const client = connect(session, "first"); + t.after(() => client.close()); + + assert.deepEqual(field(client.event("hello"), "codex"), { busy: true, threadId: "thread-2", turnId: "turn-1" }); + + session.setCodexState({ busy: false }); + assert.deepEqual(client.event("codex_state"), { busy: false, threadId: "thread-2", turnId: "turn-1" }); +}); + +test("a bound client remains the tool target while focus changes", async (t) => { + const session = new CanvasSession(); + const first = connect(session, "first"); + const second = connect(session, "second"); + t.after(() => { + first.close(); + second.close(); + }); + session.updateState(snapshot("canvas-first"), "first"); + session.updateState(snapshot("canvas-second"), "second"); + session.bindClient("first"); + session.activateClient("second"); + + assert.equal(field(await session.callTool("canvas_get_state", {}), "projectId"), "canvas-first"); + const result = session.callTool("canvas_create_text_node", { text: "bound" }); + const call = first.event("tool_call"); + assert.equal(second.event("tool_call"), undefined); + session.resolveResult("first", { requestId: String(field(call, "requestId")), result: { ok: true } }); + assert.deepEqual(await result, { ok: true }); + + session.releaseClient("first"); + assert.equal(field(await session.callTool("canvas_get_state", {}), "projectId"), "canvas-second"); +}); + +test("closing the bound client falls back to the active client", async (t) => { + const session = new CanvasSession(); + const first = connect(session, "first"); + const second = connect(session, "second"); + t.after(() => { + first.close(); + second.close(); + }); + session.updateState(snapshot("canvas-first"), "first"); + session.updateState(snapshot("canvas-second"), "second"); + session.bindClient("first"); + session.activateClient("second"); + first.close(); + + assert.equal(field(await session.callTool("canvas_get_state", {}), "projectId"), "canvas-second"); + const result = session.callTool("canvas_create_text_node", { text: "fallback" }); + const call = second.event("tool_call"); + session.resolveResult("second", { requestId: String(field(call, "requestId")), result: { ok: true } }); + assert.deepEqual(await result, { ok: true }); +}); + function connect(session: CanvasSession, clientId: string) { const response = new FakeSseResponse(); session.openEvents(new URL(`http://127.0.0.1/events?clientId=${clientId}`), response as unknown as ServerResponse); diff --git a/canvas-agent/src/canvas-session.ts b/canvas-agent/src/canvas-session.ts index 1efaeea..aadc5f8 100644 --- a/canvas-agent/src/canvas-session.ts +++ b/canvas-agent/src/canvas-session.ts @@ -6,6 +6,7 @@ import { compactCanvasState, compactNode, isToolName, nextCanvasX, parseToolInpu import type { CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js"; type PendingRequest = { clientId: string; resolve: (value: unknown) => void; reject: (error: Error) => void }; +export type CodexState = { busy: boolean; threadId: string; turnId: string }; const SITE_TOOLS = new Set([ "site_navigate", @@ -26,14 +27,29 @@ export class CanvasSession { private pending = new Map(); private canvasStates = new Map(); private activeClientId = ""; + private boundClientId = ""; private focusSequence = 0; + private codexState: CodexState = { busy: false, threadId: "", turnId: "" }; private get canvasState() { - return this.canvasStates.get(this.activeClientId) || null; + return this.canvasStates.get(this.targetClientId) || null; + } + + private get targetClientId() { + return this.boundClientId || this.activeClientId; } health() { - return { ok: true, hasCanvas: Boolean(this.canvasState), clients: this.clients.size }; + return { ok: true, hasCanvas: Boolean(this.canvasState), clients: this.clients.size, codexBusy: this.codexState.busy }; + } + + get codexBusy() { + return this.codexState.busy; + } + + setCodexState(patch: Partial) { + this.codexState = { ...this.codexState, ...patch }; + this.emitAll("codex_state", this.codexState); } openEvents(url: URL, res: ServerResponse) { @@ -48,7 +64,7 @@ export class CanvasSession { this.clientFocusOrder.set(clientId, ++this.focusSequence); } } - sendEvent(res, "hello", { ok: true, clientId }); + sendEvent(res, "hello", { ok: true, clientId, codex: this.codexState }); const timer = setInterval(() => sendEvent(res, "ping", { time: Date.now() }), 15000); res.on("close", () => { clearInterval(timer); @@ -56,6 +72,7 @@ export class CanvasSession { this.clients.delete(clientId); this.clientFocusOrder.delete(clientId); this.canvasStates.delete(clientId); + if (this.boundClientId === clientId) this.boundClientId = ""; this.pending.forEach((item, requestId) => { if (item.clientId !== clientId) return; this.pending.delete(requestId); @@ -77,6 +94,15 @@ export class CanvasSession { this.clientFocusOrder.set(clientId, ++this.focusSequence); } + bindClient(clientId: string) { + if (!this.clients.has(clientId)) throw new Error("当前网页未连接"); + this.boundClientId = clientId; + } + + releaseClient(clientId: string) { + if (this.boundClientId === clientId) this.boundClientId = ""; + } + resolveResult(clientId: string, body: { requestId?: string; error?: string; result?: unknown }) { const item = body.requestId ? this.pending.get(body.requestId) : null; if (!item || !body.requestId || item.clientId !== clientId) return false; @@ -89,6 +115,10 @@ export class CanvasSession { this.clients.forEach((client) => sendEvent(client, type, payload)); } + emitThread(type: string, threadId: string, payload: Record = {}) { + this.emitAll(type, { ...payload, threadId }); + } + async callTool(name: unknown, rawInput: unknown) { if (!isToolName(name)) throw new Error(`未知工具：${String(name)}`); let tool: ToolName = name; @@ -200,7 +230,7 @@ export class CanvasSession { private async requestCanvasTool(name: ToolName, input: Record) { const requestId = crypto.randomUUID(); - const clientId = this.activeClientId; + const clientId = this.targetClientId; const client = this.clients.get(clientId); if (!client) throw new Error("当前没有已连接画布"); sendEvent(client, "tool_call", { requestId, name, input }); diff --git a/canvas-agent/src/http-server.ts b/canvas-agent/src/http-server.ts index d886236..f2d8c9d 100644 --- a/canvas-agent/src/http-server.ts +++ b/canvas-agent/src/http-server.ts @@ -2,7 +2,7 @@ import express, { type NextFunction, type Request, type Response } from "express import { DEFAULT_PORT, ensureSiteWorkspace, loadConfig, saveConfig, updateSiteWorkspace, type CanvasAgentConfig } from "./config.js"; import { CanvasSession } from "./canvas-session.js"; -import { archiveCodexThread, interruptCodexTurn, listCodexThreads, readCodexThread, resumeCodexThread, runClaudeTurn, runCodexTurn, startCodexThread, summarizeCodexThread, verifyCodexThreadWorkspace, withAgentPrompt } from "./agents.js"; +import { archiveCodexThread, interruptCodexTurn, isRecoverableThreadError, listCodexThreads, readCodexThread, resumeCodexThread, runClaudeTurn, runCodexTurn, startCodexThread, summarizeCodexThread, verifyCodexThreadWorkspace, withAgentPrompt } from "./agents.js"; import type { AgentAttachment } from "./types.js"; export function startHttpServer() { @@ -12,7 +12,16 @@ export function startHttpServer() { saveConfig(config); const session = new CanvasSession(); - const emit = (type: string, payload: unknown) => session.emitAll(type, payload); + const emit = (type: string, payload: unknown) => { + const data = payload && typeof payload === "object" && !Array.isArray(payload) ? payload as Record : { value: payload }; + const threadId = String(data.threadId || data.thread_id || ensureSiteWorkspace(config).activeThreadId || ""); + threadId ? session.emitThread(type, threadId, data) : session.emitAll(type, data); + }; + const setActiveThread = (activeThreadId: string, payload: Record = {}) => { + const workspace = updateSiteWorkspace(config, { activeThreadId: activeThreadId || undefined }); + session.emitThread("workspace_changed", activeThreadId, { ...payload, activeThreadId }); + return workspace; + }; const app = express(); app.disable("x-powered-by"); app.use(express.json({ limit: "30mb" })); @@ -52,48 +61,100 @@ export function startHttpServer() { res.json({ ok: true, workspace, ...result }); })); app.post("/agent/codex/threads/new", route(async (_req, res) => { + if (session.codexBusy) return res.status(409).json({ ok: false, error: "Codex 正在运行，请等待当前任务完成" }); const workspace = ensureSiteWorkspace(config); const thread = await startCodexThread(emit, workspace.workspacePath); const activeThreadId = String((thread as Record).id || ""); - updateSiteWorkspace(config, { activeThreadId }); - res.json({ ok: true, workspace: { ...workspace, activeThreadId }, thread: summarizeCodexThread(thread), messages: [] }); + const nextWorkspace = setActiveThread(activeThreadId, { emptyThread: true }); + res.json({ ok: true, workspace: nextWorkspace, thread: summarizeCodexThread(thread), messages: [] }); })); app.get("/agent/codex/threads/:threadId", route(async (req, res) => { const workspace = ensureSiteWorkspace(config); const threadId = routeParam(req.params.threadId); - res.json({ ok: true, workspace, ...(await readCodexThread(emit, threadId, workspace.workspacePath)) }); + try { + res.json({ ok: true, workspace, ...(await readCodexThread(emit, threadId, workspace.workspacePath)) }); + } catch (error) { + if (workspace.activeThreadId !== threadId || !isRecoverableThreadError(error)) throw error; + res.json({ ok: true, workspace, thread: { id: threadId, preview: "", cwd: workspace.workspacePath }, messages: [] }); + } })); app.post("/agent/codex/threads/:threadId/resume", route(async (req, res) => { + if (session.codexBusy) return res.status(409).json({ ok: false, error: "Codex 正在运行，请等待当前任务完成" }); const workspace = ensureSiteWorkspace(config); const threadId = routeParam(req.params.threadId); const result = await resumeCodexThread(emit, threadId, workspace.workspacePath); - updateSiteWorkspace(config, { activeThreadId: threadId }); - res.json({ ok: true, workspace: { ...workspace, activeThreadId: threadId }, ...result }); + const nextWorkspace = setActiveThread(threadId); + res.json({ ok: true, workspace: nextWorkspace, ...result }); })); app.post("/agent/codex/threads/:threadId/delete", route(async (req, res) => { + if (session.codexBusy) return res.status(409).json({ ok: false, error: "Codex 正在运行，请等待当前任务完成" }); const workspace = ensureSiteWorkspace(config); const threadId = routeParam(req.params.threadId); await archiveCodexThread(emit, threadId, workspace.workspacePath); - if (workspace.activeThreadId === threadId) updateSiteWorkspace(config, { activeThreadId: undefined }); + setActiveThread(workspace.activeThreadId === threadId ? "" : workspace.activeThreadId || ""); res.json({ ok: true }); })); app.post("/agent/codex/turn", route(async (req, res) => { + if (session.codexBusy) return res.status(409).json({ ok: false, error: "Codex 正在运行，请等待当前任务完成" }); const attachments = Array.isArray(req.body?.attachments) ? (req.body.attachments as AgentAttachment[]) : []; const workspace = ensureSiteWorkspace(config); - let threadId = String(req.body?.threadId || workspace.activeThreadId || ""); - if (!threadId) { - const thread = await startCodexThread(emit, workspace.workspacePath); - threadId = String((thread as Record).id || ""); - updateSiteWorkspace(config, { activeThreadId: threadId }); - } else if (threadId !== workspace.activeThreadId) { - await verifyCodexThreadWorkspace(emit, threadId, workspace.workspacePath); - updateSiteWorkspace(config, { activeThreadId: threadId }); + const prompt = String(req.body?.prompt || ""); + if (!prompt.trim()) return res.status(400).json({ ok: false, error: "请输入任务内容" }); + const clientId = String(req.body?.clientId || ""); + session.setCodexState({ busy: true, threadId: String(req.body?.threadId || workspace.activeThreadId || ""), turnId: "" }); + try { + let threadId = String(req.body?.threadId || workspace.activeThreadId || ""); + let turnId = ""; + if (!threadId) { + const thread = await startCodexThread(emit, workspace.workspacePath); + threadId = String((thread as Record).id || ""); + setActiveThread(threadId, { emptyThread: true }); + } else if (threadId !== workspace.activeThreadId) { + await verifyCodexThreadWorkspace(emit, threadId, workspace.workspacePath); + setActiveThread(threadId); + } + const chatMessage = { + sourceClientId: clientId, + message: { id: String(req.body?.messageId || Date.now()), role: "user", text: String(req.body?.messageText || prompt || `发送了 ${attachments.length} 张图片`) }, + }; + let chatThreadId = ""; + const turnEmit = (type: string, payload: unknown) => { + const data = payload && typeof payload === "object" && !Array.isArray(payload) ? payload as Record : { value: payload }; + session.emitThread(type, threadId, data); + }; + void runCodexTurn(withAgentPrompt(prompt), turnEmit, attachments, { + threadId, + cwd: workspace.workspacePath, + appEmit: emit, + onStart: clientId ? () => session.bindClient(clientId) : undefined, + onThread: (actualThreadId) => { + if (actualThreadId !== threadId) { + threadId = actualThreadId; + setActiveThread(threadId, { emptyThread: true }); + } + session.setCodexState({ busy: true, threadId, turnId: "" }); + if (chatThreadId !== threadId) { + chatThreadId = threadId; + session.emitThread("chat_message", threadId, chatMessage); + } + }, + onTurn: (actualTurnId) => { + turnId = actualTurnId; + session.setCodexState({ busy: true, threadId, turnId }); + }, + onFinish: () => { + if (clientId) session.releaseClient(clientId); + session.setCodexState({ busy: false, threadId, turnId }); + }, + }); + res.json({ ok: true, threadId }); + } catch (error) { + session.setCodexState({ busy: false, threadId: String(req.body?.threadId || workspace.activeThreadId || ""), turnId: "" }); + throw error; } - void runCodexTurn(withAgentPrompt(String(req.body?.prompt || "")), emit, attachments, { threadId, cwd: workspace.workspacePath }); - res.json({ ok: true, threadId }); })); - app.post("/agent/codex/interrupt", (_req, res) => { - const ok = interruptCodexTurn(); + app.post("/agent/codex/interrupt", (req, res) => { + const ok = interruptCodexTurn(String(req.body?.threadId || "")); res.json({ ok }); }); app.post("/agent/claude/turn", (req, res) => { diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index 875286e..809e130 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -6,4 +6,6 @@ description: 当前版本已实现但仍需人工验证的变更项 # 待测试 - 全站 Agent：新增 `generation_get_status` 工具，画布生成节点可按 `nodeIds` 查询，生图和视频工作台提交后会返回 `taskId` 并可查询排队、运行、成功或失败状态；需验证查询只由当前活动标签页返回。 -- 本地 Agent 多标签页隔离：同时打开两个不同画布并连接同一个 Agent，分别聚焦标签页后通过 MCP 读取和修改画布，操作应只落在当前聚焦页面；关闭当前页面后应自动回退到仍连接的页面。 +- 本地 Agent 多标签页隔离：同时打开两个不同画布并连接同一个 Agent，分别聚焦标签页后通过 MCP 读取和修改画布，操作应只落在当前聚焦页面；网页面板发起的整个 Codex turn 应固定操作发起页面，即使中途聚焦另一标签页也不能切换目标；关闭当前页面后应回退到最近聚焦且仍连接的页面，其他页面回传同一请求结果应被拒绝。 +- 本地 Agent 多标签页会话同步：所有标签页共享同一个站点级 Codex 活跃线程；任一页面发送消息、新建、恢复或删除会话后，其他页面应同步活跃线程和聊天记录；Agent 输出仅显示在事件所属线程，运行中不能新建、恢复、删除或再次发送任务。 +- 本地 Agent 运行状态同步：在一个标签页运行较长 Codex 任务，等待某张工具卡显示「工具完成」后再打开或刷新第二个标签页；第二个标签页应立即显示 Codex 正在运行并禁用发送，整轮结束后两个标签页同时恢复；工具卡只显示「工具完成」，整轮结束由「本轮完成」表示。 diff --git a/web/src/components/canvas/canvas-agent-chat-ui.tsx b/web/src/components/canvas/canvas-agent-chat-ui.tsx index ddfb271..3b8fbb9 100644 --- a/web/src/components/canvas/canvas-agent-chat-ui.tsx +++ b/web/src/components/canvas/canvas-agent-chat-ui.tsx @@ -303,7 +303,7 @@ function toolCardState(title: string, text: string, detail?: unknown) { if (objectField(detail, "status") === "noop" || /未生效|无需|没有找到|没有.*可|已存在/.test(raw)) return { label: "未生效", color: "#d97706", softBorder: "rgba(217,119,6,.22)", softBg: "rgba(217,119,6,.04)", icon: , isError: false }; if (/拒绝|取消/.test(raw) || lower.includes("rejected")) return { label: "拒绝执行", color: "#dc2626", softBorder: "rgba(220,38,38,.20)", softBg: "rgba(220,38,38,.04)", icon: , isError: true }; if (/失败|错误/.test(raw) || lower.includes("failed") || lower.includes("error")) return { label: "执行失败", color: "#dc2626", softBorder: "rgba(220,38,38,.20)", softBg: "rgba(220,38,38,.04)", icon: , isError: true }; - if (/完成|成功/.test(raw) || lower.includes("completed") || lower.includes("succeeded")) return { label: tool === "canvas_apply_ops" || /画布操作/.test(title) ? "已批准执行" : "执行完成", color: "#16a34a", softBorder: "rgba(22,163,74,.20)", softBg: "rgba(22,163,74,.04)", icon: , isError: false }; + if (/完成|成功/.test(raw) || lower.includes("completed") || lower.includes("succeeded")) return { label: tool === "canvas_apply_ops" || /画布操作/.test(title) ? "已批准执行" : "工具完成", color: "#16a34a", softBorder: "rgba(22,163,74,.20)", softBg: "rgba(22,163,74,.04)", icon: , isError: false }; return { label: "工具调用", color: "#2563eb", softBorder: "rgba(37,99,235,.20)", softBg: "rgba(37,99,235,.04)", icon: , isError: false }; } diff --git a/web/src/components/canvas/canvas-local-agent-panel.tsx b/web/src/components/canvas/canvas-local-agent-panel.tsx index eca3544..26bc9f4 100644 --- a/web/src/components/canvas/canvas-local-agent-panel.tsx +++ b/web/src/components/canvas/canvas-local-agent-panel.tsx @@ -27,7 +27,9 @@ const AGENT_MCP_REMOVE_COMMAND = "codex mcp remove infinite-canvas"; type AgentEventPayload = { agent?: string; type?: string; + threadId?: string; thread_id?: string; + turn_id?: string; item?: AgentEventItem; error?: { message?: string }; message?: string; @@ -40,6 +42,10 @@ type AgentWorkspace = { workspacePath: string; activeThreadId?: string }; type AgentThreadsResponse = { ok?: boolean; workspace?: AgentWorkspace; data?: AgentThreadSummary[] }; type AgentThreadResponse = { ok?: boolean; workspace?: AgentWorkspace; thread?: AgentThreadSummary; messages?: AgentChatItem[] }; type AgentConfigResponse = { ok?: boolean; url?: string; token?: string; hasToken?: boolean }; +type AgentCodexState = { busy?: boolean; threadId?: string; turnId?: string }; +type AgentHelloEvent = { ok?: boolean; clientId?: string; codex?: AgentCodexState }; +type AgentWorkspaceEvent = { activeThreadId?: string; threadId?: string; emptyThread?: boolean }; +type AgentChatEvent = { threadId?: string; sourceClientId?: string; message?: AgentChatItem }; export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { embedded?: boolean; headless?: boolean; autoConnect?: boolean }) { const theme = canvasThemes[useThemeStore((state) => state.theme)]; @@ -88,28 +94,27 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb const errorLoggedRef = useRef(false); const attachmentUrlsRef = useRef(new Set()); const clientIdRef = useRef(randomId()); + const loadThreadsSequenceRef = useRef(0); const endpoint = useMemo(() => url.trim().replace(/\/$/, ""), [url]); const urlAgentAutoConnect = searchParams.has("agentUrl") && searchParams.has("agentToken"); - const loadThreads = useCallback(async () => { + const loadThreads = useCallback(async (skipHistory = false) => { if (!connectedRef.current && !useAgentStore.getState().connected) return; + const sequence = ++loadThreadsSequenceRef.current; setAgentState({ loadingThreads: true }); try { const data = await fetchAgentJson(endpoint, token, `/agent/codex/threads`); const nextThreadId = data.workspace?.activeThreadId || ""; - setAgentState({ - threads: data.data || [], - workspacePath: data.workspace?.workspacePath || "", - activeThreadId: nextThreadId, - messages: [], - }); - if (nextThreadId) { + let nextMessages: AgentChatItem[] = []; + if (nextThreadId && !skipHistory) { const thread = await fetchAgentJson(endpoint, token, `/agent/codex/threads/${encodeURIComponent(nextThreadId)}`); - setAgentState({ messages: normalizeHistoryMessages(thread.messages || []) }); + nextMessages = normalizeHistoryMessages(thread.messages || []); } + if (sequence !== loadThreadsSequenceRef.current) return; + setAgentState({ threads: data.data || [], workspacePath: data.workspace?.workspacePath || "", activeThreadId: nextThreadId, messages: nextMessages }); } catch (error) { addEventLog("读取历史失败", error); } finally { - setAgentState({ loadingThreads: false }); + if (sequence === loadThreadsSequenceRef.current) setAgentState({ loadingThreads: false }); } }, [endpoint, setAgentState, token]); @@ -144,36 +149,69 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb localStorage.setItem("canvas-agent-url", endpoint); localStorage.setItem("canvas-agent-token", token); const clientId = clientIdRef.current; + let eventQueue = Promise.resolve(); + const enqueueEvent = (task: () => void | Promise) => { + eventQueue = eventQueue.then(task).catch((error) => addEventLog("同步会话失败", error)); + }; const source = new EventSource(`${endpoint}/events?token=${encodeURIComponent(token)}&clientId=${encodeURIComponent(clientId)}`); - source.addEventListener("hello", () => { + source.addEventListener("hello", (event) => { + const busy = Boolean(parseEventData(event)?.codex?.busy); errorLoggedRef.current = false; connectedRef.current = true; - setAgentState({ connected: true, activity: "已连接", connectError: "", silentConnect: false, messages: useAgentStore.getState().messages.filter((item) => !isConnectionErrorMessage(item)) }); + setAgentState({ connected: true, activity: busy ? "Codex 正在运行" : "已连接", waiting: busy, sending: false, connectError: "", silentConnect: false, messages: useAgentStore.getState().messages.filter((item) => !isConnectionErrorMessage(item)) }); if (!headless) message.success("本地 Agent 已连接"); void postState(endpoint, token, clientId, canvasContextRef.current?.snapshot || null); if (document.visibilityState === "visible" && document.hasFocus()) void activateAgentClient(endpoint, token, clientId); }); + source.addEventListener("codex_state", (event) => { + const data = parseEventData(event); + if (!data) return; + enqueueEvent(async () => { + const busy = Boolean(data.busy); + setAgentState({ activity: busy ? "Codex 正在运行" : "完成", waiting: busy, ...(busy ? {} : { sending: false }) }); + if (!busy) await loadThreads(); + }); + }); source.addEventListener("tool_call", (event) => { const data = parseEventData(event); if (data) void handleToolCall(endpoint, token, data); }); source.addEventListener("agent_event", (event) => { const data = parseEventData(event); - if (data) handleAgentEvent(data); + if (data) enqueueEvent(() => { + if (isCurrentThreadEvent(data)) handleAgentEvent(data); + }); + }); + source.addEventListener("workspace_changed", (event) => { + const data = parseEventData(event); + if (!data) return; + enqueueEvent(async () => { + const nextThreadId = data.activeThreadId ?? data.threadId ?? ""; + pendingToolRef.current = null; + setAgentState({ activeThreadId: nextThreadId, messages: [], pendingTool: null }); + await loadThreads(data.emptyThread); + }); + }); + source.addEventListener("chat_message", (event) => { + const data = parseEventData(event); + if (!data?.message) return; + enqueueEvent(() => { + if (!isCurrentThreadEvent(data)) return; + addMessage(data.message!); + }); }); source.addEventListener("agent_log", (event) => { const text = parseEventData<{ text?: unknown }>(event)?.text; addEventLog("日志", text, text); }); source.addEventListener("agent_error", (event) => { - const message = parseEventData<{ message?: unknown }>(event)?.message; - setAgentState({ activity: "出错", waiting: false }); - addMessage({ role: "error", title: "错误", text: normalizeText(message) }); - addEventLog("错误", message, message); - }); - source.addEventListener("agent_done", () => { - setAgentState({ activity: "完成", waiting: false, sending: false }); - void loadThreads(); + const data = parseEventData(event); + if (!data) return; + enqueueEvent(() => { + if (!isCurrentThreadEvent(data)) return; + addMessage({ role: "error", title: "错误", text: normalizeText(data.message) }); + addEventLog("错误", data.message, data.message); + }); }); source.onerror = () => { const wasConnected = connectedRef.current; @@ -194,6 +232,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb return () => { source.close(); connectedRef.current = false; + loadThreadsSequenceRef.current += 1; }; }, [enabled, endpoint, loadThreads, message, setAgentState, token]); @@ -223,27 +262,35 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb addMessage({ role: "error", title: "图片过大", text: "图片附件超过 30MB，请删减后再发送。" }); return; } - setAgentState({ activity: "发送中", sending: true, waiting: true }); - addMessage({ role: "user", text: text || "发送了图片", attachments: files }); + setAgentState({ activity: "发送中", sending: true }); + const messageId = createId(); + addMessage({ id: messageId, role: "user", text: text || "发送了图片", attachments: files }); addEventLog("用户发送", { text, attachments: files.map(({ name, type, size }) => ({ name, type, size })) }); try { - const res = await fetch(`${endpoint}/agent/codex/turn?token=${encodeURIComponent(token)}`, { + const data = await fetchAgentJson<{ threadId?: string }>(endpoint, token, "/agent/codex/turn", { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ prompt: requestPrompt, threadId: useAgentStore.getState().activeThreadId || undefined, attachments: files.map(({ name, type, dataUrl }) => ({ name, type, dataUrl })) }), + body: JSON.stringify({ + prompt: requestPrompt, + messageText: text || `发送了 ${files.length} 张图片`, + messageId, + clientId: clientIdRef.current, + threadId: useAgentStore.getState().activeThreadId || undefined, + attachments: files.map(({ name, type, dataUrl }) => ({ name, type, dataUrl })), + }), }); - if (!res.ok) throw new Error("本地 Agent 拒绝了请求"); - const data = (await res.json()) as { threadId?: string }; if (data.threadId) setAgentState({ activeThreadId: data.threadId }); - addEventLog("本地 Agent 已接收", { status: res.status }); + addEventLog("本地 Agent 已接收", { threadId: data.threadId }); files.forEach((item) => { URL.revokeObjectURL(item.url); attachmentUrlsRef.current.delete(item.url); }); setAgentState({ prompt: "", attachments: [] }); } catch (error) { - setAgentState({ activity: "发送失败", waiting: false }); - addMessage({ role: "error", title: "发送失败", text: error instanceof Error ? error.message : "发送失败" }); + const text = error instanceof Error ? error.message : "发送失败"; + const busy = text.includes("Codex 正在运行"); + setAgentState({ activity: busy ? "Codex 正在运行" : "发送失败" }); + addMessage({ role: "error", title: busy ? "任务仍在运行" : "发送失败", text }); addEventLog("发送失败", error); } finally { setAgentState({ sending: false }); @@ -254,11 +301,10 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb if (!connected || (!sending && !waiting)) return; setAgentState({ activity: "停止中" }); try { - await fetch(`${endpoint}/agent/codex/interrupt?token=${encodeURIComponent(token)}`, { method: "POST", headers: { "content-type": "application/json" } }); - setAgentState({ activity: "已停止", sending: false, waiting: false }); + await fetch(`${endpoint}/agent/codex/interrupt?token=${encodeURIComponent(token)}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ threadId: useAgentStore.getState().activeThreadId || undefined }) }); addEventLog("用户停止", {}); } catch { - setAgentState({ activity: "就绪", sending: false, waiting: false }); + setAgentState({ activity: "停止失败" }); } }; @@ -306,7 +352,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb return; } pendingToolRef.current = payload; - setAgentState({ pendingTool: payload, activity: "等待确认", waiting: false }); + setAgentState({ pendingTool: payload }); addEventLog("等待确认", payload, payload); return; } @@ -316,16 +362,13 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb const runToolCall = async (endpoint: string, token: string, payload: AgentPendingToolCall) => { if (isSiteTool(payload.name)) { try { - setAgentState({ activity: SITE_TOOL_LABELS[payload.name], waiting: true }); addEventLog(toolName(payload.name), payload, payload); const result = await runSiteTool(payload.name, payload.input || {}, navigate, { canvasSnapshot: canvasContextRef.current?.snapshot || null }); await postToolResult(endpoint, token, clientIdRef.current, { requestId: payload.requestId, result }); - setAgentState({ activity: "工具完成", waiting: true }); addEventLog(`${toolName(payload.name)}完成`, result, result); addMessage({ role: "tool", title: `${toolName(payload.name)}完成`, text: siteToolSummary(payload.name, result), detail: { requestId: payload.requestId, name: payload.name, input: payload.input, result } }); } catch (error) { const message = error instanceof Error ? error.message : "工具执行失败"; - setAgentState({ activity: "工具失败", waiting: false }); addMessage({ role: "tool", title: "工具失败", text: message, detail: payload }); await postToolResult(endpoint, token, clientIdRef.current, { requestId: payload.requestId, error: message }); } @@ -333,7 +376,6 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb } try { const input: { ops?: CanvasAgentOp[]; path?: string } = payload.input || {}; - setAgentState({ activity: payload.name === "canvas_apply_ops" ? "执行画布操作" : payload.name === "site_navigate" ? "跳转页面" : "读取画布", waiting: true }); addEventLog(toolName(payload.name), payload, payload); let result: unknown; if (payload.name === "site_navigate") { @@ -351,7 +393,6 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb result = snapshot; } await postToolResult(endpoint, token, clientIdRef.current, { requestId: payload.requestId, result }); - setAgentState({ activity: "工具完成", waiting: true }); addEventLog(`${toolName(payload.name)}完成`, result, result); addMessage({ role: "tool", @@ -361,7 +402,6 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb }); } catch (error) { const message = error instanceof Error ? error.message : "画布操作失败"; - setAgentState({ activity: "工具失败", waiting: false }); addMessage({ role: "tool", title: "工具失败", text: message, detail: payload }); await postToolResult(endpoint, token, clientIdRef.current, { requestId: payload.requestId, error: message }); } @@ -370,7 +410,6 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb const rejectPendingTool = async () => { if (!pendingTool) return; await postToolResult(endpoint, token, clientIdRef.current, { requestId: pendingTool.requestId, error: "用户取消了画布工具调用" }); - setAgentState({ activity: "已取消", waiting: false }); addMessage({ role: "tool", title: "拒绝执行", text: toolName(pendingTool.name), detail: { requestId: pendingTool.requestId, name: pendingTool.name, input: pendingTool.input } }); pendingToolRef.current = null; setAgentState({ pendingTool: null }); @@ -436,6 +475,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb }, [autoConnect, connected, enabled]); function clearAgentSession(patch: Parameters[0] = {}) { + loadThreadsSequenceRef.current += 1; setAgentState({ messages: [], threads: [], @@ -451,12 +491,11 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb } const startNewThread = async () => { - if (!connected) return; + if (!connected || sending || waiting) return; setAgentState({ loadingThreads: true }); try { const data = await fetchAgentJson(endpoint, token, "/agent/codex/threads/new", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({}) }); setAgentState({ activeThreadId: data.thread?.id || data.workspace?.activeThreadId || "", messages: [], activeTab: "chat", activity: "新对话" }); - await loadThreads(); } catch (error) { addEventLog("新建对话失败", error); message.error(error instanceof Error ? error.message : "新建对话失败"); @@ -466,12 +505,11 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb }; const resumeThread = async (threadId: string) => { - if (!connected || !threadId) return; + if (!connected || !threadId || sending || waiting) return; setAgentState({ loadingThreads: true }); try { const data = await fetchAgentJson(endpoint, token, `/agent/codex/threads/${encodeURIComponent(threadId)}/resume`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({}) }); setAgentState({ activeThreadId: data.thread?.id || threadId, messages: normalizeHistoryMessages(data.messages || []), activeTab: "chat", activity: "已恢复会话" }); - await loadThreads(); } catch (error) { addEventLog("恢复对话失败", error); message.error(error instanceof Error ? error.message : "恢复对话失败"); @@ -481,7 +519,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb }; const deleteThread = async (threadId: string) => { - if (!connected || !threadId) return; + if (!connected || !threadId || sending || waiting) return; setAgentState({ loadingThreads: true }); try { await fetchAgentJson(endpoint, token, `/agent/codex/threads/${encodeURIComponent(threadId)}/delete`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({}) }); @@ -512,11 +550,12 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb }); }; - const addMessage = (item: Omit) => { + const addMessage = (item: Omit & { id?: string }) => { const text = normalizeText(item.text); if (!text && !item.attachments?.length) return; - const next = { ...item, id: `${Date.now()}-${Math.random()}`, text }; + const next = { ...item, id: item.id || `${Date.now()}-${Math.random()}`, text } as AgentChatItem; const currentMessages = useAgentStore.getState().messages; + if (currentMessages.some((message) => message.id === next.id)) return; if (next.streamId) { const index = currentMessages.findIndex((message) => message.streamId === next.streamId); if (index >= 0) { @@ -541,15 +580,8 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb const handleAgentEvent = (event: AgentEventPayload) => { if (shouldLogAgentEvent(event)) addEventLog(eventTitle(event), event, event); if (event.type === "thread.started" && event.thread_id) setAgentState({ activeThreadId: event.thread_id }); - const nextActivity = activityText(event); - if (nextActivity) setAgentState({ activity: nextActivity }); - if (event.type === "turn.started") setAgentState({ waiting: true }); - if (event.type === "turn.completed" || event.type === "turn.failed" || event.type === "error") setAgentState({ waiting: false, sending: false }); const item = formatAgentEvent(event); - if (item) { - if (item.role === "error") setAgentState({ waiting: false, sending: false }); - addMessage(item); - } + if (item) addMessage(item); }; const content = ( @@ -569,7 +601,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb }} right={ <> - @@ -596,6 +628,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb activeThreadId={activeThreadId} workspacePath={workspacePath} loading={loadingThreads} + busy={sending || waiting} connected={connected} onRefresh={() => void loadThreads()} onNewThread={() => void startNewThread()} @@ -878,6 +911,7 @@ function AgentHistoryView({ activeThreadId, workspacePath, loading, + busy, connected, onRefresh, onNewThread, @@ -889,6 +923,7 @@ function AgentHistoryView({ activeThreadId: string; workspacePath: string; loading: boolean; + busy: boolean; connected: boolean; onRefresh: () => void; onNewThread: () => void; @@ -913,7 +948,7 @@ function AgentHistoryView({ - @@ -937,11 +972,11 @@ function AgentHistoryView({
{formatThreadTime(thread.updatedAt || thread.createdAt)} - -
@@ -1005,6 +1040,11 @@ function parseEventData(event: Event) { } } +function isCurrentThreadEvent(event: { threadId?: string; thread_id?: string }) { + const threadId = event.threadId || event.thread_id || ""; + return Boolean(threadId) && threadId === useAgentStore.getState().activeThreadId; +} + function formatLogText(logs: AgentEventLog[], context: AgentLogContext) { const head = [ "Infinite Canvas Agent 诊断日志", @@ -1045,17 +1085,6 @@ function usageText(event: AgentEventPayload) { return undefined; } -function activityText(event: AgentEventPayload) { - const name = event.type || ""; - if (name === "thread.started") return "已创建会话"; - if (name === "turn.started") return "思考中"; - if (name === "turn.completed") return "完成"; - if (name === "turn.failed" || name === "error") return "出错"; - if (name === "item.started") return isMcpToolItem(event.item) ? `调用${toolName(String(event.item?.tool || ""))}` : "执行步骤"; - if (name === "item.completed") return isMcpToolItem(event.item) ? "工具完成" : "更新消息"; - return ""; -} - function eventTitle(event: AgentEventPayload) { const item = event.item; if (event.type === "thread.started") return "已创建 Codex 会话"; From 8fb8d0b224eba5bb9693c2bd91d15d8b8c1e39bd Mon Sep 17 00:00:00 2001 From: yu <2799341137@qq.com> Date: Sun, 19 Jul 2026 18:17:57 +0800 Subject: [PATCH 3/3] fix(agent): add uploaded images to canvas flows --- CHANGELOG.md | 1 + canvas-agent/src/agents.ts | 1 + canvas-agent/src/canvas-session.test.ts | 47 ++++++++++++ canvas-agent/src/canvas-session.ts | 74 ++++++++++++++++++- canvas-agent/src/config.ts | 2 +- canvas-agent/src/http-server.ts | 17 ++++- canvas-agent/src/schemas.ts | 3 + canvas-agent/src/types.ts | 2 +- docs/content/docs/progress/pending-test.mdx | 1 + .../canvas/canvas-local-agent-panel.tsx | 62 ++++++++++++++-- web/src/stores/use-agent-store.ts | 2 +- 11 files changed, 199 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5143415..aa1d0a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ + [修复] 本地 Agent 请求与发起标签页强绑定，校验工具结果归属，并在页面关闭后回退到最近聚焦页面。 + [修复] 多标签页共享 Codex 会话改由 Agent 广播同步，按线程过滤消息和状态，并禁止运行中切换会话。 + [修复] 本地 Agent 统一广播 Codex 运行状态，新连接标签页会同步禁用发送，并明确区分工具完成与本轮完成。 ++ [修复] 本地 Agent 上传的图片附件可创建为画布图片节点并连接生成流程，不再出现参考图空节点。 + [新增] 提示词来源增加自定义脚本抓取功能、支持新增来源。 ## v0.9.0 - 2026-07-17 diff --git a/canvas-agent/src/agents.ts b/canvas-agent/src/agents.ts index e215bf8..fb23c5b 100644 --- a/canvas-agent/src/agents.ts +++ b/canvas-agent/src/agents.ts @@ -482,6 +482,7 @@ function toolName(name: string) { if (name === "canvas_get_state") return "读取画布"; if (name === "canvas_get_selection") return "读取选区"; if (name === "canvas_export_snapshot") return "导出快照"; + if (name === "canvas_create_attachment_nodes") return "添加附件图片"; if (name === "canvas_create_text_node") return "创建文本"; if (name === "canvas_create_image_prompt_flow") return "创建生图流程"; if (name === "canvas_create_generation_flow") return "创建生成流程"; diff --git a/canvas-agent/src/canvas-session.test.ts b/canvas-agent/src/canvas-session.test.ts index 20780ae..a6742f1 100644 --- a/canvas-agent/src/canvas-session.test.ts +++ b/canvas-agent/src/canvas-session.test.ts @@ -43,6 +43,53 @@ test("画布写操作只发送给当前激活网页", async (t) => { assert.deepEqual(await result, { ok: true }); }); +test("当前 turn 的图片附件可在发起标签页画布创建图片节点", async (t) => { + const session = new CanvasSession(); + const first = connect(session, "first"); + t.after(() => first.close()); + const dataUrl = "data:image/png;base64,aW1hZ2U="; + session.setTurnAttachments("first", [{ id: "attachment-1", name: "商品.png", type: "image/png", size: 5, width: 1200, height: 600, dataUrl }]); + session.bindClient("first"); + + const result = session.callTool("canvas_create_attachment_nodes", { attachmentIds: ["attachment-1"], x: 100, y: 200 }); + const call = first.event("tool_call"); + const input = field(call, "input") as Record; + const nodes = input.nodes as Array>; + assert.equal(field(call, "name"), "canvas_create_attachment_nodes"); + assert.equal(nodes.length, 1); + assert.equal(nodes[0].attachmentId, "attachment-1"); + assert.equal(nodes[0].title, "商品.png"); + assert.deepEqual(nodes[0].position, { x: 100, y: 200 }); + assert.equal(nodes[0].width, 640); + assert.equal(nodes[0].height, 320); + assert.equal("dataUrl" in nodes[0], false); + assert.equal(session.getTurnAttachment("first", "attachment-1").dataUrl, dataUrl); + + session.resolveResult("first", { requestId: String(field(call, "requestId")), result: { ok: true } }); + const created = (await result) as { nodes: Array<{ id: string; attachmentId: string; title: string }> }; + assert.equal(created.nodes[0].id, nodes[0].id); + assert.equal(created.nodes[0].attachmentId, "attachment-1"); + session.clearTurnAttachments("first"); + assert.throws(() => session.getTurnAttachment("first", "attachment-1"), /找不到/); +}); + +test("图片附件只允许发起 turn 的标签页读取和落入画布", async (t) => { + const session = new CanvasSession(); + const first = connect(session, "first"); + const second = connect(session, "second"); + t.after(() => { + first.close(); + second.close(); + }); + session.setTurnAttachments("first", [{ id: "attachment-1", name: "商品.png", type: "image/png", dataUrl: "data:image/png;base64,aW1hZ2U=" }]); + session.bindClient("second"); + + await assert.rejects(session.callTool("canvas_create_attachment_nodes", { attachmentIds: ["attachment-1"] }), /发起标签页/); + assert.throws(() => session.getTurnAttachment("second", "attachment-1"), /发起标签页/); + assert.equal(first.event("tool_call"), undefined); + assert.equal(second.event("tool_call"), undefined); +}); + test("tool result is accepted only from the request client", async (t) => { const session = new CanvasSession(); const first = connect(session, "first"); diff --git a/canvas-agent/src/canvas-session.ts b/canvas-agent/src/canvas-session.ts index aadc5f8..6e47e04 100644 --- a/canvas-agent/src/canvas-session.ts +++ b/canvas-agent/src/canvas-session.ts @@ -3,9 +3,10 @@ import type { ServerResponse } from "node:http"; import { type ToolName } from "./schemas.js"; import { compactCanvasState, compactNode, isToolName, nextCanvasX, parseToolInput } from "./tools.js"; -import type { CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js"; +import type { AgentAttachment, CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js"; type PendingRequest = { clientId: string; resolve: (value: unknown) => void; reject: (error: Error) => void }; +type TurnAttachment = { clientId: string; id: string; name: string; type: string; size: number; width: number; height: number; dataUrl: string }; export type CodexState = { busy: boolean; threadId: string; turnId: string }; const SITE_TOOLS = new Set([ @@ -26,6 +27,7 @@ export class CanvasSession { private clientFocusOrder = new Map(); private pending = new Map(); private canvasStates = new Map(); + private turnAttachments = new Map(); private activeClientId = ""; private boundClientId = ""; private focusSequence = 0; @@ -103,6 +105,39 @@ export class CanvasSession { if (this.boundClientId === clientId) this.boundClientId = ""; } + setTurnAttachments(clientId: string, attachments: AgentAttachment[]) { + this.turnAttachments.clear(); + return attachments.flatMap((item, index) => { + if (!item.dataUrl?.startsWith("data:image/")) return []; + const id = item.id?.trim() || `attachment-${crypto.randomUUID()}`; + const attachment: TurnAttachment = { + clientId, + id, + name: item.name?.trim() || `图片 ${index + 1}`, + type: item.type?.startsWith("image/") ? item.type : item.dataUrl.match(/^data:([^;]+)/)?.[1] || "image/png", + size: positiveNumber(item.size, 0), + width: positiveNumber(item.width, 1024), + height: positiveNumber(item.height, 1024), + dataUrl: item.dataUrl, + }; + this.turnAttachments.set(id, attachment); + return [{ id, name: attachment.name, type: attachment.type, size: attachment.size, width: attachment.width, height: attachment.height }]; + }); + } + + clearTurnAttachments(clientId?: string) { + this.turnAttachments.forEach((item, id) => { + if (!clientId || item.clientId === clientId) this.turnAttachments.delete(id); + }); + } + + getTurnAttachment(clientId: string, attachmentId: string) { + const attachment = this.turnAttachments.get(attachmentId); + if (!attachment) throw new Error(`找不到本轮图片附件：${attachmentId}`); + if (attachment.clientId !== clientId) throw new Error("图片附件不属于当前 turn 的发起标签页"); + return attachment; + } + resolveResult(clientId: string, body: { requestId?: string; error?: string; result?: unknown }) { const item = body.requestId ? this.pending.get(body.requestId) : null; if (!item || !body.requestId || item.clientId !== clientId) return false; @@ -134,6 +169,7 @@ export class CanvasSession { const ids = new Set(this.canvasState?.selectedNodeIds || []); return { nodes: (this.canvasState?.nodes || []).filter((node) => ids.has(node.id)).map(compactNode) }; } + if (tool === "canvas_create_attachment_nodes") return await this.createAttachmentNodes(input as { attachmentIds: string[]; x?: number; y?: number; gap?: number; direction?: "row" | "column" }); if (tool === "canvas_create_node") { const data = input as { nodeType: CanvasNodeType; title?: string; x?: number; y?: number; width?: number; height?: number; metadata?: Record }; input = { ops: [{ type: "add_node", nodeType: data.nodeType, title: data.title, position: { x: data.x ?? nextCanvasX(this.canvasState), y: data.y ?? 0 }, width: data.width, height: data.height, metadata: data.metadata }] }; @@ -228,6 +264,32 @@ export class CanvasSession { return await this.requestCanvasTool(tool, input); } + private async createAttachmentNodes(input: { attachmentIds: string[]; x?: number; y?: number; gap?: number; direction?: "row" | "column" }) { + const clientId = this.targetClientId; + if (!this.clients.has(clientId)) throw new Error("当前没有已连接画布"); + const attachments = input.attachmentIds.map((id) => this.getTurnAttachment(clientId, id)); + const x = Number(input.x ?? nextCanvasX(this.canvasState)); + const y = Number(input.y ?? 0); + const gap = Number(input.gap ?? 40); + const direction = input.direction || "row"; + let offset = 0; + const nodes = attachments.map((attachment) => { + const size = fitAttachmentNodeSize(attachment.width, attachment.height); + const node = { + id: `image-${crypto.randomUUID()}`, + attachmentId: attachment.id, + title: attachment.name, + position: { x: direction === "row" ? x + offset : x, y: direction === "column" ? y + offset : y }, + width: size.width, + height: size.height, + }; + offset += (direction === "row" ? size.width : size.height) + gap; + return node; + }); + await this.requestCanvasTool("canvas_create_attachment_nodes", { nodes }); + return { nodes: nodes.map(({ id, attachmentId, title }) => ({ id, attachmentId, title })) }; + } + private async requestCanvasTool(name: ToolName, input: Record) { const requestId = crypto.randomUUID(); const clientId = this.targetClientId; @@ -326,3 +388,13 @@ function findNode(state: CanvasSnapshot | null, id: string): CanvasNode | undefi function cleanRecord(value: Record) { return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined && item !== "")); } + +function positiveNumber(value: unknown, fallback: number) { + const number = Number(value); + return Number.isFinite(number) && number > 0 ? number : fallback; +} + +function fitAttachmentNodeSize(width: number, height: number) { + const scale = Math.min(1, 640 / width, 640 / height); + return { width: width * scale, height: height * scale }; +} diff --git a/canvas-agent/src/config.ts b/canvas-agent/src/config.ts index 9c43170..633df39 100644 --- a/canvas-agent/src/config.ts +++ b/canvas-agent/src/config.ts @@ -7,7 +7,7 @@ export const DEFAULT_PORT = 17371; export const CONFIG_DIR = path.join(os.homedir(), ".infinite-canvas"); export const CONFIG_FILE = path.join(CONFIG_DIR, "canvas-agent.json"); export const VERSION = readPackageVersion(); -export const AGENT_PROMPT = "你正在帮助用户操作 Infinite Canvas 网站。切换网站页面用 site_navigate，可跳 / (首页)、/canvas (我的画布)、/canvas/:id (指定画布)、/image、/video、/prompts、/assets、/config。需要改动画布时优先使用已配置的 infinite-canvas MCP 工具：先 canvas_get_state 读取当前画布，再根据任务使用 canvas_create_text_node、canvas_generate_text、canvas_generate_image、canvas_generate_video、canvas_generate_audio、canvas_create_generation_flow、canvas_create_config_node、canvas_run_generation、canvas_update_node、canvas_connect_nodes 等通用工具；复杂批量改动再用 canvas_apply_ops，删除连线可用 delete_connections。若当前不在画布页，画布工具会报错，需先用 site_navigate 打开画布。想了解或打开用户已有画布，用 canvas_list_projects 获取画布清单和 id，再用 site_navigate 跳 /canvas/:id 打开。生图工作台可用 workbench_image_get_config 看可选项、workbench_image_generate 填提示词并生成；视频创作台对应 workbench_video_get_config 与 workbench_video_generate；用 prompts_search 分页搜索提示词库；用 assets_list 查看「我的素材」、assets_add 新增文本或图片素材。需要生成内容时直接调用对应生成工具，不要绑定特定业务场景。不要模拟鼠标点击，不要要求用户手动复制 JSON。"; +export const AGENT_PROMPT = "你正在帮助用户操作 Infinite Canvas 网站。切换网站页面用 site_navigate，可跳 / (首页)、/canvas (我的画布)、/canvas/:id (指定画布)、/image、/video、/prompts、/assets、/config。需要改动画布时优先使用已配置的 infinite-canvas MCP 工具：先 canvas_get_state 读取当前画布，再根据任务使用 canvas_create_text_node、canvas_generate_text、canvas_generate_image、canvas_generate_video、canvas_generate_audio、canvas_create_generation_flow、canvas_create_config_node、canvas_run_generation、canvas_update_node、canvas_connect_nodes 等通用工具；复杂批量改动再用 canvas_apply_ops，删除连线可用 delete_connections。本轮若有用户上传的图片附件，会同时给出 attachmentId；用户要求把附件放入画布或作为生成参考图时，必须先用 canvas_create_attachment_nodes 创建真实图片节点，再把返回的节点 ID 传给 canvas_create_generation_flow.referenceNodeIds，不要创建空图片占位节点。若当前不在画布页，画布工具会报错，需先用 site_navigate 打开画布。想了解或打开用户已有画布，用 canvas_list_projects 获取画布清单和 id，再用 site_navigate 跳 /canvas/:id 打开。生图工作台可用 workbench_image_get_config 看可选项、workbench_image_generate 填提示词并生成；视频创作台对应 workbench_video_get_config 与 workbench_video_generate；用 prompts_search 分页搜索提示词库；用 assets_list 查看「我的素材」、assets_add 新增文本或图片素材。需要生成内容时直接调用对应生成工具，不要绑定特定业务场景。不要模拟鼠标点击，不要要求用户手动复制 JSON。"; export type SiteWorkspaceConfig = { workspacePath: string; activeThreadId?: string; pinnedThreadIds?: string[] }; export type CanvasAgentConfig = { url: string; token: string; origins?: string[]; workspace?: SiteWorkspaceConfig }; diff --git a/canvas-agent/src/http-server.ts b/canvas-agent/src/http-server.ts index f2d8c9d..40bc768 100644 --- a/canvas-agent/src/http-server.ts +++ b/canvas-agent/src/http-server.ts @@ -50,6 +50,13 @@ export function startHttpServer() { const ok = session.resolveResult(String(req.query.clientId || ""), req.body); res.status(ok ? 200 : 409).json({ ok }); }); + app.get("/agent/attachments/:attachmentId", route(async (req, res) => { + const attachment = session.getTurnAttachment(String(req.query.clientId || ""), routeParam(req.params.attachmentId)); + const data = attachment.dataUrl.split(",", 2)[1]; + if (!data) throw new Error("图片附件内容无效"); + res.setHeader("Cache-Control", "no-store"); + res.type(attachment.type).send(Buffer.from(data, "base64")); + })); app.post("/api/tools", route(async (req, res) => res.json({ ok: true, result: await session.callTool(req.body?.name, req.body?.input || {}) }))); app.get("/agent/codex/workspace", (_req, res) => { const workspace = ensureSiteWorkspace(config); @@ -113,6 +120,7 @@ export function startHttpServer() { await verifyCodexThreadWorkspace(emit, threadId, workspace.workspacePath); setActiveThread(threadId); } + const attachmentRefs = session.setTurnAttachments(clientId, attachments); const chatMessage = { sourceClientId: clientId, message: { id: String(req.body?.messageId || Date.now()), role: "user", text: String(req.body?.messageText || prompt || `发送了 ${attachments.length} 张图片`) }, @@ -122,7 +130,7 @@ export function startHttpServer() { const data = payload && typeof payload === "object" && !Array.isArray(payload) ? payload as Record : { value: payload }; session.emitThread(type, threadId, data); }; - void runCodexTurn(withAgentPrompt(prompt), turnEmit, attachments, { + void runCodexTurn(withAgentPrompt(withAttachmentContext(prompt, attachmentRefs)), turnEmit, attachments, { threadId, cwd: workspace.workspacePath, appEmit: emit, @@ -143,6 +151,7 @@ export function startHttpServer() { session.setCodexState({ busy: true, threadId, turnId }); }, onFinish: () => { + session.clearTurnAttachments(clientId); if (clientId) session.releaseClient(clientId); session.setCodexState({ busy: false, threadId, turnId }); }, @@ -206,3 +215,9 @@ function validToken(req: Request, url: URL, token: string) { const header = req.headers["x-canvas-agent-token"]; return url.searchParams.get("token") === token || header === token || (Array.isArray(header) && header.includes(token)); } + +function withAttachmentContext(prompt: string, attachments: Array<{ id: string; name: string }>) { + if (!attachments.length) return prompt; + const list = attachments.map((item, index) => `${index + 1}. attachmentId=${item.id}, name=${JSON.stringify(item.name)}`).join("\n"); + return `${prompt}\n\n本轮可用图片附件（顺序与图片输入一致）：\n${list}\n需要把附件放入画布或作为生成参考图时，先调用 canvas_create_attachment_nodes，再使用返回的画布节点 ID 创建生成流程。`; +} diff --git a/canvas-agent/src/schemas.ts b/canvas-agent/src/schemas.ts index 479098b..b155598 100644 --- a/canvas-agent/src/schemas.ts +++ b/canvas-agent/src/schemas.ts @@ -14,6 +14,7 @@ export const toolNames = [ "canvas_export_snapshot", "canvas_apply_ops", "canvas_create_node", + "canvas_create_attachment_nodes", "canvas_create_text_node", "canvas_create_text_nodes", "canvas_create_config_node", @@ -94,6 +95,7 @@ export const toolInputSchemas = { canvas_export_snapshot: z.object({}).passthrough(), canvas_apply_ops: z.object({ ops: z.array(canvasOpSchema) }), canvas_create_node: z.object({ nodeType: nodeTypeSchema, title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), metadata: recordSchema.optional() }), + canvas_create_attachment_nodes: z.object({ attachmentIds: z.array(z.string()).min(1), x: z.number().optional(), y: z.number().optional(), gap: z.number().optional(), direction: z.enum(["row", "column"]).optional() }), canvas_create_text_node: z.object({ text: z.string().optional(), x: z.number().optional(), y: z.number().optional(), title: z.string().optional(), width: z.number().optional(), height: z.number().optional() }), canvas_create_text_nodes: z.object({ items: z.array(textNodeSchema).min(1), x: z.number().optional(), y: z.number().optional(), gap: z.number().optional(), direction: z.enum(["row", "column"]).optional() }), canvas_create_config_node: z.object({ prompt: z.string().optional(), mode: generationModeSchema.optional(), title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), autoRun: z.boolean().optional() }).merge(generationOptionsSchema), @@ -130,6 +132,7 @@ export const toolDescriptions: Record = { canvas_export_snapshot: "导出当前画布快照，用于理解布局。", canvas_apply_ops: "批量操作当前网页画布。ops 支持 add_node、update_node、delete_node、delete_connections、connect_nodes、set_viewport、select_nodes、run_generation。", canvas_create_node: "创建任意类型节点：text、image、config、video、audio。适合创建占位图、媒体占位、配置节点或自定义 metadata 节点。", + canvas_create_attachment_nodes: "把当前对话中用户上传的图片附件创建成真实画布图片节点。attachmentIds 使用本轮附件清单中的 ID；返回的节点 ID 可传给 canvas_create_generation_flow.referenceNodeIds 作为生成参考图。", canvas_create_text_node: "在当前画布创建单个文本节点。", canvas_create_text_nodes: "批量创建文本节点，适合生成标题、段落、脚本、说明等内容块。", canvas_create_config_node: "创建生成配置节点，可指定 text/image/video/audio 模式和生成参数，可选择立即触发生成。", diff --git a/canvas-agent/src/types.ts b/canvas-agent/src/types.ts index d513918..6c6f458 100644 --- a/canvas-agent/src/types.ts +++ b/canvas-agent/src/types.ts @@ -5,4 +5,4 @@ export type CanvasNode = { id: string; type: CanvasNodeType; title?: string; pos export type CanvasConnection = { id: string; fromNodeId: string; toNodeId: string }; export type CanvasSnapshot = { projectId?: string; title?: string; nodes?: CanvasNode[]; connections?: CanvasConnection[]; selectedNodeIds?: string[]; viewport?: Viewport; clientId?: string }; export type AgentEmit = (type: string, payload: unknown) => void; -export type AgentAttachment = { name?: string; type?: string; dataUrl?: string }; +export type AgentAttachment = { id?: string; name?: string; type?: string; size?: number; width?: number; height?: number; dataUrl?: string }; diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index 809e130..9e6abe8 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -9,3 +9,4 @@ description: 当前版本已实现但仍需人工验证的变更项 - 本地 Agent 多标签页隔离：同时打开两个不同画布并连接同一个 Agent，分别聚焦标签页后通过 MCP 读取和修改画布，操作应只落在当前聚焦页面；网页面板发起的整个 Codex turn 应固定操作发起页面，即使中途聚焦另一标签页也不能切换目标；关闭当前页面后应回退到最近聚焦且仍连接的页面，其他页面回传同一请求结果应被拒绝。 - 本地 Agent 多标签页会话同步：所有标签页共享同一个站点级 Codex 活跃线程；任一页面发送消息、新建、恢复或删除会话后，其他页面应同步活跃线程和聊天记录；Agent 输出仅显示在事件所属线程，运行中不能新建、恢复、删除或再次发送任务。 - 本地 Agent 运行状态同步：在一个标签页运行较长 Codex 任务，等待某张工具卡显示「工具完成」后再打开或刷新第二个标签页；第二个标签页应立即显示 Codex 正在运行并禁用发送，整轮结束后两个标签页同时恢复；工具卡只显示「工具完成」，整轮结束由「本轮完成」表示。 +- 本地 Agent 图片附件落画布：在右侧 Agent 上传参考图并要求基于商品信息创建生图流程，附件应创建为保持原比例的真实图片节点，分析提示词应创建为文本节点，二者都应连接到生成配置节点；刷新页面后参考图仍可显示并参与生成。任务中途切换到其他标签页时，附件只能写入发起任务的标签页；若发起标签页关闭，附件读取应失败且不能落入其他画布。 diff --git a/web/src/components/canvas/canvas-local-agent-panel.tsx b/web/src/components/canvas/canvas-local-agent-panel.tsx index 26bc9f4..4476b96 100644 --- a/web/src/components/canvas/canvas-local-agent-panel.tsx +++ b/web/src/components/canvas/canvas-local-agent-panel.tsx @@ -5,7 +5,11 @@ import copyToClipboard from "copy-to-clipboard"; import { Copy, FolderOpen, History, KeyRound, Link2, LoaderCircle, PlugZap, Plus, RefreshCw, Square, Terminal, Trash2 } from "lucide-react"; import { canvasThemes } from "@/lib/canvas-theme"; +import { imageMetadata } from "@/lib/canvas/canvas-node-factory"; +import { fitNodeSize } from "@/lib/canvas/canvas-node-size"; +import { readImageMeta } from "@/lib/image-utils"; import { randomId } from "@/lib/utils"; +import { uploadImage } from "@/services/image-storage"; import { useThemeStore } from "@/stores/use-theme-store"; import { useUserStore } from "@/stores/use-user-store"; import { useShallow } from "zustand/react/shallow"; @@ -276,7 +280,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb messageId, clientId: clientIdRef.current, threadId: useAgentStore.getState().activeThreadId || undefined, - attachments: files.map(({ name, type, dataUrl }) => ({ name, type, dataUrl })), + attachments: files.map(({ id, name, type, size, width, height, dataUrl }) => ({ id, name, type, size, width, height, dataUrl })), }), }); if (data.threadId) setAgentState({ activeThreadId: data.threadId }); @@ -316,9 +320,10 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb const next = await Promise.all( images.slice(0, Math.max(0, MAX_ATTACHMENTS - prev.length)).map(async (file) => { const dataUrl = await readDataUrl(file); + const meta = await readImageMeta(dataUrl); const url = URL.createObjectURL(file); attachmentUrlsRef.current.add(url); - return { id: createId(), name: file.name, type: file.type, size: file.size, url, dataUrl }; + return { id: createId(), name: file.name, type: file.type, size: file.size, width: meta.width, height: meta.height, url, dataUrl }; }), ); const merged = [...prev, ...next]; @@ -346,7 +351,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb }; const handleToolCall = async (endpoint: string, token: string, payload: AgentPendingToolCall) => { - if (confirmToolsRef.current && payload.name === "canvas_apply_ops") { + if (confirmToolsRef.current && isCanvasWriteTool(payload.name)) { if (pendingToolRef.current) { await postToolResult(endpoint, token, clientIdRef.current, { requestId: payload.requestId, error: "仍有待确认的画布工具调用" }); return; @@ -378,6 +383,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb const input: { ops?: CanvasAgentOp[]; path?: string } = payload.input || {}; addEventLog(toolName(payload.name), payload, payload); let result: unknown; + let appliedOps = input.ops || []; if (payload.name === "site_navigate") { const path = input.path || "/"; navigate(path); @@ -385,8 +391,14 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb } else if (payload.name === "canvas_apply_ops") { const context = canvasContextRef.current; if (!context) throw new Error("当前不在画布页，请先用 site_navigate 打开画布"); - result = context.applyOps(input.ops || []); + result = context.applyOps(appliedOps); void postState(endpoint, token, clientIdRef.current, result as CanvasAgentSnapshot); + } else if (payload.name === "canvas_create_attachment_nodes") { + const context = canvasContextRef.current; + if (!context) throw new Error("当前不在画布页，请先用 site_navigate 打开画布"); + appliedOps = await attachmentNodeOps(endpoint, token, clientIdRef.current, payload.input?.nodes); + result = context.applyOps(appliedOps); + await postState(endpoint, token, clientIdRef.current, result as CanvasAgentSnapshot); } else { const snapshot = canvasContextRef.current?.snapshot; if (!snapshot) throw new Error("当前不在画布页，请先用 site_navigate 打开画布"); @@ -397,7 +409,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb addMessage({ role: "tool", title: `${toolName(payload.name)}完成`, - text: payload.name === "canvas_apply_ops" ? summarizeCanvasAgentOps(input.ops || []) || "画布操作" : payload.name === "site_navigate" ? `已跳转到 ${input.path || "/"}` : "已完成", + text: appliedOps.length ? summarizeCanvasAgentOps(appliedOps) || "画布操作" : payload.name === "site_navigate" ? `已跳转到 ${input.path || "/"}` : "已完成", detail: { requestId: payload.requestId, name: payload.name, input, result }, }); } catch (error) { @@ -1113,6 +1125,7 @@ function toolName(name: string) { if (name === "canvas_get_selection") return "读取选区"; if (name === "canvas_export_snapshot") return "导出快照"; if (name === "canvas_create_node") return "创建节点"; + if (name === "canvas_create_attachment_nodes") return "添加附件图片"; if (name === "canvas_create_text_node") return "创建文本"; if (name === "canvas_create_text_nodes") return "批量创建文本"; if (name === "canvas_create_config_node") return "创建生成配置"; @@ -1220,9 +1233,7 @@ function mergeAgentText(prev: string, next: string) { } function promptWithAttachments(text: string, attachments: AgentAttachment[]) { - if (!attachments.length) return text; - const names = attachments.map((item) => item.name).join("、"); - return [text, `用户上传了 ${attachments.length} 张图片附件：${names}。`].filter(Boolean).join("\n\n"); + return text || (attachments.length ? "请处理上传的图片附件。" : ""); } function attachmentPayloadBytes(attachments: AgentAttachment[]) { @@ -1233,6 +1244,41 @@ function formatBytes(bytes: number) { return bytes > 1024 * 1024 ? `${(bytes / 1024 / 1024).toFixed(1)}MB` : `${Math.ceil(bytes / 1024)}KB`; } +function isCanvasWriteTool(name: string) { + return name === "canvas_apply_ops" || name === "canvas_create_attachment_nodes"; +} + +async function attachmentNodeOps(endpoint: string, token: string, clientId: string, value: unknown): Promise { + const nodes = Array.isArray(value) ? value : []; + if (!nodes.length) throw new Error("没有可添加的图片附件"); + return await Promise.all( + nodes.map(async (value) => { + const item = value as { id?: unknown; attachmentId?: unknown; title?: unknown; position?: unknown }; + const id = String(item.id || ""); + const attachmentId = String(item.attachmentId || ""); + if (!id || !attachmentId) throw new Error("图片附件节点参数无效"); + const res = await fetch(`${endpoint}/agent/attachments/${encodeURIComponent(attachmentId)}?token=${encodeURIComponent(token)}&clientId=${encodeURIComponent(clientId)}`); + if (!res.ok) { + const body = (await res.json().catch(() => null)) as { error?: string } | null; + throw new Error(body?.error || "读取图片附件失败"); + } + const image = await uploadImage(await res.blob()); + const size = fitNodeSize(image.width, image.height); + const position = item.position && typeof item.position === "object" ? (item.position as { x?: unknown; y?: unknown }) : {}; + return { + type: "add_node" as const, + id, + nodeType: "image" as const, + title: String(item.title || "参考图"), + position: { x: Number(position.x) || 0, y: Number(position.y) || 0 }, + width: size.width, + height: size.height, + metadata: imageMetadata(image), + }; + }), + ); +} + async function fetchAgentJson(endpoint: string, token: string, path: string, init?: RequestInit) { const url = `${endpoint}${path}${path.includes("?") ? "&" : "?"}token=${encodeURIComponent(token)}`; const res = await fetch(url, init); diff --git a/web/src/stores/use-agent-store.ts b/web/src/stores/use-agent-store.ts index 993cf96..20871f5 100644 --- a/web/src/stores/use-agent-store.ts +++ b/web/src/stores/use-agent-store.ts @@ -3,7 +3,7 @@ import { create } from "zustand"; import type { CanvasAgentOp, CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops"; export type AgentChatRole = "user" | "assistant" | "system" | "tool" | "error"; -export type AgentAttachment = { id: string; name: string; type: string; size: number; url: string; dataUrl: string }; +export type AgentAttachment = { id: string; name: string; type: string; size: number; width: number; height: number; url: string; dataUrl: string }; export type AgentChatItem = { id: string; role: AgentChatRole; title?: string; text: string; meta?: string; detail?: unknown; attachments?: AgentAttachment[]; streamId?: string }; export type AgentEventLog = { id: string; time: string; title: string; text: string; raw?: unknown }; export type AgentPendingToolCall = { requestId: string; name: string; input?: { ops?: CanvasAgentOp[]; path?: string } & Record };