mirror of
https://github.com/basketikun/infinite-canvas.git
synced 2026-07-24 15:24:06 +08:00
fix(agent): add uploaded images to canvas flows
This commit is contained in:
@@ -6,6 +6,7 @@
|
||||
+ [修复] 本地 Agent 请求与发起标签页强绑定,校验工具结果归属,并在页面关闭后回退到最近聚焦页面。
|
||||
+ [修复] 多标签页共享 Codex 会话改由 Agent 广播同步,按线程过滤消息和状态,并禁止运行中切换会话。
|
||||
+ [修复] 本地 Agent 统一广播 Codex 运行状态,新连接标签页会同步禁用发送,并明确区分工具完成与本轮完成。
|
||||
+ [修复] 本地 Agent 上传的图片附件可创建为画布图片节点并连接生成流程,不再出现参考图空节点。
|
||||
+ [新增] 提示词来源增加自定义脚本抓取功能、支持新增来源。
|
||||
|
||||
## v0.9.0 - 2026-07-17
|
||||
|
||||
@@ -482,6 +482,7 @@ function toolName(name: string) {
|
||||
if (name === "canvas_get_state") return "读取画布";
|
||||
if (name === "canvas_get_selection") return "读取选区";
|
||||
if (name === "canvas_export_snapshot") return "导出快照";
|
||||
if (name === "canvas_create_attachment_nodes") return "添加附件图片";
|
||||
if (name === "canvas_create_text_node") return "创建文本";
|
||||
if (name === "canvas_create_image_prompt_flow") return "创建生图流程";
|
||||
if (name === "canvas_create_generation_flow") return "创建生成流程";
|
||||
|
||||
@@ -43,6 +43,53 @@ test("画布写操作只发送给当前激活网页", async (t) => {
|
||||
assert.deepEqual(await result, { ok: true });
|
||||
});
|
||||
|
||||
test("当前 turn 的图片附件可在发起标签页画布创建图片节点", async (t) => {
|
||||
const session = new CanvasSession();
|
||||
const first = connect(session, "first");
|
||||
t.after(() => first.close());
|
||||
const dataUrl = "data:image/png;base64,aW1hZ2U=";
|
||||
session.setTurnAttachments("first", [{ id: "attachment-1", name: "商品.png", type: "image/png", size: 5, width: 1200, height: 600, dataUrl }]);
|
||||
session.bindClient("first");
|
||||
|
||||
const result = session.callTool("canvas_create_attachment_nodes", { attachmentIds: ["attachment-1"], x: 100, y: 200 });
|
||||
const call = first.event("tool_call");
|
||||
const input = field(call, "input") as Record<string, unknown>;
|
||||
const nodes = input.nodes as Array<Record<string, unknown>>;
|
||||
assert.equal(field(call, "name"), "canvas_create_attachment_nodes");
|
||||
assert.equal(nodes.length, 1);
|
||||
assert.equal(nodes[0].attachmentId, "attachment-1");
|
||||
assert.equal(nodes[0].title, "商品.png");
|
||||
assert.deepEqual(nodes[0].position, { x: 100, y: 200 });
|
||||
assert.equal(nodes[0].width, 640);
|
||||
assert.equal(nodes[0].height, 320);
|
||||
assert.equal("dataUrl" in nodes[0], false);
|
||||
assert.equal(session.getTurnAttachment("first", "attachment-1").dataUrl, dataUrl);
|
||||
|
||||
session.resolveResult("first", { requestId: String(field(call, "requestId")), result: { ok: true } });
|
||||
const created = (await result) as { nodes: Array<{ id: string; attachmentId: string; title: string }> };
|
||||
assert.equal(created.nodes[0].id, nodes[0].id);
|
||||
assert.equal(created.nodes[0].attachmentId, "attachment-1");
|
||||
session.clearTurnAttachments("first");
|
||||
assert.throws(() => session.getTurnAttachment("first", "attachment-1"), /找不到/);
|
||||
});
|
||||
|
||||
test("图片附件只允许发起 turn 的标签页读取和落入画布", async (t) => {
|
||||
const session = new CanvasSession();
|
||||
const first = connect(session, "first");
|
||||
const second = connect(session, "second");
|
||||
t.after(() => {
|
||||
first.close();
|
||||
second.close();
|
||||
});
|
||||
session.setTurnAttachments("first", [{ id: "attachment-1", name: "商品.png", type: "image/png", dataUrl: "data:image/png;base64,aW1hZ2U=" }]);
|
||||
session.bindClient("second");
|
||||
|
||||
await assert.rejects(session.callTool("canvas_create_attachment_nodes", { attachmentIds: ["attachment-1"] }), /发起标签页/);
|
||||
assert.throws(() => session.getTurnAttachment("second", "attachment-1"), /发起标签页/);
|
||||
assert.equal(first.event("tool_call"), undefined);
|
||||
assert.equal(second.event("tool_call"), undefined);
|
||||
});
|
||||
|
||||
test("tool result is accepted only from the request client", async (t) => {
|
||||
const session = new CanvasSession();
|
||||
const first = connect(session, "first");
|
||||
|
||||
@@ -3,9 +3,10 @@ import type { ServerResponse } from "node:http";
|
||||
|
||||
import { type ToolName } from "./schemas.js";
|
||||
import { compactCanvasState, compactNode, isToolName, nextCanvasX, parseToolInput } from "./tools.js";
|
||||
import type { CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js";
|
||||
import type { AgentAttachment, CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js";
|
||||
|
||||
type PendingRequest = { clientId: string; resolve: (value: unknown) => void; reject: (error: Error) => void };
|
||||
type TurnAttachment = { clientId: string; id: string; name: string; type: string; size: number; width: number; height: number; dataUrl: string };
|
||||
export type CodexState = { busy: boolean; threadId: string; turnId: string };
|
||||
|
||||
const SITE_TOOLS = new Set<ToolName>([
|
||||
@@ -26,6 +27,7 @@ export class CanvasSession {
|
||||
private clientFocusOrder = new Map<string, number>();
|
||||
private pending = new Map<string, PendingRequest>();
|
||||
private canvasStates = new Map<string, CanvasSnapshot>();
|
||||
private turnAttachments = new Map<string, TurnAttachment>();
|
||||
private activeClientId = "";
|
||||
private boundClientId = "";
|
||||
private focusSequence = 0;
|
||||
@@ -103,6 +105,39 @@ export class CanvasSession {
|
||||
if (this.boundClientId === clientId) this.boundClientId = "";
|
||||
}
|
||||
|
||||
setTurnAttachments(clientId: string, attachments: AgentAttachment[]) {
|
||||
this.turnAttachments.clear();
|
||||
return attachments.flatMap((item, index) => {
|
||||
if (!item.dataUrl?.startsWith("data:image/")) return [];
|
||||
const id = item.id?.trim() || `attachment-${crypto.randomUUID()}`;
|
||||
const attachment: TurnAttachment = {
|
||||
clientId,
|
||||
id,
|
||||
name: item.name?.trim() || `图片 ${index + 1}`,
|
||||
type: item.type?.startsWith("image/") ? item.type : item.dataUrl.match(/^data:([^;]+)/)?.[1] || "image/png",
|
||||
size: positiveNumber(item.size, 0),
|
||||
width: positiveNumber(item.width, 1024),
|
||||
height: positiveNumber(item.height, 1024),
|
||||
dataUrl: item.dataUrl,
|
||||
};
|
||||
this.turnAttachments.set(id, attachment);
|
||||
return [{ id, name: attachment.name, type: attachment.type, size: attachment.size, width: attachment.width, height: attachment.height }];
|
||||
});
|
||||
}
|
||||
|
||||
clearTurnAttachments(clientId?: string) {
|
||||
this.turnAttachments.forEach((item, id) => {
|
||||
if (!clientId || item.clientId === clientId) this.turnAttachments.delete(id);
|
||||
});
|
||||
}
|
||||
|
||||
getTurnAttachment(clientId: string, attachmentId: string) {
|
||||
const attachment = this.turnAttachments.get(attachmentId);
|
||||
if (!attachment) throw new Error(`找不到本轮图片附件:${attachmentId}`);
|
||||
if (attachment.clientId !== clientId) throw new Error("图片附件不属于当前 turn 的发起标签页");
|
||||
return attachment;
|
||||
}
|
||||
|
||||
resolveResult(clientId: string, body: { requestId?: string; error?: string; result?: unknown }) {
|
||||
const item = body.requestId ? this.pending.get(body.requestId) : null;
|
||||
if (!item || !body.requestId || item.clientId !== clientId) return false;
|
||||
@@ -134,6 +169,7 @@ export class CanvasSession {
|
||||
const ids = new Set(this.canvasState?.selectedNodeIds || []);
|
||||
return { nodes: (this.canvasState?.nodes || []).filter((node) => ids.has(node.id)).map(compactNode) };
|
||||
}
|
||||
if (tool === "canvas_create_attachment_nodes") return await this.createAttachmentNodes(input as { attachmentIds: string[]; x?: number; y?: number; gap?: number; direction?: "row" | "column" });
|
||||
if (tool === "canvas_create_node") {
|
||||
const data = input as { nodeType: CanvasNodeType; title?: string; x?: number; y?: number; width?: number; height?: number; metadata?: Record<string, unknown> };
|
||||
input = { ops: [{ type: "add_node", nodeType: data.nodeType, title: data.title, position: { x: data.x ?? nextCanvasX(this.canvasState), y: data.y ?? 0 }, width: data.width, height: data.height, metadata: data.metadata }] };
|
||||
@@ -228,6 +264,32 @@ export class CanvasSession {
|
||||
return await this.requestCanvasTool(tool, input);
|
||||
}
|
||||
|
||||
private async createAttachmentNodes(input: { attachmentIds: string[]; x?: number; y?: number; gap?: number; direction?: "row" | "column" }) {
|
||||
const clientId = this.targetClientId;
|
||||
if (!this.clients.has(clientId)) throw new Error("当前没有已连接画布");
|
||||
const attachments = input.attachmentIds.map((id) => this.getTurnAttachment(clientId, id));
|
||||
const x = Number(input.x ?? nextCanvasX(this.canvasState));
|
||||
const y = Number(input.y ?? 0);
|
||||
const gap = Number(input.gap ?? 40);
|
||||
const direction = input.direction || "row";
|
||||
let offset = 0;
|
||||
const nodes = attachments.map((attachment) => {
|
||||
const size = fitAttachmentNodeSize(attachment.width, attachment.height);
|
||||
const node = {
|
||||
id: `image-${crypto.randomUUID()}`,
|
||||
attachmentId: attachment.id,
|
||||
title: attachment.name,
|
||||
position: { x: direction === "row" ? x + offset : x, y: direction === "column" ? y + offset : y },
|
||||
width: size.width,
|
||||
height: size.height,
|
||||
};
|
||||
offset += (direction === "row" ? size.width : size.height) + gap;
|
||||
return node;
|
||||
});
|
||||
await this.requestCanvasTool("canvas_create_attachment_nodes", { nodes });
|
||||
return { nodes: nodes.map(({ id, attachmentId, title }) => ({ id, attachmentId, title })) };
|
||||
}
|
||||
|
||||
private async requestCanvasTool(name: ToolName, input: Record<string, unknown>) {
|
||||
const requestId = crypto.randomUUID();
|
||||
const clientId = this.targetClientId;
|
||||
@@ -326,3 +388,13 @@ function findNode(state: CanvasSnapshot | null, id: string): CanvasNode | undefi
|
||||
function cleanRecord(value: Record<string, unknown>) {
|
||||
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined && item !== ""));
|
||||
}
|
||||
|
||||
function positiveNumber(value: unknown, fallback: number) {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) && number > 0 ? number : fallback;
|
||||
}
|
||||
|
||||
function fitAttachmentNodeSize(width: number, height: number) {
|
||||
const scale = Math.min(1, 640 / width, 640 / height);
|
||||
return { width: width * scale, height: height * scale };
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ export const DEFAULT_PORT = 17371;
|
||||
export const CONFIG_DIR = path.join(os.homedir(), ".infinite-canvas");
|
||||
export const CONFIG_FILE = path.join(CONFIG_DIR, "canvas-agent.json");
|
||||
export const VERSION = readPackageVersion();
|
||||
export const AGENT_PROMPT = "你正在帮助用户操作 Infinite Canvas 网站。切换网站页面用 site_navigate,可跳 / (首页)、/canvas (我的画布)、/canvas/:id (指定画布)、/image、/video、/prompts、/assets、/config。需要改动画布时优先使用已配置的 infinite-canvas MCP 工具:先 canvas_get_state 读取当前画布,再根据任务使用 canvas_create_text_node、canvas_generate_text、canvas_generate_image、canvas_generate_video、canvas_generate_audio、canvas_create_generation_flow、canvas_create_config_node、canvas_run_generation、canvas_update_node、canvas_connect_nodes 等通用工具;复杂批量改动再用 canvas_apply_ops,删除连线可用 delete_connections。若当前不在画布页,画布工具会报错,需先用 site_navigate 打开画布。想了解或打开用户已有画布,用 canvas_list_projects 获取画布清单和 id,再用 site_navigate 跳 /canvas/:id 打开。生图工作台可用 workbench_image_get_config 看可选项、workbench_image_generate 填提示词并生成;视频创作台对应 workbench_video_get_config 与 workbench_video_generate;用 prompts_search 分页搜索提示词库;用 assets_list 查看「我的素材」、assets_add 新增文本或图片素材。需要生成内容时直接调用对应生成工具,不要绑定特定业务场景。不要模拟鼠标点击,不要要求用户手动复制 JSON。";
|
||||
export const AGENT_PROMPT = "你正在帮助用户操作 Infinite Canvas 网站。切换网站页面用 site_navigate,可跳 / (首页)、/canvas (我的画布)、/canvas/:id (指定画布)、/image、/video、/prompts、/assets、/config。需要改动画布时优先使用已配置的 infinite-canvas MCP 工具:先 canvas_get_state 读取当前画布,再根据任务使用 canvas_create_text_node、canvas_generate_text、canvas_generate_image、canvas_generate_video、canvas_generate_audio、canvas_create_generation_flow、canvas_create_config_node、canvas_run_generation、canvas_update_node、canvas_connect_nodes 等通用工具;复杂批量改动再用 canvas_apply_ops,删除连线可用 delete_connections。本轮若有用户上传的图片附件,会同时给出 attachmentId;用户要求把附件放入画布或作为生成参考图时,必须先用 canvas_create_attachment_nodes 创建真实图片节点,再把返回的节点 ID 传给 canvas_create_generation_flow.referenceNodeIds,不要创建空图片占位节点。若当前不在画布页,画布工具会报错,需先用 site_navigate 打开画布。想了解或打开用户已有画布,用 canvas_list_projects 获取画布清单和 id,再用 site_navigate 跳 /canvas/:id 打开。生图工作台可用 workbench_image_get_config 看可选项、workbench_image_generate 填提示词并生成;视频创作台对应 workbench_video_get_config 与 workbench_video_generate;用 prompts_search 分页搜索提示词库;用 assets_list 查看「我的素材」、assets_add 新增文本或图片素材。需要生成内容时直接调用对应生成工具,不要绑定特定业务场景。不要模拟鼠标点击,不要要求用户手动复制 JSON。";
|
||||
|
||||
export type SiteWorkspaceConfig = { workspacePath: string; activeThreadId?: string; pinnedThreadIds?: string[] };
|
||||
export type CanvasAgentConfig = { url: string; token: string; origins?: string[]; workspace?: SiteWorkspaceConfig };
|
||||
|
||||
@@ -50,6 +50,13 @@ export function startHttpServer() {
|
||||
const ok = session.resolveResult(String(req.query.clientId || ""), req.body);
|
||||
res.status(ok ? 200 : 409).json({ ok });
|
||||
});
|
||||
app.get("/agent/attachments/:attachmentId", route(async (req, res) => {
|
||||
const attachment = session.getTurnAttachment(String(req.query.clientId || ""), routeParam(req.params.attachmentId));
|
||||
const data = attachment.dataUrl.split(",", 2)[1];
|
||||
if (!data) throw new Error("图片附件内容无效");
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
res.type(attachment.type).send(Buffer.from(data, "base64"));
|
||||
}));
|
||||
app.post("/api/tools", route(async (req, res) => res.json({ ok: true, result: await session.callTool(req.body?.name, req.body?.input || {}) })));
|
||||
app.get("/agent/codex/workspace", (_req, res) => {
|
||||
const workspace = ensureSiteWorkspace(config);
|
||||
@@ -113,6 +120,7 @@ export function startHttpServer() {
|
||||
await verifyCodexThreadWorkspace(emit, threadId, workspace.workspacePath);
|
||||
setActiveThread(threadId);
|
||||
}
|
||||
const attachmentRefs = session.setTurnAttachments(clientId, attachments);
|
||||
const chatMessage = {
|
||||
sourceClientId: clientId,
|
||||
message: { id: String(req.body?.messageId || Date.now()), role: "user", text: String(req.body?.messageText || prompt || `发送了 ${attachments.length} 张图片`) },
|
||||
@@ -122,7 +130,7 @@ export function startHttpServer() {
|
||||
const data = payload && typeof payload === "object" && !Array.isArray(payload) ? payload as Record<string, unknown> : { value: payload };
|
||||
session.emitThread(type, threadId, data);
|
||||
};
|
||||
void runCodexTurn(withAgentPrompt(prompt), turnEmit, attachments, {
|
||||
void runCodexTurn(withAgentPrompt(withAttachmentContext(prompt, attachmentRefs)), turnEmit, attachments, {
|
||||
threadId,
|
||||
cwd: workspace.workspacePath,
|
||||
appEmit: emit,
|
||||
@@ -143,6 +151,7 @@ export function startHttpServer() {
|
||||
session.setCodexState({ busy: true, threadId, turnId });
|
||||
},
|
||||
onFinish: () => {
|
||||
session.clearTurnAttachments(clientId);
|
||||
if (clientId) session.releaseClient(clientId);
|
||||
session.setCodexState({ busy: false, threadId, turnId });
|
||||
},
|
||||
@@ -206,3 +215,9 @@ function validToken(req: Request, url: URL, token: string) {
|
||||
const header = req.headers["x-canvas-agent-token"];
|
||||
return url.searchParams.get("token") === token || header === token || (Array.isArray(header) && header.includes(token));
|
||||
}
|
||||
|
||||
function withAttachmentContext(prompt: string, attachments: Array<{ id: string; name: string }>) {
|
||||
if (!attachments.length) return prompt;
|
||||
const list = attachments.map((item, index) => `${index + 1}. attachmentId=${item.id}, name=${JSON.stringify(item.name)}`).join("\n");
|
||||
return `${prompt}\n\n本轮可用图片附件(顺序与图片输入一致):\n${list}\n需要把附件放入画布或作为生成参考图时,先调用 canvas_create_attachment_nodes,再使用返回的画布节点 ID 创建生成流程。`;
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ export const toolNames = [
|
||||
"canvas_export_snapshot",
|
||||
"canvas_apply_ops",
|
||||
"canvas_create_node",
|
||||
"canvas_create_attachment_nodes",
|
||||
"canvas_create_text_node",
|
||||
"canvas_create_text_nodes",
|
||||
"canvas_create_config_node",
|
||||
@@ -94,6 +95,7 @@ export const toolInputSchemas = {
|
||||
canvas_export_snapshot: z.object({}).passthrough(),
|
||||
canvas_apply_ops: z.object({ ops: z.array(canvasOpSchema) }),
|
||||
canvas_create_node: z.object({ nodeType: nodeTypeSchema, title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), metadata: recordSchema.optional() }),
|
||||
canvas_create_attachment_nodes: z.object({ attachmentIds: z.array(z.string()).min(1), x: z.number().optional(), y: z.number().optional(), gap: z.number().optional(), direction: z.enum(["row", "column"]).optional() }),
|
||||
canvas_create_text_node: z.object({ text: z.string().optional(), x: z.number().optional(), y: z.number().optional(), title: z.string().optional(), width: z.number().optional(), height: z.number().optional() }),
|
||||
canvas_create_text_nodes: z.object({ items: z.array(textNodeSchema).min(1), x: z.number().optional(), y: z.number().optional(), gap: z.number().optional(), direction: z.enum(["row", "column"]).optional() }),
|
||||
canvas_create_config_node: z.object({ prompt: z.string().optional(), mode: generationModeSchema.optional(), title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), autoRun: z.boolean().optional() }).merge(generationOptionsSchema),
|
||||
@@ -130,6 +132,7 @@ export const toolDescriptions: Record<ToolName, string> = {
|
||||
canvas_export_snapshot: "导出当前画布快照,用于理解布局。",
|
||||
canvas_apply_ops: "批量操作当前网页画布。ops 支持 add_node、update_node、delete_node、delete_connections、connect_nodes、set_viewport、select_nodes、run_generation。",
|
||||
canvas_create_node: "创建任意类型节点:text、image、config、video、audio。适合创建占位图、媒体占位、配置节点或自定义 metadata 节点。",
|
||||
canvas_create_attachment_nodes: "把当前对话中用户上传的图片附件创建成真实画布图片节点。attachmentIds 使用本轮附件清单中的 ID;返回的节点 ID 可传给 canvas_create_generation_flow.referenceNodeIds 作为生成参考图。",
|
||||
canvas_create_text_node: "在当前画布创建单个文本节点。",
|
||||
canvas_create_text_nodes: "批量创建文本节点,适合生成标题、段落、脚本、说明等内容块。",
|
||||
canvas_create_config_node: "创建生成配置节点,可指定 text/image/video/audio 模式和生成参数,可选择立即触发生成。",
|
||||
|
||||
@@ -5,4 +5,4 @@ export type CanvasNode = { id: string; type: CanvasNodeType; title?: string; pos
|
||||
export type CanvasConnection = { id: string; fromNodeId: string; toNodeId: string };
|
||||
export type CanvasSnapshot = { projectId?: string; title?: string; nodes?: CanvasNode[]; connections?: CanvasConnection[]; selectedNodeIds?: string[]; viewport?: Viewport; clientId?: string };
|
||||
export type AgentEmit = (type: string, payload: unknown) => void;
|
||||
export type AgentAttachment = { name?: string; type?: string; dataUrl?: string };
|
||||
export type AgentAttachment = { id?: string; name?: string; type?: string; size?: number; width?: number; height?: number; dataUrl?: string };
|
||||
|
||||
@@ -9,3 +9,4 @@ description: 当前版本已实现但仍需人工验证的变更项
|
||||
- 本地 Agent 多标签页隔离:同时打开两个不同画布并连接同一个 Agent,分别聚焦标签页后通过 MCP 读取和修改画布,操作应只落在当前聚焦页面;网页面板发起的整个 Codex turn 应固定操作发起页面,即使中途聚焦另一标签页也不能切换目标;关闭当前页面后应回退到最近聚焦且仍连接的页面,其他页面回传同一请求结果应被拒绝。
|
||||
- 本地 Agent 多标签页会话同步:所有标签页共享同一个站点级 Codex 活跃线程;任一页面发送消息、新建、恢复或删除会话后,其他页面应同步活跃线程和聊天记录;Agent 输出仅显示在事件所属线程,运行中不能新建、恢复、删除或再次发送任务。
|
||||
- 本地 Agent 运行状态同步:在一个标签页运行较长 Codex 任务,等待某张工具卡显示「工具完成」后再打开或刷新第二个标签页;第二个标签页应立即显示 Codex 正在运行并禁用发送,整轮结束后两个标签页同时恢复;工具卡只显示「工具完成」,整轮结束由「本轮完成」表示。
|
||||
- 本地 Agent 图片附件落画布:在右侧 Agent 上传参考图并要求基于商品信息创建生图流程,附件应创建为保持原比例的真实图片节点,分析提示词应创建为文本节点,二者都应连接到生成配置节点;刷新页面后参考图仍可显示并参与生成。任务中途切换到其他标签页时,附件只能写入发起任务的标签页;若发起标签页关闭,附件读取应失败且不能落入其他画布。
|
||||
|
||||
@@ -5,7 +5,11 @@ import copyToClipboard from "copy-to-clipboard";
|
||||
import { Copy, FolderOpen, History, KeyRound, Link2, LoaderCircle, PlugZap, Plus, RefreshCw, Square, Terminal, Trash2 } from "lucide-react";
|
||||
|
||||
import { canvasThemes } from "@/lib/canvas-theme";
|
||||
import { imageMetadata } from "@/lib/canvas/canvas-node-factory";
|
||||
import { fitNodeSize } from "@/lib/canvas/canvas-node-size";
|
||||
import { readImageMeta } from "@/lib/image-utils";
|
||||
import { randomId } from "@/lib/utils";
|
||||
import { uploadImage } from "@/services/image-storage";
|
||||
import { useThemeStore } from "@/stores/use-theme-store";
|
||||
import { useUserStore } from "@/stores/use-user-store";
|
||||
import { useShallow } from "zustand/react/shallow";
|
||||
@@ -276,7 +280,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb
|
||||
messageId,
|
||||
clientId: clientIdRef.current,
|
||||
threadId: useAgentStore.getState().activeThreadId || undefined,
|
||||
attachments: files.map(({ name, type, dataUrl }) => ({ name, type, dataUrl })),
|
||||
attachments: files.map(({ id, name, type, size, width, height, dataUrl }) => ({ id, name, type, size, width, height, dataUrl })),
|
||||
}),
|
||||
});
|
||||
if (data.threadId) setAgentState({ activeThreadId: data.threadId });
|
||||
@@ -316,9 +320,10 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb
|
||||
const next = await Promise.all(
|
||||
images.slice(0, Math.max(0, MAX_ATTACHMENTS - prev.length)).map(async (file) => {
|
||||
const dataUrl = await readDataUrl(file);
|
||||
const meta = await readImageMeta(dataUrl);
|
||||
const url = URL.createObjectURL(file);
|
||||
attachmentUrlsRef.current.add(url);
|
||||
return { id: createId(), name: file.name, type: file.type, size: file.size, url, dataUrl };
|
||||
return { id: createId(), name: file.name, type: file.type, size: file.size, width: meta.width, height: meta.height, url, dataUrl };
|
||||
}),
|
||||
);
|
||||
const merged = [...prev, ...next];
|
||||
@@ -346,7 +351,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb
|
||||
};
|
||||
|
||||
const handleToolCall = async (endpoint: string, token: string, payload: AgentPendingToolCall) => {
|
||||
if (confirmToolsRef.current && payload.name === "canvas_apply_ops") {
|
||||
if (confirmToolsRef.current && isCanvasWriteTool(payload.name)) {
|
||||
if (pendingToolRef.current) {
|
||||
await postToolResult(endpoint, token, clientIdRef.current, { requestId: payload.requestId, error: "仍有待确认的画布工具调用" });
|
||||
return;
|
||||
@@ -378,6 +383,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb
|
||||
const input: { ops?: CanvasAgentOp[]; path?: string } = payload.input || {};
|
||||
addEventLog(toolName(payload.name), payload, payload);
|
||||
let result: unknown;
|
||||
let appliedOps = input.ops || [];
|
||||
if (payload.name === "site_navigate") {
|
||||
const path = input.path || "/";
|
||||
navigate(path);
|
||||
@@ -385,8 +391,14 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb
|
||||
} else if (payload.name === "canvas_apply_ops") {
|
||||
const context = canvasContextRef.current;
|
||||
if (!context) throw new Error("当前不在画布页,请先用 site_navigate 打开画布");
|
||||
result = context.applyOps(input.ops || []);
|
||||
result = context.applyOps(appliedOps);
|
||||
void postState(endpoint, token, clientIdRef.current, result as CanvasAgentSnapshot);
|
||||
} else if (payload.name === "canvas_create_attachment_nodes") {
|
||||
const context = canvasContextRef.current;
|
||||
if (!context) throw new Error("当前不在画布页,请先用 site_navigate 打开画布");
|
||||
appliedOps = await attachmentNodeOps(endpoint, token, clientIdRef.current, payload.input?.nodes);
|
||||
result = context.applyOps(appliedOps);
|
||||
await postState(endpoint, token, clientIdRef.current, result as CanvasAgentSnapshot);
|
||||
} else {
|
||||
const snapshot = canvasContextRef.current?.snapshot;
|
||||
if (!snapshot) throw new Error("当前不在画布页,请先用 site_navigate 打开画布");
|
||||
@@ -397,7 +409,7 @@ export function CanvasLocalAgentPanel({ embedded, headless, autoConnect }: { emb
|
||||
addMessage({
|
||||
role: "tool",
|
||||
title: `${toolName(payload.name)}完成`,
|
||||
text: payload.name === "canvas_apply_ops" ? summarizeCanvasAgentOps(input.ops || []) || "画布操作" : payload.name === "site_navigate" ? `已跳转到 ${input.path || "/"}` : "已完成",
|
||||
text: appliedOps.length ? summarizeCanvasAgentOps(appliedOps) || "画布操作" : payload.name === "site_navigate" ? `已跳转到 ${input.path || "/"}` : "已完成",
|
||||
detail: { requestId: payload.requestId, name: payload.name, input, result },
|
||||
});
|
||||
} catch (error) {
|
||||
@@ -1113,6 +1125,7 @@ function toolName(name: string) {
|
||||
if (name === "canvas_get_selection") return "读取选区";
|
||||
if (name === "canvas_export_snapshot") return "导出快照";
|
||||
if (name === "canvas_create_node") return "创建节点";
|
||||
if (name === "canvas_create_attachment_nodes") return "添加附件图片";
|
||||
if (name === "canvas_create_text_node") return "创建文本";
|
||||
if (name === "canvas_create_text_nodes") return "批量创建文本";
|
||||
if (name === "canvas_create_config_node") return "创建生成配置";
|
||||
@@ -1220,9 +1233,7 @@ function mergeAgentText(prev: string, next: string) {
|
||||
}
|
||||
|
||||
function promptWithAttachments(text: string, attachments: AgentAttachment[]) {
|
||||
if (!attachments.length) return text;
|
||||
const names = attachments.map((item) => item.name).join("、");
|
||||
return [text, `用户上传了 ${attachments.length} 张图片附件:${names}。`].filter(Boolean).join("\n\n");
|
||||
return text || (attachments.length ? "请处理上传的图片附件。" : "");
|
||||
}
|
||||
|
||||
function attachmentPayloadBytes(attachments: AgentAttachment[]) {
|
||||
@@ -1233,6 +1244,41 @@ function formatBytes(bytes: number) {
|
||||
return bytes > 1024 * 1024 ? `${(bytes / 1024 / 1024).toFixed(1)}MB` : `${Math.ceil(bytes / 1024)}KB`;
|
||||
}
|
||||
|
||||
function isCanvasWriteTool(name: string) {
|
||||
return name === "canvas_apply_ops" || name === "canvas_create_attachment_nodes";
|
||||
}
|
||||
|
||||
async function attachmentNodeOps(endpoint: string, token: string, clientId: string, value: unknown): Promise<CanvasAgentOp[]> {
|
||||
const nodes = Array.isArray(value) ? value : [];
|
||||
if (!nodes.length) throw new Error("没有可添加的图片附件");
|
||||
return await Promise.all(
|
||||
nodes.map(async (value) => {
|
||||
const item = value as { id?: unknown; attachmentId?: unknown; title?: unknown; position?: unknown };
|
||||
const id = String(item.id || "");
|
||||
const attachmentId = String(item.attachmentId || "");
|
||||
if (!id || !attachmentId) throw new Error("图片附件节点参数无效");
|
||||
const res = await fetch(`${endpoint}/agent/attachments/${encodeURIComponent(attachmentId)}?token=${encodeURIComponent(token)}&clientId=${encodeURIComponent(clientId)}`);
|
||||
if (!res.ok) {
|
||||
const body = (await res.json().catch(() => null)) as { error?: string } | null;
|
||||
throw new Error(body?.error || "读取图片附件失败");
|
||||
}
|
||||
const image = await uploadImage(await res.blob());
|
||||
const size = fitNodeSize(image.width, image.height);
|
||||
const position = item.position && typeof item.position === "object" ? (item.position as { x?: unknown; y?: unknown }) : {};
|
||||
return {
|
||||
type: "add_node" as const,
|
||||
id,
|
||||
nodeType: "image" as const,
|
||||
title: String(item.title || "参考图"),
|
||||
position: { x: Number(position.x) || 0, y: Number(position.y) || 0 },
|
||||
width: size.width,
|
||||
height: size.height,
|
||||
metadata: imageMetadata(image),
|
||||
};
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchAgentJson<T>(endpoint: string, token: string, path: string, init?: RequestInit) {
|
||||
const url = `${endpoint}${path}${path.includes("?") ? "&" : "?"}token=${encodeURIComponent(token)}`;
|
||||
const res = await fetch(url, init);
|
||||
|
||||
@@ -3,7 +3,7 @@ import { create } from "zustand";
|
||||
import type { CanvasAgentOp, CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops";
|
||||
|
||||
export type AgentChatRole = "user" | "assistant" | "system" | "tool" | "error";
|
||||
export type AgentAttachment = { id: string; name: string; type: string; size: number; url: string; dataUrl: string };
|
||||
export type AgentAttachment = { id: string; name: string; type: string; size: number; width: number; height: number; url: string; dataUrl: string };
|
||||
export type AgentChatItem = { id: string; role: AgentChatRole; title?: string; text: string; meta?: string; detail?: unknown; attachments?: AgentAttachment[]; streamId?: string };
|
||||
export type AgentEventLog = { id: string; time: string; title: string; text: string; raw?: unknown };
|
||||
export type AgentPendingToolCall = { requestId: string; name: string; input?: { ops?: CanvasAgentOp[]; path?: string } & Record<string, unknown> };
|
||||
|
||||
Reference in New Issue
Block a user