mirror of
https://github.com/basketikun/infinite-canvas.git
synced 2026-08-05 00:34:22 +08:00
fix(agent): add uploaded images to canvas flows
This commit is contained in:
yu
@@ -482,6 +482,7 @@ function toolName(name: string) {
|
||||
if (name === "canvas_get_state") return "读取画布";
|
||||
if (name === "canvas_get_selection") return "读取选区";
|
||||
if (name === "canvas_export_snapshot") return "导出快照";
|
||||
if (name === "canvas_create_attachment_nodes") return "添加附件图片";
|
||||
if (name === "canvas_create_text_node") return "创建文本";
|
||||
if (name === "canvas_create_image_prompt_flow") return "创建生图流程";
|
||||
if (name === "canvas_create_generation_flow") return "创建生成流程";
|
||||
|
||||
@@ -43,6 +43,53 @@ test("画布写操作只发送给当前激活网页", async (t) => {
|
||||
assert.deepEqual(await result, { ok: true });
|
||||
});
|
||||
|
||||
test("当前 turn 的图片附件可在发起标签页画布创建图片节点", async (t) => {
|
||||
const session = new CanvasSession();
|
||||
const first = connect(session, "first");
|
||||
t.after(() => first.close());
|
||||
const dataUrl = "data:image/png;base64,aW1hZ2U=";
|
||||
session.setTurnAttachments("first", [{ id: "attachment-1", name: "商品.png", type: "image/png", size: 5, width: 1200, height: 600, dataUrl }]);
|
||||
session.bindClient("first");
|
||||
|
||||
const result = session.callTool("canvas_create_attachment_nodes", { attachmentIds: ["attachment-1"], x: 100, y: 200 });
|
||||
const call = first.event("tool_call");
|
||||
const input = field(call, "input") as Record<string, unknown>;
|
||||
const nodes = input.nodes as Array<Record<string, unknown>>;
|
||||
assert.equal(field(call, "name"), "canvas_create_attachment_nodes");
|
||||
assert.equal(nodes.length, 1);
|
||||
assert.equal(nodes[0].attachmentId, "attachment-1");
|
||||
assert.equal(nodes[0].title, "商品.png");
|
||||
assert.deepEqual(nodes[0].position, { x: 100, y: 200 });
|
||||
assert.equal(nodes[0].width, 640);
|
||||
assert.equal(nodes[0].height, 320);
|
||||
assert.equal("dataUrl" in nodes[0], false);
|
||||
assert.equal(session.getTurnAttachment("first", "attachment-1").dataUrl, dataUrl);
|
||||
|
||||
session.resolveResult("first", { requestId: String(field(call, "requestId")), result: { ok: true } });
|
||||
const created = (await result) as { nodes: Array<{ id: string; attachmentId: string; title: string }> };
|
||||
assert.equal(created.nodes[0].id, nodes[0].id);
|
||||
assert.equal(created.nodes[0].attachmentId, "attachment-1");
|
||||
session.clearTurnAttachments("first");
|
||||
assert.throws(() => session.getTurnAttachment("first", "attachment-1"), /找不到/);
|
||||
});
|
||||
|
||||
test("图片附件只允许发起 turn 的标签页读取和落入画布", async (t) => {
|
||||
const session = new CanvasSession();
|
||||
const first = connect(session, "first");
|
||||
const second = connect(session, "second");
|
||||
t.after(() => {
|
||||
first.close();
|
||||
second.close();
|
||||
});
|
||||
session.setTurnAttachments("first", [{ id: "attachment-1", name: "商品.png", type: "image/png", dataUrl: "data:image/png;base64,aW1hZ2U=" }]);
|
||||
session.bindClient("second");
|
||||
|
||||
await assert.rejects(session.callTool("canvas_create_attachment_nodes", { attachmentIds: ["attachment-1"] }), /发起标签页/);
|
||||
assert.throws(() => session.getTurnAttachment("second", "attachment-1"), /发起标签页/);
|
||||
assert.equal(first.event("tool_call"), undefined);
|
||||
assert.equal(second.event("tool_call"), undefined);
|
||||
});
|
||||
|
||||
test("tool result is accepted only from the request client", async (t) => {
|
||||
const session = new CanvasSession();
|
||||
const first = connect(session, "first");
|
||||
|
||||
@@ -3,9 +3,10 @@ import type { ServerResponse } from "node:http";
|
||||
|
||||
import { type ToolName } from "./schemas.js";
|
||||
import { compactCanvasState, compactNode, isToolName, nextCanvasX, parseToolInput } from "./tools.js";
|
||||
import type { CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js";
|
||||
import type { AgentAttachment, CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js";
|
||||
|
||||
type PendingRequest = { clientId: string; resolve: (value: unknown) => void; reject: (error: Error) => void };
|
||||
type TurnAttachment = { clientId: string; id: string; name: string; type: string; size: number; width: number; height: number; dataUrl: string };
|
||||
export type CodexState = { busy: boolean; threadId: string; turnId: string };
|
||||
|
||||
const SITE_TOOLS = new Set<ToolName>([
|
||||
@@ -26,6 +27,7 @@ export class CanvasSession {
|
||||
private clientFocusOrder = new Map<string, number>();
|
||||
private pending = new Map<string, PendingRequest>();
|
||||
private canvasStates = new Map<string, CanvasSnapshot>();
|
||||
private turnAttachments = new Map<string, TurnAttachment>();
|
||||
private activeClientId = "";
|
||||
private boundClientId = "";
|
||||
private focusSequence = 0;
|
||||
@@ -103,6 +105,39 @@ export class CanvasSession {
|
||||
if (this.boundClientId === clientId) this.boundClientId = "";
|
||||
}
|
||||
|
||||
setTurnAttachments(clientId: string, attachments: AgentAttachment[]) {
|
||||
this.turnAttachments.clear();
|
||||
return attachments.flatMap((item, index) => {
|
||||
if (!item.dataUrl?.startsWith("data:image/")) return [];
|
||||
const id = item.id?.trim() || `attachment-${crypto.randomUUID()}`;
|
||||
const attachment: TurnAttachment = {
|
||||
clientId,
|
||||
id,
|
||||
name: item.name?.trim() || `图片 ${index + 1}`,
|
||||
type: item.type?.startsWith("image/") ? item.type : item.dataUrl.match(/^data:([^;]+)/)?.[1] || "image/png",
|
||||
size: positiveNumber(item.size, 0),
|
||||
width: positiveNumber(item.width, 1024),
|
||||
height: positiveNumber(item.height, 1024),
|
||||
dataUrl: item.dataUrl,
|
||||
};
|
||||
this.turnAttachments.set(id, attachment);
|
||||
return [{ id, name: attachment.name, type: attachment.type, size: attachment.size, width: attachment.width, height: attachment.height }];
|
||||
});
|
||||
}
|
||||
|
||||
clearTurnAttachments(clientId?: string) {
|
||||
this.turnAttachments.forEach((item, id) => {
|
||||
if (!clientId || item.clientId === clientId) this.turnAttachments.delete(id);
|
||||
});
|
||||
}
|
||||
|
||||
getTurnAttachment(clientId: string, attachmentId: string) {
|
||||
const attachment = this.turnAttachments.get(attachmentId);
|
||||
if (!attachment) throw new Error(`找不到本轮图片附件：${attachmentId}`);
|
||||
if (attachment.clientId !== clientId) throw new Error("图片附件不属于当前 turn 的发起标签页");
|
||||
return attachment;
|
||||
}
|
||||
|
||||
resolveResult(clientId: string, body: { requestId?: string; error?: string; result?: unknown }) {
|
||||
const item = body.requestId ? this.pending.get(body.requestId) : null;
|
||||
if (!item || !body.requestId || item.clientId !== clientId) return false;
|
||||
@@ -134,6 +169,7 @@ export class CanvasSession {
|
||||
const ids = new Set(this.canvasState?.selectedNodeIds || []);
|
||||
return { nodes: (this.canvasState?.nodes || []).filter((node) => ids.has(node.id)).map(compactNode) };
|
||||
}
|
||||
if (tool === "canvas_create_attachment_nodes") return await this.createAttachmentNodes(input as { attachmentIds: string[]; x?: number; y?: number; gap?: number; direction?: "row" | "column" });
|
||||
if (tool === "canvas_create_node") {
|
||||
const data = input as { nodeType: CanvasNodeType; title?: string; x?: number; y?: number; width?: number; height?: number; metadata?: Record<string, unknown> };
|
||||
input = { ops: [{ type: "add_node", nodeType: data.nodeType, title: data.title, position: { x: data.x ?? nextCanvasX(this.canvasState), y: data.y ?? 0 }, width: data.width, height: data.height, metadata: data.metadata }] };
|
||||
@@ -228,6 +264,32 @@ export class CanvasSession {
|
||||
return await this.requestCanvasTool(tool, input);
|
||||
}
|
||||
|
||||
private async createAttachmentNodes(input: { attachmentIds: string[]; x?: number; y?: number; gap?: number; direction?: "row" | "column" }) {
|
||||
const clientId = this.targetClientId;
|
||||
if (!this.clients.has(clientId)) throw new Error("当前没有已连接画布");
|
||||
const attachments = input.attachmentIds.map((id) => this.getTurnAttachment(clientId, id));
|
||||
const x = Number(input.x ?? nextCanvasX(this.canvasState));
|
||||
const y = Number(input.y ?? 0);
|
||||
const gap = Number(input.gap ?? 40);
|
||||
const direction = input.direction || "row";
|
||||
let offset = 0;
|
||||
const nodes = attachments.map((attachment) => {
|
||||
const size = fitAttachmentNodeSize(attachment.width, attachment.height);
|
||||
const node = {
|
||||
id: `image-${crypto.randomUUID()}`,
|
||||
attachmentId: attachment.id,
|
||||
title: attachment.name,
|
||||
position: { x: direction === "row" ? x + offset : x, y: direction === "column" ? y + offset : y },
|
||||
width: size.width,
|
||||
height: size.height,
|
||||
};
|
||||
offset += (direction === "row" ? size.width : size.height) + gap;
|
||||
return node;
|
||||
});
|
||||
await this.requestCanvasTool("canvas_create_attachment_nodes", { nodes });
|
||||
return { nodes: nodes.map(({ id, attachmentId, title }) => ({ id, attachmentId, title })) };
|
||||
}
|
||||
|
||||
private async requestCanvasTool(name: ToolName, input: Record<string, unknown>) {
|
||||
const requestId = crypto.randomUUID();
|
||||
const clientId = this.targetClientId;
|
||||
@@ -326,3 +388,13 @@ function findNode(state: CanvasSnapshot | null, id: string): CanvasNode | undefi
|
||||
function cleanRecord(value: Record<string, unknown>) {
|
||||
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined && item !== ""));
|
||||
}
|
||||
|
||||
function positiveNumber(value: unknown, fallback: number) {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) && number > 0 ? number : fallback;
|
||||
}
|
||||
|
||||
function fitAttachmentNodeSize(width: number, height: number) {
|
||||
const scale = Math.min(1, 640 / width, 640 / height);
|
||||
return { width: width * scale, height: height * scale };
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ export const DEFAULT_PORT = 17371;
|
||||
export const CONFIG_DIR = path.join(os.homedir(), ".infinite-canvas");
|
||||
export const CONFIG_FILE = path.join(CONFIG_DIR, "canvas-agent.json");
|
||||
export const VERSION = readPackageVersion();
|
||||
export const AGENT_PROMPT = "你正在帮助用户操作 Infinite Canvas 网站。切换网站页面用 site_navigate，可跳 / (首页)、/canvas (我的画布)、/canvas/:id (指定画布)、/image、/video、/prompts、/assets、/config。需要改动画布时优先使用已配置的 infinite-canvas MCP 工具：先 canvas_get_state 读取当前画布，再根据任务使用 canvas_create_text_node、canvas_generate_text、canvas_generate_image、canvas_generate_video、canvas_generate_audio、canvas_create_generation_flow、canvas_create_config_node、canvas_run_generation、canvas_update_node、canvas_connect_nodes 等通用工具；复杂批量改动再用 canvas_apply_ops，删除连线可用 delete_connections。若当前不在画布页，画布工具会报错，需先用 site_navigate 打开画布。想了解或打开用户已有画布，用 canvas_list_projects 获取画布清单和 id，再用 site_navigate 跳 /canvas/:id 打开。生图工作台可用 workbench_image_get_config 看可选项、workbench_image_generate 填提示词并生成；视频创作台对应 workbench_video_get_config 与 workbench_video_generate；用 prompts_search 分页搜索提示词库；用 assets_list 查看「我的素材」、assets_add 新增文本或图片素材。需要生成内容时直接调用对应生成工具，不要绑定特定业务场景。不要模拟鼠标点击，不要要求用户手动复制 JSON。";
|
||||
export const AGENT_PROMPT = "你正在帮助用户操作 Infinite Canvas 网站。切换网站页面用 site_navigate，可跳 / (首页)、/canvas (我的画布)、/canvas/:id (指定画布)、/image、/video、/prompts、/assets、/config。需要改动画布时优先使用已配置的 infinite-canvas MCP 工具：先 canvas_get_state 读取当前画布，再根据任务使用 canvas_create_text_node、canvas_generate_text、canvas_generate_image、canvas_generate_video、canvas_generate_audio、canvas_create_generation_flow、canvas_create_config_node、canvas_run_generation、canvas_update_node、canvas_connect_nodes 等通用工具；复杂批量改动再用 canvas_apply_ops，删除连线可用 delete_connections。本轮若有用户上传的图片附件，会同时给出 attachmentId；用户要求把附件放入画布或作为生成参考图时，必须先用 canvas_create_attachment_nodes 创建真实图片节点，再把返回的节点 ID 传给 canvas_create_generation_flow.referenceNodeIds，不要创建空图片占位节点。若当前不在画布页，画布工具会报错，需先用 site_navigate 打开画布。想了解或打开用户已有画布，用 canvas_list_projects 获取画布清单和 id，再用 site_navigate 跳 /canvas/:id 打开。生图工作台可用 workbench_image_get_config 看可选项、workbench_image_generate 填提示词并生成；视频创作台对应 workbench_video_get_config 与 workbench_video_generate；用 prompts_search 分页搜索提示词库；用 assets_list 查看「我的素材」、assets_add 新增文本或图片素材。需要生成内容时直接调用对应生成工具，不要绑定特定业务场景。不要模拟鼠标点击，不要要求用户手动复制 JSON。";
|
||||
|
||||
export type SiteWorkspaceConfig = { workspacePath: string; activeThreadId?: string; pinnedThreadIds?: string[] };
|
||||
export type CanvasAgentConfig = { url: string; token: string; origins?: string[]; workspace?: SiteWorkspaceConfig };
|
||||
|
||||
@@ -50,6 +50,13 @@ export function startHttpServer() {
|
||||
const ok = session.resolveResult(String(req.query.clientId || ""), req.body);
|
||||
res.status(ok ? 200 : 409).json({ ok });
|
||||
});
|
||||
app.get("/agent/attachments/:attachmentId", route(async (req, res) => {
|
||||
const attachment = session.getTurnAttachment(String(req.query.clientId || ""), routeParam(req.params.attachmentId));
|
||||
const data = attachment.dataUrl.split(",", 2)[1];
|
||||
if (!data) throw new Error("图片附件内容无效");
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
res.type(attachment.type).send(Buffer.from(data, "base64"));
|
||||
}));
|
||||
app.post("/api/tools", route(async (req, res) => res.json({ ok: true, result: await session.callTool(req.body?.name, req.body?.input || {}) })));
|
||||
app.get("/agent/codex/workspace", (_req, res) => {
|
||||
const workspace = ensureSiteWorkspace(config);
|
||||
@@ -113,6 +120,7 @@ export function startHttpServer() {
|
||||
await verifyCodexThreadWorkspace(emit, threadId, workspace.workspacePath);
|
||||
setActiveThread(threadId);
|
||||
}
|
||||
const attachmentRefs = session.setTurnAttachments(clientId, attachments);
|
||||
const chatMessage = {
|
||||
sourceClientId: clientId,
|
||||
message: { id: String(req.body?.messageId || Date.now()), role: "user", text: String(req.body?.messageText || prompt || `发送了 ${attachments.length} 张图片`) },
|
||||
@@ -122,7 +130,7 @@ export function startHttpServer() {
|
||||
const data = payload && typeof payload === "object" && !Array.isArray(payload) ? payload as Record<string, unknown> : { value: payload };
|
||||
session.emitThread(type, threadId, data);
|
||||
};
|
||||
void runCodexTurn(withAgentPrompt(prompt), turnEmit, attachments, {
|
||||
void runCodexTurn(withAgentPrompt(withAttachmentContext(prompt, attachmentRefs)), turnEmit, attachments, {
|
||||
threadId,
|
||||
cwd: workspace.workspacePath,
|
||||
appEmit: emit,
|
||||
@@ -143,6 +151,7 @@ export function startHttpServer() {
|
||||
session.setCodexState({ busy: true, threadId, turnId });
|
||||
},
|
||||
onFinish: () => {
|
||||
session.clearTurnAttachments(clientId);
|
||||
if (clientId) session.releaseClient(clientId);
|
||||
session.setCodexState({ busy: false, threadId, turnId });
|
||||
},
|
||||
@@ -206,3 +215,9 @@ function validToken(req: Request, url: URL, token: string) {
|
||||
const header = req.headers["x-canvas-agent-token"];
|
||||
return url.searchParams.get("token") === token || header === token || (Array.isArray(header) && header.includes(token));
|
||||
}
|
||||
|
||||
function withAttachmentContext(prompt: string, attachments: Array<{ id: string; name: string }>) {
|
||||
if (!attachments.length) return prompt;
|
||||
const list = attachments.map((item, index) => `${index + 1}. attachmentId=${item.id}, name=${JSON.stringify(item.name)}`).join("\n");
|
||||
return `${prompt}\n\n本轮可用图片附件（顺序与图片输入一致）：\n${list}\n需要把附件放入画布或作为生成参考图时，先调用 canvas_create_attachment_nodes，再使用返回的画布节点 ID 创建生成流程。`;
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ export const toolNames = [
|
||||
"canvas_export_snapshot",
|
||||
"canvas_apply_ops",
|
||||
"canvas_create_node",
|
||||
"canvas_create_attachment_nodes",
|
||||
"canvas_create_text_node",
|
||||
"canvas_create_text_nodes",
|
||||
"canvas_create_config_node",
|
||||
@@ -94,6 +95,7 @@ export const toolInputSchemas = {
|
||||
canvas_export_snapshot: z.object({}).passthrough(),
|
||||
canvas_apply_ops: z.object({ ops: z.array(canvasOpSchema) }),
|
||||
canvas_create_node: z.object({ nodeType: nodeTypeSchema, title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), metadata: recordSchema.optional() }),
|
||||
canvas_create_attachment_nodes: z.object({ attachmentIds: z.array(z.string()).min(1), x: z.number().optional(), y: z.number().optional(), gap: z.number().optional(), direction: z.enum(["row", "column"]).optional() }),
|
||||
canvas_create_text_node: z.object({ text: z.string().optional(), x: z.number().optional(), y: z.number().optional(), title: z.string().optional(), width: z.number().optional(), height: z.number().optional() }),
|
||||
canvas_create_text_nodes: z.object({ items: z.array(textNodeSchema).min(1), x: z.number().optional(), y: z.number().optional(), gap: z.number().optional(), direction: z.enum(["row", "column"]).optional() }),
|
||||
canvas_create_config_node: z.object({ prompt: z.string().optional(), mode: generationModeSchema.optional(), title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), autoRun: z.boolean().optional() }).merge(generationOptionsSchema),
|
||||
@@ -130,6 +132,7 @@ export const toolDescriptions: Record<ToolName, string> = {
|
||||
canvas_export_snapshot: "导出当前画布快照，用于理解布局。",
|
||||
canvas_apply_ops: "批量操作当前网页画布。ops 支持 add_node、update_node、delete_node、delete_connections、connect_nodes、set_viewport、select_nodes、run_generation。",
|
||||
canvas_create_node: "创建任意类型节点：text、image、config、video、audio。适合创建占位图、媒体占位、配置节点或自定义 metadata 节点。",
|
||||
canvas_create_attachment_nodes: "把当前对话中用户上传的图片附件创建成真实画布图片节点。attachmentIds 使用本轮附件清单中的 ID；返回的节点 ID 可传给 canvas_create_generation_flow.referenceNodeIds 作为生成参考图。",
|
||||
canvas_create_text_node: "在当前画布创建单个文本节点。",
|
||||
canvas_create_text_nodes: "批量创建文本节点，适合生成标题、段落、脚本、说明等内容块。",
|
||||
canvas_create_config_node: "创建生成配置节点，可指定 text/image/video/audio 模式和生成参数，可选择立即触发生成。",
|
||||
|
||||
@@ -5,4 +5,4 @@ export type CanvasNode = { id: string; type: CanvasNodeType; title?: string; pos
|
||||
export type CanvasConnection = { id: string; fromNodeId: string; toNodeId: string };
|
||||
export type CanvasSnapshot = { projectId?: string; title?: string; nodes?: CanvasNode[]; connections?: CanvasConnection[]; selectedNodeIds?: string[]; viewport?: Viewport; clientId?: string };
|
||||
export type AgentEmit = (type: string, payload: unknown) => void;
|
||||
export type AgentAttachment = { name?: string; type?: string; dataUrl?: string };
|
||||
export type AgentAttachment = { id?: string; name?: string; type?: string; size?: number; width?: number; height?: number; dataUrl?: string };
|
||||
|
||||
Reference in New Issue
Block a user