feat(plugin): add Codex App plugin support and local agent integration

2026-08-01 22:21:15 +08:00 · 2026-06-23 17:35:43 +08:00
parent 8cbe00e348
commit bd0ad0aebf
18 changed files with 518 additions and 28 deletions
@@ -2,7 +2,7 @@

 import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
 import type { ChangeEvent as ReactChangeEvent, DragEvent as ReactDragEvent, MouseEvent as ReactMouseEvent, PointerEvent as ReactPointerEvent } from "react";
-import { useParams, useRouter } from "next/navigation";
+import { useParams, useRouter, useSearchParams } from "next/navigation";
 import { BookOpen, Bot, Home, ImageIcon, Images, List, Menu, Music2, Plus, Redo2, Settings2, Trash2, Undo2, Upload, Video } from "lucide-react";
 import { saveAs } from "file-saver";

@@ -42,6 +42,8 @@ import { CanvasNodePromptPanel, type CanvasNodeGenerationMode } from "../compone
 import { CanvasToolbar } from "../components/canvas-toolbar";
 import { AssetPickerModal, type InsertAssetPayload } from "../components/asset-picker-modal";
 import { CanvasZoomControls } from "../components/canvas-zoom-controls";
+import { CanvasLocalAgentPanel } from "../components/canvas-local-agent-panel";
+import { useCanvasAgentStore } from "../stores/use-canvas-agent-store";
 import { useCanvasStore } from "../stores/use-canvas-store";
 import { applyCanvasAgentOps, type CanvasAgentOp, type CanvasAgentSnapshot } from "../utils/canvas-agent-ops";
 import { buildCanvasResourceReferences, buildNodeMentionReferences } from "../utils/canvas-resource-references";
@@ -219,7 +221,11 @@ function InfiniteCanvasPage() {
    const { message, modal } = App.useApp();
    const params = useParams<{ id: string }>();
    const router = useRouter();
+    const searchParams = useSearchParams();
    const projectId = params.id;
+    const localAgentConnected = useCanvasAgentStore((state) => state.connected);
+    const localAgentActivity = useCanvasAgentStore((state) => state.activity);
+    const localAgentEnabled = useCanvasAgentStore((state) => state.enabled);
    const containerRef = useRef<HTMLDivElement>(null);
    const imageInputRef = useRef<HTMLInputElement>(null);
    const uploadTargetRef = useRef<{ nodeId?: string; position?: Position } | null>(null);
@@ -302,6 +308,8 @@ function InfiniteCanvasPage() {
    const [assistantClosing, setAssistantClosing] = useState(false);
    const [agentMode, setAgentMode] = useState<CanvasAgentMode>("online");
    const [agentUndoSnapshot, setAgentUndoSnapshot] = useState<CanvasAgentSnapshot | null>(null);
+    const codexAutoConnect = ["new", "recent", "choose"].includes(searchParams.get("mode") || "");
+    const codexCompactAgent = codexAutoConnect && searchParams.has("agentUrl");
    const [titleEditing, setTitleEditing] = useState(false);
    const [titleDraft, setTitleDraft] = useState("");
    const [historyState, setHistoryState] = useState({ canUndo: false, canRedo: false });
@@ -424,6 +432,15 @@ function InfiniteCanvasPage() {
        void restore();
    }, [hydrated, openProject, projectId, router]);

+    useEffect(() => {
+        if (!projectLoaded || !["new", "recent", "choose"].includes(searchParams.get("mode") || "")) return;
+        if (searchParams.has("agentUrl")) {
+            setAgentMode("local");
+            return;
+        }
+        openAgent("local");
+    }, [projectLoaded, searchParams]);
+
    useEffect(() => {
        if (!projectLoaded || applyingHistoryRef.current || historyPausedRef.current) return;
        const next = createHistoryEntry();
@@ -2494,6 +2511,7 @@ function InfiniteCanvasPage() {
                    onUndo={undoCanvas}
                    onRedo={redoCanvas}
                    agentOpen={assistantOpen}
+                    compactAgentStatus={codexCompactAgent ? { connected: localAgentConnected, enabled: localAgentEnabled, activity: localAgentActivity } : undefined}
                    onToggleAgent={() => (assistantOpen ? closeAgent() : openAgent())}
                />

@@ -2775,6 +2793,7 @@ function InfiniteCanvasPage() {
                </Modal>

                <AssetPickerModal open={assetPickerOpen} onInsert={handleAssetInsert} onClose={() => setAssetPickerOpen(false)} />
+                {codexCompactAgent && !assistantMounted ? <CanvasLocalAgentPanel headless snapshot={agentSnapshot} canUndoOps={Boolean(agentUndoSnapshot)} onApplyOps={applyAgentOps} onUndoOps={undoAgentOps} autoConnect={codexAutoConnect} /> : null}
            </section>
            {assistantMounted ? (
                <CanvasAssistantPanel
@@ -2791,6 +2810,7 @@ function InfiniteCanvasPage() {
                    onPasteImage={pasteAssistantImage}
                    agentMode={agentMode}
                    onAgentModeChange={setAgentMode}
+                    autoConnectLocal={codexAutoConnect}
                    closing={assistantClosing}
                    onCollapse={closeAgent}
                />
@@ -2817,6 +2837,7 @@ function CanvasTopBar({
    onUndo,
    onRedo,
    agentOpen,
+    compactAgentStatus,
    onToggleAgent,
 }: {
    title: string;
@@ -2836,6 +2857,7 @@ function CanvasTopBar({
    onUndo: () => void;
    onRedo: () => void;
    agentOpen: boolean;
+    compactAgentStatus?: { connected: boolean; enabled: boolean; activity: string };
    onToggleAgent: () => void;
 }) {
    const colorTheme = useThemeStore((state) => state.theme);
@@ -2907,6 +2929,7 @@ function CanvasTopBar({
                </div>

                <div className="pointer-events-auto flex items-center gap-1.5">
+                    {compactAgentStatus ? <CompactAgentStatus status={compactAgentStatus} onClick={onToggleAgent} /> : null}
                    <UserStatusActions
                        variant="canvas"
                        onOpenShortcuts={() => setShortcutsOpen(true)}
@@ -2953,6 +2976,25 @@ function MenuLabel({ text, shortcut }: { text: string; shortcut: string }) {
    );
 }

+function CompactAgentStatus({ status, onClick }: { status: { connected: boolean; enabled: boolean; activity: string }; onClick: () => void }) {
+    const colorTheme = useThemeStore((state) => state.theme);
+    const theme = canvasThemes[colorTheme];
+    const label = status.connected ? "已连接到本地 Codex" : status.enabled ? status.activity || "连接中" : "正在连接本地 Codex";
+    const dotColor = status.connected ? "#22c55e" : status.enabled ? "#f59e0b" : theme.node.muted;
+    return (
+        <button
+            type="button"
+            className="flex h-10 items-center gap-2 rounded-xl px-3 text-sm font-medium transition hover:opacity-85"
+            style={{ background: theme.toolbar.panel, color: theme.node.text, boxShadow: "0 10px 30px rgba(28,25,23,.10)" }}
+            onClick={onClick}
+            title="打开本地 Codex 面板"
+        >
+            <span className="size-2 rounded-full" style={{ background: dotColor }} />
+            <span className="max-w-[180px] truncate">{label}</span>
+        </button>
+    );
+}
+
 function Shortcut({ keys, value }: { keys: string[]; value: string }) {
    return (
        <div className="grid grid-cols-[minmax(0,1fr)_120px] items-center gap-6 rounded-lg px-1 py-1.5">
@@ -137,11 +137,12 @@ type CanvasAssistantPanelProps = {
    onPasteImage: (file: File) => void;
    agentMode: CanvasAgentMode;
    onAgentModeChange: (mode: CanvasAgentMode) => void;
+    autoConnectLocal?: boolean;
    closing: boolean;
    onCollapse: () => void;
 };

-export function CanvasAssistantPanel({ nodes, selectedNodeIds, snapshot, sessions, activeSessionId, onSelectNodeIds, onSessionsChange, onApplyOps, canUndoOps, onUndoOps, onPasteImage, agentMode, onAgentModeChange, closing, onCollapse }: CanvasAssistantPanelProps) {
+export function CanvasAssistantPanel({ nodes, selectedNodeIds, snapshot, sessions, activeSessionId, onSelectNodeIds, onSessionsChange, onApplyOps, canUndoOps, onUndoOps, onPasteImage, agentMode, onAgentModeChange, autoConnectLocal, closing, onCollapse }: CanvasAssistantPanelProps) {
    const theme = canvasThemes[useThemeStore((state) => state.theme)];
    const user = useUserStore((state) => state.user);
    const effectiveConfig = useEffectiveConfig();
@@ -660,6 +661,7 @@ export function CanvasAssistantPanel({ nodes, selectedNodeIds, snapshot, session
                        canUndoOps={canUndoOps}
                        onApplyOps={onApplyOps}
                        onUndoOps={onUndoOps}
+                        autoConnect={autoConnectLocal}
                    />
                ) : (
                    onlineContent
@@ -1,6 +1,7 @@
 "use client";

 import { useCallback, useEffect, useMemo, useRef, useState, type PointerEvent as ReactPointerEvent } from "react";
+import { useSearchParams } from "next/navigation";
 import { App, Button, Input, Segmented, Tooltip } from "antd";
 import copyToClipboard from "copy-to-clipboard";
 import { Copy, FolderOpen, History, KeyRound, Link2, LoaderCircle, PlugZap, Plus, RefreshCw, RotateCcw, Terminal, Trash2 } from "lucide-react";
@@ -16,11 +17,11 @@ import { AgentChatComposer, AgentChatMessage, AgentPanelTabs, AgentPendingToolCa
 const PANEL_MOTION_SECONDS = 0.5;
 const MAX_ATTACHMENTS = 6;
 const MAX_ATTACHMENT_PAYLOAD_BYTES = 28 * 1024 * 1024;
+const DEFAULT_AGENT_URL = "http://127.0.0.1:17371";
 const AGENT_CONNECT_STEPS = [
-    { title: "1. 本机已安装并登录 Codex", text: "先确认本机终端里的 Codex 可以正常使用。", command: "codex --version" },
-    { title: "2. 安装 Canvas Agent", text: "推荐全局安装，后续可以直接运行 canvas-agent。", command: "npm i -g @basketikun/canvas-agent" },
-    { title: "3. 启动本地 Agent", text: "启动后终端会输出 Local URL 和 Connect token。", command: "canvas-agent" },
-    { title: "4. 回到网页连接", text: "把终端输出的地址和 token 填到下面，点击连接。" },
+    { title: "安装 Codex 插件", text: "在 Codex app 安装 Infinite Canvas 插件后，首次使用插件会自动启动本地 Agent。" },
+    { title: "打开画布连接", text: "回到这里点击连接，网页会自动读取本机 Agent 配置。" },
+    { title: "手动启动备用", text: "如果自动发现失败，再运行下面命令。", command: "npx -y @basketikun/canvas-agent" },
 ];

 type AgentEventPayload = {
@@ -38,11 +39,13 @@ type AgentLogContext = { endpoint: string; connected: boolean; enabled: boolean;
 type AgentWorkspace = { canvasId: string; workspacePath: string; activeThreadId?: string };
 type AgentThreadsResponse = { ok?: boolean; workspace?: AgentWorkspace; data?: AgentThreadSummary[] };
 type AgentThreadResponse = { ok?: boolean; workspace?: AgentWorkspace; thread?: AgentThreadSummary; messages?: AgentChatItem[] };
+type AgentConfigResponse = { ok?: boolean; url?: string; token?: string; hasToken?: boolean };

-export function CanvasLocalAgentPanel({ snapshot, canUndoOps, collapsed, embedded, onApplyOps, onUndoOps }: { snapshot: CanvasAgentSnapshot; canUndoOps: boolean; collapsed?: boolean; embedded?: boolean; onApplyOps: (ops: CanvasAgentOp[]) => unknown; onUndoOps: () => CanvasAgentSnapshot | null }) {
+export function CanvasLocalAgentPanel({ snapshot, canUndoOps, collapsed, embedded, headless, autoConnect, onApplyOps, onUndoOps }: { snapshot: CanvasAgentSnapshot; canUndoOps: boolean; collapsed?: boolean; embedded?: boolean; headless?: boolean; autoConnect?: boolean; onApplyOps: (ops: CanvasAgentOp[]) => unknown; onUndoOps: () => CanvasAgentSnapshot | null }) {
    const theme = canvasThemes[useThemeStore((state) => state.theme)];
    const user = useUserStore((state) => state.user);
    const { message, modal } = App.useApp();
+    const searchParams = useSearchParams();
    const { width, url, token, connected, enabled, prompt, attachments, sending, waiting, messages, eventLogs, threads, activeThreadId, workspacePath, loadingThreads, activeTab, confirmTools, activity, connectError, pendingTool, setAgentState, addMessage: pushMessage, addEventLog: pushEventLog, clearEventLogs } = useCanvasAgentStore();
    const [resizing, setResizing] = useState(false);
    const listRef = useRef<HTMLDivElement>(null);
@@ -50,11 +53,13 @@ export function CanvasLocalAgentPanel({ snapshot, canUndoOps, collapsed, embedde
    const confirmToolsRef = useRef(confirmTools);
    const pendingToolRef = useRef<AgentPendingToolCall | null>(null);
    const onApplyOpsRef = useRef(onApplyOps);
+    const autoConnectRef = useRef(false);
    const connectedRef = useRef(false);
    const errorLoggedRef = useRef(false);
    const attachmentUrlsRef = useRef(new Set<string>());
    const clientIdRef = useRef(typeof crypto === "undefined" ? `${Date.now()}` : crypto.randomUUID());
    const endpoint = useMemo(() => url.trim().replace(/\/$/, ""), [url]);
+    const urlAgentAutoConnect = searchParams.has("agentUrl") && searchParams.has("agentToken");
    const loadThreads = useCallback(async () => {
        const projectId = snapshotRef.current.projectId;
        if ((!connectedRef.current && !useCanvasAgentStore.getState().connected) || !projectId) return;
@@ -106,7 +111,7 @@ export function CanvasLocalAgentPanel({ snapshot, canUndoOps, collapsed, embedde
            errorLoggedRef.current = false;
            connectedRef.current = true;
            setAgentState({ connected: true, activity: "已连接", connectError: "", messages: useCanvasAgentStore.getState().messages.filter((item) => !isConnectionErrorMessage(item)) });
-            message.success("本地 Agent 已连接");
+            if (!headless) message.success("本地 Agent 已连接");
            void postState(endpoint, token, clientId, snapshotRef.current);
        });
        source.addEventListener("tool_call", (event) => {
@@ -136,7 +141,7 @@ export function CanvasLocalAgentPanel({ snapshot, canUndoOps, collapsed, embedde
            const text = wasConnected ? "本地 Agent 连接失败或已断开" : "连接失败，请检查地址和 token";
            if (!errorLoggedRef.current || wasConnected) {
                addEventLog(wasConnected ? "连接断开" : "连接失败", { endpoint, error: text });
-                message.error(text);
+                if (!headless) message.error(text);
            }
            errorLoggedRef.current = true;
            connectedRef.current = false;
@@ -288,36 +293,51 @@ export function CanvasLocalAgentPanel({ snapshot, canUndoOps, collapsed, embedde
        if (connected) void postState(endpoint, token, clientIdRef.current, restored);
    };

-    const toggleAgentConnection = () => {
+    const toggleAgentConnection = async () => {
        if (enabled) {
            clearAgentSession({ enabled: false, connected: false, activity: "离线", connectError: "" });
            return;
        }
-        if (!endpoint) {
+        const urlToken = searchParams.get("agentToken") || "";
+        const urlEndpoint = searchParams.get("agentUrl") || "";
+        const discovered = urlToken ? null : await discoverAgentConfig(endpoint || DEFAULT_AGENT_URL);
+        const nextEndpoint = (urlEndpoint || discovered?.url || endpoint || DEFAULT_AGENT_URL).trim().replace(/\/$/, "");
+        const nextToken = (urlToken || token.trim() || discovered?.token || "").trim();
+        if (!nextEndpoint) {
            const text = "请填写本地 Agent 地址";
            setAgentState({ connectError: text });
-            message.warning(text);
+            if (!headless) message.warning(text);
            return;
        }
-        if (!token.trim()) {
-            const text = "请填写 Agent token";
+        if (!nextToken) {
+            const text = "没有发现本地 Agent，请先在 Codex 使用插件或手动启动 Canvas Agent";
            setAgentState({ connectError: text });
-            message.warning(text);
+            if (!headless) message.warning(text);
            return;
        }
        try {
-            const parsed = new URL(endpoint);
+            const parsed = new URL(nextEndpoint);
            if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new Error("invalid protocol");
        } catch {
            const text = "本地 Agent 地址格式不正确";
            setAgentState({ connectError: text });
-            message.warning(text);
+            if (!headless) message.warning(text);
            return;
        }
        errorLoggedRef.current = false;
-        setAgentState({ url: endpoint, token: token.trim(), enabled: true, connected: false, activity: "连接中", connectError: "", activeTab: "setup" });
+        setAgentState({ url: nextEndpoint, token: nextToken, enabled: true, connected: false, activity: "连接中", connectError: "", activeTab: "setup" });
    };

+    useEffect(() => {
+        if (urlAgentAutoConnect && confirmTools) setAgentState({ confirmTools: false });
+    }, [confirmTools, setAgentState, urlAgentAutoConnect]);
+
+    useEffect(() => {
+        if (!autoConnect || autoConnectRef.current || enabled || connected) return;
+        autoConnectRef.current = true;
+        void toggleAgentConnection();
+    }, [autoConnect, connected, enabled]);
+
    function clearAgentSession(patch: Parameters<typeof setAgentState>[0] = {}) {
        setAgentState({
            messages: [],
@@ -544,6 +564,7 @@ export function CanvasLocalAgentPanel({ snapshot, canUndoOps, collapsed, embedde
        </>
    );

+    if (headless) return null;
    if (embedded) return content;

    return (
@@ -624,7 +645,7 @@ function AgentConnectView({ theme, url, token, enabled, connected, activity, con
                <div>
                    <div className="text-base font-semibold leading-6">连接本地 Agent</div>
                    <div className="mt-1 text-xs leading-5" style={{ color: theme.node.muted }}>
-                        本地服务只监听 127.0.0.1，画布通过地址和 token 连接到你电脑上的 Codex。
+                        安装 Codex 插件后，画布会优先自动连接本机 Agent。
                    </div>
                </div>
                <div className="space-y-2">
@@ -657,7 +678,7 @@ function AgentConnectView({ theme, url, token, enabled, connected, activity, con
                                </span>
                            </div>
                            <div className="mt-1 text-xs leading-5" style={{ color: theme.node.muted }}>
-                                填入终端输出的 Local URL 和 Connect token。
+                                默认自动读取 Local URL 和 Connect token，失败时再手动填写。
                            </div>
                        </div>
                        <Button className="!h-8 !px-3" type={enabled ? "default" : "primary"} icon={<PlugZap className="size-4" />} onClick={onToggleEnabled}>
@@ -679,7 +700,7 @@ function AgentConnectView({ theme, url, token, enabled, connected, activity, con
                                连接 Token
                                <span className="font-normal opacity-70">Connect token</span>
                            </span>
-                            <Input.Password size="large" prefix={<KeyRound className="mr-1 size-4" style={{ color: theme.node.faint }} />} value={token} onChange={(event) => onTokenChange(event.target.value)} placeholder="终端输出的 Connect token" />
+                            <Input.Password size="large" prefix={<KeyRound className="mr-1 size-4" style={{ color: theme.node.faint }} />} value={token} onChange={(event) => onTokenChange(event.target.value)} placeholder="自动发现，或手动填入 Connect token" />
                        </label>
                        {connectError ? (
                            <div className="rounded-md border px-2.5 py-2 text-xs leading-5" style={{ borderColor: "rgba(220,38,38,.35)", color: "#dc2626" }}>
@@ -971,6 +992,17 @@ async function fetchAgentJson<T>(endpoint: string, token: string, path: string,
    return data;
 }

+async function discoverAgentConfig(endpoint: string) {
+    try {
+        const res = await fetch(`${endpoint}/config`);
+        if (!res.ok) return null;
+        const data = (await res.json()) as AgentConfigResponse;
+        return data.ok ? data : null;
+    } catch {
+        return null;
+    }
+}
+
 function normalizeHistoryMessages(messages: AgentChatItem[]) {
    return messages
        .map((item, index) => ({
@@ -1,7 +1,7 @@
 "use client";

 import { Check, Download, Pencil, Trash2, X } from "lucide-react";
-import { useRouter } from "next/navigation";
+import { useRouter, useSearchParams } from "next/navigation";
 import { Button, Input } from "antd";

 import { useCanvasStore, type CanvasProject } from "../stores/use-canvas-store";
@@ -10,6 +10,7 @@ import { exportCanvasProjects } from "../utils/canvas-export";

 export function CanvasProjectCard({ project }: { project: CanvasProject }) {
    const router = useRouter();
+    const searchParams = useSearchParams();
    const renameProject = useCanvasStore((state) => state.renameProject);
    const selectedIds = useCanvasUiStore((state) => state.selectedProjectIds);
    const editingId = useCanvasUiStore((state) => state.editingProjectId);
@@ -21,7 +22,7 @@ export function CanvasProjectCard({ project }: { project: CanvasProject }) {
    const setDeleteIds = useCanvasUiStore((state) => state.setDeleteProjectIds);
    const editing = editingId === project.id;
    const selected = selectedIds.includes(project.id);
-    const open = () => router.push(`/canvas/${project.id}`);
+    const open = () => router.push(`/canvas/${project.id}${searchParams.toString() ? `?${searchParams.toString()}` : ""}`);
    const saveTitle = () => {
        renameProject(project.id, editingTitle);
        stopEditing();
@@ -1,7 +1,7 @@
 "use client";

-import { useRef } from "react";
-import { useRouter } from "next/navigation";
+import { useEffect, useRef } from "react";
+import { useRouter, useSearchParams } from "next/navigation";
 import { App, Button } from "antd";
 import { Download, FileUp, Plus } from "lucide-react";

@@ -18,7 +18,9 @@ import { exportCanvasProjects } from "./utils/canvas-export";
 export default function CanvasPage() {
    const { message } = App.useApp();
    const router = useRouter();
+    const searchParams = useSearchParams();
    const inputRef = useRef<HTMLInputElement>(null);
+    const autoOpenRef = useRef(false);
    const hydrated = useCanvasStore((state) => state.hydrated);
    const projects = useCanvasStore((state) => state.projects);
    const createProject = useCanvasStore((state) => state.createProject);
@@ -26,8 +28,11 @@ export default function CanvasPage() {
    const selectedIds = useCanvasUiStore((state) => state.selectedProjectIds);
    const setDeleteIds = useCanvasUiStore((state) => state.setDeleteProjectIds);

+    const mode = searchParams.get("mode");
+    const agentMode = mode === "new" || mode === "recent" || mode === "choose";
+    const agentQuery = agentMode ? `?${searchParams.toString()}` : "";
    const enterProject = (id: string) => {
-        router.push(`/canvas/${id}`);
+        router.push(`/canvas/${id}${agentQuery}`);
    };
    const createAndEnter = () => enterProject(createProject(`无限画布 ${projects.length + 1}`));
    const importCanvas = async (file?: File) => {
@@ -56,6 +61,14 @@ export default function CanvasPage() {
        }
    };

+    useEffect(() => {
+        if (!hydrated || autoOpenRef.current || (mode !== "new" && mode !== "recent")) return;
+        autoOpenRef.current = true;
+        enterProject(mode === "new" ? createProject(`无限画布 ${projects.length + 1}`) : projects[0]?.id || createProject(`无限画布 ${projects.length + 1}`));
+    }, [createProject, hydrated, mode, projects]);
+
+    if (hydrated && (mode === "new" || mode === "recent")) return <main className="flex h-full items-center justify-center bg-background text-sm text-stone-500">正在打开画布...</main>;
+
    return (
        <main className="h-full overflow-auto bg-background text-stone-950 dark:text-stone-100">
            <div className="mx-auto flex w-full max-w-6xl flex-col gap-8 px-6 py-10">