feat(panorama): enhance 3D panorama node with AI generation support and unified interaction toggle

2026-08-05 08:54:23 +08:00 · 2026-07-16 13:21:25 +08:00
parent 2d2c7e3d5e
commit 3f179a607a
12 changed files with 547 additions and 120 deletions
@@ -25,13 +25,14 @@ type CanvasNodePromptPanelProps = {
    onStop: (nodeId: string) => void;
    mentionReferences?: CanvasResourceReference[];
    onImageSettingsOpenChange?: (open: boolean) => void;
+    modeOverride?: CanvasNodeGenerationMode; // 插件节点用 useBuiltinPanel.mode 指定生成类型
 };

-export function CanvasNodePromptPanel({ node, isRunning, onPromptChange, onConfigChange, onGenerate, onStop, mentionReferences = [], onImageSettingsOpenChange }: CanvasNodePromptPanelProps) {
+export function CanvasNodePromptPanel({ node, isRunning, onPromptChange, onConfigChange, onGenerate, onStop, mentionReferences = [], onImageSettingsOpenChange, modeOverride }: CanvasNodePromptPanelProps) {
    const globalConfig = useEffectiveConfig();
    const openConfigDialog = useConfigStore((state) => state.openConfigDialog);
    const theme = canvasThemes[useThemeStore((state) => state.theme)];
-    const mode = defaultMode(node.type);
+    const mode = modeOverride ?? defaultMode(node.type);
    const config = buildNodeConfig(globalConfig, node, mode);
    const hasTextContent = node.type === CanvasNodeType.Text && Boolean(node.metadata?.content?.trim());
    const hasImageContent = node.type === CanvasNodeType.Image && Boolean(node.metadata?.content);
@@ -129,6 +129,11 @@ export const CanvasNode = React.memo(function CanvasNode({
    const hasAudioContent = data.type === CanvasNodeType.Audio && Boolean(data.metadata?.content);
    const isGroup = data.type === CanvasNodeType.Group;
    const isBatchRoot = data.type === CanvasNodeType.Image && Boolean(data.metadata?.isBatchRoot) && batchCount > 1;
+    // 支持「交互/移动」开关的节点:移动态(默认)内容不吃指针,拖动整块;交互态内容可操作。
+    // forceInteractive(如编辑态)强制可交互;空态(无内容)始终可交互,避免上传/生成按钮点不动。
+    const supportsInteractionToggle = Boolean(definition?.interactionToggle);
+    const forceInteractive = supportsInteractionToggle ? Boolean(definition?.forceInteractive?.(data)) : false;
+    const contentInteractive = !supportsInteractionToggle || forceInteractive || !data.metadata?.content ? true : Boolean(data.metadata?.interactive);
    const isBatchChild = data.type === CanvasNodeType.Image && Boolean(data.metadata?.batchRootId);
    const isActive = isConnectionTarget || isSelected || isFocusRelated;
    const imageBorderColor = isActive ? selectionBlue : isRelated && !isBatchChild ? theme.node.muted : "transparent";
@@ -374,6 +379,7 @@ export const CanvasNode = React.memo(function CanvasNode({
                    style={
                        {
                            background: isGroup ? "transparent" : hasImageContent || hasVideoContent ? "transparent" : theme.node.fill,
+                            pointerEvents: contentInteractive ? undefined : "none",
                            "--batch-from-x": `${batchMotion?.x || 0}px`,
                            "--batch-from-y": `${batchMotion?.y || 0}px`,
                            "--batch-from-rotate": `${6 + (batchMotion?.index || 0) * 4}deg`,
@@ -22,6 +22,9 @@ export function buildNodeContext(host: CanvasPluginHost, node: CanvasNodeData, t
        applyOps: (ops) => host.applyOps(ops),
        emit: (event, payload) => emitCanvasEvent(event, payload),
        on: (event, handler) => onCanvasEvent(event, handler),
+        ai: host.ai,
+        openPanel: () => host.openPanel(node.id),
+        closePanel: () => host.closePanel(),
        storage,
    };
 }
@@ -4,11 +4,11 @@ import { useNavigate, useParams, useSearchParams } from "react-router-dom";
 import { BookOpen, Bot, Group, Home, ImageIcon, Images, List, Menu, Music2, Plus, Redo2, Settings2, Trash2, Undo2, Upload, Video, X } from "lucide-react";
 import { saveAs } from "file-saver";

-import { requestEdit, requestGeneration, requestImageQuestion } from "@/services/api/image";
+import { requestEdit, requestGeneration, requestImageQuestion, type AiTextMessage } from "@/services/api/image";
 import { requestAudioGeneration, storeGeneratedAudio } from "@/services/api/audio";
 import { requestVideoGeneration, storeGeneratedVideo } from "@/services/api/video";
 import { DOCS_URL } from "@/constant/env";
-import { defaultConfig, type AiConfig, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store";
+import { defaultConfig, decodeChannelModel, selectableModelsByCapability, type AiConfig, type ModelCapability, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store";
 import { resolveImageUrl, uploadImage, type UploadedImage } from "@/services/image-storage";
 import { resolveMediaUrl, uploadMediaFile, type UploadedFile } from "@/services/file-storage";
 import { nanoid } from "nanoid";
@@ -49,7 +49,7 @@ import { buildNodeContext } from "@/lib/canvas/plugin-node-context";
 import { ensurePluginsLoaded } from "@/lib/canvas/plugin-loader";
 import { registerBuiltinNodes } from "@/components/canvas/nodes/builtin-nodes";
 import { CanvasPluginManagerModal } from "@/components/canvas/canvas-plugin-manager-modal";
-import type { CanvasPluginHost } from "@/types/canvas-plugin";
+import type { CanvasNodeToolbarItem, CanvasPluginAi, CanvasPluginHost } from "@/types/canvas-plugin";
 import {
    CanvasNodeType,
    type CanvasAssistantImage,
@@ -819,6 +819,47 @@ function InfiniteCanvasPage() {
    }, [agentSnapshot, applyAgentOps, agentUndoSnapshot, setAgentCanvasContext, undoAgentOps]);

    // 提供给插件节点的宿主能力(节点无关,方法接收 nodeId)
+    const pluginAi = useMemo<CanvasPluginAi>(() => {
+        // 把插件传入的参考图(dataURL 或 URL)整理成宿主生成 API 需要的 ReferenceImage[]
+        const toReferences = (refs?: string[]): ReferenceImage[] =>
+            (refs || []).filter(Boolean).map((src, index) => ({ id: `plugin-ref-${index}`, name: `ref-${index}.png`, type: "image/png", dataUrl: src }));
+        // AI 配置未就绪:弹出配置弹窗并抛错,交由插件 catch 处理
+        const ensureReady = (config: AiConfig) => {
+            if (!isAiConfigReady(config, config.model)) {
+                openConfigDialog(true);
+                throw new Error("AI 配置未就绪,请先在设置里配置模型与密钥");
+            }
+        };
+        return {
+            generateImage: async (prompt, options) => {
+                const config = { ...buildGenerationConfig(effectiveConfig, undefined, "image"), count: String(options?.count || 1), ...(options?.model ? { model: options.model } : {}), ...(options?.size ? { size: options.size } : {}) };
+                ensureReady(config);
+                const references = toReferences(options?.references);
+                const items = references.length
+                    ? await requestEdit(config, prompt, references, undefined, { signal: options?.signal })
+                    : await requestGeneration(config, prompt, { signal: options?.signal });
+                return { images: items.map((item) => item.dataUrl) };
+            },
+            generateVideo: async (prompt, options) => {
+                const config = { ...buildGenerationConfig(effectiveConfig, undefined, "video"), ...(options?.model ? { model: options.model } : {}), ...(options?.size ? { size: options.size } : {}), ...(options?.seconds ? { videoSeconds: options.seconds } : {}) };
+                ensureReady(config);
+                const file = await storeGeneratedVideo(await requestVideoGeneration(config, prompt, toReferences(options?.references), [], [], { signal: options?.signal }));
+                return { url: file.url, mimeType: file.mimeType, width: file.width, height: file.height, durationMs: file.durationMs };
+            },
+            generateText: async (prompt, options) => {
+                const config = { ...buildGenerationConfig(effectiveConfig, undefined, "text"), ...(options?.model ? { model: options.model } : {}) };
+                ensureReady(config);
+                const messages: AiTextMessage[] = [...(options?.system ? [{ role: "system" as const, content: options.system }] : []), { role: "user" as const, content: prompt }];
+                const text = await requestImageQuestion(config, messages, (delta) => options?.onDelta?.(delta), { signal: options?.signal });
+                return { text };
+            },
+            // 列出某能力下用户已配置的模型;label 取编码值中的模型名(去掉 channel 前缀)
+            listModels: (capability) =>
+                selectableModelsByCapability(effectiveConfig, capability as ModelCapability | undefined).map((value) => ({ value, label: decodeChannelModel(value)?.model || value })),
+            defaultModel: (capability) => buildGenerationConfig(effectiveConfig, undefined, capability).model,
+        };
+    }, [effectiveConfig, isAiConfigReady, openConfigDialog]);
+
    const pluginHost = useMemo<CanvasPluginHost>(
        () => ({
            getNode: (id) => nodesRef.current.find((node) => node.id === id) || null,
@@ -829,8 +870,11 @@ function InfiniteCanvasPage() {
            updateNode: (nodeId, patch) => setNodes((prev) => prev.map((node) => (node.id === nodeId ? { ...node, ...patch } : node))),
            updateMetadata: (nodeId, patch) => setNodes((prev) => prev.map((node) => (node.id === nodeId ? { ...node, metadata: { ...node.metadata, ...patch } } : node))),
            applyOps: (ops) => applyAgentOps(ops),
+            ai: pluginAi,
+            openPanel: (nodeId) => setDialogNodeId(nodeId),
+            closePanel: () => setDialogNodeId(null),
        }),
-        [applyAgentOps],
+        [applyAgentOps, pluginAi],
    );

    const renderPluginPanel = useCallback(
@@ -843,6 +887,28 @@ function InfiniteCanvasPage() {
        [pluginHost, theme],
    );

+    // 组装节点悬浮工具条按钮:插件自定义 toolbar +(声明 interactionToggle 时)宿主自动注入的「交互 ⇄ 移动」开关
+    const buildNodeToolbarItems = useCallback(
+        (node: CanvasNodeData): CanvasNodeToolbarItem[] => {
+            const definition = getNodeDefinition(node.type);
+            const ctx = buildNodeContext(pluginHost, node, theme, viewportRef.current.k);
+            const custom = definition?.toolbar?.(ctx) || [];
+            // 仅在节点有内容(展示态)且非强制交互态(如编辑态)时提供「交互/移动」开关
+            if (!definition?.interactionToggle || !node.metadata?.content || definition.forceInteractive?.(node)) return custom;
+            const interactive = Boolean(node.metadata?.interactive);
+            const toggle: CanvasNodeToolbarItem = {
+                id: "node-interaction-toggle",
+                title: interactive ? "当前:交互中。点击切回「移动」——拖动可移动节点" : "当前:可移动。点击切到「交互」——可操作节点内容(如转动全景)",
+                label: interactive ? "移动" : "交互",
+                icon: interactive ? "✋" : "🖐",
+                active: interactive,
+                onClick: () => pluginHost.updateMetadata(node.id, { interactive: !interactive }),
+            };
+            return [toggle, ...custom];
+        },
+        [pluginHost, theme],
+    );
+
    // 启动时加载已安装的远程插件
    useEffect(() => {
        void ensurePluginsLoaded();
@@ -865,8 +931,15 @@ function InfiniteCanvasPage() {
            setSelectedConnectionId(null);
            const definition = getNodeDefinition(type);
            // 纯展示型插件节点(hidePanel)不弹面板;插件自定义 Panel 需显式 autoOpenPanel 才在新建时打开;
+            // 声明了 useBuiltinPanel 的插件节点复用内置生成面板,新建即打开(与图片节点一致);
            // 内置的图片/视频/配置类节点保持原有「新建即打开生图面板」行为。
-            const wantsPanel = definition?.hidePanel ? false : definition?.Panel ? Boolean(definition.autoOpenPanel) : isBuiltinType(type) && type !== CanvasNodeType.Text && type !== CanvasNodeType.Audio && type !== CanvasNodeType.Group;
+            const wantsPanel = definition?.hidePanel
+                ? false
+                : definition?.Panel
+                  ? Boolean(definition.autoOpenPanel)
+                  : definition?.useBuiltinPanel
+                    ? true
+                    : isBuiltinType(type) && type !== CanvasNodeType.Text && type !== CanvasNodeType.Audio && type !== CanvasNodeType.Group;
            if (wantsPanel) setDialogNodeId(newNode.id);
        },
        [effectiveConfig.canvasImageCount, effectiveConfig.count, effectiveConfig.imageModel, effectiveConfig.model, effectiveConfig.size, getCanvasCenter],
@@ -2080,6 +2153,45 @@ function InfiniteCanvasPage() {
                return;
            }

+            // 插件节点声明了 useBuiltinPanel.writeBackToSelf:复用内置面板生成,但结果写回节点自身。
+            // 目前支持 image 模式(全景等展示型节点),前缀由 useBuiltinPanel.promptPrefix 指定。
+            const builtinPanel = sourceNode ? getNodeDefinition(sourceNode.type)?.useBuiltinPanel : undefined;
+            if (sourceNode && builtinPanel?.writeBackToSelf && builtinPanel.mode === "image") {
+                const scene = prompt.trim();
+                if (!scene) return;
+                setRunningNodeId(nodeId);
+                const controller = startGenerationRequest(nodeId, nodeId, nodeId);
+                setNodes((prev) => prev.map((node) => (node.id === nodeId ? { ...node, metadata: { ...node.metadata, prompt: scene, status: NODE_STATUS_LOADING, errorDetails: undefined } } : node)));
+                try {
+                    const fullPrompt = (builtinPanel.promptPrefix || "") + scene;
+                    // 上游图片节点作为参考图(图生图);无上游则纯文生图
+                    const upstreamNodes = connectionsRef.current
+                        .filter((conn) => conn.toNodeId === nodeId)
+                        .map((conn) => nodesRef.current.find((node) => node.id === conn.fromNodeId))
+                        .filter((node): node is CanvasNodeData => Boolean(node));
+                    const refs = upstreamNodes.flatMap((up) =>
+                        typeof up.metadata?.content === "string" && up.metadata.content && up.type !== sourceNode.type
+                            ? [{ id: up.id, name: `${up.title || up.id}.png`, type: up.metadata.mimeType || "image/png", dataUrl: up.metadata.content, storageKey: up.metadata.storageKey }]
+                            : [],
+                    );
+                    const image = refs.length
+                        ? await requestEdit({ ...generationConfig, count: "1" }, fullPrompt, refs, undefined, { signal: controller.signal }).then((items) => items[0])
+                        : await requestGeneration({ ...generationConfig, count: "1" }, fullPrompt, { signal: controller.signal }).then((items) => items[0]);
+                    const uploaded = await uploadImage(image.dataUrl);
+                    setNodes((prev) => prev.map((node) => (node.id === nodeId ? { ...node, metadata: { ...node.metadata, ...imageMetadata(uploaded), prompt: scene, model: generationConfig.model, status: NODE_STATUS_SUCCESS, errorDetails: undefined } } : node)));
+                    setDialogNodeId(null);
+                } catch (error) {
+                    if (!isGenerationCanceled(error)) {
+                        const errorDetails = error instanceof Error ? error.message : "生成失败";
+                        message.error(errorDetails);
+                        setNodes((prev) => prev.map((node) => (node.id === nodeId ? { ...node, metadata: { ...node.metadata, status: NODE_STATUS_ERROR, errorDetails } } : node)));
+                    }
+                } finally {
+                    finishGenerationRequest(nodeId, controller);
+                }
+                return;
+            }
+
            setRunningNodeId(nodeId);
            const runController = startGenerationRequest(nodeId, nodeId, nodeId);
            const sourceTextContent = sourceNode?.type === CanvasNodeType.Text ? sourceNode.metadata?.content?.trim() || "" : "";
@@ -2714,6 +2826,7 @@ function InfiniteCanvasPage() {
                                        onConfigChange={handleConfigNodeChange}
                                        onGenerate={handleGenerateNode}
                                        onStop={confirmStopGeneration}
+                                        modeOverride={getNodeDefinition(panelNode.type)?.useBuiltinPanel?.mode}
                                        onImageSettingsOpenChange={(open) => {
                                            setNodeImageSettingsOpen(open);
                                            if (open) setToolbarNodeId(null);
@@ -2790,7 +2903,7 @@ function InfiniteCanvasPage() {
                <CanvasNodeHoverToolbar
                    node={isNodeDragging || nodeImageSettingsOpen ? null : toolbarNode}
                    viewport={viewport}
-                    extraTools={toolbarNode ? getNodeDefinition(toolbarNode.type)?.toolbar?.(buildNodeContext(pluginHost, toolbarNode, theme, viewport.k)) : undefined}
+                    extraTools={toolbarNode ? buildNodeToolbarItems(toolbarNode) : undefined}
                    onKeep={keepNodeToolbar}
                    onLeave={hideNodeToolbar}
                    onInfo={(node) => setInfoNodeId(node.id)}
@@ -8,6 +8,25 @@ import type { CanvasResourceKind } from "@/lib/canvas/canvas-resource-references
 // 插件节点作为上游输入被消费时输出的资源
 export type CanvasNodeResource = { kind: CanvasResourceKind; text?: string; url?: string };

+// --- AI 生成能力(生图/生视频/生文本),由宿主注入,复用宿主模型/密钥配置 ---
+export type GenerateOptions = { signal?: AbortSignal; references?: string[]; model?: string };
+export type GenerateImageOptions = GenerateOptions & { count?: number; size?: string };
+export type GenerateImageResult = { images: string[] };
+export type GenerateVideoOptions = GenerateOptions & { size?: string; seconds?: string };
+export type GenerateVideoResult = { url: string; mimeType: string; width?: number; height?: number; durationMs?: number };
+export type GenerateTextOptions = { signal?: AbortSignal; model?: string; system?: string; onDelta?: (text: string) => void };
+export type GenerateTextResult = { text: string };
+export type PluginModelCapability = "image" | "video" | "text" | "audio";
+export type ModelOption = { value: string; label: string };
+
+export type CanvasPluginAi = {
+    generateImage: (prompt: string, options?: GenerateImageOptions) => Promise<GenerateImageResult>;
+    generateVideo: (prompt: string, options?: GenerateVideoOptions) => Promise<GenerateVideoResult>;
+    generateText: (prompt: string, options?: GenerateTextOptions) => Promise<GenerateTextResult>;
+    listModels: (capability?: PluginModelCapability) => ModelOption[];
+    defaultModel: (capability: PluginModelCapability) => string;
+};
+
 // 节点自带的工具栏按钮(追加到 hover 工具栏尾部)
 export type CanvasNodeToolbarItem = {
    id: string;
@@ -39,6 +58,11 @@ export type CanvasNodeContext = {
    // 节点间/插件间通信
    emit: (event: string, payload?: unknown) => void;
    on: (event: string, handler: (payload: unknown) => void) => () => void;
+    // AI 生成能力(生图/生视频/生文本),复用宿主模型配置
+    ai: CanvasPluginAi;
+    // 打开/关闭本节点下方的自定义 Panel(需在节点定义里提供 Panel)
+    openPanel: () => void;
+    closePanel: () => void;
    // 插件私有持久化,命名空间隔离
    storage: PluginStorage;
 };
@@ -59,6 +83,18 @@ export type CanvasPluginHost = {
    updateNode: (nodeId: string, patch: Partial<Pick<CanvasNodeData, "title" | "width" | "height">>) => void;
    updateMetadata: (nodeId: string, patch: CanvasNodeMetadata) => void;
    applyOps: (ops: CanvasAgentOp[]) => void;
+    // AI 生成能力,复用画布页面当前的模型/密钥配置
+    ai: CanvasPluginAi;
+    // 打开/关闭指定节点下方的自定义 Panel
+    openPanel: (nodeId: string) => void;
+    closePanel: () => void;
+};
+
+// 复用宿主内置生成面板的配置(见 SDK CanvasBuiltinPanelConfig)
+export type CanvasBuiltinPanelConfig = {
+    mode: "image" | "video" | "text" | "audio";
+    promptPrefix?: string;
+    writeBackToSelf?: boolean;
 };

 // 节点类型定义:内置节点与插件节点统一走这套结构
@@ -74,6 +110,11 @@ export type CanvasNodeDefinition = {
    hasSourceHandle?: boolean; // 右侧输出连接点,默认 true
    hidePanel?: boolean; // 为 true 时:点击/新建不弹出下方面板(含内置生图面板),纯展示型节点用
    autoOpenPanel?: boolean; // 为 true 时:单击节点自动打开自定义 Panel(默认仅内置节点单击自动打开)
+    useBuiltinPanel?: CanvasBuiltinPanelConfig; // 复用宿主内置生成面板(与自定义 Panel 二选一)
+    // 为 true 时:宿主自动提供「交互 ⇄ 移动」工具条开关,并按 metadata.interactive 控制内容层指针事件
+    interactionToggle?: boolean;
+    // 配合 interactionToggle:返回 true 表示内容强制可交互(如编辑态),忽略 interactive 并隐藏开关
+    forceInteractive?: (node: CanvasNodeData) => boolean;
    keepAspectRatio?: (node: CanvasNodeData) => boolean;
    resource?: (node: CanvasNodeData) => CanvasNodeResource | null;
    // 渲染:内置节点由 canvas-node 内部渲染器负责,可不提供 Content
@@ -61,6 +61,7 @@ export type CanvasNodeMetadata = {
    bytes?: number;
    durationMs?: number;
    groupId?: string;
+    interactive?: boolean; // 插件节点「交互 ⇄ 移动」开关状态(见 CanvasNodeDefinition.interactionToggle)
 };

 export type CanvasNodeData = {