From 8f19a18c353952987ec0a81cc9d7d96580a1e7ff Mon Sep 17 00:00:00 2001 From: HouYunFei <1844025705@qq.com> Date: Tue, 2 Jun 2026 17:50:36 +0800 Subject: [PATCH] feat(audio): add audio generation capabilities with configurable settings and UI enhancements --- CHANGELOG.md | 1 + docs/content/docs/progress/pending-test.mdx | 1 + handler/ai.go | 4 + router/router.go | 1 + .../(user)/canvas/[id]/canvas-client-page.tsx | 57 ++++++++- .../canvas-audio-settings-popover.tsx | 116 ++++++++++++++++++ .../components/canvas-config-node-panel.tsx | 33 ++++- .../components/canvas-node-prompt-panel.tsx | 32 ++++- web/src/app/(user)/canvas/types.ts | 6 +- web/src/components/audio-settings-panel.tsx | 102 +++++++++++++++ .../components/layout/app-config-modal.tsx | 23 +++- web/src/lib/audio-generation.ts | 61 +++++++++ web/src/services/api/audio.ts | 91 ++++++++++++++ web/src/stores/use-config-store.ts | 18 ++- 14 files changed, 528 insertions(+), 18 deletions(-) create mode 100644 web/src/app/(user)/canvas/components/canvas-audio-settings-popover.tsx create mode 100644 web/src/components/audio-settings-panel.tsx create mode 100644 web/src/lib/audio-generation.ts create mode 100644 web/src/services/api/audio.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 7654d07..f57577e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ + [新增] 新增文档站点页面。 + [优化] 优化画布连线交互。 ++ [优化] 优化模型选择用户偏好。 ## v0.2.0 - 2026-06-01 diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index a527394..00994e3 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -6,6 +6,7 @@ description: 当前版本已实现但仍需人工验证的变更项 # 待测试 - 配置弹窗改为“配置与用户偏好”并放大为可滚动弹窗；本地直连支持配置生图、视频、文本、音频四类可选模型列表和默认模型，新建画布生图和配置节点会读取“画布默认生图张数”，需要验证远程渠道和本地直连模型列表都能正确显示。 +- 画布音频节点底部生成面板改为音频提示词、音频模型下拉和 OpenAI Speech 参数设置，支持 `voice`、`response_format`、`speed`、`instructions` 并通过 `/audio/speech` 生成音频节点；需要验证本地直连和云端渠道的生成、重试、下载和刷新恢复。 - 画布左上角菜单和右上角状态栏新增“文档”入口，会使用 `NEXT_PUBLIC_DOC_URL` 配置的地址并在新标签打开文档站；需要验证登录和未登录状态下顶部入口都可见。 - 文档站搜索改为中英文混合 tokenizer，中文正文、标题和短语会按中文词、单字、二元和三元片段建立索引；需要验证 `/api/search` 和搜索弹窗能命中文档中的中文关键词。 - 文档站改为 Next.js standalone server 输出，新增 `docs/Dockerfile`、`docs/docker-compose.yml` 和 `docs/docker-compose.local.yml` 独立运行入口；需要验证文档页、搜索接口和 LLM 文本接口在文档站容器中可访问。 diff --git a/handler/ai.go b/handler/ai.go index 26d5d7d..f480520 100644 --- a/handler/ai.go +++ b/handler/ai.go @@ -26,6 +26,10 @@ func AIChatCompletions(w http.ResponseWriter, r *http.Request) { proxyAIRequest(w, r, "/chat/completions") } +func AIAudioSpeech(w http.ResponseWriter, r *http.Request) { + proxyAIRequest(w, r, "/audio/speech") +} + func AIVideos(w http.ResponseWriter, r *http.Request) { proxyAIRequest(w, r, "/videos") } diff --git a/router/router.go b/router/router.go index 6493903..ae9fb7b 100644 --- a/router/router.go +++ b/router/router.go @@ -32,6 +32,7 @@ func New() *gin.Engine { v1.POST("/images/generations", gin.WrapF(handler.AIImagesGenerations)) v1.POST("/images/edits", gin.WrapF(handler.AIImagesEdits)) v1.POST("/chat/completions", gin.WrapF(handler.AIChatCompletions)) + v1.POST("/audio/speech", gin.WrapF(handler.AIAudioSpeech)) v1.POST("/videos", gin.WrapF(handler.AIVideos)) v1.POST("/media/references", gin.WrapF(handler.UploadReferenceMedia)) v1.GET("/videos/:id", func(c *gin.Context) { diff --git a/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx b/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx index 733970d..7906904 100644 --- a/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx +++ b/web/src/app/(user)/canvas/[id]/canvas-client-page.tsx @@ -7,6 +7,7 @@ import { BookOpen, Home, ImageIcon, Images, List, Menu, MessageSquare, Music2, P import { saveAs } from "file-saver"; import { requestEdit, requestGeneration, requestImageQuestion } from "@/services/api/image"; +import { requestAudioGeneration, storeGeneratedAudio } from "@/services/api/audio"; import { requestVideoGeneration, storeGeneratedVideo } from "@/services/api/video"; import { DOCS_URL } from "@/constant/env"; import { defaultConfig, type AiConfig, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store"; @@ -1633,7 +1634,7 @@ function InfiniteCanvasPage() { ); const effectivePrompt = generationContext.prompt.trim(); const markSourceStatus = sourceNode?.type !== CanvasNodeType.Image && !editingTextNode; - if (!effectivePrompt && mode === "text") { + if (!effectivePrompt && (mode === "text" || mode === "audio")) { setRunningNodeId(null); return; } @@ -1819,6 +1820,28 @@ function InfiniteCanvasPage() { return; } + if (mode === "audio") { + const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Audio]; + const isEmptyAudioNode = sourceNode?.type === CanvasNodeType.Audio && !sourceNode.metadata?.content; + const audioId = isEmptyAudioNode ? nodeId : nanoid(); + const parent = sourceNode?.position || { x: 0, y: 0 }; + const audioNode: CanvasNodeData = { + id: audioId, + type: CanvasNodeType.Audio, + title: effectivePrompt.slice(0, 32) || "Generated Audio", + position: isEmptyAudioNode ? sourceNode.position : { x: parent.x + (sourceNode?.width || spec.width) + 96, y: parent.y + ((sourceNode?.height || spec.height) - spec.height) / 2 }, + width: isEmptyAudioNode ? sourceNode.width : spec.width, + height: isEmptyAudioNode ? sourceNode.height : spec.height, + metadata: { prompt: effectivePrompt, status: NODE_STATUS_LOADING, ...buildAudioGenerationMetadata(generationConfig) }, + }; + pendingChildIds = [audioId]; + setNodes((prev) => (isEmptyAudioNode ? prev.map((node) => (node.id === nodeId ? { ...node, ...audioNode } : node)) : [...prev.map((node) => (node.id === nodeId ? { ...node, metadata: { ...node.metadata, status: NODE_STATUS_SUCCESS } } : node)), audioNode])); + if (!isEmptyAudioNode) setConnections((prev) => [...prev, { id: nanoid(), fromNodeId: nodeId, toNodeId: audioId }]); + const audio = await storeGeneratedAudio(await requestAudioGeneration(generationConfig, effectivePrompt), generationConfig.audioFormat); + setNodes((prev) => prev.map((node) => (node.id === audioId ? { ...node, metadata: { ...node.metadata, ...audioMetadata(audio), prompt: effectivePrompt, ...buildAudioGenerationMetadata(generationConfig) } } : node))); + return; + } + let streamed = ""; const isConfigNode = sourceNode?.type === CanvasNodeType.Config; const textCount = isConfigNode ? getGenerationCount(generationConfig.count) : 1; @@ -1895,7 +1918,7 @@ function InfiniteCanvasPage() { size: savedImageMetadata.size || effectiveConfig.size, count: "1", } - : { ...buildGenerationConfig(effectiveConfig, sourceNode, node.type === CanvasNodeType.Text ? "text" : node.type === CanvasNodeType.Video ? "video" : "image"), count: "1" }; + : { ...buildGenerationConfig(effectiveConfig, sourceNode, node.type === CanvasNodeType.Text ? "text" : node.type === CanvasNodeType.Video ? "video" : node.type === CanvasNodeType.Audio ? "audio" : "image"), count: "1" }; if (!isAiConfigReady(generationConfig, generationConfig.model)) { openConfigDialog(true); return; @@ -1938,6 +1961,11 @@ function InfiniteCanvasPage() { setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, width: videoSize.width, height: videoSize.height, position: { x: item.position.x + item.width / 2 - videoSize.width / 2, y: item.position.y + item.height / 2 - videoSize.height / 2 }, metadata: { ...item.metadata, ...videoMetadata(video), prompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, generateAudio: generationConfig.videoGenerateAudio, watermark: generationConfig.videoWatermark } } : item))); return; } + if (node.type === CanvasNodeType.Audio) { + const audio = await storeGeneratedAudio(await requestAudioGeneration(generationConfig, prompt), generationConfig.audioFormat); + setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, metadata: { ...item.metadata, ...audioMetadata(audio), prompt, ...buildAudioGenerationMetadata(generationConfig) } } : item))); + return; + } const image = useReferenceImages ? await requestEdit(generationConfig, prompt, retryImages).then((items) => items[0]) : await requestGeneration(generationConfig, prompt).then((items) => items[0]); const uploadedImage = await uploadImage(image.dataUrl); @@ -2571,7 +2599,12 @@ function imageExtension(dataUrl: string) { } function audioExtension(mimeType?: string) { - return mimeType?.includes("wav") ? "wav" : "mp3"; + if (mimeType?.includes("wav")) return "wav"; + if (mimeType?.includes("opus")) return "opus"; + if (mimeType?.includes("aac")) return "aac"; + if (mimeType?.includes("flac")) return "flac"; + if (mimeType?.includes("pcm")) return "pcm"; + return "mp3"; } function imageMetadata(image: UploadedImage): CanvasNodeMetadata { @@ -2597,6 +2630,16 @@ function buildImageGenerationMetadata(type: CanvasImageGenerationType, config: A }; } +function buildAudioGenerationMetadata(config: AiConfig): CanvasNodeMetadata { + return { + model: config.model, + audioVoice: config.audioVoice, + audioFormat: config.audioFormat, + audioSpeed: config.audioSpeed, + audioInstructions: config.audioInstructions, + }; +} + function referenceUrl(image: ReferenceImage) { return image.storageKey || image.url || (!image.dataUrl.startsWith("data:") ? image.dataUrl : undefined); } @@ -2697,16 +2740,20 @@ function getInputSummary(inputs: NodeGenerationInput[]) { } function buildGenerationConfig(config: AiConfig, node: CanvasNodeData | undefined, mode: CanvasNodeGenerationMode): AiConfig { - const defaultModel = mode === "image" ? config.imageModel : mode === "video" ? config.videoModel : config.textModel; + const defaultModel = mode === "image" ? config.imageModel : mode === "video" ? config.videoModel : mode === "audio" ? config.audioModel : config.textModel; return { ...config, - model: node?.metadata?.model || defaultModel || config.model || defaultConfig.model, + model: node?.metadata?.model || defaultModel || (mode === "audio" ? defaultConfig.audioModel : config.model || defaultConfig.model), quality: node?.metadata?.quality || config.quality || defaultConfig.quality, size: node?.metadata?.size || config.size || defaultConfig.size, videoSeconds: node?.metadata?.seconds || config.videoSeconds || defaultConfig.videoSeconds, vquality: node?.metadata?.vquality || config.vquality || defaultConfig.vquality, videoGenerateAudio: node?.metadata?.generateAudio || config.videoGenerateAudio || defaultConfig.videoGenerateAudio, videoWatermark: node?.metadata?.watermark || config.videoWatermark || defaultConfig.videoWatermark, + audioVoice: node?.metadata?.audioVoice || config.audioVoice || defaultConfig.audioVoice, + audioFormat: node?.metadata?.audioFormat || config.audioFormat || defaultConfig.audioFormat, + audioSpeed: node?.metadata?.audioSpeed || config.audioSpeed || defaultConfig.audioSpeed, + audioInstructions: node?.metadata?.audioInstructions || config.audioInstructions || defaultConfig.audioInstructions, count: String(node?.metadata?.count || (mode === "image" ? config.canvasImageCount || config.count : config.count) || defaultConfig.count), }; } diff --git a/web/src/app/(user)/canvas/components/canvas-audio-settings-popover.tsx b/web/src/app/(user)/canvas/components/canvas-audio-settings-popover.tsx new file mode 100644 index 0000000..6bdc7fa --- /dev/null +++ b/web/src/app/(user)/canvas/components/canvas-audio-settings-popover.tsx @@ -0,0 +1,116 @@ +"use client"; + +import { useEffect, useRef, useState, type RefObject } from "react"; +import { createPortal } from "react-dom"; +import { Settings2 } from "lucide-react"; +import { Button } from "antd"; + +import { AudioSettingsPanel } from "@/components/audio-settings-panel"; +import { audioFormatLabel, audioSpeedLabel, audioVoiceLabel } from "@/lib/audio-generation"; +import { canvasThemes } from "@/lib/canvas-theme"; +import { useThemeStore } from "@/stores/use-theme-store"; +import type { AiConfig } from "@/stores/use-config-store"; + +export type CanvasAudioSettingKey = "audioVoice" | "audioFormat" | "audioSpeed" | "audioInstructions"; + +type CanvasAudioSettingsPopoverProps = { + config: AiConfig; + onConfigChange: (key: CanvasAudioSettingKey, value: string) => void; + buttonClassName?: string; + placement?: "topLeft" | "top" | "topRight" | "bottomLeft" | "bottom" | "bottomRight"; +}; + +export function CanvasAudioSettingsPopover({ config, onConfigChange, buttonClassName, placement = "topLeft" }: CanvasAudioSettingsPopoverProps) { + const theme = canvasThemes[useThemeStore((state) => state.theme)]; + const buttonRef = useRef(null); + const panelRef = useRef(null); + const [open, setOpen] = useState(false); + const [buttonRect, setButtonRect] = useState(null); + + useEffect(() => { + if (!open) return; + const syncPosition = () => setButtonRect(buttonRef.current?.getBoundingClientRect() || null); + const closeOnOutsidePointer = (event: PointerEvent) => { + const target = event.target; + if (!(target instanceof Node)) return; + if (buttonRef.current?.contains(target) || panelRef.current?.contains(target)) return; + setOpen(false); + }; + + syncPosition(); + window.addEventListener("resize", syncPosition); + window.addEventListener("scroll", syncPosition, true); + window.addEventListener("pointerdown", closeOnOutsidePointer, true); + return () => { + window.removeEventListener("resize", syncPosition); + window.removeEventListener("scroll", syncPosition, true); + window.removeEventListener("pointerdown", closeOnOutsidePointer, true); + }; + }, [open]); + + const panel = open && buttonRect ? : null; + + return ( + <> + + + + {panel} + + ); +} + +function AudioSettingsPortal({ + buttonRect, + panelRef, + placement, + theme, + config, + onConfigChange, +}: { + buttonRect: DOMRect; + panelRef: RefObject; + placement: CanvasAudioSettingsPopoverProps["placement"]; + theme: (typeof canvasThemes)[keyof typeof canvasThemes]; + config: AiConfig; + onConfigChange: (key: CanvasAudioSettingKey, value: string) => void; +}) { + const width = 356; + const gap = 8; + const margin = 12; + const alignRight = placement?.endsWith("Right"); + const alignCenter = placement === "top" || placement === "bottom"; + const left = alignCenter ? buttonRect.left + buttonRect.width / 2 - width / 2 : alignRight ? buttonRect.right - width : buttonRect.left; + const topPlacement = placement?.startsWith("top"); + const style = { + position: "fixed", + zIndex: 1200, + width, + left: Math.max(margin, Math.min(window.innerWidth - width - margin, left)), + ...(topPlacement ? { bottom: window.innerHeight - buttonRect.top + gap, maxHeight: Math.max(260, buttonRect.top - margin * 2) } : { top: buttonRect.bottom + gap, maxHeight: Math.max(260, window.innerHeight - buttonRect.bottom - margin * 2) }), + background: theme.toolbar.panel, + borderRadius: 18, + boxShadow: "0 18px 54px rgba(28, 25, 23, 0.16)", + padding: 18, + overflowY: "auto", + color: theme.node.text, + } as const; + + return createPortal( +
event.stopPropagation()} + onMouseDown={(event) => event.stopPropagation()} + onClick={(event) => event.stopPropagation()} + > + onConfigChange(key, value)} theme={theme} className="space-y-4" /> +
, + document.body, + ); +} diff --git a/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx b/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx index 31d76d7..e31bf07 100644 --- a/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx +++ b/web/src/app/(user)/canvas/components/canvas-config-node-panel.tsx @@ -13,6 +13,7 @@ import { imageReferenceLabel } from "@/lib/image-reference-prompt"; import { seedanceReferenceLabel } from "@/lib/seedance-video"; import { useThemeStore } from "@/stores/use-theme-store"; import { CanvasImageSettingsPopover } from "./canvas-image-settings-popover"; +import { CanvasAudioSettingsPopover, type CanvasAudioSettingKey } from "./canvas-audio-settings-popover"; import { CanvasVideoSettingsPopover } from "./canvas-video-settings-popover"; import type { NodeGenerationInput } from "./canvas-node-generation"; import type { CanvasGenerationMode, CanvasNodeData, CanvasNodeMetadata } from "../types"; @@ -45,6 +46,8 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o const imageInputs = inputs.filter((input) => input.type === "image"); const videoInputs = inputs.filter((input) => input.type === "video"); const audioInputs = inputs.filter((input) => input.type === "audio"); + const hasAnyInput = Boolean(inputSummary.textCount || inputSummary.imageCount || inputSummary.videoCount || inputSummary.audioCount); + const canGenerate = mode === "audio" ? inputSummary.textCount > 0 : hasAnyInput; const moveInput = (input: NodeGenerationInput, offset: number) => { const sameTypeInputs = inputs.filter((item) => item.type === input.type); @@ -108,6 +111,15 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o ), }, + { + value: "audio", + label: ( + + + 音频 + + ), + }, ]} /> @@ -124,19 +136,21 @@ export function CanvasConfigNodePanel({ node, isRunning, inputSummary, inputs, o -
event.stopPropagation()}> +
event.stopPropagation()}> onConfigChange(node.id, { model })} capability={mode} onMissingConfig={() => openConfigDialog(true)} fullWidth /> {mode === "video" ? ( onConfigChange(node.id, videoConfigPatch(key, value))} /> ) : mode === "image" ? ( onConfigChange(node.id, key === "count" ? { count: Number(value) || 1 } : { [key]: value })} /> + ) : mode === "audio" ? ( + onConfigChange(node.id, audioConfigPatch(key, value))} /> ) : null}