admin/infinite-canvas
1
0
Fork 0
mirror of https://github.com/basketikun/infinite-canvas.git synced 2026-08-06 01:14:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(audio): add audio generation capabilities with configurable settings and UI enhancements

Browse Source
This commit is contained in:
HouYunFei
2026-06-02 17:50:36 +08:00
parent 38ec654748
commit 8f19a18c35
14 changed files with 528 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/src/app/(user)/canvas/[id]/canvas-client-page.tsx
+52 -5
View File
@@ -7,6 +7,7 @@ import { BookOpen, Home, ImageIcon, Images, List, Menu, MessageSquare, Music2, P
import { saveAs } from "file-saver";
import { requestEdit, requestGeneration, requestImageQuestion } from "@/services/api/image";
import { requestAudioGeneration, storeGeneratedAudio } from "@/services/api/audio";
import { requestVideoGeneration, storeGeneratedVideo } from "@/services/api/video";
import { DOCS_URL } from "@/constant/env";
import { defaultConfig, type AiConfig, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store";
@@ -1633,7 +1634,7 @@ function InfiniteCanvasPage() {
);
const effectivePrompt = generationContext.prompt.trim();
const markSourceStatus = sourceNode?.type !== CanvasNodeType.Image && !editingTextNode;
if (!effectivePrompt && mode === "text") {
if (!effectivePrompt && (mode === "text" || mode === "audio")) {
setRunningNodeId(null);
return;
}
@@ -1819,6 +1820,28 @@ function InfiniteCanvasPage() {
return;
}
if (mode === "audio") {
const spec = NODE_DEFAULT_SIZE[CanvasNodeType.Audio];
const isEmptyAudioNode = sourceNode?.type === CanvasNodeType.Audio && !sourceNode.metadata?.content;
const audioId = isEmptyAudioNode ? nodeId : nanoid();
const parent = sourceNode?.position || { x: 0, y: 0 };
const audioNode: CanvasNodeData = {
id: audioId,
type: CanvasNodeType.Audio,
title: effectivePrompt.slice(0, 32) || "Generated Audio",
position: isEmptyAudioNode ? sourceNode.position : { x: parent.x + (sourceNode?.width || spec.width) + 96, y: parent.y + ((sourceNode?.height || spec.height) - spec.height) / 2 },
width: isEmptyAudioNode ? sourceNode.width : spec.width,
height: isEmptyAudioNode ? sourceNode.height : spec.height,
metadata: { prompt: effectivePrompt, status: NODE_STATUS_LOADING, ...buildAudioGenerationMetadata(generationConfig) },
};
pendingChildIds = [audioId];
setNodes((prev) => (isEmptyAudioNode ? prev.map((node) => (node.id === nodeId ? { ...node, ...audioNode } : node)) : [...prev.map((node) => (node.id === nodeId ? { ...node, metadata: { ...node.metadata, status: NODE_STATUS_SUCCESS } } : node)), audioNode]));
if (!isEmptyAudioNode) setConnections((prev) => [...prev, { id: nanoid(), fromNodeId: nodeId, toNodeId: audioId }]);
const audio = await storeGeneratedAudio(await requestAudioGeneration(generationConfig, effectivePrompt), generationConfig.audioFormat);
setNodes((prev) => prev.map((node) => (node.id === audioId ? { ...node, metadata: { ...node.metadata, ...audioMetadata(audio), prompt: effectivePrompt, ...buildAudioGenerationMetadata(generationConfig) } } : node)));
return;
}
let streamed = "";
const isConfigNode = sourceNode?.type === CanvasNodeType.Config;
const textCount = isConfigNode ? getGenerationCount(generationConfig.count) : 1;
@@ -1895,7 +1918,7 @@ function InfiniteCanvasPage() {
size: savedImageMetadata.size || effectiveConfig.size,
count: "1",
}
: { ...buildGenerationConfig(effectiveConfig, sourceNode, node.type === CanvasNodeType.Text ? "text" : node.type === CanvasNodeType.Video ? "video" : "image"), count: "1" };
: { ...buildGenerationConfig(effectiveConfig, sourceNode, node.type === CanvasNodeType.Text ? "text" : node.type === CanvasNodeType.Video ? "video" : node.type === CanvasNodeType.Audio ? "audio" : "image"), count: "1" };
if (!isAiConfigReady(generationConfig, generationConfig.model)) {
openConfigDialog(true);
return;
@@ -1938,6 +1961,11 @@ function InfiniteCanvasPage() {
setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, width: videoSize.width, height: videoSize.height, position: { x: item.position.x + item.width / 2 - videoSize.width / 2, y: item.position.y + item.height / 2 - videoSize.height / 2 }, metadata: { ...item.metadata, ...videoMetadata(video), prompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, generateAudio: generationConfig.videoGenerateAudio, watermark: generationConfig.videoWatermark } } : item)));
return;
}
if (node.type === CanvasNodeType.Audio) {
const audio = await storeGeneratedAudio(await requestAudioGeneration(generationConfig, prompt), generationConfig.audioFormat);
setNodes((prev) => prev.map((item) => (item.id === node.id ? { ...item, metadata: { ...item.metadata, ...audioMetadata(audio), prompt, ...buildAudioGenerationMetadata(generationConfig) } } : item)));
return;
}
const image = useReferenceImages ? await requestEdit(generationConfig, prompt, retryImages).then((items) => items[0]) : await requestGeneration(generationConfig, prompt).then((items) => items[0]);
const uploadedImage = await uploadImage(image.dataUrl);
@@ -2571,7 +2599,12 @@ function imageExtension(dataUrl: string) {
}
function audioExtension(mimeType?: string) {
return mimeType?.includes("wav") ? "wav" : "mp3";
if (mimeType?.includes("wav")) return "wav";
if (mimeType?.includes("opus")) return "opus";
if (mimeType?.includes("aac")) return "aac";
if (mimeType?.includes("flac")) return "flac";
if (mimeType?.includes("pcm")) return "pcm";
return "mp3";
}
function imageMetadata(image: UploadedImage): CanvasNodeMetadata {
@@ -2597,6 +2630,16 @@ function buildImageGenerationMetadata(type: CanvasImageGenerationType, config: A
};
}
function buildAudioGenerationMetadata(config: AiConfig): CanvasNodeMetadata {
return {
model: config.model,
audioVoice: config.audioVoice,
audioFormat: config.audioFormat,
audioSpeed: config.audioSpeed,
audioInstructions: config.audioInstructions,
};
}
function referenceUrl(image: ReferenceImage) {
return image.storageKey || image.url || (!image.dataUrl.startsWith("data:") ? image.dataUrl : undefined);
}
@@ -2697,16 +2740,20 @@ function getInputSummary(inputs: NodeGenerationInput[]) {
}
function buildGenerationConfig(config: AiConfig, node: CanvasNodeData | undefined, mode: CanvasNodeGenerationMode): AiConfig {
const defaultModel = mode === "image" ? config.imageModel : mode === "video" ? config.videoModel : config.textModel;
const defaultModel = mode === "image" ? config.imageModel : mode === "video" ? config.videoModel : mode === "audio" ? config.audioModel : config.textModel;
return {
...config,
model: node?.metadata?.model || defaultModel || config.model || defaultConfig.model,
model: node?.metadata?.model || defaultModel || (mode === "audio" ? defaultConfig.audioModel : config.model || defaultConfig.model),
quality: node?.metadata?.quality || config.quality || defaultConfig.quality,
size: node?.metadata?.size || config.size || defaultConfig.size,
videoSeconds: node?.metadata?.seconds || config.videoSeconds || defaultConfig.videoSeconds,
vquality: node?.metadata?.vquality || config.vquality || defaultConfig.vquality,
videoGenerateAudio: node?.metadata?.generateAudio || config.videoGenerateAudio || defaultConfig.videoGenerateAudio,
videoWatermark: node?.metadata?.watermark || config.videoWatermark || defaultConfig.videoWatermark,
audioVoice: node?.metadata?.audioVoice || config.audioVoice || defaultConfig.audioVoice,
audioFormat: node?.metadata?.audioFormat || config.audioFormat || defaultConfig.audioFormat,
audioSpeed: node?.metadata?.audioSpeed || config.audioSpeed || defaultConfig.audioSpeed,
audioInstructions: node?.metadata?.audioInstructions || config.audioInstructions || defaultConfig.audioInstructions,
count: String(node?.metadata?.count || (mode === "image" ? config.canvasImageCount || config.count : config.count) || defaultConfig.count),
};
}