import axios from "axios"; import { buildApiUrl, resolveModelRequestConfig, resolveModelScript, type AiConfig, type ModelChannel } from "@/stores/use-config-store"; import { normalizePluginImages, runModelPlugin } from "./model-plugin"; import { nanoid } from "nanoid"; import { dataUrlToFile } from "@/lib/image-utils"; import { buildImageReferencePromptText } from "@/lib/image-reference-prompt"; import { imageToDataUrl } from "@/services/image-storage"; import type { ReferenceImage } from "@/types/image"; export type AiTextMessage = { role: "system" | "user" | "assistant"; content: string | Array<{ type: "text"; text: string } | { type: "image_url"; image_url: { url: string } }>; }; type ResponseToolCall = { id: string; type: "function"; function: { name: string; arguments: string }; thoughtSignature?: string; }; type ResponseInputMessage = | AiTextMessage | { type: "function_call"; call_id: string; name: string; arguments: string; thoughtSignature?: string } | { role: "tool"; tool_call_id: string; content: string }; type ResponseFunctionTool = { type: "function"; function: { name: string; description?: string; parameters: Record; strict?: boolean; }; }; type ToolResponseResult = { content: string; toolCalls: ResponseToolCall[]; }; type ToolChoice = "auto" | "required" | { type: "function"; name: string }; type ResponseMessageContent = AiTextMessage["content"] | string; type ResponseInputContent = { type: "input_text"; text: string } | { type: "input_image"; image_url: string }; type ResponseInputItem = | { role: "system" | "user" | "assistant"; content: string | ResponseInputContent[] } | { type: "function_call"; call_id: string; name: string; arguments: string } | { type: "function_call_output"; call_id: string; output: string }; type ResponseApiToolDefinition = { type: "function"; name: string; description?: string; parameters: Record; strict?: boolean; }; type ResponseApiOutputItem = | { type?: "message"; content?: Array<{ type?: string; text?: string }> } | { type?: "function_call"; id?: string; call_id?: string; name?: string; arguments?: string }; type ResponseApiPayload = { id?: string; output?: ResponseApiOutputItem[]; output_text?: string; error?: { message?: string }; code?: number; msg?: string; }; type ResponseStreamState = { buffer: string; text: string; payload?: ResponseApiPayload; error?: string }; type ImageApiResponse = { data?: Array>; error?: { message?: string }; code?: number; msg?: string; }; type GeminiPart = { text?: string; inlineData?: { mimeType?: string; data?: string }; inline_data?: { mime_type?: string; mimeType?: string; data?: string }; fileData?: { mimeType?: string; fileUri?: string }; functionCall?: { id?: string; name?: string; args?: Record }; functionResponse?: { id?: string; name?: string; response?: Record }; thoughtSignature?: string; thought_signature?: string; }; type GeminiContent = { role?: "user" | "model"; parts: GeminiPart[] }; type GeminiPayload = { candidates?: Array<{ content?: { parts?: GeminiPart[] }; finishReason?: string }>; models?: Array<{ name?: string }>; error?: { message?: string }; promptFeedback?: { blockReason?: string }; }; type GeminiStreamState = { buffer: string; text: string; toolCalls: ResponseToolCall[]; error?: string }; type RequestOptions = { signal?: AbortSignal }; const QUALITY_BASE: Record = { low: 1024, medium: 2048, high: 2880, standard: 1024, hd: 2048, }; const QUALITY_ALIASES: Record = { "1k": "low", "2k": "medium", "4k": "high", }; const DEFAULT_IMAGE_SHORT_SIDE = 1024; const IMAGE_SIZE_STEP = 16; const IMAGE_MIN_PIXELS = 655360; const IMAGE_MAX_PIXELS = 8294400; const IMAGE_MAX_EDGE = 3840; const IMAGE_MAX_RATIO = 3; const IMAGE_OUTPUT_FORMAT = "png"; const GEMINI_SUPPORTED_RATIOS = ["1:1", "1:4", "1:8", "2:3", "3:2", "3:4", "4:1", "4:3", "4:5", "5:4", "8:1", "9:16", "16:9", "21:9"]; const GEMINI_IMAGE_SIZE_BY_QUALITY: Record = { low: "1K", medium: "2K", high: "4K", standard: "1K", hd: "2K" }; function normalizeQuality(quality: string) { const value = quality.trim().toLowerCase(); const normalized = QUALITY_ALIASES[value] || value; return QUALITY_BASE[normalized] ? normalized : undefined; } /** Only "transparent" is forwarded; any other value (incl. empty) means keep the default opaque background. */ function normalizeBackground(background: string | undefined) { return background?.trim().toLowerCase() === "transparent" ? "transparent" : undefined; } /** Map "quality + ratio" to an explicit pixel dimension like "3840x2160". */ function resolveSize(quality: string | undefined, ratio: string): string { const parsedRatio = parseImageRatio(ratio); const basePixels = quality ? QUALITY_BASE[quality] : undefined; const isLandscape = parsedRatio.width >= parsedRatio.height; const longRatio = isLandscape ? parsedRatio.width / parsedRatio.height : parsedRatio.height / parsedRatio.width; let longSide: number; let shortSide: number; if (basePixels) { const targetPixels = basePixels * basePixels; const longSideRaw = Math.sqrt(targetPixels * longRatio); longSide = Math.floor(longSideRaw / IMAGE_SIZE_STEP) * IMAGE_SIZE_STEP; shortSide = Math.round(longSide / longRatio / IMAGE_SIZE_STEP) * IMAGE_SIZE_STEP; } else { shortSide = DEFAULT_IMAGE_SHORT_SIDE; longSide = Math.round((shortSide * longRatio) / IMAGE_SIZE_STEP) * IMAGE_SIZE_STEP; } const width = isLandscape ? longSide : shortSide; const height = isLandscape ? shortSide : longSide; validateImageSize(width, height); return `${width}x${height}`; } function parseRatioValue(value: string) { const parts = value.split(":"); if (parts.length !== 2) throw new Error("图像尺寸格式不支持,请使用 auto、9:16 或 1024x1024"); const w = Number(parts[0]); const h = Number(parts[1]); if (!Number.isFinite(w) || !Number.isFinite(h) || w <= 0 || h <= 0) throw new Error("图像比例必须是正数,例如 9:16"); return { width: w, height: h }; } function parseImageRatio(value: string) { const ratio = parseRatioValue(value); if (Math.max(ratio.width, ratio.height) / Math.min(ratio.width, ratio.height) > IMAGE_MAX_RATIO) throw new Error("图像宽高比不能超过 3:1,请调整尺寸"); return ratio; } function parseImageDimensions(value: string) { const match = value.match(/^(\d+)x(\d+)$/i); if (!match) return null; return { width: Number(match[1]), height: Number(match[2]) }; } function validateImageSize(width: number, height: number) { if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) throw new Error("图像尺寸必须是正整数,例如 1024x1024"); if (width % IMAGE_SIZE_STEP !== 0 || height % IMAGE_SIZE_STEP !== 0) throw new Error("图像尺寸的宽高必须是 16 的倍数,请调整尺寸"); if (Math.max(width, height) > IMAGE_MAX_EDGE) throw new Error("图像尺寸最长边不能超过 3840px,请调整尺寸"); if (Math.max(width, height) / Math.min(width, height) > IMAGE_MAX_RATIO) throw new Error("图像宽高比不能超过 3:1,请调整尺寸"); const pixels = width * height; if (pixels < IMAGE_MIN_PIXELS || pixels > IMAGE_MAX_PIXELS) throw new Error("图像总像素需在 655360 到 8294400 之间,请调整尺寸"); } function resolveRequestSize(quality: string | undefined, size: string) { const value = size.trim(); if (!value || value.toLowerCase() === "auto") return undefined; const dimensions = parseImageDimensions(value); if (dimensions) { validateImageSize(dimensions.width, dimensions.height); return `${dimensions.width}x${dimensions.height}`; } if (value.includes(":")) return resolveSize(quality, value); throw new Error("图像尺寸格式不支持,请使用 auto、9:16 或 1024x1024"); } function resolveGeminiImageConfig(config: AiConfig) { const value = config.size.trim(); const dimensions = parseImageDimensions(value); const ratio = dimensions ? `${dimensions.width}:${dimensions.height}` : value; const aspectRatio = value && value.toLowerCase() !== "auto" ? closestGeminiAspectRatio(ratio) : undefined; const imageSize = supportsGeminiImageSize(config.model) ? resolveGeminiImageSize(config.quality, dimensions) : undefined; const image = { ...(aspectRatio ? { aspectRatio } : {}), ...(imageSize ? { imageSize } : {}) }; return Object.keys(image).length ? { responseFormat: { image } } : {}; } function closestGeminiAspectRatio(value: string) { const ratio = parseImageRatio(value); const target = ratio.width / ratio.height; return GEMINI_SUPPORTED_RATIOS.reduce((best, item) => { const current = parseRatioValue(item); const bestRatio = parseRatioValue(best); return Math.abs(current.width / current.height - target) < Math.abs(bestRatio.width / bestRatio.height - target) ? item : best; }); } function resolveGeminiImageSize(quality: string, dimensions: { width: number; height: number } | null) { const normalizedQuality = normalizeQuality(quality); if (normalizedQuality) return GEMINI_IMAGE_SIZE_BY_QUALITY[normalizedQuality]; if (!dimensions) return undefined; const edge = Math.max(dimensions.width, dimensions.height); if (edge <= 768) return "512"; if (edge <= 1536) return "1K"; if (edge <= 3072) return "2K"; return "4K"; } function supportsGeminiImageSize(model: string) { const value = model.toLowerCase(); return value.includes("gemini-3") || value.includes("3.1") || value.includes("3-pro"); } function resolveImageDataUrl(item: Record) { if (typeof item.b64_json === "string" && item.b64_json) { return `data:image/png;base64,${item.b64_json}`; } if (typeof item.url === "string" && item.url) { return item.url; } return null; } function parseImagePayload(payload: ImageApiResponse) { if (typeof payload.code === "number" && payload.code !== 0) { throw new Error(payload.msg || "请求失败"); } // 支持 data / images / results 三种返回字段(兼容不同 API) const imageList = payload.data || (payload as Record).images as Array> | undefined || (payload as Record).results as Array> | undefined || []; const images = imageList .map(resolveImageDataUrl) .filter((value): value is string => Boolean(value)) .map((dataUrl) => ({ id: nanoid(), dataUrl })); if (images.length === 0) { // 尝试检查是否有返回了但格式不被识别的数据 const rawKeys = Object.keys(payload).filter((k) => k !== "code" && k !== "msg" && k !== "error"); throw new Error(rawKeys.length > 0 ? `接口返回了未知格式的数据(字段:${rawKeys.join("、")}),请检查模型或接口兼容性` : "接口没有返回图片,请检查提示词是否触发安全审核或模型是否支持该操作"); } return images; } function readApiErrorMessage(value: unknown): string { if (!value) return ""; if (typeof value === "string") { // 可能是 JSON 字符串(如 error.message 被序列化)或纯文本错误 try { const parsed = JSON.parse(value); const inner = readApiErrorMessage(parsed) || value; // 如果 JSON 解析后得到 "{}" 这种空对象,返回原始字符串 if (inner === value && typeof parsed === "object" && Object.keys(parsed).length === 0) return ""; return inner; } catch { // 检查是否是 HTML 错误页面 if (/<[a-z][\s\S]*>/i.test(value)) return `服务返回了 HTML 错误页面(${value.slice(0, 80)}...)`; return value; } } if (typeof value !== "object") return ""; const payload = value as { msg?: unknown; message?: unknown; error?: unknown; detail?: unknown }; // error 可能是字符串或含 message 的对象 const errorMsg = typeof payload.error === "string" ? payload.error : (payload.error as { message?: unknown })?.message; return ( readApiErrorMessage(payload.msg) || readApiErrorMessage(payload.message) || readApiErrorMessage(errorMsg) || readApiErrorMessage(payload.detail) || "" ); } function readAxiosError(error: unknown, fallback: string) { if (axios.isCancel(error)) return "请求已取消"; if (axios.isAxiosError(error)) { const responseData = error.response?.data; // 优先从响应体提取业务错误 const apiMsg = readApiErrorMessage(responseData); if (apiMsg) return apiMsg; // 响应体无法提取时用 HTTP 状态推断 const statusMsg = readStatusError(error.response?.status, fallback); if (statusMsg) return statusMsg; // 最后用 axios 自身的错误文本 return error.message || fallback; } if (error instanceof DOMException && error.name === "AbortError") return "请求已取消"; return error instanceof Error ? readApiErrorMessage(error.message) || error.message : fallback; } function readStatusError(status: number | undefined, fallback: string) { if (status === 401 || status === 403) return "鉴权失败,请检查 API Key、套餐权限或模型权限"; if (status === 429) return "请求被限流或额度不足,请稍后重试"; if (status === 404) return "接口地址不存在(404),请检查 Base URL 和模型选择"; if (status === 502) return "网关错误(502),接口服务暂时不可用,请稍后重试"; if (status === 503) return "服务繁忙(503),请稍后重试"; return status ? `请求失败(HTTP ${status}),请检查 Base URL 和 API Key 是否正确` : fallback; } function withSystemPrompt(config: AiConfig, prompt: string) { const systemPrompt = config.systemPrompt.trim(); return systemPrompt ? `${systemPrompt}\n\n${prompt}` : prompt; } function aiApiUrl(config: AiConfig, path: string) { return buildApiUrl(config.baseUrl, path); } function aiHeaders(config: AiConfig, contentType?: string) { return { Authorization: `Bearer ${config.apiKey}`, ...(contentType ? { "Content-Type": contentType } : {}), }; } function geminiBaseUrl(config: Pick) { const normalizedBaseUrl = config.baseUrl.trim().replace(/\/+$/, ""); const lowerBaseUrl = normalizedBaseUrl.toLowerCase(); return lowerBaseUrl.endsWith("/v1") || lowerBaseUrl.endsWith("/v1beta") ? normalizedBaseUrl : `${normalizedBaseUrl}/v1beta`; } function geminiModelName(model: string) { return model.trim().replace(/^models\//, ""); } function geminiApiUrl(config: Pick, action?: "generateContent" | "streamGenerateContent") { const baseUrl = geminiBaseUrl(config); if (!action) return `${baseUrl}/models`; return `${baseUrl}/models/${encodeURIComponent(geminiModelName(config.model))}:${action}`; } function geminiHeaders(config: Pick) { return { "x-goog-api-key": config.apiKey, "Content-Type": "application/json", }; } function withSystemMessage(config: AiConfig, messages: T[]): ResponseInputMessage[] { const systemPrompt = config.systemPrompt.trim(); return systemPrompt ? [{ role: "system" as const, content: systemPrompt }, ...messages] : messages; } function toResponseInput(messages: ResponseInputMessage[]): ResponseInputItem[] { return messages.flatMap((message): ResponseInputItem[] => { if ("type" in message) return [message]; if (message.role === "tool") return [{ type: "function_call_output", call_id: message.tool_call_id, output: message.content }]; return [{ role: message.role, content: toResponseContent(message.content || "") }]; }); } function toResponseContent(content: ResponseMessageContent): string | ResponseInputContent[] { if (!Array.isArray(content)) return String(content || ""); return content.map((item) => (item.type === "text" ? { type: "input_text" as const, text: item.text } : { type: "input_image" as const, image_url: item.image_url.url })); } function toResponseTool(tool: ResponseFunctionTool): ResponseApiToolDefinition { return { type: "function", name: tool.function.name, description: tool.function.description, parameters: tool.function.parameters, strict: tool.function.strict, }; } function parseToolResponse(payload: ResponseApiPayload): ToolResponseResult { const output = payload.output || []; const content = payload.output_text || output .flatMap((item) => (item.type === "message" ? item.content || [] : [])) .map((item) => item.text || "") .join(""); const toolCalls = output .filter((item): item is Extract => item.type === "function_call") .map((item) => ({ id: item.call_id || item.id || "", type: "function" as const, function: { name: item.name || "", arguments: item.arguments || "{}" }, })) .filter((item) => item.id && item.function.name); return { content, toolCalls }; } function isRecord(value: unknown): value is Record { return Boolean(value && typeof value === "object" && !Array.isArray(value)); } function responseErrorMessage(value: unknown) { if (!isRecord(value)) return ""; const error = isRecord(value.error) ? value.error : undefined; const response = isRecord(value.response) ? value.response : undefined; const responseError = response && isRecord(response.error) ? response.error : undefined; return stringValue(value.msg) || stringValue(error?.message) || stringValue(responseError?.message); } function stringValue(value: unknown) { return typeof value === "string" ? value : ""; } function validateResponsePayload(payload: ResponseApiPayload) { if (typeof payload.code === "number" && payload.code !== 0) throw new Error(payload.msg || "请求失败"); if (payload.error?.message) throw new Error(payload.error.message); } function validateGeminiPayload(payload: GeminiPayload) { if (payload.error?.message) throw new Error(payload.error.message); if (payload.promptFeedback?.blockReason) throw new Error(`Gemini 拒绝了本次请求:${payload.promptFeedback.blockReason}`); } async function readFetchError(response: Response, fallback: string) { const text = await response.text(); if (!text) return readStatusError(response.status, fallback); try { return responseErrorMessage(JSON.parse(text)) || readStatusError(response.status, fallback); } catch { return text.slice(0, 300) || readStatusError(response.status, fallback); } } function consumeResponseStreamBlock(block: string, state: ResponseStreamState, onDelta?: (text: string) => void) { const data = block .split(/\r?\n/) .filter((line) => line.startsWith("data:")) .map((line) => line.slice(5).replace(/^ /, "")) .join("\n") .trim(); if (!data || data === "[DONE]") return; const event = JSON.parse(data) as Record; const type = stringValue(event.type); const errorMessage = responseErrorMessage(event); if (errorMessage) state.error = errorMessage; if (type === "response.output_text.delta" && typeof event.delta === "string") { state.text += event.delta; onDelta?.(state.text); } if (type === "response.output_text.done" && !state.text && typeof event.text === "string") { state.text = event.text; onDelta?.(state.text); } if (type === "response.completed" && isRecord(event.response)) { state.payload = event.response as ResponseApiPayload; } else if (Array.isArray(event.output)) { state.payload = event as ResponseApiPayload; } } function consumeResponseStreamText(state: ResponseStreamState, text: string, onDelta?: (text: string) => void, flush = false) { state.buffer += text; for (;;) { const match = state.buffer.match(/\r?\n\r?\n/); if (!match) break; const index = match.index ?? 0; consumeResponseStreamBlock(state.buffer.slice(0, index), state, onDelta); state.buffer = state.buffer.slice(index + match[0].length); } if (flush && state.buffer.trim()) { consumeResponseStreamBlock(state.buffer, state, onDelta); state.buffer = ""; } } async function requestStreamingResponse(config: AiConfig, body: Record, onDelta?: (text: string) => void, options?: RequestOptions): Promise { const response = await fetch(aiApiUrl(config, "/responses"), { method: "POST", headers: { ...aiHeaders(config, "application/json"), Accept: "text/event-stream" }, body: JSON.stringify({ ...body, stream: true }), signal: options?.signal, }); if (!response.ok) throw new Error(await readFetchError(response, "请求失败")); if (!response.body) { const payload = (await response.json()) as ResponseApiPayload; validateResponsePayload(payload); return parseToolResponse(payload); } const reader = response.body.getReader(); const decoder = new TextDecoder(); const state: ResponseStreamState = { buffer: "", text: "" }; for (;;) { const { done, value } = await reader.read(); if (done) break; consumeResponseStreamText(state, decoder.decode(value, { stream: true }), onDelta); if (state.error) throw new Error(state.error); } consumeResponseStreamText(state, decoder.decode(), onDelta, true); if (state.error) throw new Error(state.error); if (!state.payload) return { content: state.text, toolCalls: [] }; validateResponsePayload(state.payload); const result = parseToolResponse(state.payload); return { ...result, content: state.text || result.content }; } function toGeminiBody(config: AiConfig, messages: ResponseInputMessage[], extra?: Record) { const systemText = [ config.systemPrompt.trim(), ...messages.flatMap((message) => (!("type" in message) && message.role === "system" ? [geminiTextContent(message.content)] : [])), ] .filter(Boolean) .join("\n\n"); const contents = toGeminiContents(messages.filter((message) => ("type" in message ? true : message.role !== "system"))); return { contents, ...(systemText ? { systemInstruction: { parts: [{ text: systemText }] } } : {}), ...extra, }; } function toGeminiContents(messages: ResponseInputMessage[]): GeminiContent[] { const callNameById = new Map(); return messages.flatMap((message): GeminiContent[] => { if ("type" in message) { callNameById.set(message.call_id, message.name); return [{ role: "model", parts: [{ functionCall: { id: message.call_id, name: message.name, args: jsonObject(message.arguments) }, ...(message.thoughtSignature ? { thoughtSignature: message.thoughtSignature } : {}) }] }]; } if (message.role === "tool") { const name = callNameById.get(message.tool_call_id) || "tool_result"; return [{ role: "user", parts: [{ functionResponse: { id: message.tool_call_id, name, response: { result: jsonValue(message.content) } } }] }]; } return [{ role: message.role === "assistant" ? "model" : "user", parts: toGeminiParts(message.content) }]; }); } function toGeminiParts(content: ResponseMessageContent): GeminiPart[] { if (!Array.isArray(content)) return [{ text: String(content || "") }]; return content.map((item) => (item.type === "text" ? { text: item.text } : toGeminiImagePart(item.image_url.url))); } function toGeminiImagePart(url: string): GeminiPart { const match = url.match(/^data:([^;,]+);base64,(.+)$/); if (match) return { inlineData: { mimeType: match[1], data: match[2] } }; return { fileData: { fileUri: url, mimeType: "image/png" } }; } function geminiTextContent(content: ResponseMessageContent) { if (!Array.isArray(content)) return String(content || ""); return content.map((item) => (item.type === "text" ? item.text : item.image_url.url)).join("\n"); } function jsonObject(value: string): Record { const parsed = jsonValue(value); return isRecord(parsed) ? parsed : {}; } function jsonValue(value: string): unknown { try { return JSON.parse(value); } catch { return value; } } function toGeminiToolOptions(tools: ResponseFunctionTool[], toolChoice: ToolChoice) { if (!tools.length) return {}; const functionDeclarations = tools.map((tool) => ({ name: tool.function.name, description: tool.function.description, parameters: tool.function.parameters, })); const functionCallingConfig = typeof toolChoice === "object" ? { mode: "ANY", allowedFunctionNames: [toolChoice.name] } : { mode: toolChoice === "required" ? "ANY" : "AUTO" }; return { tools: [{ functionDeclarations }], toolConfig: { functionCallingConfig }, }; } async function requestGeminiStreamingResponse(config: AiConfig, body: Record, onDelta?: (text: string) => void, options?: RequestOptions): Promise { const response = await fetch(`${geminiApiUrl(config, "streamGenerateContent")}?alt=sse`, { method: "POST", headers: geminiHeaders(config), body: JSON.stringify(body), signal: options?.signal, }); if (!response.ok) throw new Error(await readFetchError(response, "请求失败")); if (!response.body) { const payload = (await response.json()) as GeminiPayload; return parseGeminiToolResponse(payload); } const reader = response.body.getReader(); const decoder = new TextDecoder(); const state: GeminiStreamState = { buffer: "", text: "", toolCalls: [] }; for (;;) { const { done, value } = await reader.read(); if (done) break; consumeGeminiStreamText(state, decoder.decode(value, { stream: true }), onDelta); if (state.error) throw new Error(state.error); } consumeGeminiStreamText(state, decoder.decode(), onDelta, true); if (state.error) throw new Error(state.error); return { content: state.text, toolCalls: state.toolCalls }; } function consumeGeminiStreamText(state: GeminiStreamState, text: string, onDelta?: (text: string) => void, flush = false) { state.buffer += text; for (;;) { const match = state.buffer.match(/\r?\n\r?\n/); if (!match) break; const index = match.index ?? 0; consumeGeminiStreamBlock(state.buffer.slice(0, index), state, onDelta); state.buffer = state.buffer.slice(index + match[0].length); } if (flush && state.buffer.trim()) { consumeGeminiStreamBlock(state.buffer, state, onDelta); state.buffer = ""; } } function consumeGeminiStreamBlock(block: string, state: GeminiStreamState, onDelta?: (text: string) => void) { const data = block .split(/\r?\n/) .filter((line) => line.startsWith("data:")) .map((line) => line.slice(5).replace(/^ /, "")) .join("\n") .trim(); if (!data || data === "[DONE]") return; const result = parseGeminiToolResponse(JSON.parse(data) as GeminiPayload); if (result.content) { state.text += result.content; onDelta?.(state.text); } state.toolCalls.push(...result.toolCalls); } function parseGeminiToolResponse(payload: GeminiPayload): ToolResponseResult { validateGeminiPayload(payload); const parts = payload.candidates?.flatMap((candidate) => candidate.content?.parts || []) || []; const content = parts.map((part) => part.text || "").join(""); const toolCalls = parts .map((part) => part.functionCall) .filter((call): call is NonNullable => Boolean(call?.name)) .map((call) => { const part = parts.find((item) => item.functionCall === call); const thoughtSignature = part?.thoughtSignature || part?.thought_signature; return { id: call.id || nanoid(), type: "function" as const, function: { name: call.name || "", arguments: JSON.stringify(call.args || {}) }, ...(thoughtSignature ? { thoughtSignature } : {}), }; }); return { content, toolCalls }; } async function requestGeminiImages(config: AiConfig, prompt: string, references: ReferenceImage[], count: number, options?: RequestOptions) { const requests = Array.from({ length: count }, () => requestGeminiImagesOnce(config, prompt, references, options)); return (await Promise.all(requests)).flat(); } async function requestGeminiImagesOnce(config: AiConfig, prompt: string, references: ReferenceImage[], options?: RequestOptions) { const parts: GeminiPart[] = [{ text: prompt }]; for (const image of references) { parts.push(toGeminiImagePart(await imageToDataUrl(image))); } const response = await axios.post( geminiApiUrl(config, "generateContent"), { ...toGeminiBody(config, [{ role: "user", content: prompt }], { generationConfig: { responseModalities: ["TEXT", "IMAGE"], ...resolveGeminiImageConfig(config) } }), contents: [{ role: "user", parts }], }, { headers: geminiHeaders(config), signal: options?.signal }, ); return parseGeminiImagePayload(response.data); } function parseGeminiImagePayload(payload: GeminiPayload) { validateGeminiPayload(payload); const images = payload.candidates ?.flatMap((candidate) => candidate.content?.parts || []) .map((part) => { const inlineData = part.inlineData || (part.inline_data ? { mimeType: part.inline_data.mimeType || part.inline_data.mime_type, data: part.inline_data.data } : undefined); if (inlineData?.data) return `data:${inlineData.mimeType || "image/png"};base64,${inlineData.data}`; return part.fileData?.fileUri || null; }) .filter((value): value is string => Boolean(value)) .map((dataUrl) => ({ id: nanoid(), dataUrl })) || []; if (!images.length) throw new Error("Gemini 接口没有返回图片"); return images; } export async function requestGeneration(config: AiConfig, prompt: string, options?: RequestOptions) { const requestConfig = resolveModelRequestConfig(config, config.model || config.imageModel); const n = Math.max(1, Math.min(15, Math.floor(Math.abs(Number(config.count)) || 1))); const script = resolveModelScript(config, config.model || config.imageModel); if (script) { const quality = normalizeQuality(config.quality); const requestSize = resolveRequestSize(quality, config.size); const background = normalizeBackground(config.background); try { const result = await runModelPlugin({ capability: "image", script, config: requestConfig, prompt: withSystemPrompt(requestConfig, prompt), images: [], params: { size: requestSize, quality, count: n, ...(background ? { background } : {}) }, signal: options?.signal, }); return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl })); } catch (error) { throw new Error(readAxiosError(error, "请求失败")); } } if (requestConfig.apiFormat === "gemini") { try { return await requestGeminiImages(requestConfig, prompt, [], n, options); } catch (error) { throw new Error(readAxiosError(error, "请求失败")); } } const quality = normalizeQuality(config.quality); const requestSize = resolveRequestSize(quality, config.size); const background = normalizeBackground(config.background); try { const response = await axios.post( aiApiUrl(requestConfig, "/images/generations"), { model: requestConfig.model, prompt: withSystemPrompt(requestConfig, prompt), n, ...(quality ? { quality } : {}), ...(requestSize ? { size: requestSize } : {}), ...(background ? { background } : {}), response_format: "b64_json", output_format: IMAGE_OUTPUT_FORMAT, }, { headers: aiHeaders(requestConfig, "application/json"), signal: options?.signal, }, ); const images = parseImagePayload(response.data); return images; } catch (error) { throw new Error(readAxiosError(error, "请求失败")); } } /** Seedream 系列模型和火山方舟 Ark 接口:图生图使用 /images/generations JSON 接口传 base64 参考图 */ function isSeedreamRequestEdit(model: string, baseUrl: string) { const modelLower = model.toLowerCase(); const baseUrlLower = baseUrl.toLowerCase(); return modelLower.includes("seedream") || modelLower.includes("doubao-seedream") || isArkApiUrl(baseUrlLower); } function isArkApiUrl(baseUrlLower: string) { return baseUrlLower.includes("ark.cn-beijing.volces.com"); } export async function requestEdit(config: AiConfig, prompt: string, references: ReferenceImage[], mask?: ReferenceImage, options?: RequestOptions) { const requestConfig = resolveModelRequestConfig(config, config.model || config.imageModel); const n = Math.max(1, Math.min(15, Math.floor(Math.abs(Number(config.count)) || 1))); const requestPrompt = buildImageReferencePromptText(prompt, references); const script = resolveModelScript(config, config.model || config.imageModel); if (script) { const quality = normalizeQuality(config.quality); const requestSize = resolveRequestSize(quality, config.size); const background = normalizeBackground(config.background); const refs = await Promise.all(references.map((image) => imageToDataUrl(image))); try { const result = await runModelPlugin({ capability: "image", script, config: requestConfig, prompt: withSystemPrompt(requestConfig, requestPrompt), images: refs, params: { size: requestSize, quality, count: n, ...(background ? { background } : {}) }, signal: options?.signal, }); return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl })); } catch (error) { throw new Error(readAxiosError(error, "请求失败")); } } if (requestConfig.apiFormat === "gemini") { if (mask) throw new Error("Gemini 调用格式暂不支持蒙版编辑"); try { return await requestGeminiImages(requestConfig, requestPrompt, references, n, options); } catch (error) { throw new Error(readAxiosError(error, "请求失败")); } } // Seedream / 火山方舟 Ark 图生图:使用 /images/generations JSON 接口 // 不支持 OpenAI 的 /images/edits multipart 接口 if (isSeedreamRequestEdit(requestConfig.model, requestConfig.baseUrl)) { if (mask) throw new Error("蒙版编辑暂不支持该模型,请使用其他渠道"); const quality = normalizeQuality(config.quality); const requestSize = resolveRequestSize(quality, config.size); const background = normalizeBackground(config.background); const refs = await Promise.all(references.map(async (image) => { const dataUrl = await imageToDataUrl(image); // 火山方舟 image 参数需要完整的 data URL 格式 return dataUrl; })); try { const response = await axios.post( aiApiUrl(requestConfig, "/images/generations"), { model: requestConfig.model, prompt: withSystemPrompt(requestConfig, requestPrompt), n, response_format: "b64_json", output_format: IMAGE_OUTPUT_FORMAT, image: refs, // Seedream 图生图:base64 参考图数组 ...(quality ? { quality } : {}), ...(requestSize ? { size: requestSize } : {}), ...(background ? { background } : {}), }, { headers: aiHeaders(requestConfig, "application/json"), signal: options?.signal, }, ); return parseImagePayload(response.data); } catch (error) { throw new Error(readAxiosError(error, "请求失败")); } } // 标准 OpenAI 图生图:/images/edits multipart/form-data const quality = normalizeQuality(config.quality); const requestSize = resolveRequestSize(quality, config.size); const background = normalizeBackground(config.background); const formData = new FormData(); formData.set("model", requestConfig.model); formData.set("prompt", withSystemPrompt(requestConfig, requestPrompt)); formData.set("n", String(n)); formData.set("response_format", "b64_json"); formData.set("output_format", IMAGE_OUTPUT_FORMAT); if (quality) { formData.set("quality", quality); } if (requestSize) { formData.set("size", requestSize); } if (background) { formData.set("background", background); } const files = await Promise.all(references.map(async (image) => dataUrlToFile({ ...image, dataUrl: await imageToDataUrl(image) }))); files.forEach((file) => formData.append("image", file)); if (mask) formData.set("mask", dataUrlToFile(mask)); try { const response = await axios.post(aiApiUrl(requestConfig, "/images/edits"), formData, { headers: aiHeaders(requestConfig), signal: options?.signal }); const images = parseImagePayload(response.data); return images; } catch (error) { throw new Error(readAxiosError(error, "请求失败")); } } export async function requestImageQuestion(config: AiConfig, messages: AiTextMessage[], onDelta: (text: string) => void, options?: RequestOptions) { const requestConfig = resolveModelRequestConfig(config, config.model || config.textModel); const script = resolveModelScript(config, config.model || config.textModel); if (script) { try { const answer = await runModelPlugin({ capability: "text", script, config: requestConfig, messages: withSystemMessage(requestConfig, messages), signal: options?.signal, onDelta, }); const text = String(answer ?? "").trim() || "没有返回内容"; if (text === "没有返回内容") onDelta(text); return text; } catch (error) { throw new Error(readAxiosError(error, "请求失败")); } } try { if (requestConfig.apiFormat === "gemini") { const answer = (await requestGeminiStreamingResponse(requestConfig, toGeminiBody(requestConfig, messages), onDelta, options)).content || "没有返回内容"; if (answer === "没有返回内容") onDelta(answer); return answer; } const answer = (await requestStreamingResponse(requestConfig, { model: requestConfig.model, input: toResponseInput(withSystemMessage(requestConfig, messages)), }, onDelta, options)).content || "没有返回内容"; if (answer === "没有返回内容") onDelta(answer); return answer; } catch (error) { throw new Error(readAxiosError(error, "请求失败")); } } export async function fetchImageModels(config: Pick) { try { if (config.apiFormat === "gemini") { const response = await axios.get(geminiApiUrl({ ...defaultGeminiConfig, ...config }), { headers: geminiHeaders({ ...defaultGeminiConfig, ...config }) }); validateGeminiPayload(response.data); return (response.data.models || []) .map((model) => model.name?.replace(/^models\//, "")) .filter((id): id is string => Boolean(id)) .sort((a, b) => a.localeCompare(b)); } const response = await axios.get<{ data?: Array<{ id?: string }>; error?: { message?: string } }>(buildApiUrl(config.baseUrl, "/models"), { headers: { Authorization: `Bearer ${config.apiKey}`, }, }); return (response.data.data || []) .map((model) => model.id) .filter((id): id is string => Boolean(id)) .sort((a, b) => a.localeCompare(b)); } catch (error) { throw new Error(readAxiosError(error, "读取模型失败")); } } export async function fetchChannelModels(channel: ModelChannel) { return fetchImageModels({ baseUrl: channel.baseUrl, apiKey: channel.apiKey, apiFormat: channel.apiFormat }); } const defaultGeminiConfig: Pick = { baseUrl: "https://generativelanguage.googleapis.com", apiKey: "", apiFormat: "gemini", model: "", systemPrompt: "", };