mirror of
https://github.com/basketikun/infinite-canvas.git
synced 2026-07-27 18:14:31 +08:00
yuki
957 lines
42 KiB
TypeScript
957 lines
42 KiB
TypeScript
import axios from "axios";
|
||
|
||
import { buildApiUrl, resolveModelRequestConfig, resolveModelScript, type AiConfig, type ModelChannel } from "@/stores/use-config-store";
|
||
import { normalizePluginImages, runModelPlugin } from "./model-plugin";
|
||
import { nanoid } from "nanoid";
|
||
import { dataUrlToFile } from "@/lib/image-utils";
|
||
import { buildImageReferencePromptText } from "@/lib/image-reference-prompt";
|
||
import { imageToDataUrl } from "@/services/image-storage";
|
||
import type { ReferenceImage } from "@/types/image";
|
||
|
||
export type AiTextMessage = {
|
||
role: "system" | "user" | "assistant";
|
||
content: string | Array<{ type: "text"; text: string } | { type: "image_url"; image_url: { url: string } }>;
|
||
};
|
||
|
||
type ResponseToolCall = {
|
||
id: string;
|
||
type: "function";
|
||
function: { name: string; arguments: string };
|
||
thoughtSignature?: string;
|
||
};
|
||
|
||
type ResponseInputMessage =
|
||
| AiTextMessage
|
||
| { type: "function_call"; call_id: string; name: string; arguments: string; thoughtSignature?: string }
|
||
| { role: "tool"; tool_call_id: string; content: string };
|
||
|
||
type ResponseFunctionTool = {
|
||
type: "function";
|
||
function: {
|
||
name: string;
|
||
description?: string;
|
||
parameters: Record<string, unknown>;
|
||
strict?: boolean;
|
||
};
|
||
};
|
||
|
||
type ToolResponseResult = {
|
||
content: string;
|
||
toolCalls: ResponseToolCall[];
|
||
};
|
||
|
||
type ToolChoice = "auto" | "required" | { type: "function"; name: string };
|
||
type ResponseMessageContent = AiTextMessage["content"] | string;
|
||
type ResponseInputContent = { type: "input_text"; text: string } | { type: "input_image"; image_url: string };
|
||
type ResponseInputItem =
|
||
| { role: "system" | "user" | "assistant"; content: string | ResponseInputContent[] }
|
||
| { type: "function_call"; call_id: string; name: string; arguments: string }
|
||
| { type: "function_call_output"; call_id: string; output: string };
|
||
type ResponseApiToolDefinition = {
|
||
type: "function";
|
||
name: string;
|
||
description?: string;
|
||
parameters: Record<string, unknown>;
|
||
strict?: boolean;
|
||
};
|
||
type ResponseApiOutputItem =
|
||
| { type?: "message"; content?: Array<{ type?: string; text?: string }> }
|
||
| { type?: "function_call"; id?: string; call_id?: string; name?: string; arguments?: string };
|
||
type ResponseApiPayload = {
|
||
id?: string;
|
||
output?: ResponseApiOutputItem[];
|
||
output_text?: string;
|
||
error?: { message?: string };
|
||
code?: number;
|
||
msg?: string;
|
||
};
|
||
type ResponseStreamState = { buffer: string; text: string; payload?: ResponseApiPayload; error?: string };
|
||
|
||
type ImageApiResponse = {
|
||
data?: Array<Record<string, unknown>>;
|
||
error?: { message?: string };
|
||
code?: number;
|
||
msg?: string;
|
||
};
|
||
type GeminiPart = {
|
||
text?: string;
|
||
inlineData?: { mimeType?: string; data?: string };
|
||
inline_data?: { mime_type?: string; mimeType?: string; data?: string };
|
||
fileData?: { mimeType?: string; fileUri?: string };
|
||
functionCall?: { id?: string; name?: string; args?: Record<string, unknown> };
|
||
functionResponse?: { id?: string; name?: string; response?: Record<string, unknown> };
|
||
thoughtSignature?: string;
|
||
thought_signature?: string;
|
||
};
|
||
type GeminiContent = { role?: "user" | "model"; parts: GeminiPart[] };
|
||
type GeminiPayload = {
|
||
candidates?: Array<{ content?: { parts?: GeminiPart[] }; finishReason?: string }>;
|
||
models?: Array<{ name?: string }>;
|
||
error?: { message?: string };
|
||
promptFeedback?: { blockReason?: string };
|
||
};
|
||
type GeminiStreamState = { buffer: string; text: string; toolCalls: ResponseToolCall[]; error?: string };
|
||
type RequestOptions = { signal?: AbortSignal };
|
||
|
||
const QUALITY_BASE: Record<string, number> = {
|
||
low: 1024,
|
||
medium: 2048,
|
||
high: 2880,
|
||
standard: 1024,
|
||
hd: 2048,
|
||
};
|
||
const QUALITY_ALIASES: Record<string, string> = {
|
||
"1k": "low",
|
||
"2k": "medium",
|
||
"4k": "high",
|
||
};
|
||
const DEFAULT_IMAGE_SHORT_SIDE = 1024;
|
||
const IMAGE_SIZE_STEP = 16;
|
||
const IMAGE_MIN_PIXELS = 655360;
|
||
const IMAGE_MAX_PIXELS = 8294400;
|
||
const IMAGE_MAX_EDGE = 3840;
|
||
const IMAGE_MAX_RATIO = 3;
|
||
const IMAGE_OUTPUT_FORMAT = "png";
|
||
|
||
const GEMINI_SUPPORTED_RATIOS = ["1:1", "1:4", "1:8", "2:3", "3:2", "3:4", "4:1", "4:3", "4:5", "5:4", "8:1", "9:16", "16:9", "21:9"];
|
||
const GEMINI_IMAGE_SIZE_BY_QUALITY: Record<string, string> = { low: "1K", medium: "2K", high: "4K", standard: "1K", hd: "2K" };
|
||
|
||
function normalizeQuality(quality: string) {
|
||
const value = quality.trim().toLowerCase();
|
||
const normalized = QUALITY_ALIASES[value] || value;
|
||
return QUALITY_BASE[normalized] ? normalized : undefined;
|
||
}
|
||
|
||
/** Only "transparent" is forwarded; any other value (incl. empty) means keep the default opaque background. */
|
||
function normalizeBackground(background: string | undefined) {
|
||
return background?.trim().toLowerCase() === "transparent" ? "transparent" : undefined;
|
||
}
|
||
|
||
/** Map "quality + ratio" to an explicit pixel dimension like "3840x2160". */
|
||
function resolveSize(quality: string | undefined, ratio: string): string {
|
||
const parsedRatio = parseImageRatio(ratio);
|
||
const basePixels = quality ? QUALITY_BASE[quality] : undefined;
|
||
const isLandscape = parsedRatio.width >= parsedRatio.height;
|
||
const longRatio = isLandscape ? parsedRatio.width / parsedRatio.height : parsedRatio.height / parsedRatio.width;
|
||
let longSide: number;
|
||
let shortSide: number;
|
||
|
||
if (basePixels) {
|
||
const targetPixels = basePixels * basePixels;
|
||
const longSideRaw = Math.sqrt(targetPixels * longRatio);
|
||
longSide = Math.floor(longSideRaw / IMAGE_SIZE_STEP) * IMAGE_SIZE_STEP;
|
||
shortSide = Math.round(longSide / longRatio / IMAGE_SIZE_STEP) * IMAGE_SIZE_STEP;
|
||
} else {
|
||
shortSide = DEFAULT_IMAGE_SHORT_SIDE;
|
||
longSide = Math.round((shortSide * longRatio) / IMAGE_SIZE_STEP) * IMAGE_SIZE_STEP;
|
||
}
|
||
|
||
const width = isLandscape ? longSide : shortSide;
|
||
const height = isLandscape ? shortSide : longSide;
|
||
validateImageSize(width, height);
|
||
return `${width}x${height}`;
|
||
}
|
||
|
||
function parseRatioValue(value: string) {
|
||
const parts = value.split(":");
|
||
if (parts.length !== 2) throw new Error("图像尺寸格式不支持，请使用 auto、9:16 或 1024x1024");
|
||
const w = Number(parts[0]);
|
||
const h = Number(parts[1]);
|
||
if (!Number.isFinite(w) || !Number.isFinite(h) || w <= 0 || h <= 0) throw new Error("图像比例必须是正数，例如 9:16");
|
||
return { width: w, height: h };
|
||
}
|
||
|
||
function parseImageRatio(value: string) {
|
||
const ratio = parseRatioValue(value);
|
||
if (Math.max(ratio.width, ratio.height) / Math.min(ratio.width, ratio.height) > IMAGE_MAX_RATIO) throw new Error("图像宽高比不能超过 3:1，请调整尺寸");
|
||
return ratio;
|
||
}
|
||
|
||
function parseImageDimensions(value: string) {
|
||
const match = value.match(/^(\d+)x(\d+)$/i);
|
||
if (!match) return null;
|
||
return { width: Number(match[1]), height: Number(match[2]) };
|
||
}
|
||
|
||
function validateImageSize(width: number, height: number) {
|
||
if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) throw new Error("图像尺寸必须是正整数，例如 1024x1024");
|
||
if (width % IMAGE_SIZE_STEP !== 0 || height % IMAGE_SIZE_STEP !== 0) throw new Error("图像尺寸的宽高必须是 16 的倍数，请调整尺寸");
|
||
if (Math.max(width, height) > IMAGE_MAX_EDGE) throw new Error("图像尺寸最长边不能超过 3840px，请调整尺寸");
|
||
if (Math.max(width, height) / Math.min(width, height) > IMAGE_MAX_RATIO) throw new Error("图像宽高比不能超过 3:1，请调整尺寸");
|
||
const pixels = width * height;
|
||
if (pixels < IMAGE_MIN_PIXELS || pixels > IMAGE_MAX_PIXELS) throw new Error("图像总像素需在 655360 到 8294400 之间，请调整尺寸");
|
||
}
|
||
|
||
function resolveRequestSize(quality: string | undefined, size: string) {
|
||
const value = size.trim();
|
||
if (!value || value.toLowerCase() === "auto") return undefined;
|
||
const dimensions = parseImageDimensions(value);
|
||
if (dimensions) {
|
||
validateImageSize(dimensions.width, dimensions.height);
|
||
return `${dimensions.width}x${dimensions.height}`;
|
||
}
|
||
if (value.includes(":")) return resolveSize(quality, value);
|
||
throw new Error("图像尺寸格式不支持，请使用 auto、9:16 或 1024x1024");
|
||
}
|
||
|
||
function resolveGeminiImageConfig(config: AiConfig) {
|
||
const value = config.size.trim();
|
||
const dimensions = parseImageDimensions(value);
|
||
const ratio = dimensions ? `${dimensions.width}:${dimensions.height}` : value;
|
||
const aspectRatio = value && value.toLowerCase() !== "auto" ? closestGeminiAspectRatio(ratio) : undefined;
|
||
const imageSize = supportsGeminiImageSize(config.model) ? resolveGeminiImageSize(config.quality, dimensions) : undefined;
|
||
const image = { ...(aspectRatio ? { aspectRatio } : {}), ...(imageSize ? { imageSize } : {}) };
|
||
return Object.keys(image).length ? { responseFormat: { image } } : {};
|
||
}
|
||
|
||
function closestGeminiAspectRatio(value: string) {
|
||
const ratio = parseImageRatio(value);
|
||
const target = ratio.width / ratio.height;
|
||
return GEMINI_SUPPORTED_RATIOS.reduce((best, item) => {
|
||
const current = parseRatioValue(item);
|
||
const bestRatio = parseRatioValue(best);
|
||
return Math.abs(current.width / current.height - target) < Math.abs(bestRatio.width / bestRatio.height - target) ? item : best;
|
||
});
|
||
}
|
||
|
||
function resolveGeminiImageSize(quality: string, dimensions: { width: number; height: number } | null) {
|
||
const normalizedQuality = normalizeQuality(quality);
|
||
if (normalizedQuality) return GEMINI_IMAGE_SIZE_BY_QUALITY[normalizedQuality];
|
||
if (!dimensions) return undefined;
|
||
const edge = Math.max(dimensions.width, dimensions.height);
|
||
if (edge <= 768) return "512";
|
||
if (edge <= 1536) return "1K";
|
||
if (edge <= 3072) return "2K";
|
||
return "4K";
|
||
}
|
||
|
||
function supportsGeminiImageSize(model: string) {
|
||
const value = model.toLowerCase();
|
||
return value.includes("gemini-3") || value.includes("3.1") || value.includes("3-pro");
|
||
}
|
||
|
||
function resolveImageDataUrl(item: Record<string, unknown>) {
|
||
if (typeof item.b64_json === "string" && item.b64_json) {
|
||
return `data:image/png;base64,${item.b64_json}`;
|
||
}
|
||
if (typeof item.url === "string" && item.url) {
|
||
return item.url;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function parseImagePayload(payload: ImageApiResponse) {
|
||
if (typeof payload.code === "number" && payload.code !== 0) {
|
||
throw new Error(payload.msg || "请求失败");
|
||
}
|
||
// 支持 data / images / results 三种返回字段（兼容不同 API）
|
||
const imageList = payload.data
|
||
|| (payload as Record<string, unknown>).images as Array<Record<string, unknown>> | undefined
|
||
|| (payload as Record<string, unknown>).results as Array<Record<string, unknown>> | undefined
|
||
|| [];
|
||
const images =
|
||
imageList
|
||
.map(resolveImageDataUrl)
|
||
.filter((value): value is string => Boolean(value))
|
||
.map((dataUrl) => ({ id: nanoid(), dataUrl }));
|
||
|
||
if (images.length === 0) {
|
||
// 尝试检查是否有返回了但格式不被识别的数据
|
||
const rawKeys = Object.keys(payload).filter((k) => k !== "code" && k !== "msg" && k !== "error");
|
||
throw new Error(rawKeys.length > 0
|
||
? `接口返回了未知格式的数据（字段：${rawKeys.join("、")}），请检查模型或接口兼容性`
|
||
: "接口没有返回图片，请检查提示词是否触发安全审核或模型是否支持该操作");
|
||
}
|
||
|
||
return images;
|
||
}
|
||
|
||
function readApiErrorMessage(value: unknown): string {
|
||
if (!value) return "";
|
||
if (typeof value === "string") {
|
||
// 可能是 JSON 字符串（如 error.message 被序列化）或纯文本错误
|
||
try {
|
||
const parsed = JSON.parse(value);
|
||
const inner = readApiErrorMessage(parsed) || value;
|
||
// 如果 JSON 解析后得到 "{}" 这种空对象，返回原始字符串
|
||
if (inner === value && typeof parsed === "object" && Object.keys(parsed).length === 0) return "";
|
||
return inner;
|
||
} catch {
|
||
// 检查是否是 HTML 错误页面
|
||
if (/<[a-z][\s\S]*>/i.test(value)) return `服务返回了 HTML 错误页面（${value.slice(0, 80)}...）`;
|
||
return value;
|
||
}
|
||
}
|
||
if (typeof value !== "object") return "";
|
||
const payload = value as { msg?: unknown; message?: unknown; error?: unknown; detail?: unknown };
|
||
// error 可能是字符串或含 message 的对象
|
||
const errorMsg =
|
||
typeof payload.error === "string"
|
||
? payload.error
|
||
: (payload.error as { message?: unknown })?.message;
|
||
return (
|
||
readApiErrorMessage(payload.msg) ||
|
||
readApiErrorMessage(payload.message) ||
|
||
readApiErrorMessage(errorMsg) ||
|
||
readApiErrorMessage(payload.detail) ||
|
||
""
|
||
);
|
||
}
|
||
|
||
function readAxiosError(error: unknown, fallback: string) {
|
||
if (axios.isCancel(error)) return "请求已取消";
|
||
if (axios.isAxiosError(error)) {
|
||
const responseData = error.response?.data;
|
||
// 优先从响应体提取业务错误
|
||
const apiMsg = readApiErrorMessage(responseData);
|
||
if (apiMsg) return apiMsg;
|
||
// 响应体无法提取时用 HTTP 状态推断
|
||
const statusMsg = readStatusError(error.response?.status, fallback);
|
||
if (statusMsg) return statusMsg;
|
||
// 最后用 axios 自身的错误文本
|
||
return error.message || fallback;
|
||
}
|
||
if (error instanceof DOMException && error.name === "AbortError") return "请求已取消";
|
||
return error instanceof Error ? readApiErrorMessage(error.message) || error.message : fallback;
|
||
}
|
||
|
||
function readStatusError(status: number | undefined, fallback: string) {
|
||
if (status === 401 || status === 403) return "鉴权失败，请检查 API Key、套餐权限或模型权限";
|
||
if (status === 429) return "请求被限流或额度不足，请稍后重试";
|
||
if (status === 404) return "接口地址不存在（404），请检查 Base URL 和模型选择";
|
||
if (status === 502) return "网关错误（502），接口服务暂时不可用，请稍后重试";
|
||
if (status === 503) return "服务繁忙（503），请稍后重试";
|
||
return status ? `请求失败（HTTP ${status}），请检查 Base URL 和 API Key 是否正确` : fallback;
|
||
}
|
||
|
||
function withSystemPrompt(config: AiConfig, prompt: string) {
|
||
const systemPrompt = config.systemPrompt.trim();
|
||
return systemPrompt ? `${systemPrompt}\n\n${prompt}` : prompt;
|
||
}
|
||
|
||
function aiApiUrl(config: AiConfig, path: string) {
|
||
return buildApiUrl(config.baseUrl, path);
|
||
}
|
||
|
||
function aiHeaders(config: AiConfig, contentType?: string) {
|
||
return {
|
||
Authorization: `Bearer ${config.apiKey}`,
|
||
...(contentType ? { "Content-Type": contentType } : {}),
|
||
};
|
||
}
|
||
|
||
function geminiBaseUrl(config: Pick<AiConfig, "baseUrl">) {
|
||
const normalizedBaseUrl = config.baseUrl.trim().replace(/\/+$/, "");
|
||
const lowerBaseUrl = normalizedBaseUrl.toLowerCase();
|
||
return lowerBaseUrl.endsWith("/v1") || lowerBaseUrl.endsWith("/v1beta") ? normalizedBaseUrl : `${normalizedBaseUrl}/v1beta`;
|
||
}
|
||
|
||
function geminiModelName(model: string) {
|
||
return model.trim().replace(/^models\//, "");
|
||
}
|
||
|
||
function geminiApiUrl(config: Pick<AiConfig, "baseUrl" | "model">, action?: "generateContent" | "streamGenerateContent") {
|
||
const baseUrl = geminiBaseUrl(config);
|
||
if (!action) return `${baseUrl}/models`;
|
||
return `${baseUrl}/models/${encodeURIComponent(geminiModelName(config.model))}:${action}`;
|
||
}
|
||
|
||
function geminiHeaders(config: Pick<AiConfig, "apiKey">) {
|
||
return {
|
||
"x-goog-api-key": config.apiKey,
|
||
"Content-Type": "application/json",
|
||
};
|
||
}
|
||
|
||
function withSystemMessage<T extends ResponseInputMessage>(config: AiConfig, messages: T[]): ResponseInputMessage[] {
|
||
const systemPrompt = config.systemPrompt.trim();
|
||
return systemPrompt ? [{ role: "system" as const, content: systemPrompt }, ...messages] : messages;
|
||
}
|
||
|
||
function toResponseInput(messages: ResponseInputMessage[]): ResponseInputItem[] {
|
||
return messages.flatMap((message): ResponseInputItem[] => {
|
||
if ("type" in message) return [message];
|
||
if (message.role === "tool") return [{ type: "function_call_output", call_id: message.tool_call_id, output: message.content }];
|
||
return [{ role: message.role, content: toResponseContent(message.content || "") }];
|
||
});
|
||
}
|
||
|
||
function toResponseContent(content: ResponseMessageContent): string | ResponseInputContent[] {
|
||
if (!Array.isArray(content)) return String(content || "");
|
||
return content.map((item) => (item.type === "text" ? { type: "input_text" as const, text: item.text } : { type: "input_image" as const, image_url: item.image_url.url }));
|
||
}
|
||
|
||
function toResponseTool(tool: ResponseFunctionTool): ResponseApiToolDefinition {
|
||
return {
|
||
type: "function",
|
||
name: tool.function.name,
|
||
description: tool.function.description,
|
||
parameters: tool.function.parameters,
|
||
strict: tool.function.strict,
|
||
};
|
||
}
|
||
|
||
function parseToolResponse(payload: ResponseApiPayload): ToolResponseResult {
|
||
const output = payload.output || [];
|
||
const content =
|
||
payload.output_text ||
|
||
output
|
||
.flatMap((item) => (item.type === "message" ? item.content || [] : []))
|
||
.map((item) => item.text || "")
|
||
.join("");
|
||
const toolCalls = output
|
||
.filter((item): item is Extract<ResponseApiOutputItem, { type?: "function_call" }> => item.type === "function_call")
|
||
.map((item) => ({
|
||
id: item.call_id || item.id || "",
|
||
type: "function" as const,
|
||
function: { name: item.name || "", arguments: item.arguments || "{}" },
|
||
}))
|
||
.filter((item) => item.id && item.function.name);
|
||
return { content, toolCalls };
|
||
}
|
||
|
||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||
}
|
||
|
||
function responseErrorMessage(value: unknown) {
|
||
if (!isRecord(value)) return "";
|
||
const error = isRecord(value.error) ? value.error : undefined;
|
||
const response = isRecord(value.response) ? value.response : undefined;
|
||
const responseError = response && isRecord(response.error) ? response.error : undefined;
|
||
return stringValue(value.msg) || stringValue(error?.message) || stringValue(responseError?.message);
|
||
}
|
||
|
||
function stringValue(value: unknown) {
|
||
return typeof value === "string" ? value : "";
|
||
}
|
||
|
||
function validateResponsePayload(payload: ResponseApiPayload) {
|
||
if (typeof payload.code === "number" && payload.code !== 0) throw new Error(payload.msg || "请求失败");
|
||
if (payload.error?.message) throw new Error(payload.error.message);
|
||
}
|
||
|
||
function validateGeminiPayload(payload: GeminiPayload) {
|
||
if (payload.error?.message) throw new Error(payload.error.message);
|
||
if (payload.promptFeedback?.blockReason) throw new Error(`Gemini 拒绝了本次请求：${payload.promptFeedback.blockReason}`);
|
||
}
|
||
|
||
async function readFetchError(response: Response, fallback: string) {
|
||
const text = await response.text();
|
||
if (!text) return readStatusError(response.status, fallback);
|
||
try {
|
||
return responseErrorMessage(JSON.parse(text)) || readStatusError(response.status, fallback);
|
||
} catch {
|
||
return text.slice(0, 300) || readStatusError(response.status, fallback);
|
||
}
|
||
}
|
||
|
||
function consumeResponseStreamBlock(block: string, state: ResponseStreamState, onDelta?: (text: string) => void) {
|
||
const data = block
|
||
.split(/\r?\n/)
|
||
.filter((line) => line.startsWith("data:"))
|
||
.map((line) => line.slice(5).replace(/^ /, ""))
|
||
.join("\n")
|
||
.trim();
|
||
if (!data || data === "[DONE]") return;
|
||
const event = JSON.parse(data) as Record<string, unknown>;
|
||
const type = stringValue(event.type);
|
||
const errorMessage = responseErrorMessage(event);
|
||
if (errorMessage) state.error = errorMessage;
|
||
if (type === "response.output_text.delta" && typeof event.delta === "string") {
|
||
state.text += event.delta;
|
||
onDelta?.(state.text);
|
||
}
|
||
if (type === "response.output_text.done" && !state.text && typeof event.text === "string") {
|
||
state.text = event.text;
|
||
onDelta?.(state.text);
|
||
}
|
||
if (type === "response.completed" && isRecord(event.response)) {
|
||
state.payload = event.response as ResponseApiPayload;
|
||
} else if (Array.isArray(event.output)) {
|
||
state.payload = event as ResponseApiPayload;
|
||
}
|
||
}
|
||
|
||
function consumeResponseStreamText(state: ResponseStreamState, text: string, onDelta?: (text: string) => void, flush = false) {
|
||
state.buffer += text;
|
||
for (;;) {
|
||
const match = state.buffer.match(/\r?\n\r?\n/);
|
||
if (!match) break;
|
||
const index = match.index ?? 0;
|
||
consumeResponseStreamBlock(state.buffer.slice(0, index), state, onDelta);
|
||
state.buffer = state.buffer.slice(index + match[0].length);
|
||
}
|
||
if (flush && state.buffer.trim()) {
|
||
consumeResponseStreamBlock(state.buffer, state, onDelta);
|
||
state.buffer = "";
|
||
}
|
||
}
|
||
|
||
async function requestStreamingResponse(config: AiConfig, body: Record<string, unknown>, onDelta?: (text: string) => void, options?: RequestOptions): Promise<ToolResponseResult> {
|
||
const response = await fetch(aiApiUrl(config, "/responses"), {
|
||
method: "POST",
|
||
headers: { ...aiHeaders(config, "application/json"), Accept: "text/event-stream" },
|
||
body: JSON.stringify({ ...body, stream: true }),
|
||
signal: options?.signal,
|
||
});
|
||
if (!response.ok) throw new Error(await readFetchError(response, "请求失败"));
|
||
if (!response.body) {
|
||
const payload = (await response.json()) as ResponseApiPayload;
|
||
validateResponsePayload(payload);
|
||
return parseToolResponse(payload);
|
||
}
|
||
|
||
const reader = response.body.getReader();
|
||
const decoder = new TextDecoder();
|
||
const state: ResponseStreamState = { buffer: "", text: "" };
|
||
for (;;) {
|
||
const { done, value } = await reader.read();
|
||
if (done) break;
|
||
consumeResponseStreamText(state, decoder.decode(value, { stream: true }), onDelta);
|
||
if (state.error) throw new Error(state.error);
|
||
}
|
||
consumeResponseStreamText(state, decoder.decode(), onDelta, true);
|
||
if (state.error) throw new Error(state.error);
|
||
if (!state.payload) return { content: state.text, toolCalls: [] };
|
||
validateResponsePayload(state.payload);
|
||
const result = parseToolResponse(state.payload);
|
||
return { ...result, content: state.text || result.content };
|
||
}
|
||
|
||
function toGeminiBody(config: AiConfig, messages: ResponseInputMessage[], extra?: Record<string, unknown>) {
|
||
const systemText = [
|
||
config.systemPrompt.trim(),
|
||
...messages.flatMap((message) => (!("type" in message) && message.role === "system" ? [geminiTextContent(message.content)] : [])),
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n");
|
||
const contents = toGeminiContents(messages.filter((message) => ("type" in message ? true : message.role !== "system")));
|
||
return {
|
||
contents,
|
||
...(systemText ? { systemInstruction: { parts: [{ text: systemText }] } } : {}),
|
||
...extra,
|
||
};
|
||
}
|
||
|
||
function toGeminiContents(messages: ResponseInputMessage[]): GeminiContent[] {
|
||
const callNameById = new Map<string, string>();
|
||
return messages.flatMap((message): GeminiContent[] => {
|
||
if ("type" in message) {
|
||
callNameById.set(message.call_id, message.name);
|
||
return [{ role: "model", parts: [{ functionCall: { id: message.call_id, name: message.name, args: jsonObject(message.arguments) }, ...(message.thoughtSignature ? { thoughtSignature: message.thoughtSignature } : {}) }] }];
|
||
}
|
||
if (message.role === "tool") {
|
||
const name = callNameById.get(message.tool_call_id) || "tool_result";
|
||
return [{ role: "user", parts: [{ functionResponse: { id: message.tool_call_id, name, response: { result: jsonValue(message.content) } } }] }];
|
||
}
|
||
return [{ role: message.role === "assistant" ? "model" : "user", parts: toGeminiParts(message.content) }];
|
||
});
|
||
}
|
||
|
||
function toGeminiParts(content: ResponseMessageContent): GeminiPart[] {
|
||
if (!Array.isArray(content)) return [{ text: String(content || "") }];
|
||
return content.map((item) => (item.type === "text" ? { text: item.text } : toGeminiImagePart(item.image_url.url)));
|
||
}
|
||
|
||
function toGeminiImagePart(url: string): GeminiPart {
|
||
const match = url.match(/^data:([^;,]+);base64,(.+)$/);
|
||
if (match) return { inlineData: { mimeType: match[1], data: match[2] } };
|
||
return { fileData: { fileUri: url, mimeType: "image/png" } };
|
||
}
|
||
|
||
function geminiTextContent(content: ResponseMessageContent) {
|
||
if (!Array.isArray(content)) return String(content || "");
|
||
return content.map((item) => (item.type === "text" ? item.text : item.image_url.url)).join("\n");
|
||
}
|
||
|
||
function jsonObject(value: string): Record<string, unknown> {
|
||
const parsed = jsonValue(value);
|
||
return isRecord(parsed) ? parsed : {};
|
||
}
|
||
|
||
function jsonValue(value: string): unknown {
|
||
try {
|
||
return JSON.parse(value);
|
||
} catch {
|
||
return value;
|
||
}
|
||
}
|
||
|
||
function toGeminiToolOptions(tools: ResponseFunctionTool[], toolChoice: ToolChoice) {
|
||
if (!tools.length) return {};
|
||
const functionDeclarations = tools.map((tool) => ({
|
||
name: tool.function.name,
|
||
description: tool.function.description,
|
||
parameters: tool.function.parameters,
|
||
}));
|
||
const functionCallingConfig =
|
||
typeof toolChoice === "object"
|
||
? { mode: "ANY", allowedFunctionNames: [toolChoice.name] }
|
||
: { mode: toolChoice === "required" ? "ANY" : "AUTO" };
|
||
return {
|
||
tools: [{ functionDeclarations }],
|
||
toolConfig: { functionCallingConfig },
|
||
};
|
||
}
|
||
|
||
async function requestGeminiStreamingResponse(config: AiConfig, body: Record<string, unknown>, onDelta?: (text: string) => void, options?: RequestOptions): Promise<ToolResponseResult> {
|
||
const response = await fetch(`${geminiApiUrl(config, "streamGenerateContent")}?alt=sse`, {
|
||
method: "POST",
|
||
headers: geminiHeaders(config),
|
||
body: JSON.stringify(body),
|
||
signal: options?.signal,
|
||
});
|
||
if (!response.ok) throw new Error(await readFetchError(response, "请求失败"));
|
||
if (!response.body) {
|
||
const payload = (await response.json()) as GeminiPayload;
|
||
return parseGeminiToolResponse(payload);
|
||
}
|
||
|
||
const reader = response.body.getReader();
|
||
const decoder = new TextDecoder();
|
||
const state: GeminiStreamState = { buffer: "", text: "", toolCalls: [] };
|
||
for (;;) {
|
||
const { done, value } = await reader.read();
|
||
if (done) break;
|
||
consumeGeminiStreamText(state, decoder.decode(value, { stream: true }), onDelta);
|
||
if (state.error) throw new Error(state.error);
|
||
}
|
||
consumeGeminiStreamText(state, decoder.decode(), onDelta, true);
|
||
if (state.error) throw new Error(state.error);
|
||
return { content: state.text, toolCalls: state.toolCalls };
|
||
}
|
||
|
||
function consumeGeminiStreamText(state: GeminiStreamState, text: string, onDelta?: (text: string) => void, flush = false) {
|
||
state.buffer += text;
|
||
for (;;) {
|
||
const match = state.buffer.match(/\r?\n\r?\n/);
|
||
if (!match) break;
|
||
const index = match.index ?? 0;
|
||
consumeGeminiStreamBlock(state.buffer.slice(0, index), state, onDelta);
|
||
state.buffer = state.buffer.slice(index + match[0].length);
|
||
}
|
||
if (flush && state.buffer.trim()) {
|
||
consumeGeminiStreamBlock(state.buffer, state, onDelta);
|
||
state.buffer = "";
|
||
}
|
||
}
|
||
|
||
function consumeGeminiStreamBlock(block: string, state: GeminiStreamState, onDelta?: (text: string) => void) {
|
||
const data = block
|
||
.split(/\r?\n/)
|
||
.filter((line) => line.startsWith("data:"))
|
||
.map((line) => line.slice(5).replace(/^ /, ""))
|
||
.join("\n")
|
||
.trim();
|
||
if (!data || data === "[DONE]") return;
|
||
const result = parseGeminiToolResponse(JSON.parse(data) as GeminiPayload);
|
||
if (result.content) {
|
||
state.text += result.content;
|
||
onDelta?.(state.text);
|
||
}
|
||
state.toolCalls.push(...result.toolCalls);
|
||
}
|
||
|
||
function parseGeminiToolResponse(payload: GeminiPayload): ToolResponseResult {
|
||
validateGeminiPayload(payload);
|
||
const parts = payload.candidates?.flatMap((candidate) => candidate.content?.parts || []) || [];
|
||
const content = parts.map((part) => part.text || "").join("");
|
||
const toolCalls = parts
|
||
.map((part) => part.functionCall)
|
||
.filter((call): call is NonNullable<GeminiPart["functionCall"]> => Boolean(call?.name))
|
||
.map((call) => {
|
||
const part = parts.find((item) => item.functionCall === call);
|
||
const thoughtSignature = part?.thoughtSignature || part?.thought_signature;
|
||
return {
|
||
id: call.id || nanoid(),
|
||
type: "function" as const,
|
||
function: { name: call.name || "", arguments: JSON.stringify(call.args || {}) },
|
||
...(thoughtSignature ? { thoughtSignature } : {}),
|
||
};
|
||
});
|
||
return { content, toolCalls };
|
||
}
|
||
|
||
async function requestGeminiImages(config: AiConfig, prompt: string, references: ReferenceImage[], count: number, options?: RequestOptions) {
|
||
const requests = Array.from({ length: count }, () => requestGeminiImagesOnce(config, prompt, references, options));
|
||
return (await Promise.all(requests)).flat();
|
||
}
|
||
|
||
async function requestGeminiImagesOnce(config: AiConfig, prompt: string, references: ReferenceImage[], options?: RequestOptions) {
|
||
const parts: GeminiPart[] = [{ text: prompt }];
|
||
for (const image of references) {
|
||
parts.push(toGeminiImagePart(await imageToDataUrl(image)));
|
||
}
|
||
const response = await axios.post<GeminiPayload>(
|
||
geminiApiUrl(config, "generateContent"),
|
||
{
|
||
...toGeminiBody(config, [{ role: "user", content: prompt }], { generationConfig: { responseModalities: ["TEXT", "IMAGE"], ...resolveGeminiImageConfig(config) } }),
|
||
contents: [{ role: "user", parts }],
|
||
},
|
||
{ headers: geminiHeaders(config), signal: options?.signal },
|
||
);
|
||
return parseGeminiImagePayload(response.data);
|
||
}
|
||
|
||
function parseGeminiImagePayload(payload: GeminiPayload) {
|
||
validateGeminiPayload(payload);
|
||
const images =
|
||
payload.candidates
|
||
?.flatMap((candidate) => candidate.content?.parts || [])
|
||
.map((part) => {
|
||
const inlineData = part.inlineData || (part.inline_data ? { mimeType: part.inline_data.mimeType || part.inline_data.mime_type, data: part.inline_data.data } : undefined);
|
||
if (inlineData?.data) return `data:${inlineData.mimeType || "image/png"};base64,${inlineData.data}`;
|
||
return part.fileData?.fileUri || null;
|
||
})
|
||
.filter((value): value is string => Boolean(value))
|
||
.map((dataUrl) => ({ id: nanoid(), dataUrl })) || [];
|
||
if (!images.length) throw new Error("Gemini 接口没有返回图片");
|
||
return images;
|
||
}
|
||
|
||
export async function requestGeneration(config: AiConfig, prompt: string, options?: RequestOptions) {
|
||
const requestConfig = resolveModelRequestConfig(config, config.model || config.imageModel);
|
||
const n = Math.max(1, Math.min(15, Math.floor(Math.abs(Number(config.count)) || 1)));
|
||
const script = resolveModelScript(config, config.model || config.imageModel);
|
||
if (script) {
|
||
const quality = normalizeQuality(config.quality);
|
||
const requestSize = resolveRequestSize(quality, config.size);
|
||
const background = normalizeBackground(config.background);
|
||
try {
|
||
const result = await runModelPlugin({
|
||
capability: "image",
|
||
script,
|
||
config: requestConfig,
|
||
prompt: withSystemPrompt(requestConfig, prompt),
|
||
images: [],
|
||
params: { size: requestSize, quality, count: n, ...(background ? { background } : {}) },
|
||
signal: options?.signal,
|
||
});
|
||
return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl }));
|
||
} catch (error) {
|
||
throw new Error(readAxiosError(error, "请求失败"));
|
||
}
|
||
}
|
||
if (requestConfig.apiFormat === "gemini") {
|
||
try {
|
||
return await requestGeminiImages(requestConfig, prompt, [], n, options);
|
||
} catch (error) {
|
||
throw new Error(readAxiosError(error, "请求失败"));
|
||
}
|
||
}
|
||
const quality = normalizeQuality(config.quality);
|
||
const requestSize = resolveRequestSize(quality, config.size);
|
||
const background = normalizeBackground(config.background);
|
||
try {
|
||
const response = await axios.post<ImageApiResponse>(
|
||
aiApiUrl(requestConfig, "/images/generations"),
|
||
{
|
||
model: requestConfig.model,
|
||
prompt: withSystemPrompt(requestConfig, prompt),
|
||
n,
|
||
...(quality ? { quality } : {}),
|
||
...(requestSize ? { size: requestSize } : {}),
|
||
...(background ? { background } : {}),
|
||
response_format: "b64_json",
|
||
output_format: IMAGE_OUTPUT_FORMAT,
|
||
},
|
||
{
|
||
headers: aiHeaders(requestConfig, "application/json"),
|
||
signal: options?.signal,
|
||
},
|
||
);
|
||
const images = parseImagePayload(response.data);
|
||
return images;
|
||
} catch (error) {
|
||
throw new Error(readAxiosError(error, "请求失败"));
|
||
}
|
||
}
|
||
|
||
/** Seedream 系列模型和火山方舟 Ark 接口：图生图使用 /images/generations JSON 接口传 base64 参考图 */
|
||
function isSeedreamRequestEdit(model: string, baseUrl: string) {
|
||
const modelLower = model.toLowerCase();
|
||
const baseUrlLower = baseUrl.toLowerCase();
|
||
return modelLower.includes("seedream") || modelLower.includes("doubao-seedream") || isArkApiUrl(baseUrlLower);
|
||
}
|
||
|
||
function isArkApiUrl(baseUrlLower: string) {
|
||
return baseUrlLower.includes("ark.cn-beijing.volces.com");
|
||
}
|
||
|
||
export async function requestEdit(config: AiConfig, prompt: string, references: ReferenceImage[], mask?: ReferenceImage, options?: RequestOptions) {
|
||
const requestConfig = resolveModelRequestConfig(config, config.model || config.imageModel);
|
||
const n = Math.max(1, Math.min(15, Math.floor(Math.abs(Number(config.count)) || 1)));
|
||
const requestPrompt = buildImageReferencePromptText(prompt, references);
|
||
const script = resolveModelScript(config, config.model || config.imageModel);
|
||
if (script) {
|
||
const quality = normalizeQuality(config.quality);
|
||
const requestSize = resolveRequestSize(quality, config.size);
|
||
const background = normalizeBackground(config.background);
|
||
const refs = await Promise.all(references.map((image) => imageToDataUrl(image)));
|
||
try {
|
||
const result = await runModelPlugin({
|
||
capability: "image",
|
||
script,
|
||
config: requestConfig,
|
||
prompt: withSystemPrompt(requestConfig, requestPrompt),
|
||
images: refs,
|
||
params: { size: requestSize, quality, count: n, ...(background ? { background } : {}) },
|
||
signal: options?.signal,
|
||
});
|
||
return normalizePluginImages(result).map((dataUrl) => ({ id: nanoid(), dataUrl }));
|
||
} catch (error) {
|
||
throw new Error(readAxiosError(error, "请求失败"));
|
||
}
|
||
}
|
||
if (requestConfig.apiFormat === "gemini") {
|
||
if (mask) throw new Error("Gemini 调用格式暂不支持蒙版编辑");
|
||
try {
|
||
return await requestGeminiImages(requestConfig, requestPrompt, references, n, options);
|
||
} catch (error) {
|
||
throw new Error(readAxiosError(error, "请求失败"));
|
||
}
|
||
}
|
||
|
||
// Seedream / 火山方舟 Ark 图生图：使用 /images/generations JSON 接口
|
||
// 不支持 OpenAI 的 /images/edits multipart 接口
|
||
if (isSeedreamRequestEdit(requestConfig.model, requestConfig.baseUrl)) {
|
||
if (mask) throw new Error("蒙版编辑暂不支持该模型，请使用其他渠道");
|
||
const quality = normalizeQuality(config.quality);
|
||
const requestSize = resolveRequestSize(quality, config.size);
|
||
const background = normalizeBackground(config.background);
|
||
const refs = await Promise.all(references.map(async (image) => {
|
||
const dataUrl = await imageToDataUrl(image);
|
||
// 火山方舟 image 参数需要完整的 data URL 格式
|
||
return dataUrl;
|
||
}));
|
||
try {
|
||
const response = await axios.post<ImageApiResponse>(
|
||
aiApiUrl(requestConfig, "/images/generations"),
|
||
{
|
||
model: requestConfig.model,
|
||
prompt: withSystemPrompt(requestConfig, requestPrompt),
|
||
n,
|
||
response_format: "b64_json",
|
||
output_format: IMAGE_OUTPUT_FORMAT,
|
||
image: refs, // Seedream 图生图：base64 参考图数组
|
||
...(quality ? { quality } : {}),
|
||
...(requestSize ? { size: requestSize } : {}),
|
||
...(background ? { background } : {}),
|
||
},
|
||
{
|
||
headers: aiHeaders(requestConfig, "application/json"),
|
||
signal: options?.signal,
|
||
},
|
||
);
|
||
return parseImagePayload(response.data);
|
||
} catch (error) {
|
||
throw new Error(readAxiosError(error, "请求失败"));
|
||
}
|
||
}
|
||
|
||
// 标准 OpenAI 图生图：/images/edits multipart/form-data
|
||
const quality = normalizeQuality(config.quality);
|
||
const requestSize = resolveRequestSize(quality, config.size);
|
||
const background = normalizeBackground(config.background);
|
||
const formData = new FormData();
|
||
formData.set("model", requestConfig.model);
|
||
formData.set("prompt", withSystemPrompt(requestConfig, requestPrompt));
|
||
formData.set("n", String(n));
|
||
formData.set("response_format", "b64_json");
|
||
formData.set("output_format", IMAGE_OUTPUT_FORMAT);
|
||
if (quality) {
|
||
formData.set("quality", quality);
|
||
}
|
||
if (requestSize) {
|
||
formData.set("size", requestSize);
|
||
}
|
||
if (background) {
|
||
formData.set("background", background);
|
||
}
|
||
const files = await Promise.all(references.map(async (image) => dataUrlToFile({ ...image, dataUrl: await imageToDataUrl(image) })));
|
||
files.forEach((file) => formData.append("image", file));
|
||
if (mask) formData.set("mask", dataUrlToFile(mask));
|
||
|
||
try {
|
||
const response = await axios.post<ImageApiResponse>(aiApiUrl(requestConfig, "/images/edits"), formData, { headers: aiHeaders(requestConfig), signal: options?.signal });
|
||
const images = parseImagePayload(response.data);
|
||
return images;
|
||
} catch (error) {
|
||
throw new Error(readAxiosError(error, "请求失败"));
|
||
}
|
||
}
|
||
|
||
export async function requestImageQuestion(config: AiConfig, messages: AiTextMessage[], onDelta: (text: string) => void, options?: RequestOptions) {
|
||
const requestConfig = resolveModelRequestConfig(config, config.model || config.textModel);
|
||
const script = resolveModelScript(config, config.model || config.textModel);
|
||
if (script) {
|
||
try {
|
||
const answer = await runModelPlugin<string>({
|
||
capability: "text",
|
||
script,
|
||
config: requestConfig,
|
||
messages: withSystemMessage(requestConfig, messages),
|
||
signal: options?.signal,
|
||
onDelta,
|
||
});
|
||
const text = String(answer ?? "").trim() || "没有返回内容";
|
||
if (text === "没有返回内容") onDelta(text);
|
||
return text;
|
||
} catch (error) {
|
||
throw new Error(readAxiosError(error, "请求失败"));
|
||
}
|
||
}
|
||
try {
|
||
if (requestConfig.apiFormat === "gemini") {
|
||
const answer = (await requestGeminiStreamingResponse(requestConfig, toGeminiBody(requestConfig, messages), onDelta, options)).content || "没有返回内容";
|
||
if (answer === "没有返回内容") onDelta(answer);
|
||
return answer;
|
||
}
|
||
const answer = (await requestStreamingResponse(requestConfig, {
|
||
model: requestConfig.model,
|
||
input: toResponseInput(withSystemMessage(requestConfig, messages)),
|
||
}, onDelta, options)).content || "没有返回内容";
|
||
if (answer === "没有返回内容") onDelta(answer);
|
||
return answer;
|
||
} catch (error) {
|
||
throw new Error(readAxiosError(error, "请求失败"));
|
||
}
|
||
}
|
||
|
||
export async function fetchImageModels(config: Pick<AiConfig, "baseUrl" | "apiKey" | "apiFormat">) {
|
||
try {
|
||
if (config.apiFormat === "gemini") {
|
||
const response = await axios.get<GeminiPayload>(geminiApiUrl({ ...defaultGeminiConfig, ...config }), { headers: geminiHeaders({ ...defaultGeminiConfig, ...config }) });
|
||
validateGeminiPayload(response.data);
|
||
return (response.data.models || [])
|
||
.map((model) => model.name?.replace(/^models\//, ""))
|
||
.filter((id): id is string => Boolean(id))
|
||
.sort((a, b) => a.localeCompare(b));
|
||
}
|
||
const response = await axios.get<{ data?: Array<{ id?: string }>; error?: { message?: string } }>(buildApiUrl(config.baseUrl, "/models"), {
|
||
headers: {
|
||
Authorization: `Bearer ${config.apiKey}`,
|
||
},
|
||
});
|
||
return (response.data.data || [])
|
||
.map((model) => model.id)
|
||
.filter((id): id is string => Boolean(id))
|
||
.sort((a, b) => a.localeCompare(b));
|
||
} catch (error) {
|
||
throw new Error(readAxiosError(error, "读取模型失败"));
|
||
}
|
||
}
|
||
|
||
export async function fetchChannelModels(channel: ModelChannel) {
|
||
return fetchImageModels({ baseUrl: channel.baseUrl, apiKey: channel.apiKey, apiFormat: channel.apiFormat });
|
||
}
|
||
|
||
const defaultGeminiConfig: Pick<AiConfig, "baseUrl" | "apiKey" | "apiFormat" | "model" | "systemPrompt"> = {
|
||
baseUrl: "https://generativelanguage.googleapis.com",
|
||
apiKey: "",
|
||
apiFormat: "gemini",
|
||
model: "",
|
||
systemPrompt: "",
|
||
};
|