#[cfg(test)] use crate::model_capabilities::is_confirmed_text_only_model as confirmed_text_only_model; use crate::model_capabilities::{image_input_capability_from_settings, ImageInputCapability}; use crate::provider::Provider; use crate::proxy::error::ProxyError; use serde_json::{json, Value}; pub const UNSUPPORTED_IMAGE_MARKER: &str = "[Unsupported Image]"; /// Replace image blocks before sending when the routed model is text-only. /// /// Two paths, both reached only when the caller's media-fallback switch is on: /// - explicit capability from the provider config (modelCatalog / modalities) is /// always trusted — it is declaration-driven, never a guess; /// - the confirmed text-only registry is used for proactive replacement only /// when `allow_heuristic` is true. This switch controls silent request-body /// mutation, not the capability truth advertised by the Codex model catalog. pub fn replace_images_for_text_only_model( body: &mut Value, provider: &Provider, allow_heuristic: bool, ) -> usize { if !contains_image_blocks(body) { return 0; } let model = body .get("model") .and_then(Value::as_str) .map(str::trim) .unwrap_or(""); if image_input_capability_from_settings(&provider.settings_config, model, allow_heuristic) != ImageInputCapability::Unsupported { return 0; } replace_images_in_body(body) } pub fn contains_image_blocks(body: &Value) -> bool { messages_have_image_blocks(body) || responses_input_has_image_blocks(body.get("input")) } pub fn replace_image_blocks_with_marker(body: &mut Value) -> usize { replace_images_in_body(body) } pub fn is_unsupported_image_error(error: &ProxyError) -> bool { let ProxyError::UpstreamError { status, body } = error else { return false; }; if !matches!(*status, 400 | 415 | 422 | 501) { return false; } let Some(body) = body.as_deref() else { return false; }; let message = extract_error_text(body); let message = message.to_ascii_lowercase(); // 自证性表述:这类短语本身就断言了"仅接受文本",属于模态拒绝,无需再要求 // 错误提到 image/media 等字样——火山方舟等网关的报错是 // "Model only support text input",全程不出现 image(issue #5025)。 // 国产网关的英文常缺三单 s,因此带 s / 不带 s 两种形式都要列。 const TEXT_ONLY_SELF_EVIDENT_HINTS: &[&str] = &["only support text", "only supports text"]; if TEXT_ONLY_SELF_EVIDENT_HINTS .iter() .any(|hint| message.contains(hint)) { return true; } let mentions_image = message.contains("image") || message.contains("vision") || message.contains("multimodal") || message.contains("multi-modal") || message.contains("modality") || message.contains("modalities") || message.contains("media") || message.contains("attachment"); if !mentions_image { return false; } const UNSUPPORTED_HINTS: &[&str] = &[ "unsupported", "not supported", "does not support", "doesn't support", "do not support", "don't support", "text only", "text-only", "invalid content type", "invalid message content", "unknown variant", "unknown content type", "unrecognized content type", "cannot process", "cannot handle", "can't process", "can't handle", "unable to process", ]; UNSUPPORTED_HINTS.iter().any(|hint| message.contains(hint)) } fn content_has_image_blocks(content: &Value) -> bool { let Some(blocks) = content.as_array() else { return false; }; blocks.iter().any(|block| { is_image_block_type(block.get("type").and_then(Value::as_str)) || block.get("content").is_some_and(content_has_image_blocks) }) } fn replace_images_in_body(body: &mut Value) -> usize { let message_replacements = body .get_mut("messages") .and_then(Value::as_array_mut) .map(|messages| { messages .iter_mut() .filter_map(|message| message.get_mut("content")) .map(replace_images_in_content) .sum() }) .unwrap_or(0); message_replacements + body .get_mut("input") .map(replace_images_in_responses_input) .unwrap_or(0) } fn replace_images_in_content(content: &mut Value) -> usize { replace_images_in_content_with_text_type(content, "text") } fn replace_images_in_content_with_text_type(content: &mut Value, text_type: &str) -> usize { let Some(blocks) = content.as_array_mut() else { return 0; }; let mut replaced = 0usize; for block in blocks { if is_image_block_type(block.get("type").and_then(Value::as_str)) { replace_image_block_with_text_marker(block, text_type); replaced += 1; continue; } if let Some(nested_content) = block.get_mut("content") { replaced += replace_images_in_content_with_text_type(nested_content, text_type); } } replaced } fn messages_have_image_blocks(body: &Value) -> bool { body.get("messages") .and_then(Value::as_array) .is_some_and(|messages| { messages .iter() .filter_map(|message| message.get("content")) .any(content_has_image_blocks) }) } fn responses_input_has_image_blocks(input: Option<&Value>) -> bool { match input { Some(Value::Array(items)) => items.iter().any(responses_input_item_has_image_blocks), Some(item @ Value::Object(_)) => responses_input_item_has_image_blocks(item), _ => false, } } fn responses_input_item_has_image_blocks(item: &Value) -> bool { if item.get("type").and_then(Value::as_str) == Some("input_image") { return true; } item.get("content").is_some_and(content_has_image_blocks) } fn replace_images_in_responses_input(input: &mut Value) -> usize { match input { Value::Array(items) => items .iter_mut() .map(replace_images_in_responses_input_item) .sum(), Value::Object(_) => replace_images_in_responses_input_item(input), _ => 0, } } fn replace_images_in_responses_input_item(item: &mut Value) -> usize { let mut replaced = 0usize; if item.get("type").and_then(Value::as_str) == Some("input_image") { replace_image_block_with_text_marker(item, "input_text"); replaced += 1; } if let Some(content) = item.get_mut("content") { replaced += replace_images_in_content_with_text_type(content, "input_text"); } replaced } fn is_image_block_type(block_type: Option<&str>) -> bool { matches!(block_type, Some("image" | "image_url" | "input_image")) } fn replace_image_block_with_text_marker(block: &mut Value, text_type: &str) { let cache_control = block.get("cache_control").cloned(); *block = json!({ "type": text_type, "text": UNSUPPORTED_IMAGE_MARKER }); if let (Some(cache_control), Some(object)) = (cache_control, block.as_object_mut()) { object.insert("cache_control".to_string(), cache_control); } } fn extract_error_text(body: &str) -> String { if let Ok(value) = serde_json::from_str::(body) { let candidates = [ value.pointer("/error/message"), value.pointer("/message"), value.pointer("/detail"), value.pointer("/error"), ]; if let Some(message) = candidates .into_iter() .flatten() .find_map(|value| value.as_str()) { return message.to_string(); } if let Ok(compact) = serde_json::to_string(&value) { return compact; } } body.to_string() } #[cfg(test)] mod tests { use super::*; use crate::provider::Provider; use serde_json::json; fn provider(settings_config: Value) -> Provider { Provider { id: "test".to_string(), name: "Test".to_string(), settings_config, website_url: None, category: None, created_at: None, sort_index: None, notes: None, meta: None, icon: None, icon_color: None, in_failover_queue: false, } } #[test] fn keeps_images_when_model_capability_is_unknown() { let provider = provider(json!({})); let mut body = json!({ "model": "unknown-model", "messages": [{ "role": "user", "content": [ { "type": "text", "text": "look" }, { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }); let count = replace_images_for_text_only_model(&mut body, &provider, true); assert_eq!(count, 0); assert_eq!(body["messages"][0]["content"][1]["type"], "image"); } #[test] fn confirmed_text_only_models_replace_images_before_send() { let provider = provider(json!({})); let mut body = json!({ "model": "deepseek/deepseek-v4-pro", "messages": [{ "role": "user", "content": [ { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }); let count = replace_images_for_text_only_model(&mut body, &provider, true); assert_eq!(count, 1); assert_eq!( body["messages"][0]["content"][0]["text"], UNSUPPORTED_IMAGE_MARKER ); } #[test] fn confirmed_text_only_models_replace_chat_image_url_before_send() { let provider = provider(json!({})); let mut body = json!({ "model": "deepseek-v4-flash", "messages": [{ "role": "user", "content": [ { "type": "text", "text": "look" }, { "type": "image_url", "image_url": { "url": "data:image/png;base64,abc" } } ] }] }); let count = replace_images_for_text_only_model(&mut body, &provider, true); assert_eq!(count, 1); assert_eq!(body["messages"][0]["content"][1]["type"], "text"); assert_eq!( body["messages"][0]["content"][1]["text"], UNSUPPORTED_IMAGE_MARKER ); } #[test] fn confirmed_text_only_models_replace_codex_input_image_before_send() { let provider = provider(json!({})); let mut body = json!({ "model": "deepseek-v4-flash", "input": [{ "role": "user", "content": [ { "type": "input_text", "text": "look" }, { "type": "input_image", "image_url": "data:image/png;base64,abc" } ] }] }); let count = replace_images_for_text_only_model(&mut body, &provider, true); assert_eq!(count, 1); assert_eq!(body["input"][0]["content"][1]["type"], "input_text"); assert_eq!( body["input"][0]["content"][1]["text"], UNSUPPORTED_IMAGE_MARKER ); } #[test] fn longcat_models_are_classified_text_only() { // LongCat-2.0 (like the retired Flash Chat) is a text-only model; the // preset ships it in mixed case, so the classifier must normalize first. assert!(confirmed_text_only_model("LongCat-2.0")); assert!(confirmed_text_only_model("longcat/LongCat-2.0")); assert!(confirmed_text_only_model("LongCat-Flash-Chat")); } #[test] fn explicit_text_modalities_replace_images_before_send() { let provider = provider(json!({ "models": [ { "id": "deepseek-v4-pro", "input": ["text"] } ] })); let mut body = json!({ "model": "deepseek-v4-pro", "messages": [{ "role": "user", "content": [ { "type": "text", "text": "look" }, { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }); let count = replace_images_for_text_only_model(&mut body, &provider, true); assert_eq!(count, 1); assert_eq!(body["messages"][0]["content"][0]["text"], "look"); assert_eq!(body["messages"][0]["content"][1]["type"], "text"); assert_eq!( body["messages"][0]["content"][1]["text"], UNSUPPORTED_IMAGE_MARKER ); } #[test] fn preserves_images_without_explicit_capability_even_for_unknown_models() { let provider = provider(json!({})); let mut body = json!({ "model": "unknown-model", "messages": [{ "role": "user", "content": [ { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }); let count = replace_images_for_text_only_model(&mut body, &provider, true); assert_eq!(count, 0); assert_eq!(body["messages"][0]["content"][0]["type"], "image"); } #[test] fn explicit_text_modalities_can_override_visual_model_ids() { let provider = provider(json!({ "models": [ { "id": "gpt-4o", "input": ["text"] } ] })); let mut body = json!({ "model": "gpt-4o", "messages": [{ "role": "user", "content": [ { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }); let count = replace_images_for_text_only_model(&mut body, &provider, true); assert_eq!(count, 1); assert_eq!( body["messages"][0]["content"][0]["text"], UNSUPPORTED_IMAGE_MARKER ); } #[test] fn explicit_image_modalities_preserve_model_images() { let provider = provider(json!({ "modelCatalog": { "models": [ { "model": "deepseek-v4-pro", "modalities": { "input": ["text", "image"] } } ] } })); let mut body = json!({ "model": "deepseek-v4-pro", "messages": [{ "role": "user", "content": [ { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }); let count = replace_images_for_text_only_model(&mut body, &provider, true); assert_eq!(count, 0); assert_eq!(body["messages"][0]["content"][0]["type"], "image"); } #[test] fn known_mimo_pro_replaces_but_mimo_multimodal_preserves() { let provider = provider(json!({})); let mut pro_body = json!({ "model": "xiaomi-mimo-token-plan/mimo-v2.5-pro", "messages": [{ "role": "user", "content": [ { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }); let mut multimodal_body = json!({ "model": "xiaomi-mimo-token-plan/mimo-v2.5", "messages": [{ "role": "user", "content": [ { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }); let pro_count = replace_images_for_text_only_model(&mut pro_body, &provider, true); let multimodal_count = replace_images_for_text_only_model(&mut multimodal_body, &provider, true); assert_eq!(pro_count, 1); assert_eq!(multimodal_count, 0); assert_eq!( multimodal_body["messages"][0]["content"][0]["type"], "image" ); } #[test] fn multimodal_kimi_model_is_not_on_text_only_list() { let provider = provider(json!({})); let mut body = json!({ "model": "kimi/kimi-k2.6", "messages": [{ "role": "user", "content": [ { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }); let count = replace_images_for_text_only_model(&mut body, &provider, true); assert_eq!(count, 0); assert_eq!(body["messages"][0]["content"][0]["type"], "image"); } #[test] fn confirmed_text_only_variant_replaces_images_before_send() { let provider = provider(json!({})); let mut body = json!({ "model": "therouter/qwen/qwen3-coder-480b", "messages": [{ "role": "user", "content": [ { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }); let count = replace_images_for_text_only_model(&mut body, &provider, true); assert_eq!(count, 1); assert_eq!( body["messages"][0]["content"][0]["text"], UNSUPPORTED_IMAGE_MARKER ); } #[test] fn unconditional_marker_replacement_handles_retry_path() { let mut body = json!({ "model": "xiaomi-mimo-token-plan/mimo-v2.5-pro", "messages": [{ "role": "user", "content": [ { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }); assert!(contains_image_blocks(&body)); let count = replace_image_blocks_with_marker(&mut body); assert_eq!(count, 1); assert_eq!( body["messages"][0]["content"][0]["text"], UNSUPPORTED_IMAGE_MARKER ); } #[test] fn replaces_nested_tool_result_image_blocks() { let mut body = json!({ "model": "deepseek-v4-pro", "messages": [{ "role": "user", "content": [{ "type": "tool_result", "tool_use_id": "toolu_1", "content": [ { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }] }); let count = replace_image_blocks_with_marker(&mut body); assert_eq!(count, 1); assert_eq!( body["messages"][0]["content"][0]["content"][0]["text"], UNSUPPORTED_IMAGE_MARKER ); } #[test] fn detects_unsupported_image_errors() { let error = ProxyError::UpstreamError { status: 400, body: Some( r#"{"error":{"message":"This model does not support image input"}}"#.to_string(), ), }; assert!(is_unsupported_image_error(&error)); } #[test] fn detects_text_only_errors_without_image_mention() { // 火山方舟真实报错(issue #5025):不含 image/media 等字样,且英文缺 // 三单 s——旧逻辑的 mentions_image 门与 "only supports text" 提示都拦不住。 let error = ProxyError::UpstreamError { status: 400, body: Some( r#"{"error":{"message":"Model only support text input Request id: 021783"}}"# .to_string(), ), }; assert!(is_unsupported_image_error(&error)); } #[test] fn glm_52_is_classified_text_only() { // issue #5025:火山 Coding Plan 的 GLM 5.2 是纯文本端点, // 映射链 glm-5.2[1M] 归一化后尾部为 glm-5.2。 assert!(confirmed_text_only_model("glm-5.2")); assert!(confirmed_text_only_model("GLM-5.2[1M]")); assert!(confirmed_text_only_model("zai-org/GLM-5.2")); // 未来视觉版(智谱 4v/5v 命名惯例)不能被误判为纯文本。 assert!(!confirmed_text_only_model("glm-5.2v")); } #[test] fn ignores_non_image_errors() { let error = ProxyError::UpstreamError { status: 400, body: Some(r#"{"error":{"message":"Invalid API key"}}"#.to_string()), }; assert!(!is_unsupported_image_error(&error)); } #[test] fn preserves_cache_control_when_replacing_image() { // image block 可能承载 prompt cache 断点;替换成标记时必须把 // cache_control 迁移到新的 text block,否则会断掉缓存命中。 let mut body = json!({ "model": "deepseek-v4-pro", "messages": [{ "role": "user", "content": [{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" }, "cache_control": { "type": "ephemeral" } }] }] }); let count = replace_image_blocks_with_marker(&mut body); assert_eq!(count, 1); let block = &body["messages"][0]["content"][0]; assert_eq!(block["type"], "text"); assert_eq!(block["text"], UNSUPPORTED_IMAGE_MARKER); assert_eq!(block["cache_control"]["type"], "ephemeral"); } #[test] fn detects_media_and_attachment_error_phrasings() { let media_error = ProxyError::UpstreamError { status: 400, body: Some( r#"{"error":{"message":"This model cannot process media inputs"}}"#.to_string(), ), }; assert!(is_unsupported_image_error(&media_error)); let attachment_error = ProxyError::UpstreamError { status: 422, body: Some(r#"{"message":"attachments are not supported by this model"}"#.to_string()), }; assert!(is_unsupported_image_error(&attachment_error)); } #[test] fn detects_chat_content_unknown_variant_image_url_errors() { let error = ProxyError::UpstreamError { status: 400, body: Some( r#"{"error":{"message":"Failed to deserialize the JSON body into the target type: messages[11]: unknown variant image_url, expected text"}}"# .to_string(), ), }; assert!(is_unsupported_image_error(&error)); } #[test] fn heuristic_disabled_keeps_images_for_listed_text_only_models() { // allow_heuristic = false:内置列表不再预测性剥图,避免误判多模态模型时静默丢图。 let provider = provider(json!({})); let mut body = json!({ "model": "deepseek/deepseek-v4-pro", "messages": [{ "role": "user", "content": [ { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }); let count = replace_images_for_text_only_model(&mut body, &provider, false); assert_eq!(count, 0); assert_eq!(body["messages"][0]["content"][0]["type"], "image"); } #[test] fn explicit_text_capability_replaces_even_when_heuristic_disabled() { // 显式声明 text-only 是声明驱动、零误判,即使关掉启发式也应生效。 let provider = provider(json!({ "models": [ { "id": "deepseek-v4-pro", "input": ["text"] } ] })); let mut body = json!({ "model": "deepseek-v4-pro", "messages": [{ "role": "user", "content": [ { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } ] }] }); let count = replace_images_for_text_only_model(&mut body, &provider, false); assert_eq!(count, 1); assert_eq!( body["messages"][0]["content"][0]["text"], UNSUPPORTED_IMAGE_MARKER ); } }