diff --git a/src-tauri/src/proxy/forwarder.rs b/src-tauri/src/proxy/forwarder.rs index 981dc1799..13c95bdff 100644 --- a/src-tauri/src/proxy/forwarder.rs +++ b/src-tauri/src/proxy/forwarder.rs @@ -122,6 +122,51 @@ pub struct RequestForwarder { } impl RequestForwarder { + /// 预防式 media 降级：发送前对 text-only 模型把图片块替换为标记。 + /// + /// 受 `enabled && request_media_fallback` 管辖；其中"启发式模型名单预测" + /// 再受 `request_media_heuristic` 单独管辖（显式声明 text-only 始终生效）。 + /// 返回被替换的图片块数量（0 = 未触发或开关关闭）。 + fn apply_media_prevention(&self, body: &mut Value, provider: &Provider) -> usize { + if !(self.rectifier_config.enabled && self.rectifier_config.request_media_fallback) { + return 0; + } + let replaced_images = super::media_sanitizer::replace_images_for_text_only_model( + body, + provider, + self.rectifier_config.request_media_heuristic, + ); + if replaced_images > 0 { + let model = body.get("model").and_then(Value::as_str).unwrap_or(""); + log::info!( + "[Media] Replaced {replaced_images} image block(s) with {} for text-only provider={}, model={}", + super::media_sanitizer::UNSUPPORTED_IMAGE_MARKER, + provider.id, + model + ); + } + replaced_images + } + + /// 反应式 media 重试判定：上游因图片输入报错后，是否应替换图片块并对同一供应商重试一次。 + /// + /// 受 `enabled && request_media_fallback` 管辖；不涉及 `request_media_heuristic`—— + /// 这里是上游"实测"错误后的纯恢复，不是预测，故启发式开关与它无关。 + fn media_retry_should_trigger( + &self, + adapter_name: &str, + already_retried: bool, + provider_body: &Value, + error: &ProxyError, + ) -> bool { + adapter_name == "Claude" + && self.rectifier_config.enabled + && self.rectifier_config.request_media_fallback + && !already_retried + && super::media_sanitizer::contains_image_blocks(provider_body) + && super::media_sanitizer::is_unsupported_image_error(error) + } + #[allow(clippy::too_many_arguments)] pub fn new( router: Arc, @@ -346,6 +391,7 @@ impl RequestForwarder { // —— 首家 provider 整流后被 5xx/timeout 击落时，下家仍能用整流后的请求体走整流流程 let mut rectifier_retried = false; let mut budget_rectifier_retried = false; + let mut media_rectifier_retried = false; // 上限检查：尊重用户在 AppProxyConfig.max_retries 上配置的「重试次数」。 // 放在熔断器 allow 检查之前，避免在已经超限时还占用 HalfOpen 探测名额。 @@ -477,6 +523,123 @@ impl RequestForwarder { ); let mut signature_rectifier_non_retryable_client_error = false; + if self.media_retry_should_trigger( + adapter.name(), + media_rectifier_retried, + &provider_body, + &e, + ) { + let mut media_body = provider_body.clone(); + let replaced_images = + super::media_sanitizer::replace_image_blocks_with_marker( + &mut media_body, + ); + + if replaced_images > 0 { + let _ = std::mem::replace(&mut media_rectifier_retried, true); + let model = media_body + .get("model") + .and_then(Value::as_str) + .unwrap_or(""); + log::info!( + "[{app_type_str}] [Media] Upstream rejected image input; retrying provider={} model={} with {replaced_images} image block(s) replaced by {}", + provider.id, + model, + super::media_sanitizer::UNSUPPORTED_IMAGE_MARKER + ); + + match self + .forward( + app_type, + &method, + provider, + endpoint, + &media_body, + &headers, + &extensions, + adapter.as_ref(), + ) + .await + { + Ok((response, claude_api_format)) => { + log::info!( + "[{app_type_str}] [Media] Unsupported-image retry succeeded" + ); + self.record_success_result( + &provider.id, + app_type_str, + used_half_open_permit, + ) + .await; + + { + let mut current_providers = + self.current_providers.write().await; + current_providers.insert( + app_type_str.to_string(), + (provider.id.clone(), provider.name.clone()), + ); + } + + { + let mut status = self.status.write().await; + status.success_requests += 1; + status.last_error = None; + let should_switch = + self.current_provider_id_at_start.as_str() + != provider.id.as_str(); + if should_switch { + status.failover_count += 1; + let fm = self.failover_manager.clone(); + let ah = self.app_handle.clone(); + let pid = provider.id.clone(); + let pname = provider.name.clone(); + let at = app_type_str.to_string(); + + tokio::spawn(async move { + let _ = fm + .try_switch(ah.as_ref(), &at, &pid, &pname) + .await; + }); + } + if status.total_requests > 0 { + status.success_rate = (status.success_requests as f32 + / status.total_requests as f32) + * 100.0; + } + } + + return Ok(ForwardResult { + response, + provider: provider.clone(), + claude_api_format, + connection_guard: None, + }); + } + Err(retry_err) => { + log::warn!( + "[{app_type_str}] [Media] Unsupported-image retry still failed: {retry_err}" + ); + if let Some(err) = self + .handle_rectifier_retry_failure( + retry_err, + provider, + app_type_str, + used_half_open_permit, + "media 降级", + &mut last_error, + &mut last_provider, + ) + .await + { + return Err(err); + } + continue; + } + } + } + } + if is_anthropic_provider { let error_message = extract_error_message(&e); if should_rectify_thinking_signature( @@ -1114,6 +1277,7 @@ impl RequestForwarder { provider, api_format, ); + self.apply_media_prevention(&mut mapped_body, provider); } } let needs_transform = match resolved_claude_api_format.as_deref() { @@ -3106,4 +3270,160 @@ mod tests { assert_eq!(will_replace, should_replace, "{desc}"); } } + + // ===== P3: forwarder 层 media 开关回归测试 ===== + // 验证 gate 在 forwarder 这一层的"接线"，而非 media_sanitizer 纯函数本身。 + + fn forwarder_with_rectifier(config: RectifierConfig) -> RequestForwarder { + let mut fwd = test_forwarder(Duration::from_secs(1), Duration::from_secs(1)); + fwd.rectifier_config = config; + fwd + } + + fn provider_with_settings(settings_config: Value) -> Provider { + let mut p = test_provider_with_type(Some("anthropic")); + p.settings_config = settings_config; + p + } + + fn body_with_image(model: &str) -> Value { + json!({ + "model": model, + "messages": [{ + "role": "user", + "content": [ + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }) + } + + fn image_unsupported_error() -> ProxyError { + ProxyError::UpstreamError { + status: 400, + body: Some( + r#"{"error":{"message":"This model does not support image input"}}"#.to_string(), + ), + } + } + #[test] + fn prevention_replaces_when_all_switches_on_and_model_in_heuristic_list() { + let fwd = forwarder_with_rectifier(RectifierConfig::default()); + let provider = provider_with_settings(json!({})); + let mut body = body_with_image("deepseek-v4-pro"); + + let replaced = fwd.apply_media_prevention(&mut body, &provider); + + assert_eq!(replaced, 1, "默认全开 + 名单内模型应预替换"); + assert_eq!(body["messages"][0]["content"][0]["type"], "text"); + } + + #[test] + fn prevention_skipped_when_media_fallback_off() { + // 关闭 request_media_fallback：即使名单命中也不预替换。 + let fwd = forwarder_with_rectifier(RectifierConfig { + request_media_fallback: false, + ..RectifierConfig::default() + }); + let provider = provider_with_settings(json!({})); + let mut body = body_with_image("deepseek-v4-pro"); + + let replaced = fwd.apply_media_prevention(&mut body, &provider); + + assert_eq!(replaced, 0); + assert_eq!(body["messages"][0]["content"][0]["type"], "image"); + } + + #[test] + fn prevention_skipped_when_master_switch_off() { + let fwd = forwarder_with_rectifier(RectifierConfig { + enabled: false, + ..RectifierConfig::default() + }); + let provider = provider_with_settings(json!({})); + let mut body = body_with_image("deepseek-v4-pro"); + + assert_eq!(fwd.apply_media_prevention(&mut body, &provider), 0); + assert_eq!(body["messages"][0]["content"][0]["type"], "image"); + } + + #[test] + fn prevention_heuristic_off_skips_list_but_keeps_explicit_text_only() { + // 关闭 request_media_heuristic：名单预测失效，但显式声明 text-only 仍预替换。 + let fwd = forwarder_with_rectifier(RectifierConfig { + request_media_heuristic: false, + ..RectifierConfig::default() + }); + + // (a) 名单内模型、无显式声明 → 不再预替换 + let bare_provider = provider_with_settings(json!({})); + let mut list_body = body_with_image("deepseek-v4-pro"); + assert_eq!( + fwd.apply_media_prevention(&mut list_body, &bare_provider), + 0, + "heuristic 关闭后名单模型不应被预替换" + ); + assert_eq!(list_body["messages"][0]["content"][0]["type"], "image"); + + // (b) 显式声明 text-only → 仍预替换（声明驱动，不受 heuristic 开关影响） + let declared_provider = provider_with_settings(json!({ + "models": [ { "id": "some-text-model", "input": ["text"] } ] + })); + let mut declared_body = body_with_image("some-text-model"); + assert_eq!( + fwd.apply_media_prevention(&mut declared_body, &declared_provider), + 1, + "显式 text-only 即使关闭 heuristic 也应预替换" + ); + assert_eq!(declared_body["messages"][0]["content"][0]["type"], "text"); + } + + #[test] + fn reactive_triggers_when_all_switches_on() { + let fwd = forwarder_with_rectifier(RectifierConfig::default()); + let body = body_with_image("any-model"); + assert!(fwd.media_retry_should_trigger("Claude", false, &body, &image_unsupported_error())); + } + + #[test] + fn reactive_skipped_when_media_fallback_off() { + // 关闭 request_media_fallback：上游报图片错误也不触发兜底重试。 + let fwd = forwarder_with_rectifier(RectifierConfig { + request_media_fallback: false, + ..RectifierConfig::default() + }); + let body = body_with_image("any-model"); + assert!(!fwd.media_retry_should_trigger( + "Claude", + false, + &body, + &image_unsupported_error() + )); + } + + #[test] + fn reactive_skipped_when_master_switch_off() { + let fwd = forwarder_with_rectifier(RectifierConfig { + enabled: false, + ..RectifierConfig::default() + }); + let body = body_with_image("any-model"); + assert!(!fwd.media_retry_should_trigger( + "Claude", + false, + &body, + &image_unsupported_error() + )); + } + + #[test] + fn reactive_unaffected_by_heuristic_switch() { + // 关闭 request_media_heuristic 不影响反应式兜底——它是上游实测错误后的恢复，不是预测。 + let fwd = forwarder_with_rectifier(RectifierConfig { + request_media_heuristic: false, + ..RectifierConfig::default() + }); + let body = body_with_image("any-model"); + assert!(fwd.media_retry_should_trigger("Claude", false, &body, &image_unsupported_error())); + } } diff --git a/src-tauri/src/proxy/media_sanitizer.rs b/src-tauri/src/proxy/media_sanitizer.rs new file mode 100644 index 000000000..509b83e28 --- /dev/null +++ b/src-tauri/src/proxy/media_sanitizer.rs @@ -0,0 +1,703 @@ +use crate::provider::Provider; +use crate::proxy::error::ProxyError; +use serde_json::{json, Value}; + +pub const UNSUPPORTED_IMAGE_MARKER: &str = "[Unsupported Image]"; + +/// Replace image blocks before sending when the routed model is text-only. +/// +/// Two paths, both reached only when the caller's media-fallback switch is on: +/// - explicit capability from the provider config (modelCatalog / modalities) is +/// always trusted — it is declaration-driven, never a guess; +/// - the curated `known_text_only_model` list is a heuristic *prediction* and only +/// runs when `allow_heuristic` is true, so a mislabeled multimodal model cannot +/// have its images silently stripped when the user opts out. +pub fn replace_images_for_text_only_model( + body: &mut Value, + provider: &Provider, + allow_heuristic: bool, +) -> usize { + if !contains_image_blocks(body) { + return 0; + } + + let model = body + .get("model") + .and_then(Value::as_str) + .map(str::trim) + .unwrap_or(""); + + match explicit_model_image_support(provider, model) { + Some(true) => return 0, + Some(false) => return replace_images_in_body(body), + None => {} + } + + if !allow_heuristic || !known_text_only_model(model) { + return 0; + } + + replace_images_in_body(body) +} + +pub fn contains_image_blocks(body: &Value) -> bool { + body.get("messages") + .and_then(Value::as_array) + .is_some_and(|messages| { + messages + .iter() + .filter_map(|message| message.get("content")) + .any(content_has_image_blocks) + }) +} + +pub fn replace_image_blocks_with_marker(body: &mut Value) -> usize { + replace_images_in_body(body) +} + +pub fn is_unsupported_image_error(error: &ProxyError) -> bool { + let ProxyError::UpstreamError { status, body } = error else { + return false; + }; + + if !matches!(*status, 400 | 415 | 422 | 501) { + return false; + } + + let Some(body) = body.as_deref() else { + return false; + }; + + let message = extract_error_text(body); + let message = message.to_ascii_lowercase(); + let mentions_image = message.contains("image") + || message.contains("vision") + || message.contains("multimodal") + || message.contains("multi-modal") + || message.contains("modality") + || message.contains("modalities") + || message.contains("media") + || message.contains("attachment"); + + if !mentions_image { + return false; + } + + const UNSUPPORTED_HINTS: &[&str] = &[ + "unsupported", + "not supported", + "does not support", + "doesn't support", + "do not support", + "don't support", + "only supports text", + "text only", + "text-only", + "invalid content type", + "invalid message content", + "unknown content type", + "unrecognized content type", + "cannot process", + "cannot handle", + "can't process", + "can't handle", + "unable to process", + ]; + + UNSUPPORTED_HINTS.iter().any(|hint| message.contains(hint)) +} + +fn content_has_image_blocks(content: &Value) -> bool { + let Some(blocks) = content.as_array() else { + return false; + }; + + blocks.iter().any(|block| { + block.get("type").and_then(Value::as_str) == Some("image") + || block.get("content").is_some_and(content_has_image_blocks) + }) +} + +fn replace_images_in_body(body: &mut Value) -> usize { + let Some(messages) = body.get_mut("messages").and_then(Value::as_array_mut) else { + return 0; + }; + + messages + .iter_mut() + .filter_map(|message| message.get_mut("content")) + .map(replace_images_in_content) + .sum() +} + +fn replace_images_in_content(content: &mut Value) -> usize { + let Some(blocks) = content.as_array_mut() else { + return 0; + }; + + let mut replaced = 0usize; + for block in blocks { + if block.get("type").and_then(Value::as_str) == Some("image") { + let cache_control = block.get("cache_control").cloned(); + *block = json!({ + "type": "text", + "text": UNSUPPORTED_IMAGE_MARKER + }); + if let (Some(cache_control), Some(object)) = (cache_control, block.as_object_mut()) { + object.insert("cache_control".to_string(), cache_control); + } + replaced += 1; + continue; + } + + if let Some(nested_content) = block.get_mut("content") { + replaced += replace_images_in_content(nested_content); + } + } + + replaced +} + +fn explicit_model_image_support(provider: &Provider, model: &str) -> Option { + let settings = &provider.settings_config; + [ + settings + .get("modelCatalog") + .and_then(|catalog| catalog.get("models")), + settings.get("modelCatalog"), + settings.get("models"), + ] + .into_iter() + .flatten() + .find_map(|value| explicit_model_image_support_in_value(value, model)) +} + +fn known_text_only_model(model: &str) -> bool { + let normalized = normalize_model_id(model); + let tail = normalized.rsplit('/').next().unwrap_or(normalized.as_str()); + + const EXACT_TAILS: &[&str] = &[ + "ark-code-latest", + "deepseek-chat", + "deepseek-reasoner", + "deepseek-v4-flash", + "deepseek-v4-pro", + "glm-5.1", + "kat-coder", + "kat-coder-pro", + "kat-coder-pro v1", + "kat-coder-pro v2", + "kat-coder-pro-v1", + "kat-coder-pro-v2", + "ling-2.5-1t", + "longcat-flash-chat", + "mimo-v2.5-pro", + "us.deepseek.r1-v1", + ]; + + const TAIL_PREFIXES: &[&str] = &["minimax-m2.7", "qwen3-coder", "step-3.5-flash"]; + + EXACT_TAILS.contains(&tail) || TAIL_PREFIXES.iter().any(|prefix| tail.starts_with(prefix)) +} + +fn explicit_model_image_support_in_value(value: &Value, model: &str) -> Option { + if let Some(models) = value.as_array() { + return models.iter().find_map(|entry| { + model_entry_matches(entry, None, model).then(|| explicit_image_support(entry))? + }); + } + + let object = value.as_object()?; + object.iter().find_map(|(key, entry)| { + model_entry_matches(entry, Some(key), model).then(|| explicit_image_support(entry))? + }) +} + +fn explicit_image_support(entry: &Value) -> Option { + if let Some(value) = entry + .get("supportsImage") + .or_else(|| entry.get("supports_image")) + .or_else(|| entry.get("vision")) + .and_then(Value::as_bool) + { + return Some(value); + } + + [ + entry.get("input"), + entry.pointer("/modalities/input"), + entry.get("input_modalities"), + entry.get("inputModalities"), + ] + .into_iter() + .flatten() + .find_map(input_modalities_support_image) +} + +fn input_modalities_support_image(value: &Value) -> Option { + let modalities = value.as_array()?; + Some(modalities.iter().any(|item| { + item.as_str() + .map(str::trim) + .is_some_and(|item| item.eq_ignore_ascii_case("image")) + })) +} + +fn extract_error_text(body: &str) -> String { + if let Ok(value) = serde_json::from_str::(body) { + let candidates = [ + value.pointer("/error/message"), + value.pointer("/message"), + value.pointer("/detail"), + value.pointer("/error"), + ]; + if let Some(message) = candidates + .into_iter() + .flatten() + .find_map(|value| value.as_str()) + { + return message.to_string(); + } + + if let Ok(compact) = serde_json::to_string(&value) { + return compact; + } + } + + body.to_string() +} + +fn model_entry_matches(entry: &Value, key: Option<&str>, model: &str) -> bool { + key.is_some_and(|key| model_ids_match(key, model)) + || ["model", "id", "name"] + .into_iter() + .filter_map(|field| entry.get(field).and_then(Value::as_str)) + .any(|candidate| model_ids_match(candidate, model)) +} + +fn model_ids_match(candidate: &str, model: &str) -> bool { + let candidate = normalize_model_id(candidate); + let model = normalize_model_id(model); + if candidate.is_empty() || model.is_empty() { + return false; + } + if candidate == model { + return true; + } + + let candidate_tail = candidate.rsplit('/').next().unwrap_or(candidate.as_str()); + let model_tail = model.rsplit('/').next().unwrap_or(model.as_str()); + candidate_tail == model_tail || candidate == model_tail || candidate_tail == model +} + +fn normalize_model_id(value: &str) -> String { + let mut normalized = value + .trim() + .trim_start_matches("models/") + .trim() + .to_ascii_lowercase(); + if let Some(stripped) = + normalized.strip_suffix(crate::claude_desktop_config::ONE_M_CONTEXT_MARKER) + { + normalized = stripped.trim().to_string(); + } + normalized +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::provider::Provider; + use serde_json::json; + + fn provider(settings_config: Value) -> Provider { + Provider { + id: "test".to_string(), + name: "Test".to_string(), + settings_config, + website_url: None, + category: None, + created_at: None, + sort_index: None, + notes: None, + meta: None, + icon: None, + icon_color: None, + in_failover_queue: false, + } + } + + #[test] + fn keeps_images_when_model_capability_is_unknown() { + let provider = provider(json!({})); + let mut body = json!({ + "model": "unknown-model", + "messages": [{ + "role": "user", + "content": [ + { "type": "text", "text": "look" }, + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }); + + let count = replace_images_for_text_only_model(&mut body, &provider, true); + + assert_eq!(count, 0); + assert_eq!(body["messages"][0]["content"][1]["type"], "image"); + } + + #[test] + fn known_text_only_models_replace_images_before_send() { + let provider = provider(json!({})); + let mut body = json!({ + "model": "deepseek/deepseek-v4-pro", + "messages": [{ + "role": "user", + "content": [ + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }); + + let count = replace_images_for_text_only_model(&mut body, &provider, true); + + assert_eq!(count, 1); + assert_eq!( + body["messages"][0]["content"][0]["text"], + UNSUPPORTED_IMAGE_MARKER + ); + } + + #[test] + fn explicit_text_modalities_replace_images_before_send() { + let provider = provider(json!({ + "models": [ + { "id": "deepseek-v4-pro", "input": ["text"] } + ] + })); + let mut body = json!({ + "model": "deepseek-v4-pro", + "messages": [{ + "role": "user", + "content": [ + { "type": "text", "text": "look" }, + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }); + + let count = replace_images_for_text_only_model(&mut body, &provider, true); + + assert_eq!(count, 1); + assert_eq!(body["messages"][0]["content"][0]["text"], "look"); + assert_eq!(body["messages"][0]["content"][1]["type"], "text"); + assert_eq!( + body["messages"][0]["content"][1]["text"], + UNSUPPORTED_IMAGE_MARKER + ); + } + + #[test] + fn preserves_images_without_explicit_capability_even_for_unknown_models() { + let provider = provider(json!({})); + let mut body = json!({ + "model": "unknown-model", + "messages": [{ + "role": "user", + "content": [ + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }); + + let count = replace_images_for_text_only_model(&mut body, &provider, true); + + assert_eq!(count, 0); + assert_eq!(body["messages"][0]["content"][0]["type"], "image"); + } + + #[test] + fn explicit_text_modalities_can_override_visual_model_ids() { + let provider = provider(json!({ + "models": [ + { "id": "gpt-4o", "input": ["text"] } + ] + })); + let mut body = json!({ + "model": "gpt-4o", + "messages": [{ + "role": "user", + "content": [ + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }); + + let count = replace_images_for_text_only_model(&mut body, &provider, true); + + assert_eq!(count, 1); + assert_eq!( + body["messages"][0]["content"][0]["text"], + UNSUPPORTED_IMAGE_MARKER + ); + } + + #[test] + fn explicit_image_modalities_preserve_model_images() { + let provider = provider(json!({ + "modelCatalog": { + "models": [ + { "model": "deepseek-v4-pro", "modalities": { "input": ["text", "image"] } } + ] + } + })); + let mut body = json!({ + "model": "deepseek-v4-pro", + "messages": [{ + "role": "user", + "content": [ + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }); + + let count = replace_images_for_text_only_model(&mut body, &provider, true); + + assert_eq!(count, 0); + assert_eq!(body["messages"][0]["content"][0]["type"], "image"); + } + + #[test] + fn known_mimo_pro_replaces_but_mimo_multimodal_preserves() { + let provider = provider(json!({})); + let mut pro_body = json!({ + "model": "xiaomi-mimo-token-plan/mimo-v2.5-pro", + "messages": [{ + "role": "user", + "content": [ + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }); + let mut multimodal_body = json!({ + "model": "xiaomi-mimo-token-plan/mimo-v2.5", + "messages": [{ + "role": "user", + "content": [ + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }); + + let pro_count = replace_images_for_text_only_model(&mut pro_body, &provider, true); + let multimodal_count = + replace_images_for_text_only_model(&mut multimodal_body, &provider, true); + + assert_eq!(pro_count, 1); + assert_eq!(multimodal_count, 0); + assert_eq!( + multimodal_body["messages"][0]["content"][0]["type"], + "image" + ); + } + + #[test] + fn multimodal_kimi_model_is_not_on_text_only_list() { + let provider = provider(json!({})); + let mut body = json!({ + "model": "kimi/kimi-k2.6", + "messages": [{ + "role": "user", + "content": [ + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }); + + let count = replace_images_for_text_only_model(&mut body, &provider, true); + + assert_eq!(count, 0); + assert_eq!(body["messages"][0]["content"][0]["type"], "image"); + } + + #[test] + fn known_text_only_prefixes_replace_images_before_send() { + let provider = provider(json!({})); + let mut body = json!({ + "model": "therouter/qwen/qwen3-coder-480b", + "messages": [{ + "role": "user", + "content": [ + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }); + + let count = replace_images_for_text_only_model(&mut body, &provider, true); + + assert_eq!(count, 1); + assert_eq!( + body["messages"][0]["content"][0]["text"], + UNSUPPORTED_IMAGE_MARKER + ); + } + + #[test] + fn unconditional_marker_replacement_handles_retry_path() { + let mut body = json!({ + "model": "xiaomi-mimo-token-plan/mimo-v2.5-pro", + "messages": [{ + "role": "user", + "content": [ + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }); + + assert!(contains_image_blocks(&body)); + let count = replace_image_blocks_with_marker(&mut body); + + assert_eq!(count, 1); + assert_eq!( + body["messages"][0]["content"][0]["text"], + UNSUPPORTED_IMAGE_MARKER + ); + } + + #[test] + fn replaces_nested_tool_result_image_blocks() { + let mut body = json!({ + "model": "deepseek-v4-pro", + "messages": [{ + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": [ + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }] + }); + + let count = replace_image_blocks_with_marker(&mut body); + + assert_eq!(count, 1); + assert_eq!( + body["messages"][0]["content"][0]["content"][0]["text"], + UNSUPPORTED_IMAGE_MARKER + ); + } + + #[test] + fn detects_unsupported_image_errors() { + let error = ProxyError::UpstreamError { + status: 400, + body: Some( + r#"{"error":{"message":"This model does not support image input"}}"#.to_string(), + ), + }; + + assert!(is_unsupported_image_error(&error)); + } + + #[test] + fn ignores_non_image_errors() { + let error = ProxyError::UpstreamError { + status: 400, + body: Some(r#"{"error":{"message":"Invalid API key"}}"#.to_string()), + }; + + assert!(!is_unsupported_image_error(&error)); + } + + #[test] + fn preserves_cache_control_when_replacing_image() { + // image block 可能承载 prompt cache 断点；替换成标记时必须把 + // cache_control 迁移到新的 text block，否则会断掉缓存命中。 + let mut body = json!({ + "model": "deepseek-v4-pro", + "messages": [{ + "role": "user", + "content": [{ + "type": "image", + "source": { "type": "base64", "media_type": "image/png", "data": "abc" }, + "cache_control": { "type": "ephemeral" } + }] + }] + }); + + let count = replace_image_blocks_with_marker(&mut body); + + assert_eq!(count, 1); + let block = &body["messages"][0]["content"][0]; + assert_eq!(block["type"], "text"); + assert_eq!(block["text"], UNSUPPORTED_IMAGE_MARKER); + assert_eq!(block["cache_control"]["type"], "ephemeral"); + } + + #[test] + fn detects_media_and_attachment_error_phrasings() { + let media_error = ProxyError::UpstreamError { + status: 400, + body: Some( + r#"{"error":{"message":"This model cannot process media inputs"}}"#.to_string(), + ), + }; + assert!(is_unsupported_image_error(&media_error)); + + let attachment_error = ProxyError::UpstreamError { + status: 422, + body: Some(r#"{"message":"attachments are not supported by this model"}"#.to_string()), + }; + assert!(is_unsupported_image_error(&attachment_error)); + } + + #[test] + fn heuristic_disabled_keeps_images_for_listed_text_only_models() { + // allow_heuristic = false：内置列表不再预测性剥图，避免误判多模态模型时静默丢图。 + let provider = provider(json!({})); + let mut body = json!({ + "model": "deepseek/deepseek-v4-pro", + "messages": [{ + "role": "user", + "content": [ + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }); + + let count = replace_images_for_text_only_model(&mut body, &provider, false); + + assert_eq!(count, 0); + assert_eq!(body["messages"][0]["content"][0]["type"], "image"); + } + + #[test] + fn explicit_text_capability_replaces_even_when_heuristic_disabled() { + // 显式声明 text-only 是声明驱动、零误判，即使关掉启发式也应生效。 + let provider = provider(json!({ + "models": [ + { "id": "deepseek-v4-pro", "input": ["text"] } + ] + })); + let mut body = json!({ + "model": "deepseek-v4-pro", + "messages": [{ + "role": "user", + "content": [ + { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } } + ] + }] + }); + + let count = replace_images_for_text_only_model(&mut body, &provider, false); + + assert_eq!(count, 1); + assert_eq!( + body["messages"][0]["content"][0]["text"], + UNSUPPORTED_IMAGE_MARKER + ); + } +} diff --git a/src-tauri/src/proxy/mod.rs b/src-tauri/src/proxy/mod.rs index 6dd7bd0d9..585703c63 100644 --- a/src-tauri/src/proxy/mod.rs +++ b/src-tauri/src/proxy/mod.rs @@ -19,6 +19,7 @@ pub mod http_client; pub mod hyper_client; pub(crate) mod json_canonical; pub mod log_codes; +pub mod media_sanitizer; pub mod model_mapper; pub mod provider_router; pub mod providers; diff --git a/src-tauri/src/proxy/thinking_budget_rectifier.rs b/src-tauri/src/proxy/thinking_budget_rectifier.rs index aa0836878..9fc6a3955 100644 --- a/src-tauri/src/proxy/thinking_budget_rectifier.rs +++ b/src-tauri/src/proxy/thinking_budget_rectifier.rs @@ -148,6 +148,8 @@ mod tests { enabled: true, request_thinking_signature: true, request_thinking_budget: true, + request_media_fallback: true, + request_media_heuristic: true, } } @@ -156,6 +158,8 @@ mod tests { enabled: true, request_thinking_signature: true, request_thinking_budget: false, + request_media_fallback: true, + request_media_heuristic: true, } } @@ -164,6 +168,8 @@ mod tests { enabled: false, request_thinking_signature: true, request_thinking_budget: true, + request_media_fallback: true, + request_media_heuristic: true, } } diff --git a/src-tauri/src/proxy/thinking_rectifier.rs b/src-tauri/src/proxy/thinking_rectifier.rs index b43b4d8ea..e7082e7ba 100644 --- a/src-tauri/src/proxy/thinking_rectifier.rs +++ b/src-tauri/src/proxy/thinking_rectifier.rs @@ -251,6 +251,8 @@ mod tests { enabled: true, request_thinking_signature: true, request_thinking_budget: true, + request_media_fallback: true, + request_media_heuristic: true, } } @@ -259,6 +261,8 @@ mod tests { enabled: true, request_thinking_signature: false, request_thinking_budget: false, + request_media_fallback: true, + request_media_heuristic: true, } } @@ -267,6 +271,8 @@ mod tests { enabled: false, request_thinking_signature: true, request_thinking_budget: true, + request_media_fallback: true, + request_media_heuristic: true, } } diff --git a/src-tauri/src/proxy/types.rs b/src-tauri/src/proxy/types.rs index 68474400b..f2dfc5800 100644 --- a/src-tauri/src/proxy/types.rs +++ b/src-tauri/src/proxy/types.rs @@ -213,6 +213,19 @@ pub struct RectifierConfig { /// 处理错误：budget_tokens + thinking 相关约束 #[serde(default = "default_true")] pub request_thinking_budget: bool, + /// 请求整流：不支持的图片降级（默认开启） + /// + /// 上游拒绝图片输入时，把图片块替换为 [Unsupported Image] 标记， + /// 让对话不中断。总开关，管辖「显式声明 text-only」与「上游报错后兜底」两条事实驱动路径。 + #[serde(default = "default_true")] + pub request_media_fallback: bool, + /// 请求整流：启发式 text-only 模型名匹配（默认开启） + /// + /// 在模型未声明能力时，按内置模型名列表预测性地剥离图片（发送前）。 + /// 受 request_media_fallback 管辖；单独关闭后仅保留「显式声明」与「上游兜底」， + /// 避免内置列表把多模态模型误判成 text-only 而静默剥图。 + #[serde(default = "default_true")] + pub request_media_heuristic: bool, } fn default_true() -> bool { @@ -229,6 +242,8 @@ impl Default for RectifierConfig { enabled: true, request_thinking_signature: true, request_thinking_budget: true, + request_media_fallback: true, + request_media_heuristic: true, } } } @@ -386,6 +401,14 @@ mod tests { config.request_thinking_budget, "thinking budget 整流器默认应为 true" ); + assert!( + config.request_media_fallback, + "media 降级总开关默认应为 true" + ); + assert!( + config.request_media_heuristic, + "启发式 text-only 模型识别默认应为 true" + ); } #[test] @@ -396,6 +419,14 @@ mod tests { assert!(config.enabled); assert!(config.request_thinking_signature); assert!(config.request_thinking_budget); + assert!( + config.request_media_fallback, + "缺 requestMediaFallback 时应回退默认值 true" + ); + assert!( + config.request_media_heuristic, + "缺 requestMediaHeuristic 时应回退默认值 true" + ); } #[test] @@ -419,6 +450,19 @@ mod tests { assert!(config.request_thinking_budget); } + #[test] + fn test_rectifier_config_serde_media_explicit_false() { + // 验证 media 两字段显式 false 时被如实反序列化（用户主动关闭须生效，不能被默认值覆盖） + let json = r#"{"requestMediaFallback": false, "requestMediaHeuristic": false}"#; + let config: RectifierConfig = serde_json::from_str(json).unwrap(); + assert!(!config.request_media_fallback); + assert!(!config.request_media_heuristic); + // 其余字段仍走默认 true + assert!(config.enabled); + assert!(config.request_thinking_signature); + assert!(config.request_thinking_budget); + } + #[test] fn test_log_config_default() { let config = LogConfig::default(); diff --git a/src/components/settings/RectifierConfigPanel.tsx b/src/components/settings/RectifierConfigPanel.tsx index 6b72c029d..2f9d5579c 100644 --- a/src/components/settings/RectifierConfigPanel.tsx +++ b/src/components/settings/RectifierConfigPanel.tsx @@ -15,6 +15,8 @@ export function RectifierConfigPanel() { enabled: true, requestThinkingSignature: true, requestThinkingBudget: true, + requestMediaFallback: true, + requestMediaHeuristic: true, }); const [optimizerConfig, setOptimizerConfig] = useState({ enabled: false, @@ -111,6 +113,36 @@ export function RectifierConfigPanel() { } /> +
+
+ +

+ {t("settings.advanced.rectifier.mediaFallbackDescription")} +

+
+ + handleChange({ requestMediaFallback: checked }) + } + /> +
+
+
+ +

+ {t("settings.advanced.rectifier.mediaHeuristicDescription")} +

+
+ + handleChange({ requestMediaHeuristic: checked }) + } + /> +
diff --git a/src/i18n/locales/en.json b/src/i18n/locales/en.json index 4eb63d814..063f10551 100644 --- a/src/i18n/locales/en.json +++ b/src/i18n/locales/en.json @@ -367,7 +367,11 @@ "thinkingSignature": "Thinking Signature Rectification", "thinkingSignatureDescription": "When an Anthropic-type provider returns thinking signature incompatibility or illegal request errors, automatically removes incompatible thinking-related blocks and retries once with the same provider", "thinkingBudget": "Thinking Budget Rectification", - "thinkingBudgetDescription": "When an Anthropic-type provider returns budget_tokens constraint errors (such as at least 1024), automatically normalizes thinking to enabled, sets thinking budget to 32000, and raises max_tokens to 64000 if needed, then retries once" + "thinkingBudgetDescription": "When an Anthropic-type provider returns budget_tokens constraint errors (such as at least 1024), automatically normalizes thinking to enabled, sets thinking budget to 32000, and raises max_tokens to 64000 if needed, then retries once", + "mediaFallback": "Unsupported Image Fallback", + "mediaFallbackDescription": "When a provider does not support image input (it declares text-only capability, or the upstream returns an unsupported-image error), automatically replaces image blocks with an [Unsupported Image] marker so the conversation is not interrupted", + "mediaHeuristic": "Heuristic Text-Only Model Detection", + "mediaHeuristicDescription": "Predictively strips images before sending based on a built-in model-name list. When off, only the declared-text-only and upstream-error fallback paths remain, preventing the built-in list from misclassifying a multimodal model as text-only and silently dropping images" }, "optimizer": { "title": "Bedrock Request Optimizer", diff --git a/src/i18n/locales/ja.json b/src/i18n/locales/ja.json index abe7560c3..de28dd402 100644 --- a/src/i18n/locales/ja.json +++ b/src/i18n/locales/ja.json @@ -367,7 +367,11 @@ "thinkingSignature": "Thinking 署名整流", "thinkingSignatureDescription": "Anthropic タイプのプロバイダーが thinking 署名の非互換性や不正なリクエストエラーを返した場合、互換性のない thinking 関連ブロックを自動削除し、同じプロバイダーで 1 回リトライします", "thinkingBudget": "Thinking Budget 整流", - "thinkingBudgetDescription": "Anthropic タイプのプロバイダーが budget_tokens 制約エラー（例: 1024 以上）を返した場合、thinking を enabled に正規化し、thinking 予算を 32000 に設定し、必要に応じて max_tokens を 64000 に引き上げて 1 回リトライします" + "thinkingBudgetDescription": "Anthropic タイプのプロバイダーが budget_tokens 制約エラー（例: 1024 以上）を返した場合、thinking を enabled に正規化し、thinking 予算を 32000 に設定し、必要に応じて max_tokens を 64000 に引き上げて 1 回リトライします", + "mediaFallback": "非対応画像フォールバック", + "mediaFallbackDescription": "プロバイダーが画像入力に対応していない場合（テキストのみの能力が宣言されている、または上流が画像非対応エラーを返した場合）、画像ブロックを [Unsupported Image] マーカーに自動置換し、会話を中断させません", + "mediaHeuristic": "ヒューリスティックなテキスト専用モデル判定", + "mediaHeuristicDescription": "送信前に内蔵モデル名リストに基づいて画像を予測的に削除します。オフにすると「宣言済みテキスト専用」と「上流エラー時のフォールバック」のみが残り、内蔵リストがマルチモーダルモデルをテキスト専用と誤判定して画像を静かに削除するのを防ぎます" }, "optimizer": { "title": "Bedrock リクエストオプティマイザー", diff --git a/src/i18n/locales/zh-TW.json b/src/i18n/locales/zh-TW.json index b344ad8e2..210ad69fd 100644 --- a/src/i18n/locales/zh-TW.json +++ b/src/i18n/locales/zh-TW.json @@ -367,7 +367,11 @@ "thinkingSignature": "Thinking 簽章整流", "thinkingSignatureDescription": "當 Anthropic 類型供應商回傳 thinking 簽章不相容或非法請求等錯誤時，自動移除不相容的 thinking 相關區塊，並對同一供應商重試一次", "thinkingBudget": "Thinking Budget 整流", - "thinkingBudgetDescription": "當 Anthropic 類型供應商回傳 budget_tokens 限制錯誤（如至少 1024）時，自動將 thinking 規範為 enabled，並將 budget 設為 32000，同時在需要時將 max_tokens 設為 64000，然後重試一次" + "thinkingBudgetDescription": "當 Anthropic 類型供應商回傳 budget_tokens 限制錯誤（如至少 1024）時，自動將 thinking 規範為 enabled，並將 budget 設為 32000，同時在需要時將 max_tokens 設為 64000，然後重試一次", + "mediaFallback": "不支援圖片降級", + "mediaFallbackDescription": "當供應商不支援圖片輸入時（已宣告純文字能力，或上游回傳不支援圖片的錯誤），自動將圖片區塊替換為 [Unsupported Image] 標記，使對話不中斷", + "mediaHeuristic": "啟發式純文字模型辨識", + "mediaHeuristicDescription": "傳送前依內建模型名稱清單預測性地移除圖片。關閉後僅保留「已宣告純文字」與「上游錯誤兜底」兩條事實驅動路徑，避免內建清單將多模態模型誤判為純文字而靜默丟失圖片" }, "optimizer": { "title": "Bedrock 請求最佳化工具", diff --git a/src/i18n/locales/zh.json b/src/i18n/locales/zh.json index b8bba478a..057958377 100644 --- a/src/i18n/locales/zh.json +++ b/src/i18n/locales/zh.json @@ -367,7 +367,11 @@ "thinkingSignature": "Thinking 签名整流", "thinkingSignatureDescription": "当 Anthropic 类型供应商返回 thinking 签名不兼容或非法请求等错误时，自动移除不兼容的 thinking 相关块并对同一供应商重试一次", "thinkingBudget": "Thinking Budget 整流", - "thinkingBudgetDescription": "当 Anthropic 类型供应商返回 budget_tokens 约束错误（如至少 1024）时，自动将 thinking 规范为 enabled 并将 budget 设为 32000，同时在需要时将 max_tokens 设为 64000，然后重试一次" + "thinkingBudgetDescription": "当 Anthropic 类型供应商返回 budget_tokens 约束错误（如至少 1024）时，自动将 thinking 规范为 enabled 并将 budget 设为 32000，同时在需要时将 max_tokens 设为 64000，然后重试一次", + "mediaFallback": "不支持图片降级", + "mediaFallbackDescription": "当供应商不支持图片输入时（已声明纯文本能力，或上游返回不支持图片的错误），自动将图片块替换为 [Unsupported Image] 标记，使对话不中断", + "mediaHeuristic": "启发式纯文本模型识别", + "mediaHeuristicDescription": "发送前按内置模型名列表预测性剥离图片。关闭后仅保留「已声明纯文本」与「上游报错兜底」两条事实驱动路径，避免内置列表把多模态模型误判为纯文本而静默丢图" }, "optimizer": { "title": "Bedrock 请求优化器", diff --git a/src/lib/api/settings.ts b/src/lib/api/settings.ts index 45e77806a..c33e0052c 100644 --- a/src/lib/api/settings.ts +++ b/src/lib/api/settings.ts @@ -265,6 +265,8 @@ export interface RectifierConfig { enabled: boolean; requestThinkingSignature: boolean; requestThinkingBudget: boolean; + requestMediaFallback: boolean; + requestMediaHeuristic: boolean; } export interface OptimizerConfig {