diff --git a/src-tauri/src/proxy/forwarder.rs b/src-tauri/src/proxy/forwarder.rs index d24784f7f..3a8f815b9 100644 --- a/src-tauri/src/proxy/forwarder.rs +++ b/src-tauri/src/proxy/forwarder.rs @@ -159,7 +159,7 @@ impl RequestForwarder { provider_body: &Value, error: &ProxyError, ) -> bool { - adapter_name == "Claude" + matches!(adapter_name, "Claude" | "Codex") && self.rectifier_config.enabled && self.rectifier_config.request_media_fallback && !already_retried @@ -1321,7 +1321,7 @@ impl RequestForwarder { }; // 转换请求体(如果需要) - let request_body = if codex_responses_to_chat { + let mut request_body = if codex_responses_to_chat { let mut mapped_body = mapped_body; let restored = self .codex_chat_history @@ -1359,6 +1359,10 @@ impl RequestForwarder { mapped_body }; + if matches!(app_type, AppType::Codex) { + self.apply_media_prevention(&mut request_body, provider); + } + // 过滤私有参数(以 `_` 开头的字段),防止内部信息泄露到上游 // 默认使用空白名单,过滤所有 _ 前缀字段 let filtered_body = prepare_upstream_request_body(request_body); @@ -3298,6 +3302,18 @@ mod tests { }) } + fn body_with_codex_input_image(model: &str) -> Value { + json!({ + "model": model, + "input": [{ + "role": "user", + "content": [ + { "type": "input_image", "image_url": "data:image/png;base64,abc" } + ] + }] + }) + } + fn image_unsupported_error() -> ProxyError { ProxyError::UpstreamError { status: 400, @@ -3385,6 +3401,21 @@ mod tests { assert!(fwd.media_retry_should_trigger("Claude", false, &body, &image_unsupported_error())); } + #[test] + fn reactive_triggers_for_codex_image_url_deserialize_errors() { + let fwd = forwarder_with_rectifier(RectifierConfig::default()); + let body = body_with_codex_input_image("deepseek-v4-flash"); + let error = ProxyError::UpstreamError { + status: 400, + body: Some( + r#"{"error":{"message":"Failed to deserialize the JSON body into the target type: messages[11]: unknown variant image_url, expected text"}}"# + .to_string(), + ), + }; + + assert!(fwd.media_retry_should_trigger("Codex", false, &body, &error)); + } + #[test] fn reactive_skipped_when_media_fallback_off() { // 关闭 request_media_fallback:上游报图片错误也不触发兜底重试。 diff --git a/src-tauri/src/proxy/media_sanitizer.rs b/src-tauri/src/proxy/media_sanitizer.rs index 509b83e28..ad30fd2fc 100644 --- a/src-tauri/src/proxy/media_sanitizer.rs +++ b/src-tauri/src/proxy/media_sanitizer.rs @@ -41,14 +41,7 @@ pub fn replace_images_for_text_only_model( } pub fn contains_image_blocks(body: &Value) -> bool { - body.get("messages") - .and_then(Value::as_array) - .is_some_and(|messages| { - messages - .iter() - .filter_map(|message| message.get("content")) - .any(content_has_image_blocks) - }) + messages_have_image_blocks(body) || responses_input_has_image_blocks(body.get("input")) } pub fn replace_image_blocks_with_marker(body: &mut Value) -> usize { @@ -95,6 +88,7 @@ pub fn is_unsupported_image_error(error: &ProxyError) -> bool { "text-only", "invalid content type", "invalid message content", + "unknown variant", "unknown content type", "unrecognized content type", "cannot process", @@ -113,51 +107,124 @@ fn content_has_image_blocks(content: &Value) -> bool { }; blocks.iter().any(|block| { - block.get("type").and_then(Value::as_str) == Some("image") + is_image_block_type(block.get("type").and_then(Value::as_str)) || block.get("content").is_some_and(content_has_image_blocks) }) } fn replace_images_in_body(body: &mut Value) -> usize { - let Some(messages) = body.get_mut("messages").and_then(Value::as_array_mut) else { - return 0; - }; + let message_replacements = body + .get_mut("messages") + .and_then(Value::as_array_mut) + .map(|messages| { + messages + .iter_mut() + .filter_map(|message| message.get_mut("content")) + .map(replace_images_in_content) + .sum() + }) + .unwrap_or(0); - messages - .iter_mut() - .filter_map(|message| message.get_mut("content")) - .map(replace_images_in_content) - .sum() + message_replacements + + body + .get_mut("input") + .map(replace_images_in_responses_input) + .unwrap_or(0) } fn replace_images_in_content(content: &mut Value) -> usize { + replace_images_in_content_with_text_type(content, "text") +} + +fn replace_images_in_content_with_text_type(content: &mut Value, text_type: &str) -> usize { let Some(blocks) = content.as_array_mut() else { return 0; }; let mut replaced = 0usize; for block in blocks { - if block.get("type").and_then(Value::as_str) == Some("image") { - let cache_control = block.get("cache_control").cloned(); - *block = json!({ - "type": "text", - "text": UNSUPPORTED_IMAGE_MARKER - }); - if let (Some(cache_control), Some(object)) = (cache_control, block.as_object_mut()) { - object.insert("cache_control".to_string(), cache_control); - } + if is_image_block_type(block.get("type").and_then(Value::as_str)) { + replace_image_block_with_text_marker(block, text_type); replaced += 1; continue; } if let Some(nested_content) = block.get_mut("content") { - replaced += replace_images_in_content(nested_content); + replaced += replace_images_in_content_with_text_type(nested_content, text_type); } } replaced } +fn messages_have_image_blocks(body: &Value) -> bool { + body.get("messages") + .and_then(Value::as_array) + .is_some_and(|messages| { + messages + .iter() + .filter_map(|message| message.get("content")) + .any(content_has_image_blocks) + }) +} + +fn responses_input_has_image_blocks(input: Option<&Value>) -> bool { + match input { + Some(Value::Array(items)) => items.iter().any(responses_input_item_has_image_blocks), + Some(item @ Value::Object(_)) => responses_input_item_has_image_blocks(item), + _ => false, + } +} + +fn responses_input_item_has_image_blocks(item: &Value) -> bool { + if item.get("type").and_then(Value::as_str) == Some("input_image") { + return true; + } + + item.get("content").is_some_and(content_has_image_blocks) +} + +fn replace_images_in_responses_input(input: &mut Value) -> usize { + match input { + Value::Array(items) => items + .iter_mut() + .map(replace_images_in_responses_input_item) + .sum(), + Value::Object(_) => replace_images_in_responses_input_item(input), + _ => 0, + } +} + +fn replace_images_in_responses_input_item(item: &mut Value) -> usize { + let mut replaced = 0usize; + + if item.get("type").and_then(Value::as_str) == Some("input_image") { + replace_image_block_with_text_marker(item, "input_text"); + replaced += 1; + } + + if let Some(content) = item.get_mut("content") { + replaced += replace_images_in_content_with_text_type(content, "input_text"); + } + + replaced +} + +fn is_image_block_type(block_type: Option<&str>) -> bool { + matches!(block_type, Some("image" | "image_url" | "input_image")) +} + +fn replace_image_block_with_text_marker(block: &mut Value, text_type: &str) { + let cache_control = block.get("cache_control").cloned(); + *block = json!({ + "type": text_type, + "text": UNSUPPORTED_IMAGE_MARKER + }); + if let (Some(cache_control), Some(object)) = (cache_control, block.as_object_mut()) { + object.insert("cache_control".to_string(), cache_control); + } +} + fn explicit_model_image_support(provider: &Provider, model: &str) -> Option { let settings = &provider.settings_config; [ @@ -369,6 +436,54 @@ mod tests { ); } + #[test] + fn known_text_only_models_replace_chat_image_url_before_send() { + let provider = provider(json!({})); + let mut body = json!({ + "model": "deepseek-v4-flash", + "messages": [{ + "role": "user", + "content": [ + { "type": "text", "text": "look" }, + { "type": "image_url", "image_url": { "url": "data:image/png;base64,abc" } } + ] + }] + }); + + let count = replace_images_for_text_only_model(&mut body, &provider, true); + + assert_eq!(count, 1); + assert_eq!(body["messages"][0]["content"][1]["type"], "text"); + assert_eq!( + body["messages"][0]["content"][1]["text"], + UNSUPPORTED_IMAGE_MARKER + ); + } + + #[test] + fn known_text_only_models_replace_codex_input_image_before_send() { + let provider = provider(json!({})); + let mut body = json!({ + "model": "deepseek-v4-flash", + "input": [{ + "role": "user", + "content": [ + { "type": "input_text", "text": "look" }, + { "type": "input_image", "image_url": "data:image/png;base64,abc" } + ] + }] + }); + + let count = replace_images_for_text_only_model(&mut body, &provider, true); + + assert_eq!(count, 1); + assert_eq!(body["input"][0]["content"][1]["type"], "input_text"); + assert_eq!( + body["input"][0]["content"][1]["text"], + UNSUPPORTED_IMAGE_MARKER + ); + } + #[test] fn explicit_text_modalities_replace_images_before_send() { let provider = provider(json!({ @@ -654,6 +769,19 @@ mod tests { assert!(is_unsupported_image_error(&attachment_error)); } + #[test] + fn detects_chat_content_unknown_variant_image_url_errors() { + let error = ProxyError::UpstreamError { + status: 400, + body: Some( + r#"{"error":{"message":"Failed to deserialize the JSON body into the target type: messages[11]: unknown variant image_url, expected text"}}"# + .to_string(), + ), + }; + + assert!(is_unsupported_image_error(&error)); + } + #[test] fn heuristic_disabled_keeps_images_for_listed_text_only_models() { // allow_heuristic = false:内置列表不再预测性剥图,避免误判多模态模型时静默丢图。