diff --git a/src-tauri/src/proxy/forwarder.rs b/src-tauri/src/proxy/forwarder.rs index ebed6c57c..8c9d10df3 100644 --- a/src-tauri/src/proxy/forwarder.rs +++ b/src-tauri/src/proxy/forwarder.rs @@ -4776,6 +4776,39 @@ mod tests { }) } + fn body_with_stringified_chat_tool_image() -> Value { + let content = json!({ + "content": [{ + "type": "image", + "mimeType": "image/png", + "data": "CHAT_TOOL_SENTINEL" + }] + }) + .to_string(); + json!({ + "model": "any-model", + "messages": [{ + "role": "tool", + "tool_call_id": "call_1", + "content": content + }] + }) + } + + fn body_with_gemini_image() -> Value { + json!({ + "contents": [{ + "role": "user", + "parts": [{ + "inlineData": { + "mimeType": "image/png", + "data": "GEMINI_SENTINEL" + } + }] + }] + }) + } + fn image_unsupported_error() -> ProxyError { ProxyError::UpstreamError { status: 400, @@ -4891,6 +4924,24 @@ mod tests { } } + #[test] + fn reactive_triggers_for_chat_tool_and_gemini_images() { + let fwd = forwarder_with_rectifier(RectifierConfig::default()); + + assert!(fwd.media_retry_should_trigger( + "Claude", + false, + &body_with_stringified_chat_tool_image(), + &image_unsupported_error() + )); + assert!(fwd.media_retry_should_trigger( + "Claude", + false, + &body_with_gemini_image(), + &image_unsupported_error() + )); + } + #[test] fn reactive_does_not_treat_context_limit_as_image_rejection() { let fwd = forwarder_with_rectifier(RectifierConfig::default()); diff --git a/src-tauri/src/proxy/media_sanitizer.rs b/src-tauri/src/proxy/media_sanitizer.rs index 509c45187..ddec1e64b 100644 --- a/src-tauri/src/proxy/media_sanitizer.rs +++ b/src-tauri/src/proxy/media_sanitizer.rs @@ -43,7 +43,9 @@ pub fn replace_images_for_text_only_model( } pub fn contains_image_blocks(body: &Value) -> bool { - messages_have_image_blocks(body) || responses_input_has_image_blocks(body.get("input")) + messages_have_image_blocks(body) + || responses_input_has_image_blocks(body.get("input")) + || gemini_contents_have_image_blocks(body) } pub fn replace_image_blocks_with_marker(body: &mut Value) -> usize { @@ -122,7 +124,11 @@ fn content_has_image_blocks(content: &Value) -> bool { blocks.iter().any(|block| { is_image_block_type(block.get("type").and_then(Value::as_str)) - || block.get("content").is_some_and(content_has_image_blocks) + || block.get("content").is_some_and(|nested| { + content_has_image_blocks(nested) + || (block.get("type").and_then(Value::as_str) == Some("tool_result") + && tool_output_contains_media(nested, ToolMediaScope::ImagesOnly)) + }) }) } @@ -130,13 +136,7 @@ fn replace_images_in_body(body: &mut Value) -> usize { let message_replacements = body .get_mut("messages") .and_then(Value::as_array_mut) - .map(|messages| { - messages - .iter_mut() - .filter_map(|message| message.get_mut("content")) - .map(replace_images_in_content) - .sum() - }) + .map(|messages| messages.iter_mut().map(replace_images_in_message).sum()) .unwrap_or(0); message_replacements @@ -144,6 +144,37 @@ fn replace_images_in_body(body: &mut Value) -> usize { .get_mut("input") .map(replace_images_in_responses_input) .unwrap_or(0) + + replace_images_in_gemini_contents(body) +} + +fn replace_images_in_message(message: &mut Value) -> usize { + let is_tool_message = message.get("role").and_then(Value::as_str) == Some("tool"); + let Some(content) = message.get_mut("content") else { + return 0; + }; + + if is_tool_message { + // Preserve the legacy typed-image replacement semantics first, + // including Anthropic cache_control on the replacement text block. + // The shared traversal then handles JSON strings, MCP wrappers, and + // loose data-URL shapes that the legacy recursion does not recognize. + let mut replaced = replace_images_in_content(content); + let replacement_block = json!({ + "type":"text", + "text":UNSUPPORTED_IMAGE_MARKER + }); + let mut discarded_media = Vec::new(); + replaced += strip_media_from_tool_value( + content, + &mut discarded_media, + ToolMediaScope::ImagesOnly, + &replacement_block, + UNSUPPORTED_IMAGE_MARKER, + ); + replaced + } else { + replace_images_in_content(content) + } } fn replace_images_in_content(content: &mut Value) -> usize { @@ -157,14 +188,36 @@ fn replace_images_in_content_with_text_type(content: &mut Value, text_type: &str let mut replaced = 0usize; for block in blocks { - if is_image_block_type(block.get("type").and_then(Value::as_str)) { + let block_type = block.get("type").and_then(Value::as_str); + if is_image_block_type(block_type) { replace_image_block_with_text_marker(block, text_type); replaced += 1; continue; } + let is_tool_result = block_type == Some("tool_result"); if let Some(nested_content) = block.get_mut("content") { - replaced += replace_images_in_content_with_text_type(nested_content, text_type); + if is_tool_result { + // Run the legacy typed-block replacement before the shared + // payload-aware traversal. This makes replacement a superset + // of detection and preserves cache_control on Anthropic image + // blocks, while the second pass covers alternate tool shapes. + replaced += replace_images_in_content_with_text_type(nested_content, text_type); + let replacement_block = json!({ + "type":text_type, + "text":UNSUPPORTED_IMAGE_MARKER + }); + let mut discarded_media = Vec::new(); + replaced += strip_media_from_tool_value( + nested_content, + &mut discarded_media, + ToolMediaScope::ImagesOnly, + &replacement_block, + UNSUPPORTED_IMAGE_MARKER, + ); + } else { + replaced += replace_images_in_content_with_text_type(nested_content, text_type); + } } } @@ -175,13 +228,110 @@ fn messages_have_image_blocks(body: &Value) -> bool { body.get("messages") .and_then(Value::as_array) .is_some_and(|messages| { - messages - .iter() - .filter_map(|message| message.get("content")) - .any(content_has_image_blocks) + messages.iter().any(|message| { + let Some(content) = message.get("content") else { + return false; + }; + content_has_image_blocks(content) + || (message.get("role").and_then(Value::as_str) == Some("tool") + && tool_output_contains_media(content, ToolMediaScope::ImagesOnly)) + }) }) } +fn gemini_contents_have_image_blocks(body: &Value) -> bool { + body.get("contents") + .and_then(Value::as_array) + .is_some_and(|contents| { + contents.iter().any(|content| { + content + .get("parts") + .and_then(Value::as_array) + .is_some_and(|parts| parts.iter().any(gemini_part_has_image)) + }) + }) +} + +fn gemini_part_has_image(part: &Value) -> bool { + gemini_media_payload_is_image(part.get("inlineData").or_else(|| part.get("inline_data"))) + || gemini_media_payload_is_image(part.get("fileData").or_else(|| part.get("file_data"))) + || part + .get("functionResponse") + .or_else(|| part.get("function_response")) + .and_then(|response| response.get("parts")) + .and_then(Value::as_array) + .is_some_and(|parts| parts.iter().any(gemini_part_has_image)) +} + +fn gemini_media_payload_is_image(payload: Option<&Value>) -> bool { + payload + .and_then(|payload| payload.get("mimeType").or_else(|| payload.get("mime_type"))) + .and_then(Value::as_str) + .is_some_and(|mime_type| { + mime_type + .get(..6) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/")) + }) +} + +fn replace_images_in_gemini_contents(body: &mut Value) -> usize { + body.get_mut("contents") + .and_then(Value::as_array_mut) + .map(|contents| { + contents + .iter_mut() + .filter_map(|content| content.get_mut("parts").and_then(Value::as_array_mut)) + .map(|parts| { + parts + .iter_mut() + .map(replace_images_in_gemini_part) + .sum::() + }) + .sum() + }) + .unwrap_or(0) +} + +fn replace_images_in_gemini_part(part: &mut Value) -> usize { + if gemini_media_payload_is_image(part.get("inlineData").or_else(|| part.get("inline_data"))) + || gemini_media_payload_is_image(part.get("fileData").or_else(|| part.get("file_data"))) + { + *part = json!({"text":UNSUPPORTED_IMAGE_MARKER}); + return 1; + } + + let response_key = if part.get("functionResponse").is_some() { + "functionResponse" + } else { + "function_response" + }; + let Some(function_response) = part.get_mut(response_key) else { + return 0; + }; + let Some(media_parts) = function_response + .get_mut("parts") + .and_then(Value::as_array_mut) + else { + return 0; + }; + + let before = media_parts.len(); + media_parts.retain(|media_part| !gemini_part_has_image(media_part)); + let replaced = before.saturating_sub(media_parts.len()); + if replaced > 0 { + if let Some(response) = function_response + .get_mut("response") + .and_then(Value::as_object_mut) + { + response.insert( + "cc_switch_media".to_string(), + Value::String(UNSUPPORTED_IMAGE_MARKER.to_string()), + ); + } + } + replaced +} + fn responses_input_has_image_blocks(input: Option<&Value>) -> bool { match input { Some(Value::Array(items)) => items.iter().any(responses_input_item_has_image_blocks), @@ -633,6 +783,71 @@ mod tests { ); } + #[test] + fn replaces_file_backed_tool_result_image_and_preserves_cache_control() { + let mut body = json!({ + "model": "deepseek-v4-pro", + "messages": [{ + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "toolu_file", + "content": [{ + "type": "image", + "source": { + "type": "file", + "file_id": "file_123" + }, + "cache_control": {"type": "ephemeral"} + }] + }] + }] + }); + + assert!(contains_image_blocks(&body)); + let count = replace_image_blocks_with_marker(&mut body); + let replacement = &body["messages"][0]["content"][0]["content"][0]; + + assert_eq!(count, 1); + assert_eq!(replacement["type"], "text"); + assert_eq!(replacement["text"], UNSUPPORTED_IMAGE_MARKER); + assert_eq!(replacement["cache_control"]["type"], "ephemeral"); + assert!(!body.to_string().contains("file_123")); + } + + #[test] + fn replaces_stringified_anthropic_tool_result_image_blocks() { + let content = json!({ + "content": [{ + "type": "image", + "mimeType": "image/png", + "data": "ANTHROPIC_STRING_TOOL_SENTINEL" + }] + }) + .to_string(); + let mut body = json!({ + "model": "deepseek-v4-pro", + "messages": [{ + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "toolu_1", + "content": content + }] + }] + }); + + assert!(contains_image_blocks(&body)); + let count = replace_image_blocks_with_marker(&mut body); + let rewritten = body["messages"][0]["content"][0]["content"] + .as_str() + .unwrap(); + + assert_eq!(count, 1); + assert!(rewritten.contains(UNSUPPORTED_IMAGE_MARKER)); + assert!(!rewritten.contains("ANTHROPIC_STRING_TOOL_SENTINEL")); + } + #[test] fn detects_and_replaces_responses_function_output_images() { let data_url = large_tool_data_url(); @@ -846,6 +1061,101 @@ mod tests { assert_eq!(body["messages"][1]["content"][0]["type"], "text"); } + #[test] + fn detects_and_replaces_stringified_chat_tool_image() { + let content = json!({ + "content": [{ + "type": "image", + "mimeType": "image/png", + "data": "STRINGIFIED_CHAT_TOOL_SENTINEL" + }] + }) + .to_string(); + let mut body = json!({ + "messages": [{ + "role": "tool", + "tool_call_id": "call_1", + "content": content + }] + }); + + assert!(contains_image_blocks(&body)); + let replaced = replace_image_blocks_with_marker(&mut body); + let rewritten = body["messages"][0]["content"].as_str().unwrap(); + + assert_eq!(replaced, 1); + assert!(rewritten.contains(UNSUPPORTED_IMAGE_MARKER)); + assert!(!rewritten.contains("STRINGIFIED_CHAT_TOOL_SENTINEL")); + } + + #[test] + fn detects_and_replaces_gemini_native_image_parts() { + let mut body = json!({ + "contents": [{ + "role": "user", + "parts": [ + { + "functionResponse": { + "name": "inspect", + "response": {"content": "done"} + } + }, + { + "inlineData": { + "mimeType": "image/png", + "data": "GEMINI_INLINE_SENTINEL" + } + } + ] + }] + }); + + assert!(contains_image_blocks(&body)); + let replaced = replace_image_blocks_with_marker(&mut body); + + assert_eq!(replaced, 1); + assert_eq!( + body["contents"][0]["parts"][1]["text"], + UNSUPPORTED_IMAGE_MARKER + ); + assert!(!body.to_string().contains("GEMINI_INLINE_SENTINEL")); + } + + #[test] + fn detects_and_removes_nested_gemini_function_response_media() { + let mut body = json!({ + "contents": [{ + "role": "user", + "parts": [{ + "functionResponse": { + "name": "inspect", + "response": {"content": "done"}, + "parts": [{ + "inlineData": { + "mimeType": "image/webp", + "data": "GEMINI_FUNCTION_SENTINEL" + } + }] + } + }] + }] + }); + + assert!(contains_image_blocks(&body)); + let replaced = replace_image_blocks_with_marker(&mut body); + + assert_eq!(replaced, 1); + assert!(body["contents"][0]["parts"][0]["functionResponse"]["parts"] + .as_array() + .unwrap() + .is_empty()); + assert_eq!( + body["contents"][0]["parts"][0]["functionResponse"]["response"]["cc_switch_media"], + UNSUPPORTED_IMAGE_MARKER + ); + assert!(!body.to_string().contains("GEMINI_FUNCTION_SENTINEL")); + } + #[test] fn detects_unsupported_image_errors() { let error = ProxyError::UpstreamError { diff --git a/src-tauri/src/proxy/providers/transform.rs b/src-tauri/src/proxy/providers/transform.rs index 4b948c375..648a72b00 100644 --- a/src-tauri/src/proxy/providers/transform.rs +++ b/src-tauri/src/proxy/providers/transform.rs @@ -3,7 +3,14 @@ //! 实现 Anthropic ↔ OpenAI 格式转换，用于 OpenRouter 支持 //! 参考: anthropic-proxy-rs -use crate::proxy::{error::ProxyError, json_canonical::canonical_json_string}; +use crate::proxy::{ + error::ProxyError, + json_canonical::canonical_json_string, + tool_media::{ + chat_media_part_from_tool_part, flush_pending_chat_tool_media, plan_chat_tool_output_media, + queue_chat_tool_output_media, ToolMediaScope, + }, +}; use serde_json::{json, Value}; const ANTHROPIC_BILLING_HEADER_PREFIX: &str = "x-anthropic-billing-header:"; @@ -375,6 +382,7 @@ fn convert_message_to_openai( if let Some(blocks) = content.as_array() { let mut content_parts = Vec::new(); let mut tool_calls = Vec::new(); + let mut pending_tool_media = Vec::new(); // reasoning_parts: 仅在兼容 Moonshot/Kimi/DeepSeek thinking tool-call 路径时 // 生成 reasoning_content，通用 OpenAI-compatible 路径不发送该非标准字段。 let mut reasoning_parts = Vec::new(); @@ -389,16 +397,10 @@ fn convert_message_to_openai( } } "image" => { - if let Some(source) = block.get("source") { - let media_type = source - .get("media_type") - .and_then(|m| m.as_str()) - .unwrap_or("image/png"); - let data = source.get("data").and_then(|d| d.as_str()).unwrap_or(""); - content_parts.push(json!({ - "type": "image_url", - "image_url": {"url": format!("data:{};base64,{}", media_type, data)} - })); + if let Some(image) = + chat_media_part_from_tool_part(block, ToolMediaScope::ImagesOnly) + { + content_parts.push(image); } } "tool_use" => { @@ -421,10 +423,22 @@ fn convert_message_to_openai( .and_then(|i| i.as_str()) .unwrap_or(""); let content_val = block.get("content"); - let content_str = match content_val { - Some(Value::String(s)) => s.clone(), - Some(v) => canonical_json_string(v), - None => String::new(), + let media_plan = content_val.cloned().and_then(plan_chat_tool_output_media); + let content_str = if let Some(media_plan) = media_plan { + queue_chat_tool_output_media( + &mut pending_tool_media, + tool_use_id, + media_plan.media_parts, + ); + media_plan.tool_content + } else { + // Keep the no-media representation exactly equal to + // the legacy converter for prompt-cache stability. + match content_val { + Some(Value::String(s)) => s.clone(), + Some(v) => canonical_json_string(v), + None => String::new(), + } }; result.push(json!({ "role": "tool", @@ -452,6 +466,11 @@ fn convert_message_to_openai( } } + // Chat tool messages cannot carry image parts. Keep parallel tool + // results adjacent, then present all extracted media in one user turn + // before any ordinary message content from the same Anthropic turn. + flush_pending_chat_tool_media(&mut result, &mut pending_tool_media); + // 添加带内容和/或工具调用的消息 if !content_parts.is_empty() || !tool_calls.is_empty() { let mut msg = json!({"role": role}); @@ -1133,6 +1152,164 @@ mod tests { assert_eq!(msg["content"], "Sunny, 25°C"); } + #[test] + fn test_anthropic_to_openai_no_media_tool_results_keep_legacy_representation() { + let raw_json_string = "{ \"status\": \"ok\", \"count\": 2 }"; + let input = json!({ + "model": "claude-3-opus", + "messages": [{ + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_string", + "content": raw_json_string + }, + { + "type": "tool_result", + "tool_use_id": "call_array", + "content": [{"type": "text", "text": "plain"}] + } + ] + }] + }); + + let result = anthropic_to_openai(input).unwrap(); + let messages = result["messages"].as_array().unwrap(); + + assert_eq!(messages.len(), 2); + assert_eq!(messages[0]["content"], raw_json_string); + assert_eq!( + messages[1]["content"], + canonical_json_string(&json!([{"type": "text", "text": "plain"}])) + ); + } + + #[test] + fn test_anthropic_to_openai_moves_tool_result_image_to_user_message() { + let input = json!({ + "model": "claude-3-opus", + "max_tokens": 1024, + "messages": [{ + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "call_image", + "content": [ + {"type": "text", "text": "caption"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "CLAUDE_CHAT_IMAGE_SENTINEL" + }, + "cache_control": {"type": "ephemeral"}, + "prompt_cache_breakpoint": true + } + ] + }] + }] + }); + + let result = anthropic_to_openai(input).unwrap(); + let messages = result["messages"].as_array().unwrap(); + + assert_eq!(messages.len(), 2); + assert_eq!(messages[0]["role"], "tool"); + assert_eq!(messages[0]["tool_call_id"], "call_image"); + assert!(messages[0]["content"] + .as_str() + .unwrap() + .contains("tool result media moved")); + assert!(!messages[0]["content"] + .as_str() + .unwrap() + .contains("CLAUDE_CHAT_IMAGE_SENTINEL")); + assert_eq!(messages[1]["role"], "user"); + assert_eq!( + messages[1]["content"][0]["text"], + "[cc-switch: media output of tool call call_image]" + ); + assert_eq!(messages[1]["content"][1]["type"], "image_url"); + assert!(messages[1]["content"][1].get("cache_control").is_none()); + assert!(messages[1]["content"][1] + .get("prompt_cache_breakpoint") + .is_none()); + assert_eq!( + messages[1]["content"][1]["image_url"]["url"], + "data:image/png;base64,CLAUDE_CHAT_IMAGE_SENTINEL" + ); + } + + #[test] + fn test_anthropic_to_openai_batches_parallel_tool_result_media() { + let input = json!({ + "model": "claude-3-opus", + "messages": [{ + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_1", + "content": [{ + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "ONE"} + }] + }, + { + "type": "tool_result", + "tool_use_id": "call_2", + "content": [{ + "type": "image", + "source": {"type": "base64", "media_type": "image/jpeg", "data": "TWO"} + }] + } + ] + }] + }); + + let result = anthropic_to_openai(input).unwrap(); + let messages = result["messages"].as_array().unwrap(); + + assert_eq!(messages.len(), 3); + assert_eq!(messages[0]["role"], "tool"); + assert_eq!(messages[1]["role"], "tool"); + assert_eq!(messages[2]["role"], "user"); + assert_eq!(messages[2]["content"].as_array().unwrap().len(), 4); + } + + #[test] + fn test_anthropic_to_openai_maps_remote_image_source() { + let input = json!({ + "model": "claude-3-opus", + "messages": [{ + "role": "user", + "content": [{ + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.png" + }, + "cache_control": {"type": "ephemeral"}, + "prompt_cache_breakpoint": true + }] + }] + }); + + let result = anthropic_to_openai(input).unwrap(); + assert_eq!( + result["messages"][0]["content"][0]["image_url"]["url"], + "https://example.com/image.png" + ); + assert!(result["messages"][0]["content"][0] + .get("cache_control") + .is_none()); + assert!(result["messages"][0]["content"][0] + .get("prompt_cache_breakpoint") + .is_none()); + } + #[test] fn test_openai_to_anthropic_simple() { let input = json!({ diff --git a/src-tauri/src/proxy/providers/transform_codex_anthropic.rs b/src-tauri/src/proxy/providers/transform_codex_anthropic.rs index 50b17b1bc..74959da8d 100644 --- a/src-tauri/src/proxy/providers/transform_codex_anthropic.rs +++ b/src-tauri/src/proxy/providers/transform_codex_anthropic.rs @@ -18,6 +18,9 @@ use super::transform_responses::{sanitize_anthropic_tool_use_input, TOOL_RESULT_ use crate::proxy::error::ProxyError; use crate::proxy::json_canonical::canonical_json_string; use crate::proxy::sse::{strip_sse_field, take_sse_block}; +use crate::proxy::tool_media::{ + strip_and_clamp_media_from_tool_value, ToolMediaScope, TOOL_RESULT_MEDIA_ATTACHED_MARKER, +}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use serde_json::{json, Value}; use std::collections::{BTreeMap, HashSet}; @@ -723,10 +726,12 @@ struct ToolResultContent { fn tool_result_content_from_responses_item(item: &Value) -> ToolResultContent { match item.get("output") { - Some(Value::String(text)) => ToolResultContent { - content: json!(text), - is_error: false, - }, + Some(text @ Value::String(_)) => { + alternate_image_tool_result_content(text).unwrap_or_else(|| ToolResultContent { + content: text.clone(), + is_error: false, + }) + } Some(Value::Array(parts)) => { let mut content = Vec::new(); let mut is_error = false; @@ -761,10 +766,26 @@ fn tool_result_content_from_responses_item(item: &Value) -> ToolResultContent { })); } } - _ => content.push(json!({ - "type":"text", - "text":canonical_json_string(part) - })), + _ => { + if let Some(alternate) = alternate_image_tool_result_content(part) { + is_error |= alternate.is_error; + match alternate.content { + Value::Array(mut blocks) => content.append(&mut blocks), + Value::String(text) => { + content.push(json!({"type":"text","text":text})) + } + other => content.push(json!({ + "type":"text", + "text":canonical_json_string(&other) + })), + } + } else { + content.push(json!({ + "type":"text", + "text":canonical_json_string(part) + })); + } + } } } ToolResultContent { @@ -772,10 +793,12 @@ fn tool_result_content_from_responses_item(item: &Value) -> ToolResultContent { is_error, } } - Some(value) => ToolResultContent { - content: json!(canonical_json_string(value)), - is_error: false, - }, + Some(value) => { + alternate_image_tool_result_content(value).unwrap_or_else(|| ToolResultContent { + content: json!(canonical_json_string(value)), + is_error: false, + }) + } None => ToolResultContent { content: json!(canonical_json_string(item)), is_error: false, @@ -783,6 +806,96 @@ fn tool_result_content_from_responses_item(item: &Value) -> ToolResultContent { } } +/// Convert image-bearing tool-output variants that are not native Responses +/// content blocks. The shared traversal recognizes JSON strings, MCP image +/// blocks, Anthropic image blocks, Chat image_url blocks, nested `content` +/// wrappers, and whole image data URLs. +fn alternate_image_tool_result_content(value: &Value) -> Option { + let mut cleaned = value.clone(); + let replacement_block = json!({ + "type":"input_text", + "text":TOOL_RESULT_MEDIA_ATTACHED_MARKER + }); + let mut chat_media_parts = Vec::new(); + let replaced = strip_and_clamp_media_from_tool_value( + &mut cleaned, + &mut chat_media_parts, + ToolMediaScope::ImagesOnly, + &replacement_block, + TOOL_RESULT_MEDIA_ATTACHED_MARKER, + ); + if replaced == 0 { + return None; + } + + let mut content = Vec::new(); + let mut is_error = false; + append_sanitized_tool_result_value(&cleaned, &mut content, &mut is_error); + content.extend( + chat_media_parts + .iter() + .filter_map(image_block_from_input_image), + ); + + Some(ToolResultContent { + content: Value::Array(content), + is_error, + }) +} + +fn append_sanitized_tool_result_value( + value: &Value, + content: &mut Vec, + is_error: &mut bool, +) { + match value { + Value::String(text) => { + if text == TOOL_RESULT_ERROR_MARKER { + *is_error = true; + } else if !text.is_empty() { + content.push(json!({"type":"text","text":text})); + } + } + Value::Array(parts) => { + for part in parts { + match part.get("type").and_then(Value::as_str) { + Some("input_text" | "output_text" | "text") => { + if let Some(text) = part.get("text").and_then(Value::as_str) { + if text == TOOL_RESULT_ERROR_MARKER { + *is_error = true; + } else { + content.push(json!({"type":"text","text":text})); + } + } + } + _ => content.push(json!({ + "type":"text", + "text":canonical_json_string(part) + })), + } + } + } + Value::Object(object) + if matches!( + object.get("type").and_then(Value::as_str), + Some("input_text" | "output_text" | "text") + ) => + { + if let Some(text) = object.get("text").and_then(Value::as_str) { + if text == TOOL_RESULT_ERROR_MARKER { + *is_error = true; + } else { + content.push(json!({"type":"text","text":text})); + } + } + } + other => content.push(json!({ + "type":"text", + "text":canonical_json_string(other) + })), + } +} + /// Ensures the first message is a user: compacted/resumed sessions may start with /// assistant or function_call, but Anthropic requires the first to be user, else 400. /// An empty array is not handled (the caller decides whether to error). @@ -1071,8 +1184,12 @@ fn image_block_from_input_image(part: &Value) -> Option { .or_else(|| v.get("url").and_then(|u| u.as_str()).map(str::to_string)) })?; - if let Some(rest) = url.strip_prefix("data:") { + if url + .get(..5) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")) + { // data:;base64, + let rest = &url[5..]; let (meta, data) = rest.split_once(',')?; let media_type = meta.split(';').next().unwrap_or("image/png"); Some(json!({ @@ -1083,7 +1200,13 @@ fn image_block_from_input_image(part: &Value) -> Option { "data": data } })) - } else if url.starts_with("http://") || url.starts_with("https://") { + } else if url + .get(..7) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("http://")) + || url + .get(..8) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("https://")) + { Some(json!({ "type": "image", "source": { "type": "url", "url": url } @@ -2555,6 +2678,80 @@ mod tests { assert_eq!(content[1]["type"], "image"); } + #[test] + fn test_alternate_mcp_tool_image_is_not_stringified_for_anthropic() { + let response = responses_request_to_anthropic( + json!({ + "model": "c", + "input": [ + {"type": "function_call", "call_id": "c1", "name": "inspect", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "c1", "output": [{ + "type": "image", + "mimeType": "image/webp", + "data": "MCP_ANTHROPIC_IMAGE_SENTINEL" + }]} + ] + }), + 4096, + ) + .unwrap(); + let content = &response["messages"][2]["content"][0]["content"]; + + assert_eq!(content[0]["type"], "text"); + assert!(!content[0]["text"] + .as_str() + .unwrap() + .contains("MCP_ANTHROPIC_IMAGE_SENTINEL")); + assert_eq!(content[1]["type"], "image"); + assert_eq!(content[1]["source"]["media_type"], "image/webp"); + assert_eq!(content[1]["source"]["data"], "MCP_ANTHROPIC_IMAGE_SENTINEL"); + } + + #[test] + fn test_json_string_nested_tool_image_is_not_text_for_anthropic() { + let residual_base64 = "A".repeat(20_000); + let encoded_output = json!({ + "content": [ + {"type": "input_text", "text": "caption"}, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,STRING_IMAGE_SENTINEL" + } + }, + {"type": "video", "data": residual_base64} + ] + }) + .to_string(); + let response = responses_request_to_anthropic( + json!({ + "model": "c", + "input": [ + {"type": "function_call", "call_id": "c1", "name": "inspect", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "c1", "output": encoded_output} + ] + }), + 4096, + ) + .unwrap(); + let content = response["messages"][2]["content"][0]["content"] + .as_array() + .unwrap(); + let image = content + .iter() + .find(|block| block["type"] == "image") + .expect("stringified tool image should become an Anthropic image block"); + + assert_eq!(image["source"]["data"], "STRING_IMAGE_SENTINEL"); + assert!(content + .iter() + .filter_map(|block| block.get("text").and_then(Value::as_str)) + .all(|text| !text.contains("STRING_IMAGE_SENTINEL"))); + let serialized = response.to_string(); + assert!(serialized.contains("[cc-switch: omitted 20000 bytes]")); + assert!(!serialized.contains(&"A".repeat(64))); + } + #[test] fn test_structured_tool_output_restores_error_file_and_unknown_parts() { let response = responses_request_to_anthropic( diff --git a/src-tauri/src/proxy/providers/transform_codex_chat.rs b/src-tauri/src/proxy/providers/transform_codex_chat.rs index 65217ab0f..5a0efe8bd 100644 --- a/src-tauri/src/proxy/providers/transform_codex_chat.rs +++ b/src-tauri/src/proxy/providers/transform_codex_chat.rs @@ -17,8 +17,9 @@ use crate::proxy::{ short_sha256_hex, }, tool_media::{ - chat_file_from_input_file, clamp_base64ish_strings, strip_media_from_tool_value, - whole_string_image_data_url, ToolMediaScope, + chat_file_from_input_file, flush_pending_chat_tool_media, plan_chat_tool_output_media, + queue_chat_tool_output_media, strip_and_clamp_media_from_tool_value, ToolMediaScope, + TOOL_RESULT_MEDIA_MOVED_MARKER, }, }; use serde_json::{json, Value}; @@ -46,9 +47,6 @@ const CUSTOM_TOOL_INPUT_FIELD: &str = "input"; const CHAT_TOOL_NAME_MAX_LEN: usize = 64; const CUSTOM_TOOL_INPUT_DESCRIPTION: &str = "Raw string input for the original custom tool. Preserve formatting exactly and follow the original tool definition embedded in the description."; const CUSTOM_TOOL_PRESERVED_METADATA_HEADING: &str = "Original tool definition:"; -const TOOL_RESULT_MEDIA_MOVED_MARKER: &str = - "[cc-switch: tool result media moved to the following user message]"; - #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum CodexToolKind { Function, @@ -545,70 +543,6 @@ fn instruction_text(value: &Value) -> String { } } -struct ToolOutputMediaPlan { - tool_content: String, - media_parts: Vec, -} - -fn plan_tool_output_media(mut output: Value) -> Option { - let replacement_block = json!({ - "type": "text", - "text": TOOL_RESULT_MEDIA_MOVED_MARKER - }); - let mut media_parts = Vec::new(); - let replaced = strip_media_from_tool_value( - &mut output, - &mut media_parts, - ToolMediaScope::AllSupported, - &replacement_block, - TOOL_RESULT_MEDIA_MOVED_MARKER, - ); - if replaced == 0 { - return None; - } - - // Only media-bearing outputs enter this path. Remove any residual large - // data/base64 scalar that was not itself a recognized content block while - // preserving ordinary long OCR/log/source text. - clamp_base64ish_strings(&mut output); - Some(ToolOutputMediaPlan { - tool_content: canonical_json_string(&output), - media_parts, - }) -} - -fn plan_raw_tool_output_data_url(output: &str) -> Option { - whole_string_image_data_url(output).map(|media_part| ToolOutputMediaPlan { - // A raw tool-output string remains a raw tool-output string. Do not add - // JSON string-literal quotes around the replacement marker. - tool_content: TOOL_RESULT_MEDIA_MOVED_MARKER.to_string(), - media_parts: vec![media_part], - }) -} - -fn queue_tool_output_media(pending_media: &mut Vec, call_id: &str, media_parts: Vec) { - if media_parts.is_empty() { - return; - } - - pending_media.push(json!({ - "type": "text", - "text": format!("[cc-switch: media output of tool call {call_id}]") - })); - pending_media.extend(media_parts); -} - -fn flush_pending_media(messages: &mut Vec, pending_media: &mut Vec) { - if pending_media.is_empty() { - return; - } - - messages.push(json!({ - "role": "user", - "content": std::mem::take(pending_media) - })); -} - fn append_responses_input_as_chat_messages( input: &Value, messages: &mut Vec, @@ -656,7 +590,7 @@ fn append_responses_input_as_chat_messages( // If a later assistant tool-call batch was accumulated after an earlier // media-bearing result, the synthetic user media belongs before that next // assistant turn. - flush_pending_media(messages, &mut pending_media); + flush_pending_chat_tool_media(messages, &mut pending_media); flush_pending_tool_calls( messages, &mut pending_tool_calls, @@ -712,21 +646,12 @@ fn append_responses_item_as_chat_message( last_assistant_index, ); let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or(""); - let media_plan = match item.get("output") { - Some(Value::String(output)) => { - plan_raw_tool_output_data_url(output).or_else(|| { - serde_json::from_str::(output.trim()) - .ok() - .and_then(plan_tool_output_media) - }) - } - Some(output @ (Value::Array(_) | Value::Object(_))) => { - plan_tool_output_media(output.clone()) - } - _ => None, - }; + let media_plan = item + .get("output") + .cloned() + .and_then(plan_chat_tool_output_media); let output = if let Some(media_plan) = media_plan { - queue_tool_output_media(pending_media, call_id, media_plan.media_parts); + queue_chat_tool_output_media(pending_media, call_id, media_plan.media_parts); media_plan.tool_content } else { // Cache-sensitive no-media fallback: keep these expressions @@ -761,7 +686,7 @@ fn append_responses_item_as_chat_message( let replaced = transformed_item .get_mut("output") .map(|output| { - strip_media_from_tool_value( + strip_and_clamp_media_from_tool_value( output, &mut media_parts, ToolMediaScope::AllSupported, @@ -771,10 +696,7 @@ fn append_responses_item_as_chat_message( }) .unwrap_or(0); let output = if replaced > 0 { - if let Some(output) = transformed_item.get_mut("output") { - clamp_base64ish_strings(output); - } - queue_tool_output_media(pending_media, call_id, media_parts); + queue_chat_tool_output_media(pending_media, call_id, media_parts); canonical_json_string(&transformed_item) } else { // Preserve the legacy whole-item representation exactly. @@ -807,7 +729,7 @@ fn append_responses_item_as_chat_message( // `flush_pending_tool_calls` intentionally returns early when // there is no new assistant batch. A previous tool result may // still have media waiting, so flush it before this new message. - flush_pending_media(messages, pending_media); + flush_pending_chat_tool_media(messages, pending_media); let role = item .get("role") .and_then(|v| v.as_str()) @@ -846,7 +768,7 @@ fn append_responses_item_as_chat_message( pending_reasoning, last_assistant_index, ); - flush_pending_media(messages, pending_media); + flush_pending_chat_tool_media(messages, pending_media); let message = responses_message_item_to_chat_message( item, pending_reasoning, @@ -876,7 +798,7 @@ fn append_responses_item_as_chat_message( pending_reasoning, last_assistant_index, ); - flush_pending_media(messages, pending_media); + flush_pending_chat_tool_media(messages, pending_media); let message = responses_message_item_to_chat_message( item, pending_reasoning, @@ -916,7 +838,7 @@ fn flush_pending_tool_calls( // Media from the preceding tool-result batch must be presented before a // new assistant tool-call turn. Consecutive outputs do not enter here // because `pending_tool_calls` is empty after the first output. - flush_pending_media(messages, pending_media); + flush_pending_chat_tool_media(messages, pending_media); let mut message = json!({ "role": "assistant", "content": null, @@ -3625,6 +3547,46 @@ mod tests { } } + #[test] + fn responses_request_to_chat_clamps_stringified_custom_output_residual_base64() { + let encoded_output = json!({ + "content": [ + { + "type": "input_image", + "image_url": "data:image/png;base64,CUSTOM_STRING_IMAGE_SENTINEL" + }, + { + "type": "video", + "data": "A".repeat(20_000) + } + ] + }) + .to_string(); + let result = convert_test_input(vec![ + json!({ + "type": "custom_tool_call", + "call_id": "call_custom_string", + "name": "render", + "input": "draw" + }), + json!({ + "type": "custom_tool_call_output", + "call_id": "call_custom_string", + "status": "completed", + "output": encoded_output + }), + ]); + let messages = result_messages(&result); + let tool_item: Value = + serde_json::from_str(messages[1]["content"].as_str().unwrap()).unwrap(); + let rewritten = tool_item["output"].as_str().unwrap(); + + assert!(rewritten.contains("[cc-switch: omitted 20000 bytes]")); + assert!(!rewritten.contains(&"A".repeat(64))); + assert!(!rewritten.contains("CUSTOM_STRING_IMAGE_SENTINEL")); + assert_eq!(messages[2]["content"][1]["type"], "image_url"); + } + #[test] fn responses_request_to_chat_rejects_false_positive_media_shapes() { let outputs = [ @@ -3741,22 +3703,21 @@ mod tests { let data_url = large_test_image_data_url(); let long_text = format!("{}end", "ordinary OCR text with spaces. ".repeat(3_500)); let residual_base64 = "A".repeat(20_000); + let encoded_output = json!([ + {"type": "input_image", "image_url": data_url.clone()}, + {"type": "text", "text": long_text.clone()}, + {"type": "video", "data": residual_base64} + ]) + .to_string(); let result = convert_test_input(vec![ test_function_call("call_clamp"), - test_function_output( - "call_clamp", - json!([ - {"type": "input_image", "image_url": data_url.clone()}, - {"type": "text", "text": long_text.clone()}, - {"raw": residual_base64} - ]), - ), + test_function_output("call_clamp", json!(encoded_output)), ]); let tool_content_text = result["messages"][1]["content"].as_str().unwrap(); let tool_content: Value = serde_json::from_str(tool_content_text).unwrap(); assert_eq!(tool_content[1]["text"], long_text); - assert!(tool_content[2]["raw"] + assert!(tool_content[2]["data"] .as_str() .unwrap() .starts_with("[cc-switch: omitted 20000 bytes]")); diff --git a/src-tauri/src/proxy/providers/transform_gemini.rs b/src-tauri/src/proxy/providers/transform_gemini.rs index 76601c3e1..ad68704b4 100644 --- a/src-tauri/src/proxy/providers/transform_gemini.rs +++ b/src-tauri/src/proxy/providers/transform_gemini.rs @@ -7,6 +7,9 @@ use super::gemini_schema::build_gemini_function_declaration; use super::gemini_shadow::{GeminiAssistantTurn, GeminiShadowStore, GeminiToolCallMeta}; use crate::proxy::error::ProxyError; +use crate::proxy::tool_media::{ + strip_and_clamp_media_from_tool_value, ToolMediaScope, TOOL_RESULT_MEDIA_ATTACHED_MARKER, +}; use serde_json::{json, Map, Value}; use std::collections::{HashMap, HashSet}; @@ -61,6 +64,10 @@ pub fn anthropic_to_gemini_with_shadow( .and_then(|((store, provider_id), session_id)| store.get_session(provider_id, session_id)) .map(|snapshot| snapshot.turns) .unwrap_or_default(); + let supports_multimodal_function_response = body + .get("model") + .and_then(Value::as_str) + .is_some_and(is_gemini_3_series); let messages = body.get("messages").and_then(|value| value.as_array()); @@ -73,7 +80,11 @@ pub fn anthropic_to_gemini_with_shadow( } if let Some(messages) = messages { - result["contents"] = json!(convert_messages_to_contents(messages, &shadow_turns)?); + result["contents"] = json!(convert_messages_to_contents( + messages, + &shadow_turns, + supports_multimodal_function_response, + )?); } if let Some(generation_config) = build_generation_config(&body) { @@ -361,6 +372,7 @@ fn build_generation_config(body: &Value) -> Option { fn convert_messages_to_contents( messages: &[Value], shadow_turns: &[GeminiAssistantTurn], + supports_multimodal_function_response: bool, ) -> Result, ProxyError> { let mut contents = Vec::new(); let mut used_shadow_indices = HashSet::new(); @@ -446,6 +458,7 @@ fn convert_messages_to_contents( role, &mut tool_name_by_id, &thought_signature_by_id, + supports_multimodal_function_response, )? } } else { @@ -454,6 +467,7 @@ fn convert_messages_to_contents( role, &mut tool_name_by_id, &thought_signature_by_id, + supports_multimodal_function_response, )? } } else { @@ -462,6 +476,7 @@ fn convert_messages_to_contents( role, &mut tool_name_by_id, &thought_signature_by_id, + supports_multimodal_function_response, )? }; @@ -560,6 +575,7 @@ fn convert_message_content_to_parts( role: &str, tool_name_by_id: &mut std::collections::HashMap, thought_signature_by_id: &std::collections::HashMap, + supports_multimodal_function_response: bool, ) -> Result, ProxyError> { let Some(content) = content else { return Ok(Vec::new()); @@ -705,16 +721,31 @@ fn convert_message_content_to_parts( )) })?; + let (response, media_parts) = plan_gemini_tool_result(block.get("content")); + // See `tool_use` above: synthesized ids must not leak upstream. let mut function_response = json!({ "name": name, - "response": normalize_tool_result_response(block.get("content")) + "response": response }); if !tool_use_id.is_empty() && !is_synthesized_tool_call_id(tool_use_id) { function_response["id"] = json!(tool_use_id); } - parts.push(json!({ "functionResponse": function_response })); + if supports_multimodal_function_response && !media_parts.is_empty() { + function_response["parts"] = Value::Array(media_parts); + parts.push(json!({ "functionResponse": function_response })); + } else { + parts.push(json!({ "functionResponse": function_response })); + if !media_parts.is_empty() { + parts.push(json!({ + "text": format!( + "[cc-switch: media output of tool call {tool_use_id}]" + ) + })); + parts.extend(media_parts); + } + } } "thinking" | "redacted_thinking" => {} _ => {} @@ -745,6 +776,74 @@ fn normalize_tool_result_response(content: Option<&Value>) -> Value { } } +fn plan_gemini_tool_result(content: Option<&Value>) -> (Value, Vec) { + let Some(content) = content else { + return (normalize_tool_result_response(None), Vec::new()); + }; + + let mut cleaned = content.clone(); + let replacement_block = json!({ + "type":"text", + "text":TOOL_RESULT_MEDIA_ATTACHED_MARKER + }); + let mut chat_media_parts = Vec::new(); + let replaced = strip_and_clamp_media_from_tool_value( + &mut cleaned, + &mut chat_media_parts, + ToolMediaScope::InlineImagesOnly, + &replacement_block, + TOOL_RESULT_MEDIA_ATTACHED_MARKER, + ); + if replaced == 0 { + return (normalize_tool_result_response(Some(content)), Vec::new()); + } + + let mut gemini_parts = Vec::new(); + gemini_parts.extend( + chat_media_parts + .iter() + .filter_map(gemini_part_from_chat_image), + ); + + (normalize_tool_result_response(Some(&cleaned)), gemini_parts) +} + +fn is_gemini_3_series(model: &str) -> bool { + let normalized = model.trim().to_ascii_lowercase(); + normalized.starts_with("gemini-3") + || normalized + .rsplit('/') + .next() + .is_some_and(|tail| tail.starts_with("gemini-3")) +} + +fn gemini_part_from_chat_image(part: &Value) -> Option { + let image_url = part + .pointer("/image_url/url") + .and_then(Value::as_str) + .filter(|url| !url.trim().is_empty())?; + + if image_url + .get(..5) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")) + { + let rest = &image_url[5..]; + let (meta, data) = rest.split_once(',')?; + if data.is_empty() || !meta.to_ascii_lowercase().contains(";base64") { + return None; + } + let mime_type = meta.split(';').next().unwrap_or("image/png"); + return Some(json!({ + "inlineData": { + "mimeType": mime_type, + "data": data + } + })); + } + + None +} + fn shadow_parts(content: &Value) -> Option> { let mut parts = content .get("parts") @@ -1266,6 +1365,290 @@ mod tests { ); } + #[test] + fn anthropic_to_gemini_moves_mixed_tool_result_image_to_native_part() { + let input = json!({ + "messages": [ + { + "role": "assistant", + "content": [{ + "type": "tool_use", + "id": "call_image", + "name": "inspect", + "input": {} + }] + }, + { + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "call_image", + "content": [ + {"type": "text", "text": "caption"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "GEMINI_TOOL_IMAGE_SENTINEL" + } + } + ] + }] + } + ] + }); + + let result = anthropic_to_gemini(input).unwrap(); + let parts = result["contents"][1]["parts"].as_array().unwrap(); + let response = &parts[0]["functionResponse"]["response"]["content"]; + + assert!(response.as_str().unwrap().contains("caption")); + assert!(response + .as_str() + .unwrap() + .contains("tool result media attached")); + assert!(!response + .as_str() + .unwrap() + .contains("GEMINI_TOOL_IMAGE_SENTINEL")); + assert_eq!( + parts[1]["text"], + "[cc-switch: media output of tool call call_image]" + ); + assert_eq!(parts[2]["inlineData"]["mimeType"], "image/png"); + assert_eq!(parts[2]["inlineData"]["data"], "GEMINI_TOOL_IMAGE_SENTINEL"); + } + + #[test] + fn anthropic_to_gemini_clamps_json_string_residual_base64_before_serializing() { + let residual_base64 = "A".repeat(20_000); + let encoded = json!({ + "content": [ + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,GEMINI_STRING_IMAGE_SENTINEL" + } + }, + {"type": "video", "data": residual_base64} + ] + }) + .to_string(); + let input = json!({ + "messages": [ + { + "role": "assistant", + "content": [{ + "type": "tool_use", + "id": "call_string", + "name": "inspect", + "input": {} + }] + }, + { + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "call_string", + "content": encoded + }] + } + ] + }); + + let result = anthropic_to_gemini(input).unwrap(); + let serialized = result.to_string(); + + assert!(serialized.contains("[cc-switch: omitted 20000 bytes]")); + assert!(!serialized.contains(&"A".repeat(64))); + assert_eq!( + result["contents"][1]["parts"][2]["inlineData"]["data"], + "GEMINI_STRING_IMAGE_SENTINEL" + ); + } + + #[test] + fn anthropic_to_gemini_keeps_remote_tool_image_in_legacy_response() { + let remote_image = json!({ + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/tool-image.png" + } + }); + let input = json!({ + "messages": [ + { + "role": "assistant", + "content": [{ + "type": "tool_use", + "id": "call_remote", + "name": "inspect", + "input": {} + }] + }, + { + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "call_remote", + "content": [remote_image.clone()] + }] + } + ] + }); + + let result = anthropic_to_gemini(input).unwrap(); + let parts = result["contents"][1]["parts"].as_array().unwrap(); + + assert_eq!(parts.len(), 1); + assert_eq!( + parts[0]["functionResponse"]["response"]["content"][0], + remote_image + ); + assert!(!result.to_string().contains("fileData")); + assert!(!result + .to_string() + .contains(TOOL_RESULT_MEDIA_ATTACHED_MARKER)); + } + + #[test] + fn anthropic_to_gemini_does_not_strip_unconvertible_tool_data_url() { + let malformed_image = json!({ + "type": "image_url", + "image_url": { + "url": "data:image/png,NOT_BASE64" + } + }); + let input = json!({ + "messages": [ + { + "role": "assistant", + "content": [{ + "type": "tool_use", + "id": "call_malformed", + "name": "inspect", + "input": {} + }] + }, + { + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "call_malformed", + "content": [malformed_image.clone()] + }] + } + ] + }); + + let result = anthropic_to_gemini(input).unwrap(); + let parts = result["contents"][1]["parts"].as_array().unwrap(); + + assert_eq!(parts.len(), 1); + assert_eq!( + parts[0]["functionResponse"]["response"]["content"][0], + malformed_image + ); + assert!(!result.to_string().contains("inlineData")); + assert!(!result + .to_string() + .contains(TOOL_RESULT_MEDIA_ATTACHED_MARKER)); + } + + #[test] + fn anthropic_to_gemini_does_not_embed_image_only_tool_result_as_json() { + let input = json!({ + "messages": [ + { + "role": "assistant", + "content": [{ + "type": "tool_use", + "id": "call_image", + "name": "inspect", + "input": {} + }] + }, + { + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "call_image", + "content": [{ + "type": "image", + "source": { + "type": "base64", + "media_type": "image/webp", + "data": "IMAGE_ONLY_SENTINEL" + } + }] + }] + } + ] + }); + + let result = anthropic_to_gemini(input).unwrap(); + let parts = result["contents"][1]["parts"].as_array().unwrap(); + let response = &parts[0]["functionResponse"]["response"]["content"]; + + assert!(response.is_string()); + assert!(!response.as_str().unwrap().contains("IMAGE_ONLY_SENTINEL")); + assert_eq!(parts[2]["inlineData"]["mimeType"], "image/webp"); + assert_eq!(parts[2]["inlineData"]["data"], "IMAGE_ONLY_SENTINEL"); + } + + #[test] + fn anthropic_to_gemini_3_uses_multimodal_function_response_parts() { + let input = json!({ + "model": "gemini-3-pro-preview", + "messages": [ + { + "role": "assistant", + "content": [{ + "type": "tool_use", + "id": "call_image", + "name": "inspect", + "input": {} + }] + }, + { + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "call_image", + "content": [{ + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": "GEMINI_3_IMAGE_SENTINEL" + } + }] + }] + } + ] + }); + + let result = anthropic_to_gemini(input).unwrap(); + let parts = result["contents"][1]["parts"].as_array().unwrap(); + let function_response = &parts[0]["functionResponse"]; + + assert_eq!(parts.len(), 1); + assert_eq!( + function_response["parts"][0]["inlineData"]["mimeType"], + "image/jpeg" + ); + assert_eq!( + function_response["parts"][0]["inlineData"]["data"], + "GEMINI_3_IMAGE_SENTINEL" + ); + assert!(!function_response["response"]["content"] + .as_str() + .unwrap() + .contains("GEMINI_3_IMAGE_SENTINEL")); + } + #[test] fn anthropic_to_gemini_resolves_tool_result_name_from_shadow_content() { let store = GeminiShadowStore::with_limits(8, 4); diff --git a/src-tauri/src/proxy/providers/transform_responses.rs b/src-tauri/src/proxy/providers/transform_responses.rs index 916a0dbf3..e63291ced 100644 --- a/src-tauri/src/proxy/providers/transform_responses.rs +++ b/src-tauri/src/proxy/providers/transform_responses.rs @@ -8,7 +8,13 @@ //! - system prompt 使用 `instructions` 字段而非 system role message //! - usage 字段命名与 Anthropic 一致 (input_tokens/output_tokens) -use crate::proxy::{error::ProxyError, json_canonical::canonical_json_string}; +use crate::proxy::{ + error::ProxyError, + json_canonical::canonical_json_string, + tool_media::{ + strip_and_clamp_media_from_tool_value, ToolMediaScope, TOOL_RESULT_MEDIA_ATTACHED_MARKER, + }, +}; use serde_json::{json, Value}; use super::reasoning_bridge::{ @@ -80,8 +86,11 @@ fn anthropic_tool_result_to_responses_output(block: &Value) -> Value { let content = block.get("content"); if !is_error { - if let Some(Value::String(text)) = content { - return json!(text); + if let Some(text @ Value::String(_)) = content { + if let Some(output) = alternate_image_tool_result_to_responses(text) { + return Value::Array(output); + } + return text.clone(); } } @@ -92,7 +101,13 @@ fn anthropic_tool_result_to_responses_output(block: &Value) -> Value { match content { Some(Value::String(text)) => { - output.push(json!({"type":"input_text","text":text})); + if let Some(mut alternate) = + alternate_image_tool_result_to_responses(&Value::String(text.clone())) + { + output.append(&mut alternate); + } else { + output.push(json!({"type":"input_text","text":text})); + } } Some(Value::Array(blocks)) => { for part in blocks { @@ -105,6 +120,10 @@ fn anthropic_tool_result_to_responses_output(block: &Value) -> Value { Some("image") => { if let Some(image) = anthropic_image_to_responses_part(part) { output.push(image); + } else if let Some(mut alternate) = + alternate_image_tool_result_to_responses(part) + { + output.append(&mut alternate); } else { output.push(json!({ "type":"input_text", @@ -122,6 +141,77 @@ fn anthropic_tool_result_to_responses_output(block: &Value) -> Value { })); } } + _ => { + if let Some(mut alternate) = alternate_image_tool_result_to_responses(part) + { + output.append(&mut alternate); + } else { + output.push(json!({ + "type":"input_text", + "text":canonical_json_string(part) + })); + } + } + } + } + } + Some(value) => { + if let Some(mut alternate) = alternate_image_tool_result_to_responses(value) { + output.append(&mut alternate); + } else { + output.push(json!({ + "type":"input_text", + "text":canonical_json_string(value) + })); + } + } + None => {} + } + + Value::Array(output) +} + +fn alternate_image_tool_result_to_responses(value: &Value) -> Option> { + let mut cleaned = value.clone(); + let replacement_block = json!({ + "type":"input_text", + "text":TOOL_RESULT_MEDIA_ATTACHED_MARKER + }); + let mut chat_media_parts = Vec::new(); + let replaced = strip_and_clamp_media_from_tool_value( + &mut cleaned, + &mut chat_media_parts, + ToolMediaScope::ImagesOnly, + &replacement_block, + TOOL_RESULT_MEDIA_ATTACHED_MARKER, + ); + if replaced == 0 { + return None; + } + + let mut output = Vec::new(); + append_sanitized_responses_tool_value(&cleaned, &mut output); + output.extend( + chat_media_parts + .iter() + .filter_map(responses_image_from_chat_media), + ); + Some(output) +} + +fn append_sanitized_responses_tool_value(value: &Value, output: &mut Vec) { + match value { + Value::String(text) if !text.is_empty() => { + output.push(json!({"type":"input_text","text":text})); + } + Value::Array(parts) => { + for part in parts { + match part.get("type").and_then(Value::as_str) { + Some("input_text" | "output_text" | "text") => { + if let Some(text) = part.get("text").and_then(Value::as_str) { + output.push(json!({"type":"input_text","text":text})); + } + } _ => output.push(json!({ "type":"input_text", "text":canonical_json_string(part) @@ -129,14 +219,37 @@ fn anthropic_tool_result_to_responses_output(block: &Value) -> Value { } } } - Some(value) => output.push(json!({ + Value::Object(object) + if matches!( + object.get("type").and_then(Value::as_str), + Some("input_text" | "output_text" | "text") + ) => + { + if let Some(text) = object.get("text").and_then(Value::as_str) { + output.push(json!({"type":"input_text","text":text})); + } + } + Value::Null | Value::String(_) => {} + other => output.push(json!({ "type":"input_text", - "text":canonical_json_string(value) + "text":canonical_json_string(other) })), - None => {} } +} - Value::Array(output) +fn responses_image_from_chat_media(part: &Value) -> Option { + let image_url = part + .pointer("/image_url/url") + .and_then(Value::as_str) + .filter(|url| !url.trim().is_empty())?; + let mut image = json!({ + "type":"input_image", + "image_url":image_url + }); + if let Some(detail) = part.pointer("/image_url/detail") { + image["detail"] = detail.clone(); + } + Some(image) } pub(crate) fn sanitize_anthropic_tool_use_input(name: &str, input: Value) -> Value { @@ -1157,6 +1270,78 @@ mod tests { assert_eq!(output[3]["filename"], "trace.pdf"); } + #[test] + fn test_anthropic_to_responses_converts_mcp_tool_image() { + let input = json!({ + "model":"gpt-5", + "messages":[{"role":"user","content":[{ + "type":"tool_result", + "tool_use_id":"call_1", + "content":[{ + "type":"image", + "mimeType":"image/webp", + "data":"MCP_RESPONSES_IMAGE_SENTINEL" + }] + }]}] + }); + + let result = anthropic_to_responses(input, None, false, false).unwrap(); + let output = result["input"][0]["output"].as_array().unwrap(); + + assert_eq!(output[0]["type"], "input_text"); + assert!(!output[0]["text"] + .as_str() + .unwrap() + .contains("MCP_RESPONSES_IMAGE_SENTINEL")); + assert_eq!(output[1]["type"], "input_image"); + assert_eq!( + output[1]["image_url"], + "data:image/webp;base64,MCP_RESPONSES_IMAGE_SENTINEL" + ); + } + + #[test] + fn test_anthropic_to_responses_converts_json_string_tool_image() { + let residual_base64 = "A".repeat(20_000); + let encoded = json!({ + "content":[ + { + "type":"image_url", + "image_url":{"url":"data:image/png;base64,STRING_RESPONSES_SENTINEL"} + }, + {"type":"video","data":residual_base64} + ] + }) + .to_string(); + let input = json!({ + "model":"gpt-5", + "messages":[{"role":"user","content":[{ + "type":"tool_result", + "tool_use_id":"call_1", + "content":encoded + }]}] + }); + + let result = anthropic_to_responses(input, None, false, false).unwrap(); + let output = result["input"][0]["output"].as_array().unwrap(); + let image = output + .iter() + .find(|part| part["type"] == "input_image") + .expect("stringified image must stay a Responses image"); + + assert_eq!( + image["image_url"], + "data:image/png;base64,STRING_RESPONSES_SENTINEL" + ); + assert!(output + .iter() + .filter_map(|part| part.get("text").and_then(Value::as_str)) + .all(|text| !text.contains("STRING_RESPONSES_SENTINEL"))); + let serialized = result.to_string(); + assert!(serialized.contains("[cc-switch: omitted 20000 bytes]")); + assert!(!serialized.contains(&"A".repeat(64))); + } + #[test] fn test_anthropic_to_responses_thinking_discarded() { let input = json!({ diff --git a/src-tauri/src/proxy/tool_media.rs b/src-tauri/src/proxy/tool_media.rs index 0c3a907d6..03389931c 100644 --- a/src-tauri/src/proxy/tool_media.rs +++ b/src-tauri/src/proxy/tool_media.rs @@ -1,15 +1,19 @@ -//! Shared media handling for Codex tool outputs. +//! Shared media handling for tool outputs. //! -//! Responses tool outputs may carry structured media blocks. Chat Completions -//! tool messages are text-only, so the Responses→Chat bridge extracts those -//! blocks and re-emits them in a synthetic user message. The media sanitizer -//! reuses the same recognition and traversal rules when it needs to remove -//! images for a text-only upstream. +//! Responses and Anthropic tool outputs may carry structured media blocks. +//! Chat Completions tool messages are text-only, so protocol bridges extract +//! those blocks and re-emit them in a synthetic user message. The media +//! sanitizer reuses the same recognition and traversal rules when it needs to +//! remove images for a text-only upstream. use crate::proxy::json_canonical::canonical_json_string; use serde_json::{json, Map, Value}; pub(crate) const WHOLE_DATA_URL_MIN_BYTES: usize = 8 * 1024; +pub(crate) const TOOL_RESULT_MEDIA_MOVED_MARKER: &str = + "[cc-switch: tool result media moved to the following user message]"; +pub(crate) const TOOL_RESULT_MEDIA_ATTACHED_MARKER: &str = + "[cc-switch: tool result media attached as native media]"; const BASE64ISH_MIN_BYTES: usize = 16 * 1024; const MAX_MEDIA_TRAVERSAL_DEPTH: usize = 32; @@ -17,7 +21,11 @@ const MAX_MEDIA_TRAVERSAL_DEPTH: usize = 32; pub(crate) enum ToolMediaScope { /// Used by the existing image-capability sanitizer and its retry path. ImagesOnly, - /// Used by Responses→Chat conversion, where user messages can carry all + /// Used by Gemini Native `generateContent`, whose existing bridge only + /// promises inline base64 image input. Remote URLs and malformed data URLs + /// must stay in the legacy tool-result representation. + InlineImagesOnly, + /// Used by Chat conversion bridges, where user messages can carry all /// currently mapped Chat input modalities. AllSupported, } @@ -29,21 +37,95 @@ enum ToolMediaKind { Audio, } +pub(crate) struct ChatToolOutputMediaPlan { + pub(crate) tool_content: String, + pub(crate) media_parts: Vec, +} + impl ToolMediaScope { fn allows(self, kind: ToolMediaKind) -> bool { matches!(kind, ToolMediaKind::Image) || matches!(self, Self::AllSupported) } + + fn accepts_chat_part(self, part: &Value) -> bool { + !matches!(self, Self::InlineImagesOnly) || chat_image_part_has_inline_data(part) + } } -/// Convert one recognized Responses/tool media block to a Chat user content -/// part. This is the single shape-recognition entry point used by extraction. +/// Build a Chat-compatible tool-output plan without changing no-media output. +/// +/// Scalar strings remain scalar strings after replacement. This matters for a +/// raw image data URL and for JSON encoded inside a tool-output string: adding +/// another layer of JSON string quotes would change what the model sees. +pub(crate) fn plan_chat_tool_output_media(mut output: Value) -> Option { + let output_was_string = output.is_string(); + let replacement_block = json!({ + "type": "text", + "text": TOOL_RESULT_MEDIA_MOVED_MARKER + }); + let mut media_parts = Vec::new(); + let replaced = strip_and_clamp_media_from_tool_value( + &mut output, + &mut media_parts, + ToolMediaScope::AllSupported, + &replacement_block, + TOOL_RESULT_MEDIA_MOVED_MARKER, + ); + if replaced == 0 { + return None; + } + + let tool_content = if output_was_string { + output.as_str().unwrap_or_default().to_string() + } else { + canonical_json_string(&output) + }; + + Some(ChatToolOutputMediaPlan { + tool_content, + media_parts, + }) +} + +pub(crate) fn queue_chat_tool_output_media( + pending_media: &mut Vec, + call_id: &str, + media_parts: Vec, +) { + if media_parts.is_empty() { + return; + } + + pending_media.push(json!({ + "type": "text", + "text": format!("[cc-switch: media output of tool call {call_id}]") + })); + pending_media.extend(media_parts); +} + +pub(crate) fn flush_pending_chat_tool_media( + messages: &mut Vec, + pending_media: &mut Vec, +) { + if pending_media.is_empty() { + return; + } + + messages.push(json!({ + "role": "user", + "content": std::mem::take(pending_media) + })); +} + +/// Convert one recognized tool media block to a Chat user content part. This +/// is the single shape-recognition entry point used by extraction. pub(crate) fn chat_media_part_from_tool_part(part: &Value, scope: ToolMediaScope) -> Option { let kind = tool_media_kind(part)?; if !scope.allows(kind) { return None; } - match kind { + let chat_part = match kind { ToolMediaKind::Image => chat_image_part(part), ToolMediaKind::File => chat_file_from_input_file(part).map(|file| { json!({ @@ -57,7 +139,9 @@ pub(crate) fn chat_media_part_from_tool_part(part: &Value, scope: ToolMediaScope "input_audio": input_audio.clone() }) }), - } + }?; + + scope.accepts_chat_part(&chat_part).then_some(chat_part) } /// Map a Responses `input_file` block to the Chat file payload. Kept here so @@ -122,10 +206,37 @@ pub(crate) fn strip_media_from_tool_value( scope, replacement_block, replacement_text, + false, 0, ) } +/// Extract media and clamp residual large data/base64 scalars on media-bearing +/// outputs. Parseable JSON strings are clamped while still represented as a +/// JSON tree, before they are canonicalized back into their original string +/// container. +pub(crate) fn strip_and_clamp_media_from_tool_value( + value: &mut Value, + media_parts: &mut Vec, + scope: ToolMediaScope, + replacement_block: &Value, + replacement_text: &str, +) -> usize { + let replaced = strip_media_from_tool_value_at_depth( + value, + media_parts, + scope, + replacement_block, + replacement_text, + true, + 0, + ); + if replaced > 0 { + clamp_base64ish_strings(value); + } + replaced +} + /// Remove residual data/base64 payloads only after a tool output has already /// been positively identified as media-bearing. Ordinary long text is kept. pub(crate) fn clamp_base64ish_strings(value: &mut Value) { @@ -181,7 +292,7 @@ fn tool_output_contains_media_at_depth(value: &Value, scope: ToolMediaScope, dep .iter() .any(|item| tool_output_contains_media_at_depth(item, scope, depth + 1)), Value::Object(object) => { - if tool_media_kind(value).is_some_and(|kind| scope.allows(kind)) { + if chat_media_part_from_tool_part(value, scope).is_some() { return true; } @@ -199,6 +310,7 @@ fn strip_media_from_tool_value_at_depth( scope: ToolMediaScope, replacement_block: &Value, replacement_text: &str, + clamp_parsed_strings: bool, depth: usize, ) -> usize { if depth > MAX_MEDIA_TRAVERSAL_DEPTH { @@ -228,9 +340,13 @@ fn strip_media_from_tool_value_at_depth( scope, replacement_block, replacement_text, + clamp_parsed_strings, depth + 1, ); if replaced > 0 { + if clamp_parsed_strings { + clamp_base64ish_strings(&mut parsed); + } *text = canonical_json_string(&parsed); } replaced @@ -244,6 +360,7 @@ fn strip_media_from_tool_value_at_depth( scope, replacement_block, replacement_text, + clamp_parsed_strings, depth + 1, ) }) @@ -266,6 +383,7 @@ fn strip_media_from_tool_value_at_depth( scope, replacement_block, replacement_text, + clamp_parsed_strings, depth + 1, ) }) @@ -297,13 +415,9 @@ fn tool_media_kind(part: &Value) -> Option { fn chat_image_part(part: &Value) -> Option { match part.get("type").and_then(Value::as_str) { - Some("input_image" | "image_url") => { - normalized_image_url(part).map(|image_url| image_url_content_part(part, image_url)) - } - Some("image") => { - typed_image_url(part).map(|image_url| image_url_content_part(part, image_url)) - } - None => loose_data_image_url(part).map(|image_url| image_url_content_part(part, image_url)), + Some("input_image" | "image_url") => normalized_image_url(part).map(image_url_content_part), + Some("image") => typed_image_url(part).map(image_url_content_part), + None => loose_data_image_url(part).map(image_url_content_part), _ => None, } } @@ -440,19 +554,25 @@ fn typed_image_url(part: &Value) -> Option { Some(Value::Object(image_url)) } -fn image_url_content_part(source_part: &Value, image_url: Value) -> Value { +fn image_url_content_part(image_url: Value) -> Value { let mut content_part = Map::new(); content_part.insert("type".to_string(), Value::String("image_url".to_string())); content_part.insert("image_url".to_string(), image_url); - - for key in ["cache_control", "prompt_cache_breakpoint"] { - if let Some(value) = source_part.get(key) { - content_part.insert(key.to_string(), value.clone()); - } - } Value::Object(content_part) } +fn chat_image_part_has_inline_data(part: &Value) -> bool { + part.pointer("/image_url/url") + .and_then(Value::as_str) + .is_some_and(|url| { + let trimmed = url.trim(); + let Some(comma_index) = trimmed.find(',') else { + return false; + }; + comma_index + 1 < trimmed.len() && is_image_base64_data_url(trimmed) + }) +} + fn merge_top_level_detail(part: &Value, image_url: &mut Map) { if image_url.get("detail").is_none() { if let Some(detail) = part.get("detail") { @@ -529,12 +649,23 @@ mod tests { "image_url": { "url": "https://example.com/image.png", "detail": "low" - } + }, + "cache_control": {"type": "ephemeral"}, + "prompt_cache_breakpoint": true }); let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap(); - assert_eq!(mapped, part); + assert_eq!( + mapped, + json!({ + "type": "image_url", + "image_url": { + "url": "https://example.com/image.png", + "detail": "low" + } + }) + ); } #[test] @@ -625,6 +756,43 @@ mod tests { )); } + #[test] + fn inline_image_scope_rejects_remote_and_malformed_data_urls() { + let inline = json!({ + "type": "image_url", + "image_url": {"url": "data:image/png;base64,YWJj"} + }); + let remote = json!({ + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"} + }); + let missing_base64 = json!({ + "type": "image_url", + "image_url": {"url": "data:image/png,YWJj"} + }); + let empty_data = json!({ + "type": "image_url", + "image_url": {"url": "data:image/png;base64,"} + }); + + assert!(tool_output_contains_media( + &inline, + ToolMediaScope::InlineImagesOnly + )); + assert!(!tool_output_contains_media( + &remote, + ToolMediaScope::InlineImagesOnly + )); + assert!(!tool_output_contains_media( + &missing_base64, + ToolMediaScope::InlineImagesOnly + )); + assert!(!tool_output_contains_media( + &empty_data, + ToolMediaScope::InlineImagesOnly + )); + } + #[test] fn does_not_scan_embedded_data_urls_inside_plain_text() { let data_url = large_image_data_url(); @@ -693,6 +861,55 @@ mod tests { assert!(!serialized.contains("iVBORw0KGgo")); } + #[test] + fn chat_plan_keeps_scalar_tool_strings_unquoted() { + let raw_data_url = large_image_data_url(); + let raw_plan = plan_chat_tool_output_media(Value::String(raw_data_url.clone())).unwrap(); + assert_eq!(raw_plan.tool_content, TOOL_RESULT_MEDIA_MOVED_MARKER); + assert_eq!(raw_plan.media_parts[0]["image_url"]["url"], raw_data_url); + + let encoded = json!({ + "content": [{ + "type": "input_image", + "image_url": raw_data_url + }] + }) + .to_string(); + let encoded_plan = plan_chat_tool_output_media(Value::String(encoded)).unwrap(); + assert!(encoded_plan.tool_content.starts_with('{')); + assert!(encoded_plan + .tool_content + .contains(TOOL_RESULT_MEDIA_MOVED_MARKER)); + assert!(!encoded_plan.tool_content.starts_with('"')); + } + + #[test] + fn chat_plan_clamps_residual_base64_inside_json_string_before_serializing() { + let residual_base64 = "A".repeat(20_000); + let encoded = json!({ + "content": [ + { + "type": "input_image", + "image_url": "data:image/png;base64,IMAGE_SENTINEL" + }, + { + "type": "video", + "data": residual_base64 + } + ] + }) + .to_string(); + + let plan = plan_chat_tool_output_media(Value::String(encoded)).unwrap(); + + assert!(plan + .tool_content + .contains("[cc-switch: omitted 20000 bytes]")); + assert!(!plan.tool_content.contains(&"A".repeat(64))); + assert!(!plan.tool_content.contains("IMAGE_SENTINEL")); + assert_eq!(plan.media_parts.len(), 1); + } + #[test] fn image_only_scope_ignores_file_and_audio() { let file = json!({"type": "input_file", "file_id": "file_1"});