From 6c9d444c8a9245800cc0482a697a0b24c69043cd Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 24 Jul 2026 10:16:57 +0800 Subject: [PATCH] fix(proxy): move Codex tool-result media out of stringified tool text The Responses->Chat conversion serialized image-bearing *_output items into role:"tool" text via canonical_json_string, so view_image results were tokenized as base64 text (~9000x inflation). Codex replays full history every turn, so sessions hit context-limit 400s and wedged (#4465, #5663). - add proxy/tool_media: shared detection/strip/clamp walker for tool output media (typed input_image / image_url / input_file / input_audio, Anthropic source and MCP data+mimeType image shapes, untyped data: image_url, whole-string bare data URLs) - transform_codex_chat: replace media blocks in place with marker text so tool content stays a plain string, and flush the extracted media as one synthetic role:"user" message after each consecutive tool batch; media-free traffic stays byte-identical to keep prompt-cache prefixes stable - media_sanitizer: detect and strip tool-output media symmetrically (including JSON-string outputs) so reactive image stripping can heal upstream modality rejections - forwarder: regression tests pinning the reactive trigger and the context-limit-400 non-trigger E2E against Kimi K3 through the proxy: the replayed turn stays ~12k input tokens with 99% cache hit, versus ~85k+ of base64 text per replay before. --- src-tauri/src/proxy/forwarder.rs | 46 + src-tauri/src/proxy/media_sanitizer.rs | 243 ++++++ src-tauri/src/proxy/mod.rs | 1 + .../proxy/providers/transform_codex_chat.rs | 785 +++++++++++++++++- src-tauri/src/proxy/tool_media.rs | 774 +++++++++++++++++ 5 files changed, 1820 insertions(+), 29 deletions(-) create mode 100644 src-tauri/src/proxy/tool_media.rs diff --git a/src-tauri/src/proxy/forwarder.rs b/src-tauri/src/proxy/forwarder.rs index 0f86857fa..ebed6c57c 100644 --- a/src-tauri/src/proxy/forwarder.rs +++ b/src-tauri/src/proxy/forwarder.rs @@ -4755,6 +4755,27 @@ mod tests { }) } + fn body_with_codex_tool_output_image(stringified: bool) -> Value { + let output = json!({ + "content": [{ + "type": "input_image", + "image_url": "data:image/png;base64,TOOL_OUTPUT_SENTINEL" + }] + }); + json!({ + "model": "any-model", + "input": [{ + "type": "function_call_output", + "call_id": "call_1", + "output": if stringified { + Value::String(output.to_string()) + } else { + output + } + }] + }) + } + fn image_unsupported_error() -> ProxyError { ProxyError::UpstreamError { status: 400, @@ -4857,6 +4878,31 @@ mod tests { assert!(fwd.media_retry_should_trigger("Codex", false, &body, &error)); } + #[test] + fn reactive_triggers_for_structured_and_stringified_codex_tool_images() { + let fwd = forwarder_with_rectifier(RectifierConfig::default()); + + for stringified in [false, true] { + let body = body_with_codex_tool_output_image(stringified); + assert!( + fwd.media_retry_should_trigger("Codex", false, &body, &image_unsupported_error()), + "tool-output image should trigger retry (stringified={stringified})" + ); + } + } + + #[test] + fn reactive_does_not_treat_context_limit_as_image_rejection() { + let fwd = forwarder_with_rectifier(RectifierConfig::default()); + let body = body_with_codex_tool_output_image(false); + let context_error = ProxyError::UpstreamError { + status: 400, + body: Some(r#"{"error":{"message":"maximum context length exceeded"}}"#.to_string()), + }; + + assert!(!fwd.media_retry_should_trigger("Codex", false, &body, &context_error)); + } + #[test] fn reactive_skipped_when_media_fallback_off() { // 关闭 request_media_fallback：上游报图片错误也不触发兜底重试。 diff --git a/src-tauri/src/proxy/media_sanitizer.rs b/src-tauri/src/proxy/media_sanitizer.rs index ec1b4ea0c..509c45187 100644 --- a/src-tauri/src/proxy/media_sanitizer.rs +++ b/src-tauri/src/proxy/media_sanitizer.rs @@ -3,6 +3,9 @@ use crate::model_capabilities::is_confirmed_text_only_model as confirmed_text_on use crate::model_capabilities::{image_input_capability_from_settings, ImageInputCapability}; use crate::provider::Provider; use crate::proxy::error::ProxyError; +use crate::proxy::tool_media::{ + strip_media_from_tool_value, tool_output_contains_media, ToolMediaScope, +}; use serde_json::{json, Value}; pub const UNSUPPORTED_IMAGE_MARKER: &str = "[Unsupported Image]"; @@ -193,6 +196,9 @@ fn responses_input_item_has_image_blocks(item: &Value) -> bool { } item.get("content").is_some_and(content_has_image_blocks) + || item + .get("output") + .is_some_and(|output| tool_output_contains_media(output, ToolMediaScope::ImagesOnly)) } fn replace_images_in_responses_input(input: &mut Value) -> usize { @@ -218,6 +224,23 @@ fn replace_images_in_responses_input_item(item: &mut Value) -> usize { replaced += replace_images_in_content_with_text_type(content, "input_text"); } + if let Some(output) = item.get_mut("output") { + // The image-capability fallback deliberately strips images only. + // Tool-output files/audio remain a known unsupported-modality gap. + let replacement_block = json!({ + "type": "input_text", + "text": UNSUPPORTED_IMAGE_MARKER + }); + let mut discarded_media = Vec::new(); + replaced += strip_media_from_tool_value( + output, + &mut discarded_media, + ToolMediaScope::ImagesOnly, + &replacement_block, + UNSUPPORTED_IMAGE_MARKER, + ); + } + replaced } @@ -283,6 +306,13 @@ mod tests { } } + fn large_tool_data_url() -> String { + format!( + "data:image/png;base64,{}", + "SANITIZER_TOOL_MEDIA_SENTINEL".repeat(400) + ) + } + #[test] fn keeps_images_when_model_capability_is_unknown() { let provider = provider(json!({})); @@ -603,6 +633,219 @@ mod tests { ); } + #[test] + fn detects_and_replaces_responses_function_output_images() { + let data_url = large_tool_data_url(); + let mut body = json!({ + "model": "text-only", + "input": [{ + "type": "function_call_output", + "call_id": "call_1", + "output": { + "content": [ + {"type": "input_text", "text": "caption"}, + {"type": "input_image", "image_url": data_url.clone()}, + {"type": "image", "mimeType": "image/webp", "data": "MCP_SENTINEL"} + ] + } + }] + }); + + assert!(contains_image_blocks(&body)); + let replaced = replace_image_blocks_with_marker(&mut body); + + assert_eq!(replaced, 2); + assert_eq!( + body["input"][0]["output"]["content"][1], + json!({"type": "input_text", "text": UNSUPPORTED_IMAGE_MARKER}) + ); + assert_eq!( + body["input"][0]["output"]["content"][2], + json!({"type": "input_text", "text": UNSUPPORTED_IMAGE_MARKER}) + ); + assert!(!body.to_string().contains(&data_url)); + assert!(!body.to_string().contains("MCP_SENTINEL")); + } + + #[test] + fn proactive_text_only_sanitizer_covers_responses_tool_outputs() { + let provider = provider(json!({ + "models": [{"id": "text-model", "input": ["text"]}] + })); + let mut body = json!({ + "model": "text-model", + "input": [{ + "type": "function_call_output", + "call_id": "call_1", + "output": [{ + "type": "input_image", + "image_url": "data:image/png;base64,PROACTIVE_SENTINEL" + }] + }] + }); + + let replaced = replace_images_for_text_only_model(&mut body, &provider, true); + + assert_eq!(replaced, 1); + assert_eq!(body["input"][0]["output"][0]["type"], "input_text"); + assert!(!body.to_string().contains("PROACTIVE_SENTINEL")); + } + + #[test] + fn detects_and_replaces_json_string_tool_output_symmetrically() { + let data_url = large_tool_data_url(); + let output = json!({ + "content": [{ + "type": "input_image", + "image_url": data_url.clone() + }] + }) + .to_string(); + let mut body = json!({ + "input": [{ + "type": "function_call_output", + "call_id": "call_string", + "output": output + }] + }); + + assert!(contains_image_blocks(&body)); + let replaced = replace_image_blocks_with_marker(&mut body); + + assert_eq!(replaced, 1); + let rewritten = body["input"][0]["output"].as_str().unwrap(); + assert!(rewritten.contains(UNSUPPORTED_IMAGE_MARKER)); + assert!(!rewritten.contains(&data_url)); + let parsed: Value = serde_json::from_str(rewritten).unwrap(); + assert_eq!(parsed["content"][0]["type"], "input_text"); + } + + #[test] + fn detects_and_replaces_whole_string_tool_image_data_url() { + let data_url = large_tool_data_url(); + let mut body = json!({ + "input": [{ + "type": "function_call_output", + "call_id": "call_raw", + "output": data_url.clone() + }] + }); + + assert!(contains_image_blocks(&body)); + let replaced = replace_image_blocks_with_marker(&mut body); + + assert_eq!(replaced, 1); + assert_eq!( + body["input"][0]["output"], + Value::String(UNSUPPORTED_IMAGE_MARKER.to_string()) + ); + assert!(!body.to_string().contains(&data_url)); + } + + #[test] + fn detects_and_replaces_custom_tool_output_images() { + let mut body = json!({ + "input": [{ + "type": "custom_tool_call_output", + "call_id": "call_custom", + "status": "completed", + "output": [{ + "type": "image_url", + "image_url": {"url": "https://example.com/render.png"} + }] + }] + }); + + assert!(contains_image_blocks(&body)); + let replaced = replace_image_blocks_with_marker(&mut body); + + assert_eq!(replaced, 1); + assert_eq!(body["input"][0]["status"], "completed"); + assert_eq!(body["input"][0]["output"][0]["type"], "input_text"); + } + + #[test] + fn ignores_no_media_and_untyped_remote_tool_outputs() { + let mut body = json!({ + "input": [ + { + "type": "function_call_output", + "call_id": "call_text", + "output": {"content": [{"type": "text", "text": "ordinary result"}]} + }, + { + "type": "tool_search_output", + "call_id": "call_search", + "output": { + "image_url": {"url": "https://example.com/search-thumbnail.png"} + } + } + ] + }); + let original = body.clone(); + + assert!(!contains_image_blocks(&body)); + assert_eq!(replace_image_blocks_with_marker(&mut body), 0); + assert_eq!(body, original); + } + + #[test] + fn image_retry_scope_intentionally_ignores_tool_files_and_audio() { + let mut body = json!({ + "input": [{ + "type": "function_call_output", + "call_id": "call_modalities", + "output": { + "content": [ + {"type": "input_file", "file_id": "file_1"}, + { + "type": "input_audio", + "input_audio": {"data": "AUDIO", "format": "wav"} + } + ] + } + }] + }); + let original = body.clone(); + + assert!(!contains_image_blocks(&body)); + assert_eq!(replace_image_blocks_with_marker(&mut body), 0); + assert_eq!(body, original); + } + + #[test] + fn replaces_synthetic_user_and_tool_role_chat_image_parts() { + let mut body = json!({ + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "tool media"}, + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,USER_SENTINEL"} + } + ] + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{ + "type": "image_url", + "image_url": {"url": "https://example.com/tool.png"} + }] + } + ] + }); + + assert!(contains_image_blocks(&body)); + let replaced = replace_image_blocks_with_marker(&mut body); + + assert_eq!(replaced, 2); + assert_eq!(body["messages"][0]["content"][1]["type"], "text"); + assert_eq!(body["messages"][1]["content"][0]["type"], "text"); + } + #[test] fn detects_unsupported_image_errors() { let error = ProxyError::UpstreamError { diff --git a/src-tauri/src/proxy/mod.rs b/src-tauri/src/proxy/mod.rs index cbc24404c..cd843e20a 100644 --- a/src-tauri/src/proxy/mod.rs +++ b/src-tauri/src/proxy/mod.rs @@ -33,6 +33,7 @@ pub(crate) mod switch_lock; pub mod thinking_budget_rectifier; pub mod thinking_optimizer; pub mod thinking_rectifier; +pub(crate) mod tool_media; pub(crate) mod types; pub mod usage; diff --git a/src-tauri/src/proxy/providers/transform_codex_chat.rs b/src-tauri/src/proxy/providers/transform_codex_chat.rs index 05f27f391..65217ab0f 100644 --- a/src-tauri/src/proxy/providers/transform_codex_chat.rs +++ b/src-tauri/src/proxy/providers/transform_codex_chat.rs @@ -16,6 +16,10 @@ use crate::proxy::{ canonical_json_string, canonicalize_json_string_if_parseable, canonicalize_tool_arguments, short_sha256_hex, }, + tool_media::{ + chat_file_from_input_file, clamp_base64ish_strings, strip_media_from_tool_value, + whole_string_image_data_url, ToolMediaScope, + }, }; use serde_json::{json, Value}; use std::collections::{HashMap, HashSet}; @@ -42,6 +46,8 @@ const CUSTOM_TOOL_INPUT_FIELD: &str = "input"; const CHAT_TOOL_NAME_MAX_LEN: usize = 64; const CUSTOM_TOOL_INPUT_DESCRIPTION: &str = "Raw string input for the original custom tool. Preserve formatting exactly and follow the original tool definition embedded in the description."; const CUSTOM_TOOL_PRESERVED_METADATA_HEADING: &str = "Original tool definition:"; +const TOOL_RESULT_MEDIA_MOVED_MARKER: &str = + "[cc-switch: tool result media moved to the following user message]"; #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum CodexToolKind { @@ -539,12 +545,77 @@ fn instruction_text(value: &Value) -> String { } } +struct ToolOutputMediaPlan { + tool_content: String, + media_parts: Vec, +} + +fn plan_tool_output_media(mut output: Value) -> Option { + let replacement_block = json!({ + "type": "text", + "text": TOOL_RESULT_MEDIA_MOVED_MARKER + }); + let mut media_parts = Vec::new(); + let replaced = strip_media_from_tool_value( + &mut output, + &mut media_parts, + ToolMediaScope::AllSupported, + &replacement_block, + TOOL_RESULT_MEDIA_MOVED_MARKER, + ); + if replaced == 0 { + return None; + } + + // Only media-bearing outputs enter this path. Remove any residual large + // data/base64 scalar that was not itself a recognized content block while + // preserving ordinary long OCR/log/source text. + clamp_base64ish_strings(&mut output); + Some(ToolOutputMediaPlan { + tool_content: canonical_json_string(&output), + media_parts, + }) +} + +fn plan_raw_tool_output_data_url(output: &str) -> Option { + whole_string_image_data_url(output).map(|media_part| ToolOutputMediaPlan { + // A raw tool-output string remains a raw tool-output string. Do not add + // JSON string-literal quotes around the replacement marker. + tool_content: TOOL_RESULT_MEDIA_MOVED_MARKER.to_string(), + media_parts: vec![media_part], + }) +} + +fn queue_tool_output_media(pending_media: &mut Vec, call_id: &str, media_parts: Vec) { + if media_parts.is_empty() { + return; + } + + pending_media.push(json!({ + "type": "text", + "text": format!("[cc-switch: media output of tool call {call_id}]") + })); + pending_media.extend(media_parts); +} + +fn flush_pending_media(messages: &mut Vec, pending_media: &mut Vec) { + if pending_media.is_empty() { + return; + } + + messages.push(json!({ + "role": "user", + "content": std::mem::take(pending_media) + })); +} + fn append_responses_input_as_chat_messages( input: &Value, messages: &mut Vec, tool_context: &CodexToolContext, ) -> Result<(), ProxyError> { let mut pending_tool_calls = Vec::new(); + let mut pending_media = Vec::new(); let mut pending_reasoning: Option = None; let mut last_assistant_index: Option = None; @@ -561,6 +632,7 @@ fn append_responses_input_as_chat_messages( item, messages, &mut pending_tool_calls, + &mut pending_media, &mut pending_reasoning, &mut last_assistant_index, tool_context, @@ -572,6 +644,7 @@ fn append_responses_input_as_chat_messages( input, messages, &mut pending_tool_calls, + &mut pending_media, &mut pending_reasoning, &mut last_assistant_index, tool_context, @@ -580,9 +653,14 @@ fn append_responses_input_as_chat_messages( _ => {} } + // If a later assistant tool-call batch was accumulated after an earlier + // media-bearing result, the synthetic user media belongs before that next + // assistant turn. + flush_pending_media(messages, &mut pending_media); flush_pending_tool_calls( messages, &mut pending_tool_calls, + &mut pending_media, &mut pending_reasoning, &mut last_assistant_index, ); @@ -603,6 +681,7 @@ fn append_responses_item_as_chat_message( item: &Value, messages: &mut Vec, pending_tool_calls: &mut Vec, + pending_media: &mut Vec, pending_reasoning: &mut Option, last_assistant_index: &mut Option, tool_context: &CodexToolContext, @@ -628,14 +707,35 @@ fn append_responses_item_as_chat_message( flush_pending_tool_calls( messages, pending_tool_calls, + pending_media, pending_reasoning, last_assistant_index, ); let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or(""); - let output = match item.get("output") { - Some(Value::String(s)) => canonicalize_json_string_if_parseable(s), - Some(v) => canonical_json_string(v), - None => String::new(), + let media_plan = match item.get("output") { + Some(Value::String(output)) => { + plan_raw_tool_output_data_url(output).or_else(|| { + serde_json::from_str::(output.trim()) + .ok() + .and_then(plan_tool_output_media) + }) + } + Some(output @ (Value::Array(_) | Value::Object(_))) => { + plan_tool_output_media(output.clone()) + } + _ => None, + }; + let output = if let Some(media_plan) = media_plan { + queue_tool_output_media(pending_media, call_id, media_plan.media_parts); + media_plan.tool_content + } else { + // Cache-sensitive no-media fallback: keep these expressions + // byte-for-byte equivalent to the pre-fix conversion. + match item.get("output") { + Some(Value::String(s)) => canonicalize_json_string_if_parseable(s), + Some(v) => canonical_json_string(v), + None => String::new(), + } }; messages.push(json!({ "role": "tool", @@ -647,11 +747,39 @@ fn append_responses_item_as_chat_message( flush_pending_tool_calls( messages, pending_tool_calls, + pending_media, pending_reasoning, last_assistant_index, ); let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or(""); - let output = canonical_json_string(item); + let mut transformed_item = item.clone(); + let replacement_block = json!({ + "type": "text", + "text": TOOL_RESULT_MEDIA_MOVED_MARKER + }); + let mut media_parts = Vec::new(); + let replaced = transformed_item + .get_mut("output") + .map(|output| { + strip_media_from_tool_value( + output, + &mut media_parts, + ToolMediaScope::AllSupported, + &replacement_block, + TOOL_RESULT_MEDIA_MOVED_MARKER, + ) + }) + .unwrap_or(0); + let output = if replaced > 0 { + if let Some(output) = transformed_item.get_mut("output") { + clamp_base64ish_strings(output); + } + queue_tool_output_media(pending_media, call_id, media_parts); + canonical_json_string(&transformed_item) + } else { + // Preserve the legacy whole-item representation exactly. + canonical_json_string(item) + }; messages.push(json!({ "role": "tool", "tool_call_id": call_id, @@ -672,9 +800,14 @@ fn append_responses_item_as_chat_message( flush_pending_tool_calls( messages, pending_tool_calls, + pending_media, pending_reasoning, last_assistant_index, ); + // `flush_pending_tool_calls` intentionally returns early when + // there is no new assistant batch. A previous tool result may + // still have media waiting, so flush it before this new message. + flush_pending_media(messages, pending_media); let role = item .get("role") .and_then(|v| v.as_str()) @@ -705,13 +838,15 @@ fn append_responses_item_as_chat_message( messages.push(message); } Some("message") | None => { - flush_pending_tool_calls( - messages, - pending_tool_calls, - pending_reasoning, - last_assistant_index, - ); if item.get("role").is_some() || item.get("content").is_some() { + flush_pending_tool_calls( + messages, + pending_tool_calls, + pending_media, + pending_reasoning, + last_assistant_index, + ); + flush_pending_media(messages, pending_media); let message = responses_message_item_to_chat_message( item, pending_reasoning, @@ -720,16 +855,28 @@ fn append_responses_item_as_chat_message( ); update_last_assistant_index(messages, &message, last_assistant_index); messages.push(message); + } else if pending_media.is_empty() { + // Preserve legacy no-media ordering: inert message-like items + // used to close a pending tool-call batch. + flush_pending_tool_calls( + messages, + pending_tool_calls, + pending_media, + pending_reasoning, + last_assistant_index, + ); } } _ => { - flush_pending_tool_calls( - messages, - pending_tool_calls, - pending_reasoning, - last_assistant_index, - ); if item.get("role").is_some() || item.get("content").is_some() { + flush_pending_tool_calls( + messages, + pending_tool_calls, + pending_media, + pending_reasoning, + last_assistant_index, + ); + flush_pending_media(messages, pending_media); let message = responses_message_item_to_chat_message( item, pending_reasoning, @@ -738,6 +885,16 @@ fn append_responses_item_as_chat_message( ); update_last_assistant_index(messages, &message, last_assistant_index); messages.push(message); + } else if pending_media.is_empty() { + // Preserve legacy no-media ordering without letting an inert + // unknown item flush a media-bearing result batch. + flush_pending_tool_calls( + messages, + pending_tool_calls, + pending_media, + pending_reasoning, + last_assistant_index, + ); } } } @@ -748,6 +905,7 @@ fn append_responses_item_as_chat_message( fn flush_pending_tool_calls( messages: &mut Vec, pending_tool_calls: &mut Vec, + pending_media: &mut Vec, pending_reasoning: &mut Option, last_assistant_index: &mut Option, ) { @@ -755,6 +913,10 @@ fn flush_pending_tool_calls( return; } + // Media from the preceding tool-result batch must be presented before a + // new assistant tool-call turn. Consecutive outputs do not enter here + // because `pending_tool_calls` is empty after the first output. + flush_pending_media(messages, pending_media); let mut message = json!({ "role": "assistant", "content": null, @@ -1057,18 +1219,7 @@ fn responses_content_to_chat_content(_role: &str, content: &Value) -> Value { } fn responses_input_file_to_chat_file(part: &Value) -> Option { - let mut file = serde_json::Map::new(); - let has_supported_file_ref = part.get("file_id").is_some() || part.get("file_data").is_some(); - if !has_supported_file_ref { - return None; - } - - for key in ["file_id", "file_data", "filename"] { - if let Some(value) = part.get(key) { - file.insert(key.to_string(), value.clone()); - } - } - Some(Value::Object(file)) + chat_file_from_input_file(part) } fn collect_tool_search_output_tools(value: &Value, context: &mut CodexToolContext) { @@ -1834,6 +1985,50 @@ pub fn chat_error_to_response_error(body: Option<&Value>) -> Value { #[cfg(test)] mod tests { use super::*; + use base64::{engine::general_purpose::STANDARD, Engine as _}; + + fn large_test_image_data_url() -> String { + let bytes = b"CC_SWITCH_TOOL_MEDIA_SENTINEL".repeat(400); + format!("data:image/png;base64,{}", STANDARD.encode(bytes)) + } + + fn message_roles(result: &Value) -> Vec<&str> { + result["messages"] + .as_array() + .unwrap() + .iter() + .filter_map(|message| message.get("role").and_then(Value::as_str)) + .collect() + } + + fn test_function_call(call_id: &str) -> Value { + json!({ + "type": "function_call", + "call_id": call_id, + "name": "view_image", + "arguments": "{}" + }) + } + + fn test_function_output(call_id: &str, output: Value) -> Value { + json!({ + "type": "function_call_output", + "call_id": call_id, + "output": output + }) + } + + fn convert_test_input(items: Vec) -> Value { + responses_to_chat_completions(json!({ + "model": "kimi-k3", + "input": items + })) + .unwrap() + } + + fn result_messages(result: &Value) -> &[Value] { + result["messages"].as_array().unwrap() + } #[test] fn responses_request_with_stream_injects_include_usage() { @@ -3061,6 +3256,538 @@ mod tests { assert_eq!(messages[1]["content"], "plain text result"); } + #[test] + fn responses_request_to_chat_moves_tool_image_to_synthetic_user_message() { + let data_url = large_test_image_data_url(); + let result = convert_test_input(vec![ + test_function_call("call_image"), + test_function_output( + "call_image", + json!([ + {"type": "input_text", "text": "screenshot follows"}, + {"type": "input_image", "image_url": data_url.clone()} + ]), + ), + ]); + let messages = result_messages(&result); + + assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]); + assert!(messages[1]["content"].is_string()); + let tool_content: Value = + serde_json::from_str(messages[1]["content"].as_str().unwrap()).unwrap(); + assert_eq!(tool_content[0]["text"], "screenshot follows"); + assert_eq!(tool_content[1]["type"], "text"); + assert_eq!(tool_content[1]["text"], TOOL_RESULT_MEDIA_MOVED_MARKER); + assert!(!messages[1]["content"].as_str().unwrap().contains(&data_url)); + + assert_eq!( + messages[2]["content"][0]["text"], + "[cc-switch: media output of tool call call_image]" + ); + assert_eq!(messages[2]["content"][1]["type"], "image_url"); + assert_eq!(messages[2]["content"][1]["image_url"]["url"], data_url); + } + + #[test] + fn responses_request_to_chat_groups_parallel_media_after_all_tool_outputs() { + let first_url = large_test_image_data_url(); + let second_payload = "MCP_TOOL_MEDIA_SENTINEL"; + let result = convert_test_input(vec![ + test_function_call("call_1"), + test_function_call("call_2"), + test_function_output( + "call_1", + json!({"type": "input_image", "image_url": first_url.clone()}), + ), + json!({ + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "keep outputs adjacent"}] + }), + test_function_output( + "call_2", + json!({ + "type": "image", + "mimeType": "image/webp", + "data": second_payload + }), + ), + ]); + let messages = result_messages(&result); + + assert_eq!( + message_roles(&result), + vec!["assistant", "tool", "tool", "user"] + ); + assert_eq!(messages[0]["reasoning_content"], "keep outputs adjacent"); + assert_ne!(messages[0]["reasoning_content"], "tool call"); + assert_eq!(messages[1]["tool_call_id"], "call_1"); + assert_eq!(messages[2]["tool_call_id"], "call_2"); + assert!(messages[1]["content"].is_string()); + assert!(messages[2]["content"].is_string()); + assert!(!messages[1]["content"] + .as_str() + .unwrap() + .contains(&first_url)); + assert!(!messages[2]["content"] + .as_str() + .unwrap() + .contains(second_payload)); + assert_eq!(messages[3]["content"].as_array().unwrap().len(), 4); + assert_eq!( + messages[3]["content"][3]["image_url"]["url"], + format!("data:image/webp;base64,{second_payload}") + ); + } + + #[test] + fn responses_request_to_chat_flushes_media_before_next_tool_call_batch() { + let result = convert_test_input(vec![ + test_function_call("call_1"), + test_function_output( + "call_1", + json!({ + "type": "input_image", + "image_url": large_test_image_data_url() + }), + ), + test_function_call("call_2"), + test_function_output("call_2", json!("second result")), + ]); + let messages = result_messages(&result); + + assert_eq!( + message_roles(&result), + vec!["assistant", "tool", "user", "assistant", "tool"] + ); + assert_eq!(messages[0]["tool_calls"][0]["id"], "call_1"); + assert_eq!(messages[3]["tool_calls"][0]["id"], "call_2"); + assert_eq!(messages[4]["tool_call_id"], "call_2"); + } + + #[test] + fn responses_request_to_chat_flushes_media_before_real_user_messages() { + let boundaries = [ + json!({"type": "input_text", "text": "continue"}), + json!({"type": "future_message", "role": "user", "content": "continue"}), + ]; + + for boundary in boundaries { + let result = convert_test_input(vec![ + test_function_call("call_1"), + test_function_output( + "call_1", + json!({ + "type": "input_image", + "image_url": large_test_image_data_url() + }), + ), + boundary, + ]); + let messages = result_messages(&result); + + assert_eq!( + message_roles(&result), + vec!["assistant", "tool", "user", "user"] + ); + assert!(messages[2]["content"].is_array()); + assert_eq!(messages[3]["role"], "user"); + } + } + + #[test] + fn responses_request_to_chat_handles_raw_data_url_thresholds() { + let large = large_test_image_data_url(); + let large_result = convert_test_input(vec![ + test_function_call("call_large"), + test_function_output("call_large", Value::String(large.clone())), + ]); + let large_messages = result_messages(&large_result); + + assert_eq!( + message_roles(&large_result), + vec!["assistant", "tool", "user"] + ); + assert_eq!(large_messages[1]["content"], TOOL_RESULT_MEDIA_MOVED_MARKER); + assert_eq!(large_messages[2]["content"][1]["image_url"]["url"], large); + + let small = "data:image/png;base64,YWJj"; + let small_result = convert_test_input(vec![ + test_function_call("call_small"), + test_function_output("call_small", json!(small)), + ]); + assert_eq!(message_roles(&small_result), vec!["assistant", "tool"]); + assert_eq!(small_result["messages"][1]["content"], small); + } + + #[test] + fn responses_request_to_chat_maps_supported_structured_image_shapes() { + let cases = vec![ + ( + json!({ + "type": "input_image", + "image_url": {"url": "https://example.com/input.png"}, + "detail": "high" + }), + "https://example.com/input.png", + Some("high"), + ), + ( + json!({ + "type": "image_url", + "image_url": "https://example.com/chat-string.png" + }), + "https://example.com/chat-string.png", + None, + ), + ( + json!({ + "type": "image_url", + "image_url": { + "url": "https://example.com/chat-object.png", + "detail": "low" + } + }), + "https://example.com/chat-object.png", + Some("low"), + ), + ( + json!({"image_url": "data:image/gif;base64,LOOSE_SENTINEL"}), + "data:image/gif;base64,LOOSE_SENTINEL", + None, + ), + ( + json!({ + "type": "image", + "source": { + "media_type": "image/jpeg", + "data": "ANTHROPIC_SENTINEL" + } + }), + "data:image/jpeg;base64,ANTHROPIC_SENTINEL", + None, + ), + ( + json!({ + "type": "image", + "mimeType": "image/webp", + "data": "MCP_SENTINEL" + }), + "data:image/webp;base64,MCP_SENTINEL", + None, + ), + ]; + + for (index, (output, expected_url, expected_detail)) in cases.into_iter().enumerate() { + let call_id = format!("call_shape_{index}"); + let result = convert_test_input(vec![ + test_function_call(&call_id), + test_function_output(&call_id, output), + ]); + let image = &result["messages"][2]["content"][1]; + + assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]); + assert_eq!(image["type"], "image_url"); + assert_eq!(image["image_url"]["url"], expected_url); + match expected_detail { + Some(detail) => assert_eq!(image["image_url"]["detail"], detail), + None => assert!(image["image_url"].get("detail").is_none()), + } + } + } + + #[test] + fn responses_request_to_chat_extracts_media_from_json_string_and_content_wrapper() { + let output = json!({ + "content": [ + {"type": "input_text", "text": "MCP response"}, + { + "type": "image", + "mimeType": "image/png", + "data": "STRING_MCP_SENTINEL" + } + ] + }) + .to_string(); + let result = convert_test_input(vec![ + test_function_call("call_string"), + test_function_output("call_string", Value::String(output)), + ]); + let messages = result_messages(&result); + + assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]); + let tool_content: Value = + serde_json::from_str(messages[1]["content"].as_str().unwrap()).unwrap(); + assert_eq!(tool_content["content"][0]["text"], "MCP response"); + assert_eq!( + tool_content["content"][1]["text"], + TOOL_RESULT_MEDIA_MOVED_MARKER + ); + assert!(!messages[1]["content"] + .as_str() + .unwrap() + .contains("STRING_MCP_SENTINEL")); + assert_eq!( + messages[2]["content"][1]["image_url"]["url"], + "data:image/png;base64,STRING_MCP_SENTINEL" + ); + } + + #[test] + fn responses_request_to_chat_extracts_tool_files_and_audio() { + let result = convert_test_input(vec![ + test_function_call("call_media"), + test_function_output( + "call_media", + json!({ + "content": [ + { + "type": "input_file", + "file_id": "file_123", + "filename": "report.pdf" + }, + { + "type": "input_audio", + "input_audio": {"data": "AUDIO_SENTINEL", "format": "wav"} + } + ] + }), + ), + ]); + let messages = result_messages(&result); + + assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]); + assert_eq!(messages[2]["content"][1]["type"], "file"); + assert_eq!(messages[2]["content"][1]["file"]["file_id"], "file_123"); + assert_eq!(messages[2]["content"][2]["type"], "input_audio"); + assert_eq!( + messages[2]["content"][2]["input_audio"]["data"], + "AUDIO_SENTINEL" + ); + } + + #[test] + fn responses_request_to_chat_extracts_custom_and_tool_search_output_media() { + let cases = [ + ( + json!({ + "type": "custom_tool_call", + "call_id": "call_custom", + "name": "render", + "input": "draw" + }), + json!({ + "type": "custom_tool_call_output", + "call_id": "call_custom", + "status": "completed", + "output": { + "content": [{ + "type": "input_image", + "image_url": "data:image/png;base64,CUSTOM_SENTINEL" + }] + } + }), + ), + ( + json!({ + "type": "tool_search_call", + "call_id": "call_search", + "arguments": {"query": "image tool"} + }), + json!({ + "type": "tool_search_output", + "call_id": "call_search", + "status": "completed", + "output": { + "content": [{ + "type": "image", + "mimeType": "image/png", + "data": "SEARCH_SENTINEL" + }] + } + }), + ), + ]; + + for (call, output) in cases { + let expected_type = output["type"].as_str().unwrap().to_string(); + let result = convert_test_input(vec![call, output]); + let messages = result_messages(&result); + let tool_content: Value = + serde_json::from_str(messages[1]["content"].as_str().unwrap()).unwrap(); + + assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]); + assert_eq!(tool_content["type"], expected_type); + assert_eq!(tool_content["status"], "completed"); + assert_eq!( + tool_content["output"]["content"][0]["text"], + TOOL_RESULT_MEDIA_MOVED_MARKER + ); + } + } + + #[test] + fn responses_request_to_chat_rejects_false_positive_media_shapes() { + let outputs = [ + json!({"type": "image", "name": "business metadata"}), + json!({ + "type": "image", + "mimeType": "text/plain", + "data": "NOT_AN_IMAGE" + }), + json!({ + "image_url": { + "url": "https://example.com/search-thumbnail.png" + } + }), + ]; + + for (index, output) in outputs.into_iter().enumerate() { + let call_id = format!("call_false_positive_{index}"); + let expected = canonical_json_string(&output); + let result = convert_test_input(vec![ + test_function_call(&call_id), + test_function_output(&call_id, output), + ]); + + assert_eq!(message_roles(&result), vec!["assistant", "tool"]); + assert_eq!(result["messages"][1]["content"], expected); + } + } + + #[test] + fn responses_request_to_chat_keeps_no_media_tool_output_bytes_stable() { + let cases = [ + (Some(json!("plain text")), "plain text".to_string()), + ( + Some(json!("{ \"z\": true, \"a\": [2, 1] }")), + r#"{"a":[2,1],"z":true}"#.to_string(), + ), + ( + Some(json!(["main.rs", "lib.rs"])), + r#"["main.rs","lib.rs"]"#.to_string(), + ), + ( + Some(json!({"z": true, "a": [2, 1]})), + r#"{"a":[2,1],"z":true}"#.to_string(), + ), + (Some(json!([])), "[]".to_string()), + (None, String::new()), + ]; + + for (index, (output, expected)) in cases.into_iter().enumerate() { + let call_id = format!("call_stable_{index}"); + let mut item = json!({ + "type": "function_call_output", + "call_id": call_id + }); + if let Some(output) = output { + item["output"] = output; + } + let result = convert_test_input(vec![test_function_call(&call_id), item]); + + assert_eq!(message_roles(&result), vec!["assistant", "tool"]); + assert_eq!(result["messages"][1]["content"], expected); + } + + for item in [ + json!({ + "type": "custom_tool_call_output", + "call_id": "call_custom", + "status": "completed", + "output": {"text": "unchanged"} + }), + json!({ + "type": "tool_search_output", + "call_id": "call_search", + "status": "completed", + "output": [] + }), + ] { + let expected = canonical_json_string(&item); + let result = convert_test_input(vec![item]); + assert_eq!(result["messages"][0]["content"], expected); + } + } + + #[test] + fn responses_request_to_chat_preserves_legacy_unknown_item_batch_boundary_without_media() { + let result = convert_test_input(vec![ + test_function_call("call_1"), + json!({"type": "future_metadata", "value": 1}), + json!({ + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "second batch reasoning"}] + }), + test_function_call("call_2"), + test_function_output("call_1", json!("first result")), + test_function_output("call_2", json!("second result")), + ]); + let messages = result_messages(&result); + + assert_eq!( + message_roles(&result), + vec!["assistant", "assistant", "tool", "tool"] + ); + assert_eq!(messages[0]["tool_calls"].as_array().unwrap().len(), 1); + assert_eq!(messages[0]["tool_calls"][0]["id"], "call_1"); + assert_eq!(messages[0]["reasoning_content"], "tool call"); + assert_eq!(messages[1]["tool_calls"].as_array().unwrap().len(), 1); + assert_eq!(messages[1]["tool_calls"][0]["id"], "call_2"); + assert_eq!(messages[1]["reasoning_content"], "second batch reasoning"); + } + + #[test] + fn responses_request_to_chat_clamps_only_residual_base64ish_strings() { + let data_url = large_test_image_data_url(); + let long_text = format!("{}end", "ordinary OCR text with spaces. ".repeat(3_500)); + let residual_base64 = "A".repeat(20_000); + let result = convert_test_input(vec![ + test_function_call("call_clamp"), + test_function_output( + "call_clamp", + json!([ + {"type": "input_image", "image_url": data_url.clone()}, + {"type": "text", "text": long_text.clone()}, + {"raw": residual_base64} + ]), + ), + ]); + let tool_content_text = result["messages"][1]["content"].as_str().unwrap(); + let tool_content: Value = serde_json::from_str(tool_content_text).unwrap(); + + assert_eq!(tool_content[1]["text"], long_text); + assert!(tool_content[2]["raw"] + .as_str() + .unwrap() + .starts_with("[cc-switch: omitted 20000 bytes]")); + assert!(!tool_content_text.contains(&data_url)); + assert!(!tool_content_text.contains("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + } + + #[test] + fn responses_request_to_chat_media_conversion_is_deterministic() { + let input = json!({ + "model": "kimi-k3", + "input": [ + test_function_call("call_repeat"), + test_function_output( + "call_repeat", + json!({ + "content": [{ + "type": "input_image", + "image_url": large_test_image_data_url() + }] + }) + ) + ] + }); + + let first = responses_to_chat_completions(input.clone()).unwrap(); + let second = responses_to_chat_completions(input).unwrap(); + + assert_eq!(first, second); + } + #[test] fn chat_response_to_responses_maps_text_tool_calls_and_usage() { let input = json!({ diff --git a/src-tauri/src/proxy/tool_media.rs b/src-tauri/src/proxy/tool_media.rs new file mode 100644 index 000000000..0c3a907d6 --- /dev/null +++ b/src-tauri/src/proxy/tool_media.rs @@ -0,0 +1,774 @@ +//! Shared media handling for Codex tool outputs. +//! +//! Responses tool outputs may carry structured media blocks. Chat Completions +//! tool messages are text-only, so the Responses→Chat bridge extracts those +//! blocks and re-emits them in a synthetic user message. The media sanitizer +//! reuses the same recognition and traversal rules when it needs to remove +//! images for a text-only upstream. + +use crate::proxy::json_canonical::canonical_json_string; +use serde_json::{json, Map, Value}; + +pub(crate) const WHOLE_DATA_URL_MIN_BYTES: usize = 8 * 1024; +const BASE64ISH_MIN_BYTES: usize = 16 * 1024; +const MAX_MEDIA_TRAVERSAL_DEPTH: usize = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ToolMediaScope { + /// Used by the existing image-capability sanitizer and its retry path. + ImagesOnly, + /// Used by Responses→Chat conversion, where user messages can carry all + /// currently mapped Chat input modalities. + AllSupported, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ToolMediaKind { + Image, + File, + Audio, +} + +impl ToolMediaScope { + fn allows(self, kind: ToolMediaKind) -> bool { + matches!(kind, ToolMediaKind::Image) || matches!(self, Self::AllSupported) + } +} + +/// Convert one recognized Responses/tool media block to a Chat user content +/// part. This is the single shape-recognition entry point used by extraction. +pub(crate) fn chat_media_part_from_tool_part(part: &Value, scope: ToolMediaScope) -> Option { + let kind = tool_media_kind(part)?; + if !scope.allows(kind) { + return None; + } + + match kind { + ToolMediaKind::Image => chat_image_part(part), + ToolMediaKind::File => chat_file_from_input_file(part).map(|file| { + json!({ + "type": "file", + "file": file + }) + }), + ToolMediaKind::Audio => part.get("input_audio").map(|input_audio| { + json!({ + "type": "input_audio", + "input_audio": input_audio.clone() + }) + }), + } +} + +/// Map a Responses `input_file` block to the Chat file payload. Kept here so +/// top-level content and tool-output extraction share the exact same rules. +pub(crate) fn chat_file_from_input_file(part: &Value) -> Option { + let mut file = Map::new(); + let has_supported_file_ref = part.get("file_id").is_some() || part.get("file_data").is_some(); + if !has_supported_file_ref { + return None; + } + + for key in ["file_id", "file_data", "filename"] { + if let Some(value) = part.get(key) { + file.insert(key.to_string(), value.clone()); + } + } + Some(Value::Object(file)) +} + +/// Recognize a complete image data URL stored as a scalar string. +/// +/// Only whole-string matches are accepted. Embedded data URLs in HTML/CSS/SVG +/// source are deliberately left alone. Small values remain text as well, which +/// preserves workflows that intentionally inspect tiny inline icons. +pub(crate) fn whole_string_image_data_url(value: &str) -> Option { + let trimmed = value.trim(); + if trimmed.len() < WHOLE_DATA_URL_MIN_BYTES || !is_image_base64_data_url(trimmed) { + return None; + } + + Some(json!({ + "type": "image_url", + "image_url": { + "url": trimmed + } + })) +} + +/// Read-only media detection using the same shape classifier and recursive +/// boundaries as [`strip_media_from_tool_value`]. +pub(crate) fn tool_output_contains_media(value: &Value, scope: ToolMediaScope) -> bool { + tool_output_contains_media_at_depth(value, scope, 0) +} + +/// Extract recognized media blocks and replace them in-place. +/// +/// `replacement_block` is used for structured array/object parts. A scalar +/// string that is itself a complete image data URL uses `replacement_text`, so +/// an originally plain string stays a plain string. Parseable JSON strings are +/// recursively transformed and canonicalized back into a string only when a +/// replacement actually occurred. +pub(crate) fn strip_media_from_tool_value( + value: &mut Value, + media_parts: &mut Vec, + scope: ToolMediaScope, + replacement_block: &Value, + replacement_text: &str, +) -> usize { + strip_media_from_tool_value_at_depth( + value, + media_parts, + scope, + replacement_block, + replacement_text, + 0, + ) +} + +/// Remove residual data/base64 payloads only after a tool output has already +/// been positively identified as media-bearing. Ordinary long text is kept. +pub(crate) fn clamp_base64ish_strings(value: &mut Value) { + match value { + Value::String(text) => { + let trimmed = text.trim(); + let should_omit = (trimmed.len() >= WHOLE_DATA_URL_MIN_BYTES + && trimmed + .get(..5) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))) + || looks_like_base64_payload(trimmed); + if should_omit { + let byte_len = text.len(); + *text = format!("[cc-switch: omitted {byte_len} bytes]"); + } + } + Value::Array(items) => { + for item in items { + clamp_base64ish_strings(item); + } + } + Value::Object(object) => { + for nested in object.values_mut() { + clamp_base64ish_strings(nested); + } + } + _ => {} + } +} + +fn tool_output_contains_media_at_depth(value: &Value, scope: ToolMediaScope, depth: usize) -> bool { + if depth > MAX_MEDIA_TRAVERSAL_DEPTH { + return false; + } + + match value { + Value::String(text) => { + if scope.allows(ToolMediaKind::Image) && whole_string_image_data_url(text).is_some() { + return true; + } + + let trimmed = text.trim(); + if trimmed.is_empty() { + return false; + } + serde_json::from_str::(trimmed) + .ok() + .is_some_and(|parsed| { + tool_output_contains_media_at_depth(&parsed, scope, depth + 1) + }) + } + Value::Array(items) => items + .iter() + .any(|item| tool_output_contains_media_at_depth(item, scope, depth + 1)), + Value::Object(object) => { + if tool_media_kind(value).is_some_and(|kind| scope.allows(kind)) { + return true; + } + + object.get("content").is_some_and(|content| { + tool_output_contains_media_at_depth(content, scope, depth + 1) + }) + } + _ => false, + } +} + +fn strip_media_from_tool_value_at_depth( + value: &mut Value, + media_parts: &mut Vec, + scope: ToolMediaScope, + replacement_block: &Value, + replacement_text: &str, + depth: usize, +) -> usize { + if depth > MAX_MEDIA_TRAVERSAL_DEPTH { + return 0; + } + + match value { + Value::String(text) => { + if scope.allows(ToolMediaKind::Image) { + if let Some(media_part) = whole_string_image_data_url(text) { + media_parts.push(media_part); + *text = replacement_text.to_string(); + return 1; + } + } + + let trimmed = text.trim(); + if trimmed.is_empty() { + return 0; + } + let Ok(mut parsed) = serde_json::from_str::(trimmed) else { + return 0; + }; + let replaced = strip_media_from_tool_value_at_depth( + &mut parsed, + media_parts, + scope, + replacement_block, + replacement_text, + depth + 1, + ); + if replaced > 0 { + *text = canonical_json_string(&parsed); + } + replaced + } + Value::Array(items) => items + .iter_mut() + .map(|item| { + strip_media_from_tool_value_at_depth( + item, + media_parts, + scope, + replacement_block, + replacement_text, + depth + 1, + ) + }) + .sum(), + Value::Object(_) => { + if let Some(media_part) = chat_media_part_from_tool_part(value, scope) { + media_parts.push(media_part); + *value = replacement_block.clone(); + return 1; + } + + value + .as_object_mut() + .expect("object match arm must remain an object") + .get_mut("content") + .map(|content| { + strip_media_from_tool_value_at_depth( + content, + media_parts, + scope, + replacement_block, + replacement_text, + depth + 1, + ) + }) + .unwrap_or(0) + } + _ => 0, + } +} + +fn tool_media_kind(part: &Value) -> Option { + let object = part.as_object()?; + let part_type = object.get("type").and_then(Value::as_str); + + match part_type { + Some("input_image" | "image_url") if normalized_image_url(part).is_some() => { + Some(ToolMediaKind::Image) + } + Some("input_file") if part.get("file_id").is_some() || part.get("file_data").is_some() => { + Some(ToolMediaKind::File) + } + Some("input_audio") if part.get("input_audio").is_some_and(Value::is_object) => { + Some(ToolMediaKind::Audio) + } + Some("image") if typed_image_has_payload(part) => Some(ToolMediaKind::Image), + None if loose_data_image_url(part).is_some() => Some(ToolMediaKind::Image), + _ => None, + } +} + +fn chat_image_part(part: &Value) -> Option { + match part.get("type").and_then(Value::as_str) { + Some("input_image" | "image_url") => { + normalized_image_url(part).map(|image_url| image_url_content_part(part, image_url)) + } + Some("image") => { + typed_image_url(part).map(|image_url| image_url_content_part(part, image_url)) + } + None => loose_data_image_url(part).map(|image_url| image_url_content_part(part, image_url)), + _ => None, + } +} + +fn normalized_image_url(part: &Value) -> Option { + let image_url = part.get("image_url")?; + let mut object = match image_url { + Value::String(url) if !url.trim().is_empty() => { + let mut object = Map::new(); + object.insert("url".to_string(), Value::String(url.clone())); + object + } + Value::Object(object) + if object + .get("url") + .and_then(Value::as_str) + .is_some_and(|url| !url.trim().is_empty()) => + { + object.clone() + } + _ => return None, + }; + merge_top_level_detail(part, &mut object); + Some(Value::Object(object)) +} + +fn loose_data_image_url(part: &Value) -> Option { + if part.get("type").is_some() { + return None; + } + let normalized = normalized_image_url(part)?; + let url = normalized.get("url").and_then(Value::as_str)?; + if !url + .get(..5) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")) + { + return None; + } + Some(normalized) +} + +fn typed_image_has_payload(part: &Value) -> bool { + let Some(object) = part.as_object() else { + return false; + }; + + if let Some(source) = object.get("source").and_then(Value::as_object) { + if source_media_type_is_image(source) { + let has_url = source + .get("url") + .and_then(Value::as_str) + .is_some_and(|url| !url.trim().is_empty()); + let has_data = source + .get("data") + .and_then(Value::as_str) + .is_some_and(|data| !data.is_empty()); + if has_url || has_data { + return true; + } + } + } + + object + .get("data") + .and_then(Value::as_str) + .is_some_and(|data| !data.is_empty()) + && object + .get("mimeType") + .or_else(|| object.get("mime_type")) + .and_then(Value::as_str) + .is_some_and(is_image_mime_type) +} + +fn typed_image_url(part: &Value) -> Option { + let object = part.as_object()?; + + if let Some(source) = object.get("source").and_then(Value::as_object) { + if !source_media_type_is_image(source) { + return None; + } + + if let Some(url) = source + .get("url") + .and_then(Value::as_str) + .filter(|url| !url.trim().is_empty()) + { + let mut image_url = Map::new(); + image_url.insert("url".to_string(), Value::String(url.to_string())); + merge_top_level_detail(part, &mut image_url); + return Some(Value::Object(image_url)); + } + + if let Some(data) = source + .get("data") + .and_then(Value::as_str) + .filter(|data| !data.is_empty()) + { + let media_type = source + .get("media_type") + .or_else(|| source.get("mime_type")) + .or_else(|| source.get("mimeType")) + .and_then(Value::as_str) + .unwrap_or("image/png"); + let url = if data + .get(..11) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:image/")) + { + data.to_string() + } else { + format!("data:{media_type};base64,{data}") + }; + let mut image_url = Map::new(); + image_url.insert("url".to_string(), Value::String(url)); + merge_top_level_detail(part, &mut image_url); + return Some(Value::Object(image_url)); + } + } + + let data = object + .get("data") + .and_then(Value::as_str) + .filter(|data| !data.is_empty())?; + let media_type = object + .get("mimeType") + .or_else(|| object.get("mime_type")) + .and_then(Value::as_str) + .filter(|media_type| is_image_mime_type(media_type))?; + let mut image_url = Map::new(); + image_url.insert( + "url".to_string(), + Value::String(format!("data:{media_type};base64,{data}")), + ); + merge_top_level_detail(part, &mut image_url); + Some(Value::Object(image_url)) +} + +fn image_url_content_part(source_part: &Value, image_url: Value) -> Value { + let mut content_part = Map::new(); + content_part.insert("type".to_string(), Value::String("image_url".to_string())); + content_part.insert("image_url".to_string(), image_url); + + for key in ["cache_control", "prompt_cache_breakpoint"] { + if let Some(value) = source_part.get(key) { + content_part.insert(key.to_string(), value.clone()); + } + } + Value::Object(content_part) +} + +fn merge_top_level_detail(part: &Value, image_url: &mut Map) { + if image_url.get("detail").is_none() { + if let Some(detail) = part.get("detail") { + image_url.insert("detail".to_string(), detail.clone()); + } + } +} + +fn source_media_type_is_image(source: &Map) -> bool { + source + .get("media_type") + .or_else(|| source.get("mime_type")) + .or_else(|| source.get("mimeType")) + .and_then(Value::as_str) + .is_none_or(is_image_mime_type) +} + +fn is_image_mime_type(value: &str) -> bool { + value + .get(..6) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/")) +} + +fn is_image_base64_data_url(value: &str) -> bool { + let Some(comma_index) = value.find(',') else { + return false; + }; + let header = &value[..comma_index]; + let header = header.to_ascii_lowercase(); + header.starts_with("data:image/") && header.ends_with(";base64") +} + +fn looks_like_base64_payload(value: &str) -> bool { + if value.len() < BASE64ISH_MIN_BYTES { + return false; + } + + value + .bytes() + .all(|byte| matches!(byte, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'+' | b'/' | b'=')) +} + +#[cfg(test)] +mod tests { + use super::*; + use base64::{engine::general_purpose::STANDARD, Engine as _}; + + fn large_image_data_url() -> String { + format!( + "data:image/png;base64,{}", + "iVBORw0KGgoAAAANSUhEUgAAAAE".repeat(400) + ) + } + + #[test] + fn maps_input_image_and_merges_top_level_detail() { + let part = json!({ + "type": "input_image", + "image_url": "https://example.com/image.png", + "detail": "high" + }); + + let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap(); + + assert_eq!(mapped["type"], "image_url"); + assert_eq!(mapped["image_url"]["url"], "https://example.com/image.png"); + assert_eq!(mapped["image_url"]["detail"], "high"); + } + + #[test] + fn maps_already_chat_shaped_image_url() { + let part = json!({ + "type": "image_url", + "image_url": { + "url": "https://example.com/image.png", + "detail": "low" + } + }); + + let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap(); + + assert_eq!(mapped, part); + } + + #[test] + fn maps_anthropic_and_mcp_image_shapes() { + let anthropic = json!({ + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": "YWJj" + } + }); + let mcp = json!({ + "type": "image", + "mimeType": "image/webp", + "data": "ZGVm" + }); + let anthropic_url = json!({ + "type": "image", + "source": { + "url": "https://example.com/anthropic.png" + } + }); + + let anthropic = + chat_media_part_from_tool_part(&anthropic, ToolMediaScope::AllSupported).unwrap(); + let mcp = chat_media_part_from_tool_part(&mcp, ToolMediaScope::AllSupported).unwrap(); + let anthropic_url = + chat_media_part_from_tool_part(&anthropic_url, ToolMediaScope::AllSupported).unwrap(); + + assert_eq!(anthropic["image_url"]["url"], "data:image/jpeg;base64,YWJj"); + assert_eq!(mcp["image_url"]["url"], "data:image/webp;base64,ZGVm"); + assert_eq!( + anthropic_url["image_url"]["url"], + "https://example.com/anthropic.png" + ); + } + + #[test] + fn maps_anthropic_source_data_when_optional_url_is_empty() { + let part = json!({ + "type": "image", + "source": { + "url": "", + "media_type": "image/png", + "data": "YWJj" + } + }); + + let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap(); + + assert_eq!(mapped["image_url"]["url"], "data:image/png;base64,YWJj"); + } + + #[test] + fn rejects_image_metadata_and_non_image_mcp_payloads() { + let metadata = json!({"type": "image", "name": "cover"}); + let non_image = json!({ + "type": "image", + "mimeType": "text/plain", + "data": "aGVsbG8=" + }); + + assert!(chat_media_part_from_tool_part(&metadata, ToolMediaScope::AllSupported).is_none()); + assert!(chat_media_part_from_tool_part(&non_image, ToolMediaScope::AllSupported).is_none()); + } + + #[test] + fn loose_data_image_url_is_media_but_loose_remote_url_is_not() { + let data = json!({ + "image_url": { + "url": "data:application/octet-stream;base64,YWJj" + } + }); + let remote = json!({ + "image_url": { + "url": "https://example.com/search-thumbnail.png" + } + }); + + assert!(tool_output_contains_media( + &data, + ToolMediaScope::ImagesOnly + )); + assert!(!tool_output_contains_media( + &remote, + ToolMediaScope::ImagesOnly + )); + } + + #[test] + fn does_not_scan_embedded_data_urls_inside_plain_text() { + let data_url = large_image_data_url(); + let mut value = json!(format!("")); + let original = value.clone(); + let replacement = json!({"type": "text", "text": "moved"}); + let mut media = Vec::new(); + + assert!(!tool_output_contains_media( + &value, + ToolMediaScope::AllSupported + )); + assert_eq!( + strip_media_from_tool_value( + &mut value, + &mut media, + ToolMediaScope::AllSupported, + &replacement, + "moved", + ), + 0 + ); + assert!(media.is_empty()); + assert_eq!(value, original); + } + + #[test] + fn whole_string_data_url_respects_threshold() { + let large = large_image_data_url(); + let small = "data:image/png;base64,YWJj"; + + assert!(whole_string_image_data_url(&large).is_some()); + assert!(whole_string_image_data_url(small).is_none()); + } + + #[test] + fn strips_media_from_json_string_and_nested_content() { + let data_url = large_image_data_url(); + let mut value = Value::String( + json!({ + "content": [ + {"type": "input_text", "text": "caption"}, + {"type": "input_image", "image_url": data_url} + ] + }) + .to_string(), + ); + let replacement = json!({ + "type": "text", + "text": "moved" + }); + let mut media = Vec::new(); + + let replaced = strip_media_from_tool_value( + &mut value, + &mut media, + ToolMediaScope::AllSupported, + &replacement, + "moved", + ); + + assert_eq!(replaced, 1); + assert_eq!(media.len(), 1); + let serialized = value.as_str().unwrap(); + assert!(serialized.contains("\"text\":\"moved\"")); + assert!(!serialized.contains("iVBORw0KGgo")); + } + + #[test] + fn image_only_scope_ignores_file_and_audio() { + let file = json!({"type": "input_file", "file_id": "file_1"}); + let audio = json!({ + "type": "input_audio", + "input_audio": {"data": "YWJj", "format": "wav"} + }); + + assert!(!tool_output_contains_media( + &file, + ToolMediaScope::ImagesOnly + )); + assert!(!tool_output_contains_media( + &audio, + ToolMediaScope::ImagesOnly + )); + assert!(tool_output_contains_media( + &file, + ToolMediaScope::AllSupported + )); + assert!(tool_output_contains_media( + &audio, + ToolMediaScope::AllSupported + )); + } + + #[test] + fn clamp_preserves_long_text_but_removes_data_and_base64_payloads() { + let long_text = format!( + "{} with spaces and punctuation!", + "ordinary text ".repeat(9000) + ); + let data_url = large_image_data_url(); + let bytes = (0_u8..=255).cycle().take(18_000).collect::>(); + let base64 = STANDARD.encode(bytes); + let mut value = json!({ + "text": long_text, + "data_url": data_url, + "raw": base64 + }); + + clamp_base64ish_strings(&mut value); + + assert_eq!(value["text"], long_text); + assert!(value["data_url"] + .as_str() + .unwrap() + .starts_with("[cc-switch: omitted ")); + assert!(value["raw"] + .as_str() + .unwrap() + .starts_with("[cc-switch: omitted ")); + } + + #[test] + fn no_media_strip_is_byte_stable() { + let mut value = json!({ + "content": [ + {"type": "text", "text": "hello"}, + {"type": "image", "name": "business metadata"} + ] + }); + let before = canonical_json_string(&value); + let replacement = json!({"type": "text", "text": "moved"}); + let mut media = Vec::new(); + + let replaced = strip_media_from_tool_value( + &mut value, + &mut media, + ToolMediaScope::AllSupported, + &replacement, + "moved", + ); + + assert_eq!(replaced, 0); + assert!(media.is_empty()); + assert_eq!(canonical_json_string(&value), before); + } +}