fix(proxy): move Codex tool-result media out of stringified tool text

The Responses->Chat conversion serialized image-bearing *_output items
into role:"tool" text via canonical_json_string, so view_image results
were tokenized as base64 text (~9000x inflation). Codex replays full
history every turn, so sessions hit context-limit 400s and wedged
(#4465, #5663).

- add proxy/tool_media: shared detection/strip/clamp walker for tool
  output media (typed input_image / image_url / input_file /
  input_audio, Anthropic source and MCP data+mimeType image shapes,
  untyped data: image_url, whole-string bare data URLs)
- transform_codex_chat: replace media blocks in place with marker text
  so tool content stays a plain string, and flush the extracted media
  as one synthetic role:"user" message after each consecutive tool
  batch; media-free traffic stays byte-identical to keep prompt-cache
  prefixes stable
- media_sanitizer: detect and strip tool-output media symmetrically
  (including JSON-string outputs) so reactive image stripping can heal
  upstream modality rejections
- forwarder: regression tests pinning the reactive trigger and the
  context-limit-400 non-trigger

E2E against Kimi K3 through the proxy: the replayed turn stays ~12k
input tokens with 99% cache hit, versus ~85k+ of base64 text per
replay before.
This commit is contained in:
Jason
2026-07-24 10:16:57 +08:00
parent 34cbb375f0
commit 6c9d444c8a
5 changed files with 1820 additions and 29 deletions
+46
View File
@@ -4755,6 +4755,27 @@ mod tests {
})
}
fn body_with_codex_tool_output_image(stringified: bool) -> Value {
let output = json!({
"content": [{
"type": "input_image",
"image_url": "data:image/png;base64,TOOL_OUTPUT_SENTINEL"
}]
});
json!({
"model": "any-model",
"input": [{
"type": "function_call_output",
"call_id": "call_1",
"output": if stringified {
Value::String(output.to_string())
} else {
output
}
}]
})
}
fn image_unsupported_error() -> ProxyError {
ProxyError::UpstreamError {
status: 400,
@@ -4857,6 +4878,31 @@ mod tests {
assert!(fwd.media_retry_should_trigger("Codex", false, &body, &error));
}
#[test]
fn reactive_triggers_for_structured_and_stringified_codex_tool_images() {
let fwd = forwarder_with_rectifier(RectifierConfig::default());
for stringified in [false, true] {
let body = body_with_codex_tool_output_image(stringified);
assert!(
fwd.media_retry_should_trigger("Codex", false, &body, &image_unsupported_error()),
"tool-output image should trigger retry (stringified={stringified})"
);
}
}
#[test]
fn reactive_does_not_treat_context_limit_as_image_rejection() {
let fwd = forwarder_with_rectifier(RectifierConfig::default());
let body = body_with_codex_tool_output_image(false);
let context_error = ProxyError::UpstreamError {
status: 400,
body: Some(r#"{"error":{"message":"maximum context length exceeded"}}"#.to_string()),
};
assert!(!fwd.media_retry_should_trigger("Codex", false, &body, &context_error));
}
#[test]
fn reactive_skipped_when_media_fallback_off() {
// 关闭 request_media_fallback:上游报图片错误也不触发兜底重试。
+243
View File
@@ -3,6 +3,9 @@ use crate::model_capabilities::is_confirmed_text_only_model as confirmed_text_on
use crate::model_capabilities::{image_input_capability_from_settings, ImageInputCapability};
use crate::provider::Provider;
use crate::proxy::error::ProxyError;
use crate::proxy::tool_media::{
strip_media_from_tool_value, tool_output_contains_media, ToolMediaScope,
};
use serde_json::{json, Value};
pub const UNSUPPORTED_IMAGE_MARKER: &str = "[Unsupported Image]";
@@ -193,6 +196,9 @@ fn responses_input_item_has_image_blocks(item: &Value) -> bool {
}
item.get("content").is_some_and(content_has_image_blocks)
|| item
.get("output")
.is_some_and(|output| tool_output_contains_media(output, ToolMediaScope::ImagesOnly))
}
fn replace_images_in_responses_input(input: &mut Value) -> usize {
@@ -218,6 +224,23 @@ fn replace_images_in_responses_input_item(item: &mut Value) -> usize {
replaced += replace_images_in_content_with_text_type(content, "input_text");
}
if let Some(output) = item.get_mut("output") {
// The image-capability fallback deliberately strips images only.
// Tool-output files/audio remain a known unsupported-modality gap.
let replacement_block = json!({
"type": "input_text",
"text": UNSUPPORTED_IMAGE_MARKER
});
let mut discarded_media = Vec::new();
replaced += strip_media_from_tool_value(
output,
&mut discarded_media,
ToolMediaScope::ImagesOnly,
&replacement_block,
UNSUPPORTED_IMAGE_MARKER,
);
}
replaced
}
@@ -283,6 +306,13 @@ mod tests {
}
}
fn large_tool_data_url() -> String {
format!(
"data:image/png;base64,{}",
"SANITIZER_TOOL_MEDIA_SENTINEL".repeat(400)
)
}
#[test]
fn keeps_images_when_model_capability_is_unknown() {
let provider = provider(json!({}));
@@ -603,6 +633,219 @@ mod tests {
);
}
#[test]
fn detects_and_replaces_responses_function_output_images() {
let data_url = large_tool_data_url();
let mut body = json!({
"model": "text-only",
"input": [{
"type": "function_call_output",
"call_id": "call_1",
"output": {
"content": [
{"type": "input_text", "text": "caption"},
{"type": "input_image", "image_url": data_url.clone()},
{"type": "image", "mimeType": "image/webp", "data": "MCP_SENTINEL"}
]
}
}]
});
assert!(contains_image_blocks(&body));
let replaced = replace_image_blocks_with_marker(&mut body);
assert_eq!(replaced, 2);
assert_eq!(
body["input"][0]["output"]["content"][1],
json!({"type": "input_text", "text": UNSUPPORTED_IMAGE_MARKER})
);
assert_eq!(
body["input"][0]["output"]["content"][2],
json!({"type": "input_text", "text": UNSUPPORTED_IMAGE_MARKER})
);
assert!(!body.to_string().contains(&data_url));
assert!(!body.to_string().contains("MCP_SENTINEL"));
}
#[test]
fn proactive_text_only_sanitizer_covers_responses_tool_outputs() {
let provider = provider(json!({
"models": [{"id": "text-model", "input": ["text"]}]
}));
let mut body = json!({
"model": "text-model",
"input": [{
"type": "function_call_output",
"call_id": "call_1",
"output": [{
"type": "input_image",
"image_url": "data:image/png;base64,PROACTIVE_SENTINEL"
}]
}]
});
let replaced = replace_images_for_text_only_model(&mut body, &provider, true);
assert_eq!(replaced, 1);
assert_eq!(body["input"][0]["output"][0]["type"], "input_text");
assert!(!body.to_string().contains("PROACTIVE_SENTINEL"));
}
#[test]
fn detects_and_replaces_json_string_tool_output_symmetrically() {
let data_url = large_tool_data_url();
let output = json!({
"content": [{
"type": "input_image",
"image_url": data_url.clone()
}]
})
.to_string();
let mut body = json!({
"input": [{
"type": "function_call_output",
"call_id": "call_string",
"output": output
}]
});
assert!(contains_image_blocks(&body));
let replaced = replace_image_blocks_with_marker(&mut body);
assert_eq!(replaced, 1);
let rewritten = body["input"][0]["output"].as_str().unwrap();
assert!(rewritten.contains(UNSUPPORTED_IMAGE_MARKER));
assert!(!rewritten.contains(&data_url));
let parsed: Value = serde_json::from_str(rewritten).unwrap();
assert_eq!(parsed["content"][0]["type"], "input_text");
}
#[test]
fn detects_and_replaces_whole_string_tool_image_data_url() {
let data_url = large_tool_data_url();
let mut body = json!({
"input": [{
"type": "function_call_output",
"call_id": "call_raw",
"output": data_url.clone()
}]
});
assert!(contains_image_blocks(&body));
let replaced = replace_image_blocks_with_marker(&mut body);
assert_eq!(replaced, 1);
assert_eq!(
body["input"][0]["output"],
Value::String(UNSUPPORTED_IMAGE_MARKER.to_string())
);
assert!(!body.to_string().contains(&data_url));
}
#[test]
fn detects_and_replaces_custom_tool_output_images() {
let mut body = json!({
"input": [{
"type": "custom_tool_call_output",
"call_id": "call_custom",
"status": "completed",
"output": [{
"type": "image_url",
"image_url": {"url": "https://example.com/render.png"}
}]
}]
});
assert!(contains_image_blocks(&body));
let replaced = replace_image_blocks_with_marker(&mut body);
assert_eq!(replaced, 1);
assert_eq!(body["input"][0]["status"], "completed");
assert_eq!(body["input"][0]["output"][0]["type"], "input_text");
}
#[test]
fn ignores_no_media_and_untyped_remote_tool_outputs() {
let mut body = json!({
"input": [
{
"type": "function_call_output",
"call_id": "call_text",
"output": {"content": [{"type": "text", "text": "ordinary result"}]}
},
{
"type": "tool_search_output",
"call_id": "call_search",
"output": {
"image_url": {"url": "https://example.com/search-thumbnail.png"}
}
}
]
});
let original = body.clone();
assert!(!contains_image_blocks(&body));
assert_eq!(replace_image_blocks_with_marker(&mut body), 0);
assert_eq!(body, original);
}
#[test]
fn image_retry_scope_intentionally_ignores_tool_files_and_audio() {
let mut body = json!({
"input": [{
"type": "function_call_output",
"call_id": "call_modalities",
"output": {
"content": [
{"type": "input_file", "file_id": "file_1"},
{
"type": "input_audio",
"input_audio": {"data": "AUDIO", "format": "wav"}
}
]
}
}]
});
let original = body.clone();
assert!(!contains_image_blocks(&body));
assert_eq!(replace_image_blocks_with_marker(&mut body), 0);
assert_eq!(body, original);
}
#[test]
fn replaces_synthetic_user_and_tool_role_chat_image_parts() {
let mut body = json!({
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "tool media"},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64,USER_SENTINEL"}
}
]
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": [{
"type": "image_url",
"image_url": {"url": "https://example.com/tool.png"}
}]
}
]
});
assert!(contains_image_blocks(&body));
let replaced = replace_image_blocks_with_marker(&mut body);
assert_eq!(replaced, 2);
assert_eq!(body["messages"][0]["content"][1]["type"], "text");
assert_eq!(body["messages"][1]["content"][0]["type"], "text");
}
#[test]
fn detects_unsupported_image_errors() {
let error = ProxyError::UpstreamError {
+1
View File
@@ -33,6 +33,7 @@ pub(crate) mod switch_lock;
pub mod thinking_budget_rectifier;
pub mod thinking_optimizer;
pub mod thinking_rectifier;
pub(crate) mod tool_media;
pub(crate) mod types;
pub mod usage;
@@ -16,6 +16,10 @@ use crate::proxy::{
canonical_json_string, canonicalize_json_string_if_parseable, canonicalize_tool_arguments,
short_sha256_hex,
},
tool_media::{
chat_file_from_input_file, clamp_base64ish_strings, strip_media_from_tool_value,
whole_string_image_data_url, ToolMediaScope,
},
};
use serde_json::{json, Value};
use std::collections::{HashMap, HashSet};
@@ -42,6 +46,8 @@ const CUSTOM_TOOL_INPUT_FIELD: &str = "input";
const CHAT_TOOL_NAME_MAX_LEN: usize = 64;
const CUSTOM_TOOL_INPUT_DESCRIPTION: &str = "Raw string input for the original custom tool. Preserve formatting exactly and follow the original tool definition embedded in the description.";
const CUSTOM_TOOL_PRESERVED_METADATA_HEADING: &str = "Original tool definition:";
const TOOL_RESULT_MEDIA_MOVED_MARKER: &str =
"[cc-switch: tool result media moved to the following user message]";
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum CodexToolKind {
@@ -539,12 +545,77 @@ fn instruction_text(value: &Value) -> String {
}
}
struct ToolOutputMediaPlan {
tool_content: String,
media_parts: Vec<Value>,
}
fn plan_tool_output_media(mut output: Value) -> Option<ToolOutputMediaPlan> {
let replacement_block = json!({
"type": "text",
"text": TOOL_RESULT_MEDIA_MOVED_MARKER
});
let mut media_parts = Vec::new();
let replaced = strip_media_from_tool_value(
&mut output,
&mut media_parts,
ToolMediaScope::AllSupported,
&replacement_block,
TOOL_RESULT_MEDIA_MOVED_MARKER,
);
if replaced == 0 {
return None;
}
// Only media-bearing outputs enter this path. Remove any residual large
// data/base64 scalar that was not itself a recognized content block while
// preserving ordinary long OCR/log/source text.
clamp_base64ish_strings(&mut output);
Some(ToolOutputMediaPlan {
tool_content: canonical_json_string(&output),
media_parts,
})
}
fn plan_raw_tool_output_data_url(output: &str) -> Option<ToolOutputMediaPlan> {
whole_string_image_data_url(output).map(|media_part| ToolOutputMediaPlan {
// A raw tool-output string remains a raw tool-output string. Do not add
// JSON string-literal quotes around the replacement marker.
tool_content: TOOL_RESULT_MEDIA_MOVED_MARKER.to_string(),
media_parts: vec![media_part],
})
}
fn queue_tool_output_media(pending_media: &mut Vec<Value>, call_id: &str, media_parts: Vec<Value>) {
if media_parts.is_empty() {
return;
}
pending_media.push(json!({
"type": "text",
"text": format!("[cc-switch: media output of tool call {call_id}]")
}));
pending_media.extend(media_parts);
}
fn flush_pending_media(messages: &mut Vec<Value>, pending_media: &mut Vec<Value>) {
if pending_media.is_empty() {
return;
}
messages.push(json!({
"role": "user",
"content": std::mem::take(pending_media)
}));
}
fn append_responses_input_as_chat_messages(
input: &Value,
messages: &mut Vec<Value>,
tool_context: &CodexToolContext,
) -> Result<(), ProxyError> {
let mut pending_tool_calls = Vec::new();
let mut pending_media = Vec::new();
let mut pending_reasoning: Option<String> = None;
let mut last_assistant_index: Option<usize> = None;
@@ -561,6 +632,7 @@ fn append_responses_input_as_chat_messages(
item,
messages,
&mut pending_tool_calls,
&mut pending_media,
&mut pending_reasoning,
&mut last_assistant_index,
tool_context,
@@ -572,6 +644,7 @@ fn append_responses_input_as_chat_messages(
input,
messages,
&mut pending_tool_calls,
&mut pending_media,
&mut pending_reasoning,
&mut last_assistant_index,
tool_context,
@@ -580,9 +653,14 @@ fn append_responses_input_as_chat_messages(
_ => {}
}
// If a later assistant tool-call batch was accumulated after an earlier
// media-bearing result, the synthetic user media belongs before that next
// assistant turn.
flush_pending_media(messages, &mut pending_media);
flush_pending_tool_calls(
messages,
&mut pending_tool_calls,
&mut pending_media,
&mut pending_reasoning,
&mut last_assistant_index,
);
@@ -603,6 +681,7 @@ fn append_responses_item_as_chat_message(
item: &Value,
messages: &mut Vec<Value>,
pending_tool_calls: &mut Vec<Value>,
pending_media: &mut Vec<Value>,
pending_reasoning: &mut Option<String>,
last_assistant_index: &mut Option<usize>,
tool_context: &CodexToolContext,
@@ -628,14 +707,35 @@ fn append_responses_item_as_chat_message(
flush_pending_tool_calls(
messages,
pending_tool_calls,
pending_media,
pending_reasoning,
last_assistant_index,
);
let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or("");
let output = match item.get("output") {
Some(Value::String(s)) => canonicalize_json_string_if_parseable(s),
Some(v) => canonical_json_string(v),
None => String::new(),
let media_plan = match item.get("output") {
Some(Value::String(output)) => {
plan_raw_tool_output_data_url(output).or_else(|| {
serde_json::from_str::<Value>(output.trim())
.ok()
.and_then(plan_tool_output_media)
})
}
Some(output @ (Value::Array(_) | Value::Object(_))) => {
plan_tool_output_media(output.clone())
}
_ => None,
};
let output = if let Some(media_plan) = media_plan {
queue_tool_output_media(pending_media, call_id, media_plan.media_parts);
media_plan.tool_content
} else {
// Cache-sensitive no-media fallback: keep these expressions
// byte-for-byte equivalent to the pre-fix conversion.
match item.get("output") {
Some(Value::String(s)) => canonicalize_json_string_if_parseable(s),
Some(v) => canonical_json_string(v),
None => String::new(),
}
};
messages.push(json!({
"role": "tool",
@@ -647,11 +747,39 @@ fn append_responses_item_as_chat_message(
flush_pending_tool_calls(
messages,
pending_tool_calls,
pending_media,
pending_reasoning,
last_assistant_index,
);
let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or("");
let output = canonical_json_string(item);
let mut transformed_item = item.clone();
let replacement_block = json!({
"type": "text",
"text": TOOL_RESULT_MEDIA_MOVED_MARKER
});
let mut media_parts = Vec::new();
let replaced = transformed_item
.get_mut("output")
.map(|output| {
strip_media_from_tool_value(
output,
&mut media_parts,
ToolMediaScope::AllSupported,
&replacement_block,
TOOL_RESULT_MEDIA_MOVED_MARKER,
)
})
.unwrap_or(0);
let output = if replaced > 0 {
if let Some(output) = transformed_item.get_mut("output") {
clamp_base64ish_strings(output);
}
queue_tool_output_media(pending_media, call_id, media_parts);
canonical_json_string(&transformed_item)
} else {
// Preserve the legacy whole-item representation exactly.
canonical_json_string(item)
};
messages.push(json!({
"role": "tool",
"tool_call_id": call_id,
@@ -672,9 +800,14 @@ fn append_responses_item_as_chat_message(
flush_pending_tool_calls(
messages,
pending_tool_calls,
pending_media,
pending_reasoning,
last_assistant_index,
);
// `flush_pending_tool_calls` intentionally returns early when
// there is no new assistant batch. A previous tool result may
// still have media waiting, so flush it before this new message.
flush_pending_media(messages, pending_media);
let role = item
.get("role")
.and_then(|v| v.as_str())
@@ -705,13 +838,15 @@ fn append_responses_item_as_chat_message(
messages.push(message);
}
Some("message") | None => {
flush_pending_tool_calls(
messages,
pending_tool_calls,
pending_reasoning,
last_assistant_index,
);
if item.get("role").is_some() || item.get("content").is_some() {
flush_pending_tool_calls(
messages,
pending_tool_calls,
pending_media,
pending_reasoning,
last_assistant_index,
);
flush_pending_media(messages, pending_media);
let message = responses_message_item_to_chat_message(
item,
pending_reasoning,
@@ -720,16 +855,28 @@ fn append_responses_item_as_chat_message(
);
update_last_assistant_index(messages, &message, last_assistant_index);
messages.push(message);
} else if pending_media.is_empty() {
// Preserve legacy no-media ordering: inert message-like items
// used to close a pending tool-call batch.
flush_pending_tool_calls(
messages,
pending_tool_calls,
pending_media,
pending_reasoning,
last_assistant_index,
);
}
}
_ => {
flush_pending_tool_calls(
messages,
pending_tool_calls,
pending_reasoning,
last_assistant_index,
);
if item.get("role").is_some() || item.get("content").is_some() {
flush_pending_tool_calls(
messages,
pending_tool_calls,
pending_media,
pending_reasoning,
last_assistant_index,
);
flush_pending_media(messages, pending_media);
let message = responses_message_item_to_chat_message(
item,
pending_reasoning,
@@ -738,6 +885,16 @@ fn append_responses_item_as_chat_message(
);
update_last_assistant_index(messages, &message, last_assistant_index);
messages.push(message);
} else if pending_media.is_empty() {
// Preserve legacy no-media ordering without letting an inert
// unknown item flush a media-bearing result batch.
flush_pending_tool_calls(
messages,
pending_tool_calls,
pending_media,
pending_reasoning,
last_assistant_index,
);
}
}
}
@@ -748,6 +905,7 @@ fn append_responses_item_as_chat_message(
fn flush_pending_tool_calls(
messages: &mut Vec<Value>,
pending_tool_calls: &mut Vec<Value>,
pending_media: &mut Vec<Value>,
pending_reasoning: &mut Option<String>,
last_assistant_index: &mut Option<usize>,
) {
@@ -755,6 +913,10 @@ fn flush_pending_tool_calls(
return;
}
// Media from the preceding tool-result batch must be presented before a
// new assistant tool-call turn. Consecutive outputs do not enter here
// because `pending_tool_calls` is empty after the first output.
flush_pending_media(messages, pending_media);
let mut message = json!({
"role": "assistant",
"content": null,
@@ -1057,18 +1219,7 @@ fn responses_content_to_chat_content(_role: &str, content: &Value) -> Value {
}
fn responses_input_file_to_chat_file(part: &Value) -> Option<Value> {
let mut file = serde_json::Map::new();
let has_supported_file_ref = part.get("file_id").is_some() || part.get("file_data").is_some();
if !has_supported_file_ref {
return None;
}
for key in ["file_id", "file_data", "filename"] {
if let Some(value) = part.get(key) {
file.insert(key.to_string(), value.clone());
}
}
Some(Value::Object(file))
chat_file_from_input_file(part)
}
fn collect_tool_search_output_tools(value: &Value, context: &mut CodexToolContext) {
@@ -1834,6 +1985,50 @@ pub fn chat_error_to_response_error(body: Option<&Value>) -> Value {
#[cfg(test)]
mod tests {
use super::*;
use base64::{engine::general_purpose::STANDARD, Engine as _};
fn large_test_image_data_url() -> String {
let bytes = b"CC_SWITCH_TOOL_MEDIA_SENTINEL".repeat(400);
format!("data:image/png;base64,{}", STANDARD.encode(bytes))
}
fn message_roles(result: &Value) -> Vec<&str> {
result["messages"]
.as_array()
.unwrap()
.iter()
.filter_map(|message| message.get("role").and_then(Value::as_str))
.collect()
}
fn test_function_call(call_id: &str) -> Value {
json!({
"type": "function_call",
"call_id": call_id,
"name": "view_image",
"arguments": "{}"
})
}
fn test_function_output(call_id: &str, output: Value) -> Value {
json!({
"type": "function_call_output",
"call_id": call_id,
"output": output
})
}
fn convert_test_input(items: Vec<Value>) -> Value {
responses_to_chat_completions(json!({
"model": "kimi-k3",
"input": items
}))
.unwrap()
}
fn result_messages(result: &Value) -> &[Value] {
result["messages"].as_array().unwrap()
}
#[test]
fn responses_request_with_stream_injects_include_usage() {
@@ -3061,6 +3256,538 @@ mod tests {
assert_eq!(messages[1]["content"], "plain text result");
}
#[test]
fn responses_request_to_chat_moves_tool_image_to_synthetic_user_message() {
let data_url = large_test_image_data_url();
let result = convert_test_input(vec![
test_function_call("call_image"),
test_function_output(
"call_image",
json!([
{"type": "input_text", "text": "screenshot follows"},
{"type": "input_image", "image_url": data_url.clone()}
]),
),
]);
let messages = result_messages(&result);
assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]);
assert!(messages[1]["content"].is_string());
let tool_content: Value =
serde_json::from_str(messages[1]["content"].as_str().unwrap()).unwrap();
assert_eq!(tool_content[0]["text"], "screenshot follows");
assert_eq!(tool_content[1]["type"], "text");
assert_eq!(tool_content[1]["text"], TOOL_RESULT_MEDIA_MOVED_MARKER);
assert!(!messages[1]["content"].as_str().unwrap().contains(&data_url));
assert_eq!(
messages[2]["content"][0]["text"],
"[cc-switch: media output of tool call call_image]"
);
assert_eq!(messages[2]["content"][1]["type"], "image_url");
assert_eq!(messages[2]["content"][1]["image_url"]["url"], data_url);
}
#[test]
fn responses_request_to_chat_groups_parallel_media_after_all_tool_outputs() {
let first_url = large_test_image_data_url();
let second_payload = "MCP_TOOL_MEDIA_SENTINEL";
let result = convert_test_input(vec![
test_function_call("call_1"),
test_function_call("call_2"),
test_function_output(
"call_1",
json!({"type": "input_image", "image_url": first_url.clone()}),
),
json!({
"type": "reasoning",
"summary": [{"type": "summary_text", "text": "keep outputs adjacent"}]
}),
test_function_output(
"call_2",
json!({
"type": "image",
"mimeType": "image/webp",
"data": second_payload
}),
),
]);
let messages = result_messages(&result);
assert_eq!(
message_roles(&result),
vec!["assistant", "tool", "tool", "user"]
);
assert_eq!(messages[0]["reasoning_content"], "keep outputs adjacent");
assert_ne!(messages[0]["reasoning_content"], "tool call");
assert_eq!(messages[1]["tool_call_id"], "call_1");
assert_eq!(messages[2]["tool_call_id"], "call_2");
assert!(messages[1]["content"].is_string());
assert!(messages[2]["content"].is_string());
assert!(!messages[1]["content"]
.as_str()
.unwrap()
.contains(&first_url));
assert!(!messages[2]["content"]
.as_str()
.unwrap()
.contains(second_payload));
assert_eq!(messages[3]["content"].as_array().unwrap().len(), 4);
assert_eq!(
messages[3]["content"][3]["image_url"]["url"],
format!("data:image/webp;base64,{second_payload}")
);
}
#[test]
fn responses_request_to_chat_flushes_media_before_next_tool_call_batch() {
let result = convert_test_input(vec![
test_function_call("call_1"),
test_function_output(
"call_1",
json!({
"type": "input_image",
"image_url": large_test_image_data_url()
}),
),
test_function_call("call_2"),
test_function_output("call_2", json!("second result")),
]);
let messages = result_messages(&result);
assert_eq!(
message_roles(&result),
vec!["assistant", "tool", "user", "assistant", "tool"]
);
assert_eq!(messages[0]["tool_calls"][0]["id"], "call_1");
assert_eq!(messages[3]["tool_calls"][0]["id"], "call_2");
assert_eq!(messages[4]["tool_call_id"], "call_2");
}
#[test]
fn responses_request_to_chat_flushes_media_before_real_user_messages() {
let boundaries = [
json!({"type": "input_text", "text": "continue"}),
json!({"type": "future_message", "role": "user", "content": "continue"}),
];
for boundary in boundaries {
let result = convert_test_input(vec![
test_function_call("call_1"),
test_function_output(
"call_1",
json!({
"type": "input_image",
"image_url": large_test_image_data_url()
}),
),
boundary,
]);
let messages = result_messages(&result);
assert_eq!(
message_roles(&result),
vec!["assistant", "tool", "user", "user"]
);
assert!(messages[2]["content"].is_array());
assert_eq!(messages[3]["role"], "user");
}
}
#[test]
fn responses_request_to_chat_handles_raw_data_url_thresholds() {
let large = large_test_image_data_url();
let large_result = convert_test_input(vec![
test_function_call("call_large"),
test_function_output("call_large", Value::String(large.clone())),
]);
let large_messages = result_messages(&large_result);
assert_eq!(
message_roles(&large_result),
vec!["assistant", "tool", "user"]
);
assert_eq!(large_messages[1]["content"], TOOL_RESULT_MEDIA_MOVED_MARKER);
assert_eq!(large_messages[2]["content"][1]["image_url"]["url"], large);
let small = "data:image/png;base64,YWJj";
let small_result = convert_test_input(vec![
test_function_call("call_small"),
test_function_output("call_small", json!(small)),
]);
assert_eq!(message_roles(&small_result), vec!["assistant", "tool"]);
assert_eq!(small_result["messages"][1]["content"], small);
}
#[test]
fn responses_request_to_chat_maps_supported_structured_image_shapes() {
let cases = vec![
(
json!({
"type": "input_image",
"image_url": {"url": "https://example.com/input.png"},
"detail": "high"
}),
"https://example.com/input.png",
Some("high"),
),
(
json!({
"type": "image_url",
"image_url": "https://example.com/chat-string.png"
}),
"https://example.com/chat-string.png",
None,
),
(
json!({
"type": "image_url",
"image_url": {
"url": "https://example.com/chat-object.png",
"detail": "low"
}
}),
"https://example.com/chat-object.png",
Some("low"),
),
(
json!({"image_url": "data:image/gif;base64,LOOSE_SENTINEL"}),
"data:image/gif;base64,LOOSE_SENTINEL",
None,
),
(
json!({
"type": "image",
"source": {
"media_type": "image/jpeg",
"data": "ANTHROPIC_SENTINEL"
}
}),
"data:image/jpeg;base64,ANTHROPIC_SENTINEL",
None,
),
(
json!({
"type": "image",
"mimeType": "image/webp",
"data": "MCP_SENTINEL"
}),
"data:image/webp;base64,MCP_SENTINEL",
None,
),
];
for (index, (output, expected_url, expected_detail)) in cases.into_iter().enumerate() {
let call_id = format!("call_shape_{index}");
let result = convert_test_input(vec![
test_function_call(&call_id),
test_function_output(&call_id, output),
]);
let image = &result["messages"][2]["content"][1];
assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]);
assert_eq!(image["type"], "image_url");
assert_eq!(image["image_url"]["url"], expected_url);
match expected_detail {
Some(detail) => assert_eq!(image["image_url"]["detail"], detail),
None => assert!(image["image_url"].get("detail").is_none()),
}
}
}
#[test]
fn responses_request_to_chat_extracts_media_from_json_string_and_content_wrapper() {
let output = json!({
"content": [
{"type": "input_text", "text": "MCP response"},
{
"type": "image",
"mimeType": "image/png",
"data": "STRING_MCP_SENTINEL"
}
]
})
.to_string();
let result = convert_test_input(vec![
test_function_call("call_string"),
test_function_output("call_string", Value::String(output)),
]);
let messages = result_messages(&result);
assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]);
let tool_content: Value =
serde_json::from_str(messages[1]["content"].as_str().unwrap()).unwrap();
assert_eq!(tool_content["content"][0]["text"], "MCP response");
assert_eq!(
tool_content["content"][1]["text"],
TOOL_RESULT_MEDIA_MOVED_MARKER
);
assert!(!messages[1]["content"]
.as_str()
.unwrap()
.contains("STRING_MCP_SENTINEL"));
assert_eq!(
messages[2]["content"][1]["image_url"]["url"],
"data:image/png;base64,STRING_MCP_SENTINEL"
);
}
#[test]
fn responses_request_to_chat_extracts_tool_files_and_audio() {
let result = convert_test_input(vec![
test_function_call("call_media"),
test_function_output(
"call_media",
json!({
"content": [
{
"type": "input_file",
"file_id": "file_123",
"filename": "report.pdf"
},
{
"type": "input_audio",
"input_audio": {"data": "AUDIO_SENTINEL", "format": "wav"}
}
]
}),
),
]);
let messages = result_messages(&result);
assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]);
assert_eq!(messages[2]["content"][1]["type"], "file");
assert_eq!(messages[2]["content"][1]["file"]["file_id"], "file_123");
assert_eq!(messages[2]["content"][2]["type"], "input_audio");
assert_eq!(
messages[2]["content"][2]["input_audio"]["data"],
"AUDIO_SENTINEL"
);
}
#[test]
fn responses_request_to_chat_extracts_custom_and_tool_search_output_media() {
let cases = [
(
json!({
"type": "custom_tool_call",
"call_id": "call_custom",
"name": "render",
"input": "draw"
}),
json!({
"type": "custom_tool_call_output",
"call_id": "call_custom",
"status": "completed",
"output": {
"content": [{
"type": "input_image",
"image_url": "data:image/png;base64,CUSTOM_SENTINEL"
}]
}
}),
),
(
json!({
"type": "tool_search_call",
"call_id": "call_search",
"arguments": {"query": "image tool"}
}),
json!({
"type": "tool_search_output",
"call_id": "call_search",
"status": "completed",
"output": {
"content": [{
"type": "image",
"mimeType": "image/png",
"data": "SEARCH_SENTINEL"
}]
}
}),
),
];
for (call, output) in cases {
let expected_type = output["type"].as_str().unwrap().to_string();
let result = convert_test_input(vec![call, output]);
let messages = result_messages(&result);
let tool_content: Value =
serde_json::from_str(messages[1]["content"].as_str().unwrap()).unwrap();
assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]);
assert_eq!(tool_content["type"], expected_type);
assert_eq!(tool_content["status"], "completed");
assert_eq!(
tool_content["output"]["content"][0]["text"],
TOOL_RESULT_MEDIA_MOVED_MARKER
);
}
}
#[test]
fn responses_request_to_chat_rejects_false_positive_media_shapes() {
let outputs = [
json!({"type": "image", "name": "business metadata"}),
json!({
"type": "image",
"mimeType": "text/plain",
"data": "NOT_AN_IMAGE"
}),
json!({
"image_url": {
"url": "https://example.com/search-thumbnail.png"
}
}),
];
for (index, output) in outputs.into_iter().enumerate() {
let call_id = format!("call_false_positive_{index}");
let expected = canonical_json_string(&output);
let result = convert_test_input(vec![
test_function_call(&call_id),
test_function_output(&call_id, output),
]);
assert_eq!(message_roles(&result), vec!["assistant", "tool"]);
assert_eq!(result["messages"][1]["content"], expected);
}
}
#[test]
fn responses_request_to_chat_keeps_no_media_tool_output_bytes_stable() {
let cases = [
(Some(json!("plain text")), "plain text".to_string()),
(
Some(json!("{ \"z\": true, \"a\": [2, 1] }")),
r#"{"a":[2,1],"z":true}"#.to_string(),
),
(
Some(json!(["main.rs", "lib.rs"])),
r#"["main.rs","lib.rs"]"#.to_string(),
),
(
Some(json!({"z": true, "a": [2, 1]})),
r#"{"a":[2,1],"z":true}"#.to_string(),
),
(Some(json!([])), "[]".to_string()),
(None, String::new()),
];
for (index, (output, expected)) in cases.into_iter().enumerate() {
let call_id = format!("call_stable_{index}");
let mut item = json!({
"type": "function_call_output",
"call_id": call_id
});
if let Some(output) = output {
item["output"] = output;
}
let result = convert_test_input(vec![test_function_call(&call_id), item]);
assert_eq!(message_roles(&result), vec!["assistant", "tool"]);
assert_eq!(result["messages"][1]["content"], expected);
}
for item in [
json!({
"type": "custom_tool_call_output",
"call_id": "call_custom",
"status": "completed",
"output": {"text": "unchanged"}
}),
json!({
"type": "tool_search_output",
"call_id": "call_search",
"status": "completed",
"output": []
}),
] {
let expected = canonical_json_string(&item);
let result = convert_test_input(vec![item]);
assert_eq!(result["messages"][0]["content"], expected);
}
}
#[test]
fn responses_request_to_chat_preserves_legacy_unknown_item_batch_boundary_without_media() {
let result = convert_test_input(vec![
test_function_call("call_1"),
json!({"type": "future_metadata", "value": 1}),
json!({
"type": "reasoning",
"summary": [{"type": "summary_text", "text": "second batch reasoning"}]
}),
test_function_call("call_2"),
test_function_output("call_1", json!("first result")),
test_function_output("call_2", json!("second result")),
]);
let messages = result_messages(&result);
assert_eq!(
message_roles(&result),
vec!["assistant", "assistant", "tool", "tool"]
);
assert_eq!(messages[0]["tool_calls"].as_array().unwrap().len(), 1);
assert_eq!(messages[0]["tool_calls"][0]["id"], "call_1");
assert_eq!(messages[0]["reasoning_content"], "tool call");
assert_eq!(messages[1]["tool_calls"].as_array().unwrap().len(), 1);
assert_eq!(messages[1]["tool_calls"][0]["id"], "call_2");
assert_eq!(messages[1]["reasoning_content"], "second batch reasoning");
}
#[test]
fn responses_request_to_chat_clamps_only_residual_base64ish_strings() {
let data_url = large_test_image_data_url();
let long_text = format!("{}end", "ordinary OCR text with spaces. ".repeat(3_500));
let residual_base64 = "A".repeat(20_000);
let result = convert_test_input(vec![
test_function_call("call_clamp"),
test_function_output(
"call_clamp",
json!([
{"type": "input_image", "image_url": data_url.clone()},
{"type": "text", "text": long_text.clone()},
{"raw": residual_base64}
]),
),
]);
let tool_content_text = result["messages"][1]["content"].as_str().unwrap();
let tool_content: Value = serde_json::from_str(tool_content_text).unwrap();
assert_eq!(tool_content[1]["text"], long_text);
assert!(tool_content[2]["raw"]
.as_str()
.unwrap()
.starts_with("[cc-switch: omitted 20000 bytes]"));
assert!(!tool_content_text.contains(&data_url));
assert!(!tool_content_text.contains("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"));
}
#[test]
fn responses_request_to_chat_media_conversion_is_deterministic() {
let input = json!({
"model": "kimi-k3",
"input": [
test_function_call("call_repeat"),
test_function_output(
"call_repeat",
json!({
"content": [{
"type": "input_image",
"image_url": large_test_image_data_url()
}]
})
)
]
});
let first = responses_to_chat_completions(input.clone()).unwrap();
let second = responses_to_chat_completions(input).unwrap();
assert_eq!(first, second);
}
#[test]
fn chat_response_to_responses_maps_text_tool_calls_and_usage() {
let input = json!({
+774
View File
@@ -0,0 +1,774 @@
//! Shared media handling for Codex tool outputs.
//!
//! Responses tool outputs may carry structured media blocks. Chat Completions
//! tool messages are text-only, so the Responses→Chat bridge extracts those
//! blocks and re-emits them in a synthetic user message. The media sanitizer
//! reuses the same recognition and traversal rules when it needs to remove
//! images for a text-only upstream.
use crate::proxy::json_canonical::canonical_json_string;
use serde_json::{json, Map, Value};
pub(crate) const WHOLE_DATA_URL_MIN_BYTES: usize = 8 * 1024;
const BASE64ISH_MIN_BYTES: usize = 16 * 1024;
const MAX_MEDIA_TRAVERSAL_DEPTH: usize = 32;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ToolMediaScope {
/// Used by the existing image-capability sanitizer and its retry path.
ImagesOnly,
/// Used by Responses→Chat conversion, where user messages can carry all
/// currently mapped Chat input modalities.
AllSupported,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ToolMediaKind {
Image,
File,
Audio,
}
impl ToolMediaScope {
fn allows(self, kind: ToolMediaKind) -> bool {
matches!(kind, ToolMediaKind::Image) || matches!(self, Self::AllSupported)
}
}
/// Convert one recognized Responses/tool media block to a Chat user content
/// part. This is the single shape-recognition entry point used by extraction.
pub(crate) fn chat_media_part_from_tool_part(part: &Value, scope: ToolMediaScope) -> Option<Value> {
let kind = tool_media_kind(part)?;
if !scope.allows(kind) {
return None;
}
match kind {
ToolMediaKind::Image => chat_image_part(part),
ToolMediaKind::File => chat_file_from_input_file(part).map(|file| {
json!({
"type": "file",
"file": file
})
}),
ToolMediaKind::Audio => part.get("input_audio").map(|input_audio| {
json!({
"type": "input_audio",
"input_audio": input_audio.clone()
})
}),
}
}
/// Map a Responses `input_file` block to the Chat file payload. Kept here so
/// top-level content and tool-output extraction share the exact same rules.
pub(crate) fn chat_file_from_input_file(part: &Value) -> Option<Value> {
let mut file = Map::new();
let has_supported_file_ref = part.get("file_id").is_some() || part.get("file_data").is_some();
if !has_supported_file_ref {
return None;
}
for key in ["file_id", "file_data", "filename"] {
if let Some(value) = part.get(key) {
file.insert(key.to_string(), value.clone());
}
}
Some(Value::Object(file))
}
/// Recognize a complete image data URL stored as a scalar string.
///
/// Only whole-string matches are accepted. Embedded data URLs in HTML/CSS/SVG
/// source are deliberately left alone. Small values remain text as well, which
/// preserves workflows that intentionally inspect tiny inline icons.
pub(crate) fn whole_string_image_data_url(value: &str) -> Option<Value> {
let trimmed = value.trim();
if trimmed.len() < WHOLE_DATA_URL_MIN_BYTES || !is_image_base64_data_url(trimmed) {
return None;
}
Some(json!({
"type": "image_url",
"image_url": {
"url": trimmed
}
}))
}
/// Read-only media detection using the same shape classifier and recursive
/// boundaries as [`strip_media_from_tool_value`].
pub(crate) fn tool_output_contains_media(value: &Value, scope: ToolMediaScope) -> bool {
tool_output_contains_media_at_depth(value, scope, 0)
}
/// Extract recognized media blocks and replace them in-place.
///
/// `replacement_block` is used for structured array/object parts. A scalar
/// string that is itself a complete image data URL uses `replacement_text`, so
/// an originally plain string stays a plain string. Parseable JSON strings are
/// recursively transformed and canonicalized back into a string only when a
/// replacement actually occurred.
pub(crate) fn strip_media_from_tool_value(
value: &mut Value,
media_parts: &mut Vec<Value>,
scope: ToolMediaScope,
replacement_block: &Value,
replacement_text: &str,
) -> usize {
strip_media_from_tool_value_at_depth(
value,
media_parts,
scope,
replacement_block,
replacement_text,
0,
)
}
/// Remove residual data/base64 payloads only after a tool output has already
/// been positively identified as media-bearing. Ordinary long text is kept.
pub(crate) fn clamp_base64ish_strings(value: &mut Value) {
match value {
Value::String(text) => {
let trimmed = text.trim();
let should_omit = (trimmed.len() >= WHOLE_DATA_URL_MIN_BYTES
&& trimmed
.get(..5)
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")))
|| looks_like_base64_payload(trimmed);
if should_omit {
let byte_len = text.len();
*text = format!("[cc-switch: omitted {byte_len} bytes]");
}
}
Value::Array(items) => {
for item in items {
clamp_base64ish_strings(item);
}
}
Value::Object(object) => {
for nested in object.values_mut() {
clamp_base64ish_strings(nested);
}
}
_ => {}
}
}
fn tool_output_contains_media_at_depth(value: &Value, scope: ToolMediaScope, depth: usize) -> bool {
if depth > MAX_MEDIA_TRAVERSAL_DEPTH {
return false;
}
match value {
Value::String(text) => {
if scope.allows(ToolMediaKind::Image) && whole_string_image_data_url(text).is_some() {
return true;
}
let trimmed = text.trim();
if trimmed.is_empty() {
return false;
}
serde_json::from_str::<Value>(trimmed)
.ok()
.is_some_and(|parsed| {
tool_output_contains_media_at_depth(&parsed, scope, depth + 1)
})
}
Value::Array(items) => items
.iter()
.any(|item| tool_output_contains_media_at_depth(item, scope, depth + 1)),
Value::Object(object) => {
if tool_media_kind(value).is_some_and(|kind| scope.allows(kind)) {
return true;
}
object.get("content").is_some_and(|content| {
tool_output_contains_media_at_depth(content, scope, depth + 1)
})
}
_ => false,
}
}
fn strip_media_from_tool_value_at_depth(
value: &mut Value,
media_parts: &mut Vec<Value>,
scope: ToolMediaScope,
replacement_block: &Value,
replacement_text: &str,
depth: usize,
) -> usize {
if depth > MAX_MEDIA_TRAVERSAL_DEPTH {
return 0;
}
match value {
Value::String(text) => {
if scope.allows(ToolMediaKind::Image) {
if let Some(media_part) = whole_string_image_data_url(text) {
media_parts.push(media_part);
*text = replacement_text.to_string();
return 1;
}
}
let trimmed = text.trim();
if trimmed.is_empty() {
return 0;
}
let Ok(mut parsed) = serde_json::from_str::<Value>(trimmed) else {
return 0;
};
let replaced = strip_media_from_tool_value_at_depth(
&mut parsed,
media_parts,
scope,
replacement_block,
replacement_text,
depth + 1,
);
if replaced > 0 {
*text = canonical_json_string(&parsed);
}
replaced
}
Value::Array(items) => items
.iter_mut()
.map(|item| {
strip_media_from_tool_value_at_depth(
item,
media_parts,
scope,
replacement_block,
replacement_text,
depth + 1,
)
})
.sum(),
Value::Object(_) => {
if let Some(media_part) = chat_media_part_from_tool_part(value, scope) {
media_parts.push(media_part);
*value = replacement_block.clone();
return 1;
}
value
.as_object_mut()
.expect("object match arm must remain an object")
.get_mut("content")
.map(|content| {
strip_media_from_tool_value_at_depth(
content,
media_parts,
scope,
replacement_block,
replacement_text,
depth + 1,
)
})
.unwrap_or(0)
}
_ => 0,
}
}
fn tool_media_kind(part: &Value) -> Option<ToolMediaKind> {
let object = part.as_object()?;
let part_type = object.get("type").and_then(Value::as_str);
match part_type {
Some("input_image" | "image_url") if normalized_image_url(part).is_some() => {
Some(ToolMediaKind::Image)
}
Some("input_file") if part.get("file_id").is_some() || part.get("file_data").is_some() => {
Some(ToolMediaKind::File)
}
Some("input_audio") if part.get("input_audio").is_some_and(Value::is_object) => {
Some(ToolMediaKind::Audio)
}
Some("image") if typed_image_has_payload(part) => Some(ToolMediaKind::Image),
None if loose_data_image_url(part).is_some() => Some(ToolMediaKind::Image),
_ => None,
}
}
fn chat_image_part(part: &Value) -> Option<Value> {
match part.get("type").and_then(Value::as_str) {
Some("input_image" | "image_url") => {
normalized_image_url(part).map(|image_url| image_url_content_part(part, image_url))
}
Some("image") => {
typed_image_url(part).map(|image_url| image_url_content_part(part, image_url))
}
None => loose_data_image_url(part).map(|image_url| image_url_content_part(part, image_url)),
_ => None,
}
}
fn normalized_image_url(part: &Value) -> Option<Value> {
let image_url = part.get("image_url")?;
let mut object = match image_url {
Value::String(url) if !url.trim().is_empty() => {
let mut object = Map::new();
object.insert("url".to_string(), Value::String(url.clone()));
object
}
Value::Object(object)
if object
.get("url")
.and_then(Value::as_str)
.is_some_and(|url| !url.trim().is_empty()) =>
{
object.clone()
}
_ => return None,
};
merge_top_level_detail(part, &mut object);
Some(Value::Object(object))
}
fn loose_data_image_url(part: &Value) -> Option<Value> {
if part.get("type").is_some() {
return None;
}
let normalized = normalized_image_url(part)?;
let url = normalized.get("url").and_then(Value::as_str)?;
if !url
.get(..5)
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
{
return None;
}
Some(normalized)
}
fn typed_image_has_payload(part: &Value) -> bool {
let Some(object) = part.as_object() else {
return false;
};
if let Some(source) = object.get("source").and_then(Value::as_object) {
if source_media_type_is_image(source) {
let has_url = source
.get("url")
.and_then(Value::as_str)
.is_some_and(|url| !url.trim().is_empty());
let has_data = source
.get("data")
.and_then(Value::as_str)
.is_some_and(|data| !data.is_empty());
if has_url || has_data {
return true;
}
}
}
object
.get("data")
.and_then(Value::as_str)
.is_some_and(|data| !data.is_empty())
&& object
.get("mimeType")
.or_else(|| object.get("mime_type"))
.and_then(Value::as_str)
.is_some_and(is_image_mime_type)
}
fn typed_image_url(part: &Value) -> Option<Value> {
let object = part.as_object()?;
if let Some(source) = object.get("source").and_then(Value::as_object) {
if !source_media_type_is_image(source) {
return None;
}
if let Some(url) = source
.get("url")
.and_then(Value::as_str)
.filter(|url| !url.trim().is_empty())
{
let mut image_url = Map::new();
image_url.insert("url".to_string(), Value::String(url.to_string()));
merge_top_level_detail(part, &mut image_url);
return Some(Value::Object(image_url));
}
if let Some(data) = source
.get("data")
.and_then(Value::as_str)
.filter(|data| !data.is_empty())
{
let media_type = source
.get("media_type")
.or_else(|| source.get("mime_type"))
.or_else(|| source.get("mimeType"))
.and_then(Value::as_str)
.unwrap_or("image/png");
let url = if data
.get(..11)
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:image/"))
{
data.to_string()
} else {
format!("data:{media_type};base64,{data}")
};
let mut image_url = Map::new();
image_url.insert("url".to_string(), Value::String(url));
merge_top_level_detail(part, &mut image_url);
return Some(Value::Object(image_url));
}
}
let data = object
.get("data")
.and_then(Value::as_str)
.filter(|data| !data.is_empty())?;
let media_type = object
.get("mimeType")
.or_else(|| object.get("mime_type"))
.and_then(Value::as_str)
.filter(|media_type| is_image_mime_type(media_type))?;
let mut image_url = Map::new();
image_url.insert(
"url".to_string(),
Value::String(format!("data:{media_type};base64,{data}")),
);
merge_top_level_detail(part, &mut image_url);
Some(Value::Object(image_url))
}
fn image_url_content_part(source_part: &Value, image_url: Value) -> Value {
let mut content_part = Map::new();
content_part.insert("type".to_string(), Value::String("image_url".to_string()));
content_part.insert("image_url".to_string(), image_url);
for key in ["cache_control", "prompt_cache_breakpoint"] {
if let Some(value) = source_part.get(key) {
content_part.insert(key.to_string(), value.clone());
}
}
Value::Object(content_part)
}
fn merge_top_level_detail(part: &Value, image_url: &mut Map<String, Value>) {
if image_url.get("detail").is_none() {
if let Some(detail) = part.get("detail") {
image_url.insert("detail".to_string(), detail.clone());
}
}
}
fn source_media_type_is_image(source: &Map<String, Value>) -> bool {
source
.get("media_type")
.or_else(|| source.get("mime_type"))
.or_else(|| source.get("mimeType"))
.and_then(Value::as_str)
.is_none_or(is_image_mime_type)
}
fn is_image_mime_type(value: &str) -> bool {
value
.get(..6)
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/"))
}
fn is_image_base64_data_url(value: &str) -> bool {
let Some(comma_index) = value.find(',') else {
return false;
};
let header = &value[..comma_index];
let header = header.to_ascii_lowercase();
header.starts_with("data:image/") && header.ends_with(";base64")
}
fn looks_like_base64_payload(value: &str) -> bool {
if value.len() < BASE64ISH_MIN_BYTES {
return false;
}
value
.bytes()
.all(|byte| matches!(byte, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'+' | b'/' | b'='))
}
#[cfg(test)]
mod tests {
use super::*;
use base64::{engine::general_purpose::STANDARD, Engine as _};
fn large_image_data_url() -> String {
format!(
"data:image/png;base64,{}",
"iVBORw0KGgoAAAANSUhEUgAAAAE".repeat(400)
)
}
#[test]
fn maps_input_image_and_merges_top_level_detail() {
let part = json!({
"type": "input_image",
"image_url": "https://example.com/image.png",
"detail": "high"
});
let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap();
assert_eq!(mapped["type"], "image_url");
assert_eq!(mapped["image_url"]["url"], "https://example.com/image.png");
assert_eq!(mapped["image_url"]["detail"], "high");
}
#[test]
fn maps_already_chat_shaped_image_url() {
let part = json!({
"type": "image_url",
"image_url": {
"url": "https://example.com/image.png",
"detail": "low"
}
});
let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap();
assert_eq!(mapped, part);
}
#[test]
fn maps_anthropic_and_mcp_image_shapes() {
let anthropic = json!({
"type": "image",
"source": {
"type": "base64",
"media_type": "image/jpeg",
"data": "YWJj"
}
});
let mcp = json!({
"type": "image",
"mimeType": "image/webp",
"data": "ZGVm"
});
let anthropic_url = json!({
"type": "image",
"source": {
"url": "https://example.com/anthropic.png"
}
});
let anthropic =
chat_media_part_from_tool_part(&anthropic, ToolMediaScope::AllSupported).unwrap();
let mcp = chat_media_part_from_tool_part(&mcp, ToolMediaScope::AllSupported).unwrap();
let anthropic_url =
chat_media_part_from_tool_part(&anthropic_url, ToolMediaScope::AllSupported).unwrap();
assert_eq!(anthropic["image_url"]["url"], "data:image/jpeg;base64,YWJj");
assert_eq!(mcp["image_url"]["url"], "data:image/webp;base64,ZGVm");
assert_eq!(
anthropic_url["image_url"]["url"],
"https://example.com/anthropic.png"
);
}
#[test]
fn maps_anthropic_source_data_when_optional_url_is_empty() {
let part = json!({
"type": "image",
"source": {
"url": "",
"media_type": "image/png",
"data": "YWJj"
}
});
let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap();
assert_eq!(mapped["image_url"]["url"], "data:image/png;base64,YWJj");
}
#[test]
fn rejects_image_metadata_and_non_image_mcp_payloads() {
let metadata = json!({"type": "image", "name": "cover"});
let non_image = json!({
"type": "image",
"mimeType": "text/plain",
"data": "aGVsbG8="
});
assert!(chat_media_part_from_tool_part(&metadata, ToolMediaScope::AllSupported).is_none());
assert!(chat_media_part_from_tool_part(&non_image, ToolMediaScope::AllSupported).is_none());
}
#[test]
fn loose_data_image_url_is_media_but_loose_remote_url_is_not() {
let data = json!({
"image_url": {
"url": "data:application/octet-stream;base64,YWJj"
}
});
let remote = json!({
"image_url": {
"url": "https://example.com/search-thumbnail.png"
}
});
assert!(tool_output_contains_media(
&data,
ToolMediaScope::ImagesOnly
));
assert!(!tool_output_contains_media(
&remote,
ToolMediaScope::ImagesOnly
));
}
#[test]
fn does_not_scan_embedded_data_urls_inside_plain_text() {
let data_url = large_image_data_url();
let mut value = json!(format!("<html><img src=\"{data_url}\"></html>"));
let original = value.clone();
let replacement = json!({"type": "text", "text": "moved"});
let mut media = Vec::new();
assert!(!tool_output_contains_media(
&value,
ToolMediaScope::AllSupported
));
assert_eq!(
strip_media_from_tool_value(
&mut value,
&mut media,
ToolMediaScope::AllSupported,
&replacement,
"moved",
),
0
);
assert!(media.is_empty());
assert_eq!(value, original);
}
#[test]
fn whole_string_data_url_respects_threshold() {
let large = large_image_data_url();
let small = "data:image/png;base64,YWJj";
assert!(whole_string_image_data_url(&large).is_some());
assert!(whole_string_image_data_url(small).is_none());
}
#[test]
fn strips_media_from_json_string_and_nested_content() {
let data_url = large_image_data_url();
let mut value = Value::String(
json!({
"content": [
{"type": "input_text", "text": "caption"},
{"type": "input_image", "image_url": data_url}
]
})
.to_string(),
);
let replacement = json!({
"type": "text",
"text": "moved"
});
let mut media = Vec::new();
let replaced = strip_media_from_tool_value(
&mut value,
&mut media,
ToolMediaScope::AllSupported,
&replacement,
"moved",
);
assert_eq!(replaced, 1);
assert_eq!(media.len(), 1);
let serialized = value.as_str().unwrap();
assert!(serialized.contains("\"text\":\"moved\""));
assert!(!serialized.contains("iVBORw0KGgo"));
}
#[test]
fn image_only_scope_ignores_file_and_audio() {
let file = json!({"type": "input_file", "file_id": "file_1"});
let audio = json!({
"type": "input_audio",
"input_audio": {"data": "YWJj", "format": "wav"}
});
assert!(!tool_output_contains_media(
&file,
ToolMediaScope::ImagesOnly
));
assert!(!tool_output_contains_media(
&audio,
ToolMediaScope::ImagesOnly
));
assert!(tool_output_contains_media(
&file,
ToolMediaScope::AllSupported
));
assert!(tool_output_contains_media(
&audio,
ToolMediaScope::AllSupported
));
}
#[test]
fn clamp_preserves_long_text_but_removes_data_and_base64_payloads() {
let long_text = format!(
"{} with spaces and punctuation!",
"ordinary text ".repeat(9000)
);
let data_url = large_image_data_url();
let bytes = (0_u8..=255).cycle().take(18_000).collect::<Vec<_>>();
let base64 = STANDARD.encode(bytes);
let mut value = json!({
"text": long_text,
"data_url": data_url,
"raw": base64
});
clamp_base64ish_strings(&mut value);
assert_eq!(value["text"], long_text);
assert!(value["data_url"]
.as_str()
.unwrap()
.starts_with("[cc-switch: omitted "));
assert!(value["raw"]
.as_str()
.unwrap()
.starts_with("[cc-switch: omitted "));
}
#[test]
fn no_media_strip_is_byte_stable() {
let mut value = json!({
"content": [
{"type": "text", "text": "hello"},
{"type": "image", "name": "business metadata"}
]
});
let before = canonical_json_string(&value);
let replacement = json!({"type": "text", "text": "moved"});
let mut media = Vec::new();
let replaced = strip_media_from_tool_value(
&mut value,
&mut media,
ToolMediaScope::AllSupported,
&replacement,
"moved",
);
assert_eq!(replaced, 0);
assert!(media.is_empty());
assert_eq!(canonical_json_string(&value), before);
}
}