admin/CC-Switch
1
0
Fork 0
mirror of https://github.com/farion1231/cc-switch.git synced 2026-07-28 00:35:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(proxy): extend tool-result media handling to all conversion bridges

Browse Source 
Generalize the Codex Responses-to-Chat tool media mechanism from 6c9d444c
to the remaining protocol bridges, so image-bearing tool results are never
tokenized as base64 text on any conversion path:

- Claude-to-Chat: tool_result images become native image_url parts,
  batched into one synthetic user message after the tool message batch.
- Claude-to-Responses: JSON-string, MCP, and nested variants are restored
  as native input_image parts inside function_call_output.
- Codex/GrokBuild-to-Anthropic: non-standard tool images are restored as
  Anthropic image blocks (case-insensitive data:/http prefix parsing).
- Claude-to-Gemini: Gemini 3 uses multimodal functionResponse.parts;
  older models get inlineData parts in the same user turn. The new
  InlineImagesOnly scope keeps remote URLs and malformed data URLs in the
  legacy text form instead of emitting fileData the API would reject.

Shared changes in tool_media:

- Centralize the plan/queue/flush helpers used by both Chat bridges.
- strip_and_clamp_media_from_tool_value clamps residual base64 inside
  parsed JSON strings at any nesting depth before re-serialization.
- Emitted Chat parts no longer carry cache_control or
  prompt_cache_breakpoint, preserving the strip-all-cache_control
  contract that strict upstreams (GLM/Qwen) depend on.
- Media detection now requires full convertibility, keeping detection
  and extraction symmetric by construction.

The media sanitizer detects and strips the new shapes symmetrically
(Chat string tool results, Anthropic string tool results, Gemini
inlineData/fileData and functionResponse.parts); the legacy typed-block
replacement runs first so replacement stays a superset of detection and
cache_control survives on replaced Anthropic blocks.

No-media tool outputs keep byte-identical legacy representations on all
bridges to protect prompt-cache prefixes.
This commit is contained in:
Jason
2026-07-24 12:12:22 +08:00
parent 6c9d444c8a
commit 878c26f31e
8 changed files with 1666 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src-tauri/src/proxy/forwarder.rs
+51
View File
@@ -4776,6 +4776,39 @@ mod tests {
})
}
fn body_with_stringified_chat_tool_image() -> Value {
let content = json!({
"content": [{
"type": "image",
"mimeType": "image/png",
"data": "CHAT_TOOL_SENTINEL"
}]
})
.to_string();
json!({
"model": "any-model",
"messages": [{
"role": "tool",
"tool_call_id": "call_1",
"content": content
}]
})
}
fn body_with_gemini_image() -> Value {
json!({
"contents": [{
"role": "user",
"parts": [{
"inlineData": {
"mimeType": "image/png",
"data": "GEMINI_SENTINEL"
}
}]
}]
})
}
fn image_unsupported_error() -> ProxyError {
ProxyError::UpstreamError {
status: 400,
@@ -4891,6 +4924,24 @@ mod tests {
}
}
#[test]
fn reactive_triggers_for_chat_tool_and_gemini_images() {
let fwd = forwarder_with_rectifier(RectifierConfig::default());
assert!(fwd.media_retry_should_trigger(
"Claude",
false,
&body_with_stringified_chat_tool_image(),
&image_unsupported_error()
));
assert!(fwd.media_retry_should_trigger(
"Claude",
false,
&body_with_gemini_image(),
&image_unsupported_error()
));
}
#[test]
fn reactive_does_not_treat_context_limit_as_image_rejection() {
let fwd = forwarder_with_rectifier(RectifierConfig::default());
src-tauri/src/proxy/media_sanitizer.rs
+325 -15
View File
@@ -43,7 +43,9 @@ pub fn replace_images_for_text_only_model(
}
pub fn contains_image_blocks(body: &Value) -> bool {
messages_have_image_blocks(body) || responses_input_has_image_blocks(body.get("input"))
messages_have_image_blocks(body)
|| responses_input_has_image_blocks(body.get("input"))
|| gemini_contents_have_image_blocks(body)
}
pub fn replace_image_blocks_with_marker(body: &mut Value) -> usize {
@@ -122,7 +124,11 @@ fn content_has_image_blocks(content: &Value) -> bool {
blocks.iter().any(|block| {
is_image_block_type(block.get("type").and_then(Value::as_str))
|| block.get("content").is_some_and(content_has_image_blocks)
|| block.get("content").is_some_and(|nested| {
content_has_image_blocks(nested)
|| (block.get("type").and_then(Value::as_str) == Some("tool_result")
&& tool_output_contains_media(nested, ToolMediaScope::ImagesOnly))
})
})
}
@@ -130,13 +136,7 @@ fn replace_images_in_body(body: &mut Value) -> usize {
let message_replacements = body
.get_mut("messages")
.and_then(Value::as_array_mut)
.map(|messages| {
messages
.iter_mut()
.filter_map(|message| message.get_mut("content"))
.map(replace_images_in_content)
.sum()
})
.map(|messages| messages.iter_mut().map(replace_images_in_message).sum())
.unwrap_or(0);
message_replacements
@@ -144,6 +144,37 @@ fn replace_images_in_body(body: &mut Value) -> usize {
.get_mut("input")
.map(replace_images_in_responses_input)
.unwrap_or(0)
+ replace_images_in_gemini_contents(body)
}
fn replace_images_in_message(message: &mut Value) -> usize {
let is_tool_message = message.get("role").and_then(Value::as_str) == Some("tool");
let Some(content) = message.get_mut("content") else {
return 0;
};
if is_tool_message {
// Preserve the legacy typed-image replacement semantics first,
// including Anthropic cache_control on the replacement text block.
// The shared traversal then handles JSON strings, MCP wrappers, and
// loose data-URL shapes that the legacy recursion does not recognize.
let mut replaced = replace_images_in_content(content);
let replacement_block = json!({
"type":"text",
"text":UNSUPPORTED_IMAGE_MARKER
});
let mut discarded_media = Vec::new();
replaced += strip_media_from_tool_value(
content,
&mut discarded_media,
ToolMediaScope::ImagesOnly,
&replacement_block,
UNSUPPORTED_IMAGE_MARKER,
);
replaced
} else {
replace_images_in_content(content)
}
}
fn replace_images_in_content(content: &mut Value) -> usize {
@@ -157,14 +188,36 @@ fn replace_images_in_content_with_text_type(content: &mut Value, text_type: &str
let mut replaced = 0usize;
for block in blocks {
if is_image_block_type(block.get("type").and_then(Value::as_str)) {
let block_type = block.get("type").and_then(Value::as_str);
if is_image_block_type(block_type) {
replace_image_block_with_text_marker(block, text_type);
replaced += 1;
continue;
}
let is_tool_result = block_type == Some("tool_result");
if let Some(nested_content) = block.get_mut("content") {
replaced += replace_images_in_content_with_text_type(nested_content, text_type);
if is_tool_result {
// Run the legacy typed-block replacement before the shared
// payload-aware traversal. This makes replacement a superset
// of detection and preserves cache_control on Anthropic image
// blocks, while the second pass covers alternate tool shapes.
replaced += replace_images_in_content_with_text_type(nested_content, text_type);
let replacement_block = json!({
"type":text_type,
"text":UNSUPPORTED_IMAGE_MARKER
});
let mut discarded_media = Vec::new();
replaced += strip_media_from_tool_value(
nested_content,
&mut discarded_media,
ToolMediaScope::ImagesOnly,
&replacement_block,
UNSUPPORTED_IMAGE_MARKER,
);
} else {
replaced += replace_images_in_content_with_text_type(nested_content, text_type);
}
}
}
@@ -175,13 +228,110 @@ fn messages_have_image_blocks(body: &Value) -> bool {
body.get("messages")
.and_then(Value::as_array)
.is_some_and(|messages| {
messages
.iter()
.filter_map(|message| message.get("content"))
.any(content_has_image_blocks)
messages.iter().any(|message| {
let Some(content) = message.get("content") else {
return false;
};
content_has_image_blocks(content)
|| (message.get("role").and_then(Value::as_str) == Some("tool")
&& tool_output_contains_media(content, ToolMediaScope::ImagesOnly))
})
})
}
fn gemini_contents_have_image_blocks(body: &Value) -> bool {
body.get("contents")
.and_then(Value::as_array)
.is_some_and(|contents| {
contents.iter().any(|content| {
content
.get("parts")
.and_then(Value::as_array)
.is_some_and(|parts| parts.iter().any(gemini_part_has_image))
})
})
}
fn gemini_part_has_image(part: &Value) -> bool {
gemini_media_payload_is_image(part.get("inlineData").or_else(|| part.get("inline_data")))
|| gemini_media_payload_is_image(part.get("fileData").or_else(|| part.get("file_data")))
|| part
.get("functionResponse")
.or_else(|| part.get("function_response"))
.and_then(|response| response.get("parts"))
.and_then(Value::as_array)
.is_some_and(|parts| parts.iter().any(gemini_part_has_image))
}
fn gemini_media_payload_is_image(payload: Option<&Value>) -> bool {
payload
.and_then(|payload| payload.get("mimeType").or_else(|| payload.get("mime_type")))
.and_then(Value::as_str)
.is_some_and(|mime_type| {
mime_type
.get(..6)
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/"))
})
}
fn replace_images_in_gemini_contents(body: &mut Value) -> usize {
body.get_mut("contents")
.and_then(Value::as_array_mut)
.map(|contents| {
contents
.iter_mut()
.filter_map(|content| content.get_mut("parts").and_then(Value::as_array_mut))
.map(|parts| {
parts
.iter_mut()
.map(replace_images_in_gemini_part)
.sum::<usize>()
})
.sum()
})
.unwrap_or(0)
}
fn replace_images_in_gemini_part(part: &mut Value) -> usize {
if gemini_media_payload_is_image(part.get("inlineData").or_else(|| part.get("inline_data")))
|| gemini_media_payload_is_image(part.get("fileData").or_else(|| part.get("file_data")))
{
*part = json!({"text":UNSUPPORTED_IMAGE_MARKER});
return 1;
}
let response_key = if part.get("functionResponse").is_some() {
"functionResponse"
} else {
"function_response"
};
let Some(function_response) = part.get_mut(response_key) else {
return 0;
};
let Some(media_parts) = function_response
.get_mut("parts")
.and_then(Value::as_array_mut)
else {
return 0;
};
let before = media_parts.len();
media_parts.retain(|media_part| !gemini_part_has_image(media_part));
let replaced = before.saturating_sub(media_parts.len());
if replaced > 0 {
if let Some(response) = function_response
.get_mut("response")
.and_then(Value::as_object_mut)
{
response.insert(
"cc_switch_media".to_string(),
Value::String(UNSUPPORTED_IMAGE_MARKER.to_string()),
);
}
}
replaced
}
fn responses_input_has_image_blocks(input: Option<&Value>) -> bool {
match input {
Some(Value::Array(items)) => items.iter().any(responses_input_item_has_image_blocks),
@@ -633,6 +783,71 @@ mod tests {
);
}
#[test]
fn replaces_file_backed_tool_result_image_and_preserves_cache_control() {
let mut body = json!({
"model": "deepseek-v4-pro",
"messages": [{
"role": "user",
"content": [{
"type": "tool_result",
"tool_use_id": "toolu_file",
"content": [{
"type": "image",
"source": {
"type": "file",
"file_id": "file_123"
},
"cache_control": {"type": "ephemeral"}
}]
}]
}]
});
assert!(contains_image_blocks(&body));
let count = replace_image_blocks_with_marker(&mut body);
let replacement = &body["messages"][0]["content"][0]["content"][0];
assert_eq!(count, 1);
assert_eq!(replacement["type"], "text");
assert_eq!(replacement["text"], UNSUPPORTED_IMAGE_MARKER);
assert_eq!(replacement["cache_control"]["type"], "ephemeral");
assert!(!body.to_string().contains("file_123"));
}
#[test]
fn replaces_stringified_anthropic_tool_result_image_blocks() {
let content = json!({
"content": [{
"type": "image",
"mimeType": "image/png",
"data": "ANTHROPIC_STRING_TOOL_SENTINEL"
}]
})
.to_string();
let mut body = json!({
"model": "deepseek-v4-pro",
"messages": [{
"role": "user",
"content": [{
"type": "tool_result",
"tool_use_id": "toolu_1",
"content": content
}]
}]
});
assert!(contains_image_blocks(&body));
let count = replace_image_blocks_with_marker(&mut body);
let rewritten = body["messages"][0]["content"][0]["content"]
.as_str()
.unwrap();
assert_eq!(count, 1);
assert!(rewritten.contains(UNSUPPORTED_IMAGE_MARKER));
assert!(!rewritten.contains("ANTHROPIC_STRING_TOOL_SENTINEL"));
}
#[test]
fn detects_and_replaces_responses_function_output_images() {
let data_url = large_tool_data_url();
@@ -846,6 +1061,101 @@ mod tests {
assert_eq!(body["messages"][1]["content"][0]["type"], "text");
}
#[test]
fn detects_and_replaces_stringified_chat_tool_image() {
let content = json!({
"content": [{
"type": "image",
"mimeType": "image/png",
"data": "STRINGIFIED_CHAT_TOOL_SENTINEL"
}]
})
.to_string();
let mut body = json!({
"messages": [{
"role": "tool",
"tool_call_id": "call_1",
"content": content
}]
});
assert!(contains_image_blocks(&body));
let replaced = replace_image_blocks_with_marker(&mut body);
let rewritten = body["messages"][0]["content"].as_str().unwrap();
assert_eq!(replaced, 1);
assert!(rewritten.contains(UNSUPPORTED_IMAGE_MARKER));
assert!(!rewritten.contains("STRINGIFIED_CHAT_TOOL_SENTINEL"));
}
#[test]
fn detects_and_replaces_gemini_native_image_parts() {
let mut body = json!({
"contents": [{
"role": "user",
"parts": [
{
"functionResponse": {
"name": "inspect",
"response": {"content": "done"}
}
},
{
"inlineData": {
"mimeType": "image/png",
"data": "GEMINI_INLINE_SENTINEL"
}
}
]
}]
});
assert!(contains_image_blocks(&body));
let replaced = replace_image_blocks_with_marker(&mut body);
assert_eq!(replaced, 1);
assert_eq!(
body["contents"][0]["parts"][1]["text"],
UNSUPPORTED_IMAGE_MARKER
);
assert!(!body.to_string().contains("GEMINI_INLINE_SENTINEL"));
}
#[test]
fn detects_and_removes_nested_gemini_function_response_media() {
let mut body = json!({
"contents": [{
"role": "user",
"parts": [{
"functionResponse": {
"name": "inspect",
"response": {"content": "done"},
"parts": [{
"inlineData": {
"mimeType": "image/webp",
"data": "GEMINI_FUNCTION_SENTINEL"
}
}]
}
}]
}]
});
assert!(contains_image_blocks(&body));
let replaced = replace_image_blocks_with_marker(&mut body);
assert_eq!(replaced, 1);
assert!(body["contents"][0]["parts"][0]["functionResponse"]["parts"]
.as_array()
.unwrap()
.is_empty());
assert_eq!(
body["contents"][0]["parts"][0]["functionResponse"]["response"]["cc_switch_media"],
UNSUPPORTED_IMAGE_MARKER
);
assert!(!body.to_string().contains("GEMINI_FUNCTION_SENTINEL"));
}
#[test]
fn detects_unsupported_image_errors() {
let error = ProxyError::UpstreamError {
src-tauri/src/proxy/providers/transform.rs
+192 -15
View File
@@ -3,7 +3,14 @@
//! 实现 Anthropic ↔ OpenAI 格式转换，用于 OpenRouter 支持
//! 参考: anthropic-proxy-rs
use crate::proxy::{error::ProxyError, json_canonical::canonical_json_string};
use crate::proxy::{
error::ProxyError,
json_canonical::canonical_json_string,
tool_media::{
chat_media_part_from_tool_part, flush_pending_chat_tool_media, plan_chat_tool_output_media,
queue_chat_tool_output_media, ToolMediaScope,
},
};
use serde_json::{json, Value};
const ANTHROPIC_BILLING_HEADER_PREFIX: &str = "x-anthropic-billing-header:";
@@ -375,6 +382,7 @@ fn convert_message_to_openai(
if let Some(blocks) = content.as_array() {
let mut content_parts = Vec::new();
let mut tool_calls = Vec::new();
let mut pending_tool_media = Vec::new();
// reasoning_parts: 仅在兼容 Moonshot/Kimi/DeepSeek thinking tool-call 路径时
// 生成 reasoning_content，通用 OpenAI-compatible 路径不发送该非标准字段。
let mut reasoning_parts = Vec::new();
@@ -389,16 +397,10 @@ fn convert_message_to_openai(
}
}
"image" => {
if let Some(source) = block.get("source") {
let media_type = source
.get("media_type")
.and_then(|m| m.as_str())
.unwrap_or("image/png");
let data = source.get("data").and_then(|d| d.as_str()).unwrap_or("");
content_parts.push(json!({
"type": "image_url",
"image_url": {"url": format!("data:{};base64,{}", media_type, data)}
}));
if let Some(image) =
chat_media_part_from_tool_part(block, ToolMediaScope::ImagesOnly)
{
content_parts.push(image);
}
}
"tool_use" => {
@@ -421,10 +423,22 @@ fn convert_message_to_openai(
.and_then(|i| i.as_str())
.unwrap_or("");
let content_val = block.get("content");
let content_str = match content_val {
Some(Value::String(s)) => s.clone(),
Some(v) => canonical_json_string(v),
None => String::new(),
let media_plan = content_val.cloned().and_then(plan_chat_tool_output_media);
let content_str = if let Some(media_plan) = media_plan {
queue_chat_tool_output_media(
&mut pending_tool_media,
tool_use_id,
media_plan.media_parts,
);
media_plan.tool_content
} else {
// Keep the no-media representation exactly equal to
// the legacy converter for prompt-cache stability.
match content_val {
Some(Value::String(s)) => s.clone(),
Some(v) => canonical_json_string(v),
None => String::new(),
}
};
result.push(json!({
"role": "tool",
@@ -452,6 +466,11 @@ fn convert_message_to_openai(
}
}
// Chat tool messages cannot carry image parts. Keep parallel tool
// results adjacent, then present all extracted media in one user turn
// before any ordinary message content from the same Anthropic turn.
flush_pending_chat_tool_media(&mut result, &mut pending_tool_media);
// 添加带内容和/或工具调用的消息
if !content_parts.is_empty() || !tool_calls.is_empty() {
let mut msg = json!({"role": role});
@@ -1133,6 +1152,164 @@ mod tests {
assert_eq!(msg["content"], "Sunny, 25°C");
}
#[test]
fn test_anthropic_to_openai_no_media_tool_results_keep_legacy_representation() {
let raw_json_string = "{ \"status\": \"ok\", \"count\": 2 }";
let input = json!({
"model": "claude-3-opus",
"messages": [{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "call_string",
"content": raw_json_string
},
{
"type": "tool_result",
"tool_use_id": "call_array",
"content": [{"type": "text", "text": "plain"}]
}
]
}]
});
let result = anthropic_to_openai(input).unwrap();
let messages = result["messages"].as_array().unwrap();
assert_eq!(messages.len(), 2);
assert_eq!(messages[0]["content"], raw_json_string);
assert_eq!(
messages[1]["content"],
canonical_json_string(&json!([{"type": "text", "text": "plain"}]))
);
}
#[test]
fn test_anthropic_to_openai_moves_tool_result_image_to_user_message() {
let input = json!({
"model": "claude-3-opus",
"max_tokens": 1024,
"messages": [{
"role": "user",
"content": [{
"type": "tool_result",
"tool_use_id": "call_image",
"content": [
{"type": "text", "text": "caption"},
{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/png",
"data": "CLAUDE_CHAT_IMAGE_SENTINEL"
},
"cache_control": {"type": "ephemeral"},
"prompt_cache_breakpoint": true
}
]
}]
}]
});
let result = anthropic_to_openai(input).unwrap();
let messages = result["messages"].as_array().unwrap();
assert_eq!(messages.len(), 2);
assert_eq!(messages[0]["role"], "tool");
assert_eq!(messages[0]["tool_call_id"], "call_image");
assert!(messages[0]["content"]
.as_str()
.unwrap()
.contains("tool result media moved"));
assert!(!messages[0]["content"]
.as_str()
.unwrap()
.contains("CLAUDE_CHAT_IMAGE_SENTINEL"));
assert_eq!(messages[1]["role"], "user");
assert_eq!(
messages[1]["content"][0]["text"],
"[cc-switch: media output of tool call call_image]"
);
assert_eq!(messages[1]["content"][1]["type"], "image_url");
assert!(messages[1]["content"][1].get("cache_control").is_none());
assert!(messages[1]["content"][1]
.get("prompt_cache_breakpoint")
.is_none());
assert_eq!(
messages[1]["content"][1]["image_url"]["url"],
"data:image/png;base64,CLAUDE_CHAT_IMAGE_SENTINEL"
);
}
#[test]
fn test_anthropic_to_openai_batches_parallel_tool_result_media() {
let input = json!({
"model": "claude-3-opus",
"messages": [{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "call_1",
"content": [{
"type": "image",
"source": {"type": "base64", "media_type": "image/png", "data": "ONE"}
}]
},
{
"type": "tool_result",
"tool_use_id": "call_2",
"content": [{
"type": "image",
"source": {"type": "base64", "media_type": "image/jpeg", "data": "TWO"}
}]
}
]
}]
});
let result = anthropic_to_openai(input).unwrap();
let messages = result["messages"].as_array().unwrap();
assert_eq!(messages.len(), 3);
assert_eq!(messages[0]["role"], "tool");
assert_eq!(messages[1]["role"], "tool");
assert_eq!(messages[2]["role"], "user");
assert_eq!(messages[2]["content"].as_array().unwrap().len(), 4);
}
#[test]
fn test_anthropic_to_openai_maps_remote_image_source() {
let input = json!({
"model": "claude-3-opus",
"messages": [{
"role": "user",
"content": [{
"type": "image",
"source": {
"type": "url",
"url": "https://example.com/image.png"
},
"cache_control": {"type": "ephemeral"},
"prompt_cache_breakpoint": true
}]
}]
});
let result = anthropic_to_openai(input).unwrap();
assert_eq!(
result["messages"][0]["content"][0]["image_url"]["url"],
"https://example.com/image.png"
);
assert!(result["messages"][0]["content"][0]
.get("cache_control")
.is_none());
assert!(result["messages"][0]["content"][0]
.get("prompt_cache_breakpoint")
.is_none());
}
#[test]
fn test_openai_to_anthropic_simple() {
let input = json!({
src-tauri/src/proxy/providers/transform_codex_anthropic.rs
+211 -14
View File
@@ -18,6 +18,9 @@ use super::transform_responses::{sanitize_anthropic_tool_use_input, TOOL_RESULT_
use crate::proxy::error::ProxyError;
use crate::proxy::json_canonical::canonical_json_string;
use crate::proxy::sse::{strip_sse_field, take_sse_block};
use crate::proxy::tool_media::{
strip_and_clamp_media_from_tool_value, ToolMediaScope, TOOL_RESULT_MEDIA_ATTACHED_MARKER,
};
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
use serde_json::{json, Value};
use std::collections::{BTreeMap, HashSet};
@@ -723,10 +726,12 @@ struct ToolResultContent {
fn tool_result_content_from_responses_item(item: &Value) -> ToolResultContent {
match item.get("output") {
Some(Value::String(text)) => ToolResultContent {
content: json!(text),
is_error: false,
},
Some(text @ Value::String(_)) => {
alternate_image_tool_result_content(text).unwrap_or_else(|| ToolResultContent {
content: text.clone(),
is_error: false,
})
}
Some(Value::Array(parts)) => {
let mut content = Vec::new();
let mut is_error = false;
@@ -761,10 +766,26 @@ fn tool_result_content_from_responses_item(item: &Value) -> ToolResultContent {
}));
}
}
_ => content.push(json!({
"type":"text",
"text":canonical_json_string(part)
})),
_ => {
if let Some(alternate) = alternate_image_tool_result_content(part) {
is_error |= alternate.is_error;
match alternate.content {
Value::Array(mut blocks) => content.append(&mut blocks),
Value::String(text) => {
content.push(json!({"type":"text","text":text}))
}
other => content.push(json!({
"type":"text",
"text":canonical_json_string(&other)
})),
}
} else {
content.push(json!({
"type":"text",
"text":canonical_json_string(part)
}));
}
}
}
}
ToolResultContent {
@@ -772,10 +793,12 @@ fn tool_result_content_from_responses_item(item: &Value) -> ToolResultContent {
is_error,
}
}
Some(value) => ToolResultContent {
content: json!(canonical_json_string(value)),
is_error: false,
},
Some(value) => {
alternate_image_tool_result_content(value).unwrap_or_else(|| ToolResultContent {
content: json!(canonical_json_string(value)),
is_error: false,
})
}
None => ToolResultContent {
content: json!(canonical_json_string(item)),
is_error: false,
@@ -783,6 +806,96 @@ fn tool_result_content_from_responses_item(item: &Value) -> ToolResultContent {
}
}
/// Convert image-bearing tool-output variants that are not native Responses
/// content blocks. The shared traversal recognizes JSON strings, MCP image
/// blocks, Anthropic image blocks, Chat image_url blocks, nested `content`
/// wrappers, and whole image data URLs.
fn alternate_image_tool_result_content(value: &Value) -> Option<ToolResultContent> {
let mut cleaned = value.clone();
let replacement_block = json!({
"type":"input_text",
"text":TOOL_RESULT_MEDIA_ATTACHED_MARKER
});
let mut chat_media_parts = Vec::new();
let replaced = strip_and_clamp_media_from_tool_value(
&mut cleaned,
&mut chat_media_parts,
ToolMediaScope::ImagesOnly,
&replacement_block,
TOOL_RESULT_MEDIA_ATTACHED_MARKER,
);
if replaced == 0 {
return None;
}
let mut content = Vec::new();
let mut is_error = false;
append_sanitized_tool_result_value(&cleaned, &mut content, &mut is_error);
content.extend(
chat_media_parts
.iter()
.filter_map(image_block_from_input_image),
);
Some(ToolResultContent {
content: Value::Array(content),
is_error,
})
}
fn append_sanitized_tool_result_value(
value: &Value,
content: &mut Vec<Value>,
is_error: &mut bool,
) {
match value {
Value::String(text) => {
if text == TOOL_RESULT_ERROR_MARKER {
*is_error = true;
} else if !text.is_empty() {
content.push(json!({"type":"text","text":text}));
}
}
Value::Array(parts) => {
for part in parts {
match part.get("type").and_then(Value::as_str) {
Some("input_text" | "output_text" | "text") => {
if let Some(text) = part.get("text").and_then(Value::as_str) {
if text == TOOL_RESULT_ERROR_MARKER {
*is_error = true;
} else {
content.push(json!({"type":"text","text":text}));
}
}
}
_ => content.push(json!({
"type":"text",
"text":canonical_json_string(part)
})),
}
}
}
Value::Object(object)
if matches!(
object.get("type").and_then(Value::as_str),
Some("input_text" | "output_text" | "text")
) =>
{
if let Some(text) = object.get("text").and_then(Value::as_str) {
if text == TOOL_RESULT_ERROR_MARKER {
*is_error = true;
} else {
content.push(json!({"type":"text","text":text}));
}
}
}
other => content.push(json!({
"type":"text",
"text":canonical_json_string(other)
})),
}
}
/// Ensures the first message is a user: compacted/resumed sessions may start with
/// assistant or function_call, but Anthropic requires the first to be user, else 400.
/// An empty array is not handled (the caller decides whether to error).
@@ -1071,8 +1184,12 @@ fn image_block_from_input_image(part: &Value) -> Option<Value> {
.or_else(|| v.get("url").and_then(|u| u.as_str()).map(str::to_string))
})?;
if let Some(rest) = url.strip_prefix("data:") {
if url
.get(..5)
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
{
// data:<media_type>;base64,<data>
let rest = &url[5..];
let (meta, data) = rest.split_once(',')?;
let media_type = meta.split(';').next().unwrap_or("image/png");
Some(json!({
@@ -1083,7 +1200,13 @@ fn image_block_from_input_image(part: &Value) -> Option<Value> {
"data": data
}
}))
} else if url.starts_with("http://") || url.starts_with("https://") {
} else if url
.get(..7)
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("http://"))
|| url
.get(..8)
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("https://"))
{
Some(json!({
"type": "image",
"source": { "type": "url", "url": url }
@@ -2555,6 +2678,80 @@ mod tests {
assert_eq!(content[1]["type"], "image");
}
#[test]
fn test_alternate_mcp_tool_image_is_not_stringified_for_anthropic() {
let response = responses_request_to_anthropic(
json!({
"model": "c",
"input": [
{"type": "function_call", "call_id": "c1", "name": "inspect", "arguments": "{}"},
{"type": "function_call_output", "call_id": "c1", "output": [{
"type": "image",
"mimeType": "image/webp",
"data": "MCP_ANTHROPIC_IMAGE_SENTINEL"
}]}
]
}),
4096,
)
.unwrap();
let content = &response["messages"][2]["content"][0]["content"];
assert_eq!(content[0]["type"], "text");
assert!(!content[0]["text"]
.as_str()
.unwrap()
.contains("MCP_ANTHROPIC_IMAGE_SENTINEL"));
assert_eq!(content[1]["type"], "image");
assert_eq!(content[1]["source"]["media_type"], "image/webp");
assert_eq!(content[1]["source"]["data"], "MCP_ANTHROPIC_IMAGE_SENTINEL");
}
#[test]
fn test_json_string_nested_tool_image_is_not_text_for_anthropic() {
let residual_base64 = "A".repeat(20_000);
let encoded_output = json!({
"content": [
{"type": "input_text", "text": "caption"},
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,STRING_IMAGE_SENTINEL"
}
},
{"type": "video", "data": residual_base64}
]
})
.to_string();
let response = responses_request_to_anthropic(
json!({
"model": "c",
"input": [
{"type": "function_call", "call_id": "c1", "name": "inspect", "arguments": "{}"},
{"type": "function_call_output", "call_id": "c1", "output": encoded_output}
]
}),
4096,
)
.unwrap();
let content = response["messages"][2]["content"][0]["content"]
.as_array()
.unwrap();
let image = content
.iter()
.find(|block| block["type"] == "image")
.expect("stringified tool image should become an Anthropic image block");
assert_eq!(image["source"]["data"], "STRING_IMAGE_SENTINEL");
assert!(content
.iter()
.filter_map(|block| block.get("text").and_then(Value::as_str))
.all(|text| !text.contains("STRING_IMAGE_SENTINEL")));
let serialized = response.to_string();
assert!(serialized.contains("[cc-switch: omitted 20000 bytes]"));
assert!(!serialized.contains(&"A".repeat(64)));
}
#[test]
fn test_structured_tool_output_restores_error_file_and_unknown_parts() {
let response = responses_request_to_anthropic(
src-tauri/src/proxy/providers/transform_codex_chat.rs
+63 -102
View File
@@ -17,8 +17,9 @@ use crate::proxy::{
short_sha256_hex,
},
tool_media::{
chat_file_from_input_file, clamp_base64ish_strings, strip_media_from_tool_value,
whole_string_image_data_url, ToolMediaScope,
chat_file_from_input_file, flush_pending_chat_tool_media, plan_chat_tool_output_media,
queue_chat_tool_output_media, strip_and_clamp_media_from_tool_value, ToolMediaScope,
TOOL_RESULT_MEDIA_MOVED_MARKER,
},
};
use serde_json::{json, Value};
@@ -46,9 +47,6 @@ const CUSTOM_TOOL_INPUT_FIELD: &str = "input";
const CHAT_TOOL_NAME_MAX_LEN: usize = 64;
const CUSTOM_TOOL_INPUT_DESCRIPTION: &str = "Raw string input for the original custom tool. Preserve formatting exactly and follow the original tool definition embedded in the description.";
const CUSTOM_TOOL_PRESERVED_METADATA_HEADING: &str = "Original tool definition:";
const TOOL_RESULT_MEDIA_MOVED_MARKER: &str =
"[cc-switch: tool result media moved to the following user message]";
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum CodexToolKind {
Function,
@@ -545,70 +543,6 @@ fn instruction_text(value: &Value) -> String {
}
}
struct ToolOutputMediaPlan {
tool_content: String,
media_parts: Vec<Value>,
}
fn plan_tool_output_media(mut output: Value) -> Option<ToolOutputMediaPlan> {
let replacement_block = json!({
"type": "text",
"text": TOOL_RESULT_MEDIA_MOVED_MARKER
});
let mut media_parts = Vec::new();
let replaced = strip_media_from_tool_value(
&mut output,
&mut media_parts,
ToolMediaScope::AllSupported,
&replacement_block,
TOOL_RESULT_MEDIA_MOVED_MARKER,
);
if replaced == 0 {
return None;
}
// Only media-bearing outputs enter this path. Remove any residual large
// data/base64 scalar that was not itself a recognized content block while
// preserving ordinary long OCR/log/source text.
clamp_base64ish_strings(&mut output);
Some(ToolOutputMediaPlan {
tool_content: canonical_json_string(&output),
media_parts,
})
}
fn plan_raw_tool_output_data_url(output: &str) -> Option<ToolOutputMediaPlan> {
whole_string_image_data_url(output).map(|media_part| ToolOutputMediaPlan {
// A raw tool-output string remains a raw tool-output string. Do not add
// JSON string-literal quotes around the replacement marker.
tool_content: TOOL_RESULT_MEDIA_MOVED_MARKER.to_string(),
media_parts: vec![media_part],
})
}
fn queue_tool_output_media(pending_media: &mut Vec<Value>, call_id: &str, media_parts: Vec<Value>) {
if media_parts.is_empty() {
return;
}
pending_media.push(json!({
"type": "text",
"text": format!("[cc-switch: media output of tool call {call_id}]")
}));
pending_media.extend(media_parts);
}
fn flush_pending_media(messages: &mut Vec<Value>, pending_media: &mut Vec<Value>) {
if pending_media.is_empty() {
return;
}
messages.push(json!({
"role": "user",
"content": std::mem::take(pending_media)
}));
}
fn append_responses_input_as_chat_messages(
input: &Value,
messages: &mut Vec<Value>,
@@ -656,7 +590,7 @@ fn append_responses_input_as_chat_messages(
// If a later assistant tool-call batch was accumulated after an earlier
// media-bearing result, the synthetic user media belongs before that next
// assistant turn.
flush_pending_media(messages, &mut pending_media);
flush_pending_chat_tool_media(messages, &mut pending_media);
flush_pending_tool_calls(
messages,
&mut pending_tool_calls,
@@ -712,21 +646,12 @@ fn append_responses_item_as_chat_message(
last_assistant_index,
);
let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or("");
let media_plan = match item.get("output") {
Some(Value::String(output)) => {
plan_raw_tool_output_data_url(output).or_else(|| {
serde_json::from_str::<Value>(output.trim())
.ok()
.and_then(plan_tool_output_media)
})
}
Some(output @ (Value::Array(_) | Value::Object(_))) => {
plan_tool_output_media(output.clone())
}
_ => None,
};
let media_plan = item
.get("output")
.cloned()
.and_then(plan_chat_tool_output_media);
let output = if let Some(media_plan) = media_plan {
queue_tool_output_media(pending_media, call_id, media_plan.media_parts);
queue_chat_tool_output_media(pending_media, call_id, media_plan.media_parts);
media_plan.tool_content
} else {
// Cache-sensitive no-media fallback: keep these expressions
@@ -761,7 +686,7 @@ fn append_responses_item_as_chat_message(
let replaced = transformed_item
.get_mut("output")
.map(|output| {
strip_media_from_tool_value(
strip_and_clamp_media_from_tool_value(
output,
&mut media_parts,
ToolMediaScope::AllSupported,
@@ -771,10 +696,7 @@ fn append_responses_item_as_chat_message(
})
.unwrap_or(0);
let output = if replaced > 0 {
if let Some(output) = transformed_item.get_mut("output") {
clamp_base64ish_strings(output);
}
queue_tool_output_media(pending_media, call_id, media_parts);
queue_chat_tool_output_media(pending_media, call_id, media_parts);
canonical_json_string(&transformed_item)
} else {
// Preserve the legacy whole-item representation exactly.
@@ -807,7 +729,7 @@ fn append_responses_item_as_chat_message(
// `flush_pending_tool_calls` intentionally returns early when
// there is no new assistant batch. A previous tool result may
// still have media waiting, so flush it before this new message.
flush_pending_media(messages, pending_media);
flush_pending_chat_tool_media(messages, pending_media);
let role = item
.get("role")
.and_then(|v| v.as_str())
@@ -846,7 +768,7 @@ fn append_responses_item_as_chat_message(
pending_reasoning,
last_assistant_index,
);
flush_pending_media(messages, pending_media);
flush_pending_chat_tool_media(messages, pending_media);
let message = responses_message_item_to_chat_message(
item,
pending_reasoning,
@@ -876,7 +798,7 @@ fn append_responses_item_as_chat_message(
pending_reasoning,
last_assistant_index,
);
flush_pending_media(messages, pending_media);
flush_pending_chat_tool_media(messages, pending_media);
let message = responses_message_item_to_chat_message(
item,
pending_reasoning,
@@ -916,7 +838,7 @@ fn flush_pending_tool_calls(
// Media from the preceding tool-result batch must be presented before a
// new assistant tool-call turn. Consecutive outputs do not enter here
// because `pending_tool_calls` is empty after the first output.
flush_pending_media(messages, pending_media);
flush_pending_chat_tool_media(messages, pending_media);
let mut message = json!({
"role": "assistant",
"content": null,
@@ -3625,6 +3547,46 @@ mod tests {
}
}
#[test]
fn responses_request_to_chat_clamps_stringified_custom_output_residual_base64() {
let encoded_output = json!({
"content": [
{
"type": "input_image",
"image_url": "data:image/png;base64,CUSTOM_STRING_IMAGE_SENTINEL"
},
{
"type": "video",
"data": "A".repeat(20_000)
}
]
})
.to_string();
let result = convert_test_input(vec![
json!({
"type": "custom_tool_call",
"call_id": "call_custom_string",
"name": "render",
"input": "draw"
}),
json!({
"type": "custom_tool_call_output",
"call_id": "call_custom_string",
"status": "completed",
"output": encoded_output
}),
]);
let messages = result_messages(&result);
let tool_item: Value =
serde_json::from_str(messages[1]["content"].as_str().unwrap()).unwrap();
let rewritten = tool_item["output"].as_str().unwrap();
assert!(rewritten.contains("[cc-switch: omitted 20000 bytes]"));
assert!(!rewritten.contains(&"A".repeat(64)));
assert!(!rewritten.contains("CUSTOM_STRING_IMAGE_SENTINEL"));
assert_eq!(messages[2]["content"][1]["type"], "image_url");
}
#[test]
fn responses_request_to_chat_rejects_false_positive_media_shapes() {
let outputs = [
@@ -3741,22 +3703,21 @@ mod tests {
let data_url = large_test_image_data_url();
let long_text = format!("{}end", "ordinary OCR text with spaces. ".repeat(3_500));
let residual_base64 = "A".repeat(20_000);
let encoded_output = json!([
{"type": "input_image", "image_url": data_url.clone()},
{"type": "text", "text": long_text.clone()},
{"type": "video", "data": residual_base64}
])
.to_string();
let result = convert_test_input(vec![
test_function_call("call_clamp"),
test_function_output(
"call_clamp",
json!([
{"type": "input_image", "image_url": data_url.clone()},
{"type": "text", "text": long_text.clone()},
{"raw": residual_base64}
]),
),
test_function_output("call_clamp", json!(encoded_output)),
]);
let tool_content_text = result["messages"][1]["content"].as_str().unwrap();
let tool_content: Value = serde_json::from_str(tool_content_text).unwrap();
assert_eq!(tool_content[1]["text"], long_text);
assert!(tool_content[2]["raw"]
assert!(tool_content[2]["data"]
.as_str()
.unwrap()
.starts_with("[cc-switch: omitted 20000 bytes]"));
src-tauri/src/proxy/providers/transform_gemini.rs
+386 -3
View File
@@ -7,6 +7,9 @@
use super::gemini_schema::build_gemini_function_declaration;
use super::gemini_shadow::{GeminiAssistantTurn, GeminiShadowStore, GeminiToolCallMeta};
use crate::proxy::error::ProxyError;
use crate::proxy::tool_media::{
strip_and_clamp_media_from_tool_value, ToolMediaScope, TOOL_RESULT_MEDIA_ATTACHED_MARKER,
};
use serde_json::{json, Map, Value};
use std::collections::{HashMap, HashSet};
@@ -61,6 +64,10 @@ pub fn anthropic_to_gemini_with_shadow(
.and_then(|((store, provider_id), session_id)| store.get_session(provider_id, session_id))
.map(|snapshot| snapshot.turns)
.unwrap_or_default();
let supports_multimodal_function_response = body
.get("model")
.and_then(Value::as_str)
.is_some_and(is_gemini_3_series);
let messages = body.get("messages").and_then(|value| value.as_array());
@@ -73,7 +80,11 @@ pub fn anthropic_to_gemini_with_shadow(
}
if let Some(messages) = messages {
result["contents"] = json!(convert_messages_to_contents(messages, &shadow_turns)?);
result["contents"] = json!(convert_messages_to_contents(
messages,
&shadow_turns,
supports_multimodal_function_response,
)?);
}
if let Some(generation_config) = build_generation_config(&body) {
@@ -361,6 +372,7 @@ fn build_generation_config(body: &Value) -> Option<Value> {
fn convert_messages_to_contents(
messages: &[Value],
shadow_turns: &[GeminiAssistantTurn],
supports_multimodal_function_response: bool,
) -> Result<Vec<Value>, ProxyError> {
let mut contents = Vec::new();
let mut used_shadow_indices = HashSet::new();
@@ -446,6 +458,7 @@ fn convert_messages_to_contents(
role,
&mut tool_name_by_id,
&thought_signature_by_id,
supports_multimodal_function_response,
)?
}
} else {
@@ -454,6 +467,7 @@ fn convert_messages_to_contents(
role,
&mut tool_name_by_id,
&thought_signature_by_id,
supports_multimodal_function_response,
)?
}
} else {
@@ -462,6 +476,7 @@ fn convert_messages_to_contents(
role,
&mut tool_name_by_id,
&thought_signature_by_id,
supports_multimodal_function_response,
)?
};
@@ -560,6 +575,7 @@ fn convert_message_content_to_parts(
role: &str,
tool_name_by_id: &mut std::collections::HashMap<String, String>,
thought_signature_by_id: &std::collections::HashMap<String, String>,
supports_multimodal_function_response: bool,
) -> Result<Vec<Value>, ProxyError> {
let Some(content) = content else {
return Ok(Vec::new());
@@ -705,16 +721,31 @@ fn convert_message_content_to_parts(
))
})?;
let (response, media_parts) = plan_gemini_tool_result(block.get("content"));
// See `tool_use` above: synthesized ids must not leak upstream.
let mut function_response = json!({
"name": name,
"response": normalize_tool_result_response(block.get("content"))
"response": response
});
if !tool_use_id.is_empty() && !is_synthesized_tool_call_id(tool_use_id) {
function_response["id"] = json!(tool_use_id);
}
parts.push(json!({ "functionResponse": function_response }));
if supports_multimodal_function_response && !media_parts.is_empty() {
function_response["parts"] = Value::Array(media_parts);
parts.push(json!({ "functionResponse": function_response }));
} else {
parts.push(json!({ "functionResponse": function_response }));
if !media_parts.is_empty() {
parts.push(json!({
"text": format!(
"[cc-switch: media output of tool call {tool_use_id}]"
)
}));
parts.extend(media_parts);
}
}
}
"thinking" | "redacted_thinking" => {}
_ => {}
@@ -745,6 +776,74 @@ fn normalize_tool_result_response(content: Option<&Value>) -> Value {
}
}
fn plan_gemini_tool_result(content: Option<&Value>) -> (Value, Vec<Value>) {
let Some(content) = content else {
return (normalize_tool_result_response(None), Vec::new());
};
let mut cleaned = content.clone();
let replacement_block = json!({
"type":"text",
"text":TOOL_RESULT_MEDIA_ATTACHED_MARKER
});
let mut chat_media_parts = Vec::new();
let replaced = strip_and_clamp_media_from_tool_value(
&mut cleaned,
&mut chat_media_parts,
ToolMediaScope::InlineImagesOnly,
&replacement_block,
TOOL_RESULT_MEDIA_ATTACHED_MARKER,
);
if replaced == 0 {
return (normalize_tool_result_response(Some(content)), Vec::new());
}
let mut gemini_parts = Vec::new();
gemini_parts.extend(
chat_media_parts
.iter()
.filter_map(gemini_part_from_chat_image),
);
(normalize_tool_result_response(Some(&cleaned)), gemini_parts)
}
fn is_gemini_3_series(model: &str) -> bool {
let normalized = model.trim().to_ascii_lowercase();
normalized.starts_with("gemini-3")
|| normalized
.rsplit('/')
.next()
.is_some_and(|tail| tail.starts_with("gemini-3"))
}
fn gemini_part_from_chat_image(part: &Value) -> Option<Value> {
let image_url = part
.pointer("/image_url/url")
.and_then(Value::as_str)
.filter(|url| !url.trim().is_empty())?;
if image_url
.get(..5)
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
{
let rest = &image_url[5..];
let (meta, data) = rest.split_once(',')?;
if data.is_empty() || !meta.to_ascii_lowercase().contains(";base64") {
return None;
}
let mime_type = meta.split(';').next().unwrap_or("image/png");
return Some(json!({
"inlineData": {
"mimeType": mime_type,
"data": data
}
}));
}
None
}
fn shadow_parts(content: &Value) -> Option<Vec<Value>> {
let mut parts = content
.get("parts")
@@ -1266,6 +1365,290 @@ mod tests {
);
}
#[test]
fn anthropic_to_gemini_moves_mixed_tool_result_image_to_native_part() {
let input = json!({
"messages": [
{
"role": "assistant",
"content": [{
"type": "tool_use",
"id": "call_image",
"name": "inspect",
"input": {}
}]
},
{
"role": "user",
"content": [{
"type": "tool_result",
"tool_use_id": "call_image",
"content": [
{"type": "text", "text": "caption"},
{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/png",
"data": "GEMINI_TOOL_IMAGE_SENTINEL"
}
}
]
}]
}
]
});
let result = anthropic_to_gemini(input).unwrap();
let parts = result["contents"][1]["parts"].as_array().unwrap();
let response = &parts[0]["functionResponse"]["response"]["content"];
assert!(response.as_str().unwrap().contains("caption"));
assert!(response
.as_str()
.unwrap()
.contains("tool result media attached"));
assert!(!response
.as_str()
.unwrap()
.contains("GEMINI_TOOL_IMAGE_SENTINEL"));
assert_eq!(
parts[1]["text"],
"[cc-switch: media output of tool call call_image]"
);
assert_eq!(parts[2]["inlineData"]["mimeType"], "image/png");
assert_eq!(parts[2]["inlineData"]["data"], "GEMINI_TOOL_IMAGE_SENTINEL");
}
#[test]
fn anthropic_to_gemini_clamps_json_string_residual_base64_before_serializing() {
let residual_base64 = "A".repeat(20_000);
let encoded = json!({
"content": [
{
"type": "image_url",
"image_url": {
"url": "data:image/png;base64,GEMINI_STRING_IMAGE_SENTINEL"
}
},
{"type": "video", "data": residual_base64}
]
})
.to_string();
let input = json!({
"messages": [
{
"role": "assistant",
"content": [{
"type": "tool_use",
"id": "call_string",
"name": "inspect",
"input": {}
}]
},
{
"role": "user",
"content": [{
"type": "tool_result",
"tool_use_id": "call_string",
"content": encoded
}]
}
]
});
let result = anthropic_to_gemini(input).unwrap();
let serialized = result.to_string();
assert!(serialized.contains("[cc-switch: omitted 20000 bytes]"));
assert!(!serialized.contains(&"A".repeat(64)));
assert_eq!(
result["contents"][1]["parts"][2]["inlineData"]["data"],
"GEMINI_STRING_IMAGE_SENTINEL"
);
}
#[test]
fn anthropic_to_gemini_keeps_remote_tool_image_in_legacy_response() {
let remote_image = json!({
"type": "image",
"source": {
"type": "url",
"url": "https://example.com/tool-image.png"
}
});
let input = json!({
"messages": [
{
"role": "assistant",
"content": [{
"type": "tool_use",
"id": "call_remote",
"name": "inspect",
"input": {}
}]
},
{
"role": "user",
"content": [{
"type": "tool_result",
"tool_use_id": "call_remote",
"content": [remote_image.clone()]
}]
}
]
});
let result = anthropic_to_gemini(input).unwrap();
let parts = result["contents"][1]["parts"].as_array().unwrap();
assert_eq!(parts.len(), 1);
assert_eq!(
parts[0]["functionResponse"]["response"]["content"][0],
remote_image
);
assert!(!result.to_string().contains("fileData"));
assert!(!result
.to_string()
.contains(TOOL_RESULT_MEDIA_ATTACHED_MARKER));
}
#[test]
fn anthropic_to_gemini_does_not_strip_unconvertible_tool_data_url() {
let malformed_image = json!({
"type": "image_url",
"image_url": {
"url": "data:image/png,NOT_BASE64"
}
});
let input = json!({
"messages": [
{
"role": "assistant",
"content": [{
"type": "tool_use",
"id": "call_malformed",
"name": "inspect",
"input": {}
}]
},
{
"role": "user",
"content": [{
"type": "tool_result",
"tool_use_id": "call_malformed",
"content": [malformed_image.clone()]
}]
}
]
});
let result = anthropic_to_gemini(input).unwrap();
let parts = result["contents"][1]["parts"].as_array().unwrap();
assert_eq!(parts.len(), 1);
assert_eq!(
parts[0]["functionResponse"]["response"]["content"][0],
malformed_image
);
assert!(!result.to_string().contains("inlineData"));
assert!(!result
.to_string()
.contains(TOOL_RESULT_MEDIA_ATTACHED_MARKER));
}
#[test]
fn anthropic_to_gemini_does_not_embed_image_only_tool_result_as_json() {
let input = json!({
"messages": [
{
"role": "assistant",
"content": [{
"type": "tool_use",
"id": "call_image",
"name": "inspect",
"input": {}
}]
},
{
"role": "user",
"content": [{
"type": "tool_result",
"tool_use_id": "call_image",
"content": [{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/webp",
"data": "IMAGE_ONLY_SENTINEL"
}
}]
}]
}
]
});
let result = anthropic_to_gemini(input).unwrap();
let parts = result["contents"][1]["parts"].as_array().unwrap();
let response = &parts[0]["functionResponse"]["response"]["content"];
assert!(response.is_string());
assert!(!response.as_str().unwrap().contains("IMAGE_ONLY_SENTINEL"));
assert_eq!(parts[2]["inlineData"]["mimeType"], "image/webp");
assert_eq!(parts[2]["inlineData"]["data"], "IMAGE_ONLY_SENTINEL");
}
#[test]
fn anthropic_to_gemini_3_uses_multimodal_function_response_parts() {
let input = json!({
"model": "gemini-3-pro-preview",
"messages": [
{
"role": "assistant",
"content": [{
"type": "tool_use",
"id": "call_image",
"name": "inspect",
"input": {}
}]
},
{
"role": "user",
"content": [{
"type": "tool_result",
"tool_use_id": "call_image",
"content": [{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/jpeg",
"data": "GEMINI_3_IMAGE_SENTINEL"
}
}]
}]
}
]
});
let result = anthropic_to_gemini(input).unwrap();
let parts = result["contents"][1]["parts"].as_array().unwrap();
let function_response = &parts[0]["functionResponse"];
assert_eq!(parts.len(), 1);
assert_eq!(
function_response["parts"][0]["inlineData"]["mimeType"],
"image/jpeg"
);
assert_eq!(
function_response["parts"][0]["inlineData"]["data"],
"GEMINI_3_IMAGE_SENTINEL"
);
assert!(!function_response["response"]["content"]
.as_str()
.unwrap()
.contains("GEMINI_3_IMAGE_SENTINEL"));
}
#[test]
fn anthropic_to_gemini_resolves_tool_result_name_from_shadow_content() {
let store = GeminiShadowStore::with_limits(8, 4);
src-tauri/src/proxy/providers/transform_responses.rs
+193 -8
View File
@@ -8,7 +8,13 @@
//! - system prompt 使用 `instructions` 字段而非 system role message
//! - usage 字段命名与 Anthropic 一致 (input_tokens/output_tokens)
use crate::proxy::{error::ProxyError, json_canonical::canonical_json_string};
use crate::proxy::{
error::ProxyError,
json_canonical::canonical_json_string,
tool_media::{
strip_and_clamp_media_from_tool_value, ToolMediaScope, TOOL_RESULT_MEDIA_ATTACHED_MARKER,
},
};
use serde_json::{json, Value};
use super::reasoning_bridge::{
@@ -80,8 +86,11 @@ fn anthropic_tool_result_to_responses_output(block: &Value) -> Value {
let content = block.get("content");
if !is_error {
if let Some(Value::String(text)) = content {
return json!(text);
if let Some(text @ Value::String(_)) = content {
if let Some(output) = alternate_image_tool_result_to_responses(text) {
return Value::Array(output);
}
return text.clone();
}
}
@@ -92,7 +101,13 @@ fn anthropic_tool_result_to_responses_output(block: &Value) -> Value {
match content {
Some(Value::String(text)) => {
output.push(json!({"type":"input_text","text":text}));
if let Some(mut alternate) =
alternate_image_tool_result_to_responses(&Value::String(text.clone()))
{
output.append(&mut alternate);
} else {
output.push(json!({"type":"input_text","text":text}));
}
}
Some(Value::Array(blocks)) => {
for part in blocks {
@@ -105,6 +120,10 @@ fn anthropic_tool_result_to_responses_output(block: &Value) -> Value {
Some("image") => {
if let Some(image) = anthropic_image_to_responses_part(part) {
output.push(image);
} else if let Some(mut alternate) =
alternate_image_tool_result_to_responses(part)
{
output.append(&mut alternate);
} else {
output.push(json!({
"type":"input_text",
@@ -122,6 +141,77 @@ fn anthropic_tool_result_to_responses_output(block: &Value) -> Value {
}));
}
}
_ => {
if let Some(mut alternate) = alternate_image_tool_result_to_responses(part)
{
output.append(&mut alternate);
} else {
output.push(json!({
"type":"input_text",
"text":canonical_json_string(part)
}));
}
}
}
}
}
Some(value) => {
if let Some(mut alternate) = alternate_image_tool_result_to_responses(value) {
output.append(&mut alternate);
} else {
output.push(json!({
"type":"input_text",
"text":canonical_json_string(value)
}));
}
}
None => {}
}
Value::Array(output)
}
fn alternate_image_tool_result_to_responses(value: &Value) -> Option<Vec<Value>> {
let mut cleaned = value.clone();
let replacement_block = json!({
"type":"input_text",
"text":TOOL_RESULT_MEDIA_ATTACHED_MARKER
});
let mut chat_media_parts = Vec::new();
let replaced = strip_and_clamp_media_from_tool_value(
&mut cleaned,
&mut chat_media_parts,
ToolMediaScope::ImagesOnly,
&replacement_block,
TOOL_RESULT_MEDIA_ATTACHED_MARKER,
);
if replaced == 0 {
return None;
}
let mut output = Vec::new();
append_sanitized_responses_tool_value(&cleaned, &mut output);
output.extend(
chat_media_parts
.iter()
.filter_map(responses_image_from_chat_media),
);
Some(output)
}
fn append_sanitized_responses_tool_value(value: &Value, output: &mut Vec<Value>) {
match value {
Value::String(text) if !text.is_empty() => {
output.push(json!({"type":"input_text","text":text}));
}
Value::Array(parts) => {
for part in parts {
match part.get("type").and_then(Value::as_str) {
Some("input_text" | "output_text" | "text") => {
if let Some(text) = part.get("text").and_then(Value::as_str) {
output.push(json!({"type":"input_text","text":text}));
}
}
_ => output.push(json!({
"type":"input_text",
"text":canonical_json_string(part)
@@ -129,14 +219,37 @@ fn anthropic_tool_result_to_responses_output(block: &Value) -> Value {
}
}
}
Some(value) => output.push(json!({
Value::Object(object)
if matches!(
object.get("type").and_then(Value::as_str),
Some("input_text" | "output_text" | "text")
) =>
{
if let Some(text) = object.get("text").and_then(Value::as_str) {
output.push(json!({"type":"input_text","text":text}));
}
}
Value::Null | Value::String(_) => {}
other => output.push(json!({
"type":"input_text",
"text":canonical_json_string(value)
"text":canonical_json_string(other)
})),
None => {}
}
}
Value::Array(output)
fn responses_image_from_chat_media(part: &Value) -> Option<Value> {
let image_url = part
.pointer("/image_url/url")
.and_then(Value::as_str)
.filter(|url| !url.trim().is_empty())?;
let mut image = json!({
"type":"input_image",
"image_url":image_url
});
if let Some(detail) = part.pointer("/image_url/detail") {
image["detail"] = detail.clone();
}
Some(image)
}
pub(crate) fn sanitize_anthropic_tool_use_input(name: &str, input: Value) -> Value {
@@ -1157,6 +1270,78 @@ mod tests {
assert_eq!(output[3]["filename"], "trace.pdf");
}
#[test]
fn test_anthropic_to_responses_converts_mcp_tool_image() {
let input = json!({
"model":"gpt-5",
"messages":[{"role":"user","content":[{
"type":"tool_result",
"tool_use_id":"call_1",
"content":[{
"type":"image",
"mimeType":"image/webp",
"data":"MCP_RESPONSES_IMAGE_SENTINEL"
}]
}]}]
});
let result = anthropic_to_responses(input, None, false, false).unwrap();
let output = result["input"][0]["output"].as_array().unwrap();
assert_eq!(output[0]["type"], "input_text");
assert!(!output[0]["text"]
.as_str()
.unwrap()
.contains("MCP_RESPONSES_IMAGE_SENTINEL"));
assert_eq!(output[1]["type"], "input_image");
assert_eq!(
output[1]["image_url"],
"data:image/webp;base64,MCP_RESPONSES_IMAGE_SENTINEL"
);
}
#[test]
fn test_anthropic_to_responses_converts_json_string_tool_image() {
let residual_base64 = "A".repeat(20_000);
let encoded = json!({
"content":[
{
"type":"image_url",
"image_url":{"url":"data:image/png;base64,STRING_RESPONSES_SENTINEL"}
},
{"type":"video","data":residual_base64}
]
})
.to_string();
let input = json!({
"model":"gpt-5",
"messages":[{"role":"user","content":[{
"type":"tool_result",
"tool_use_id":"call_1",
"content":encoded
}]}]
});
let result = anthropic_to_responses(input, None, false, false).unwrap();
let output = result["input"][0]["output"].as_array().unwrap();
let image = output
.iter()
.find(|part| part["type"] == "input_image")
.expect("stringified image must stay a Responses image");
assert_eq!(
image["image_url"],
"data:image/png;base64,STRING_RESPONSES_SENTINEL"
);
assert!(output
.iter()
.filter_map(|part| part.get("text").and_then(Value::as_str))
.all(|text| !text.contains("STRING_RESPONSES_SENTINEL")));
let serialized = result.to_string();
assert!(serialized.contains("[cc-switch: omitted 20000 bytes]"));
assert!(!serialized.contains(&"A".repeat(64)));
}
#[test]
fn test_anthropic_to_responses_thinking_discarded() {
let input = json!({
src-tauri/src/proxy/tool_media.rs
+245 -28
View File
@@ -1,15 +1,19 @@
//! Shared media handling for Codex tool outputs.
//! Shared media handling for tool outputs.
//!
//! Responses tool outputs may carry structured media blocks. Chat Completions
//! tool messages are text-only, so the Responses→Chat bridge extracts those
//! blocks and re-emits them in a synthetic user message. The media sanitizer
//! reuses the same recognition and traversal rules when it needs to remove
//! images for a text-only upstream.
//! Responses and Anthropic tool outputs may carry structured media blocks.
//! Chat Completions tool messages are text-only, so protocol bridges extract
//! those blocks and re-emit them in a synthetic user message. The media
//! sanitizer reuses the same recognition and traversal rules when it needs to
//! remove images for a text-only upstream.
use crate::proxy::json_canonical::canonical_json_string;
use serde_json::{json, Map, Value};
pub(crate) const WHOLE_DATA_URL_MIN_BYTES: usize = 8 * 1024;
pub(crate) const TOOL_RESULT_MEDIA_MOVED_MARKER: &str =
"[cc-switch: tool result media moved to the following user message]";
pub(crate) const TOOL_RESULT_MEDIA_ATTACHED_MARKER: &str =
"[cc-switch: tool result media attached as native media]";
const BASE64ISH_MIN_BYTES: usize = 16 * 1024;
const MAX_MEDIA_TRAVERSAL_DEPTH: usize = 32;
@@ -17,7 +21,11 @@ const MAX_MEDIA_TRAVERSAL_DEPTH: usize = 32;
pub(crate) enum ToolMediaScope {
/// Used by the existing image-capability sanitizer and its retry path.
ImagesOnly,
/// Used by Responses→Chat conversion, where user messages can carry all
/// Used by Gemini Native `generateContent`, whose existing bridge only
/// promises inline base64 image input. Remote URLs and malformed data URLs
/// must stay in the legacy tool-result representation.
InlineImagesOnly,
/// Used by Chat conversion bridges, where user messages can carry all
/// currently mapped Chat input modalities.
AllSupported,
}
@@ -29,21 +37,95 @@ enum ToolMediaKind {
Audio,
}
pub(crate) struct ChatToolOutputMediaPlan {
pub(crate) tool_content: String,
pub(crate) media_parts: Vec<Value>,
}
impl ToolMediaScope {
fn allows(self, kind: ToolMediaKind) -> bool {
matches!(kind, ToolMediaKind::Image) || matches!(self, Self::AllSupported)
}
fn accepts_chat_part(self, part: &Value) -> bool {
!matches!(self, Self::InlineImagesOnly) || chat_image_part_has_inline_data(part)
}
}
/// Convert one recognized Responses/tool media block to a Chat user content
/// part. This is the single shape-recognition entry point used by extraction.
/// Build a Chat-compatible tool-output plan without changing no-media output.
///
/// Scalar strings remain scalar strings after replacement. This matters for a
/// raw image data URL and for JSON encoded inside a tool-output string: adding
/// another layer of JSON string quotes would change what the model sees.
pub(crate) fn plan_chat_tool_output_media(mut output: Value) -> Option<ChatToolOutputMediaPlan> {
let output_was_string = output.is_string();
let replacement_block = json!({
"type": "text",
"text": TOOL_RESULT_MEDIA_MOVED_MARKER
});
let mut media_parts = Vec::new();
let replaced = strip_and_clamp_media_from_tool_value(
&mut output,
&mut media_parts,
ToolMediaScope::AllSupported,
&replacement_block,
TOOL_RESULT_MEDIA_MOVED_MARKER,
);
if replaced == 0 {
return None;
}
let tool_content = if output_was_string {
output.as_str().unwrap_or_default().to_string()
} else {
canonical_json_string(&output)
};
Some(ChatToolOutputMediaPlan {
tool_content,
media_parts,
})
}
pub(crate) fn queue_chat_tool_output_media(
pending_media: &mut Vec<Value>,
call_id: &str,
media_parts: Vec<Value>,
) {
if media_parts.is_empty() {
return;
}
pending_media.push(json!({
"type": "text",
"text": format!("[cc-switch: media output of tool call {call_id}]")
}));
pending_media.extend(media_parts);
}
pub(crate) fn flush_pending_chat_tool_media(
messages: &mut Vec<Value>,
pending_media: &mut Vec<Value>,
) {
if pending_media.is_empty() {
return;
}
messages.push(json!({
"role": "user",
"content": std::mem::take(pending_media)
}));
}
/// Convert one recognized tool media block to a Chat user content part. This
/// is the single shape-recognition entry point used by extraction.
pub(crate) fn chat_media_part_from_tool_part(part: &Value, scope: ToolMediaScope) -> Option<Value> {
let kind = tool_media_kind(part)?;
if !scope.allows(kind) {
return None;
}
match kind {
let chat_part = match kind {
ToolMediaKind::Image => chat_image_part(part),
ToolMediaKind::File => chat_file_from_input_file(part).map(|file| {
json!({
@@ -57,7 +139,9 @@ pub(crate) fn chat_media_part_from_tool_part(part: &Value, scope: ToolMediaScope
"input_audio": input_audio.clone()
})
}),
}
}?;
scope.accepts_chat_part(&chat_part).then_some(chat_part)
}
/// Map a Responses `input_file` block to the Chat file payload. Kept here so
@@ -122,10 +206,37 @@ pub(crate) fn strip_media_from_tool_value(
scope,
replacement_block,
replacement_text,
false,
0,
)
}
/// Extract media and clamp residual large data/base64 scalars on media-bearing
/// outputs. Parseable JSON strings are clamped while still represented as a
/// JSON tree, before they are canonicalized back into their original string
/// container.
pub(crate) fn strip_and_clamp_media_from_tool_value(
value: &mut Value,
media_parts: &mut Vec<Value>,
scope: ToolMediaScope,
replacement_block: &Value,
replacement_text: &str,
) -> usize {
let replaced = strip_media_from_tool_value_at_depth(
value,
media_parts,
scope,
replacement_block,
replacement_text,
true,
0,
);
if replaced > 0 {
clamp_base64ish_strings(value);
}
replaced
}
/// Remove residual data/base64 payloads only after a tool output has already
/// been positively identified as media-bearing. Ordinary long text is kept.
pub(crate) fn clamp_base64ish_strings(value: &mut Value) {
@@ -181,7 +292,7 @@ fn tool_output_contains_media_at_depth(value: &Value, scope: ToolMediaScope, dep
.iter()
.any(|item| tool_output_contains_media_at_depth(item, scope, depth + 1)),
Value::Object(object) => {
if tool_media_kind(value).is_some_and(|kind| scope.allows(kind)) {
if chat_media_part_from_tool_part(value, scope).is_some() {
return true;
}
@@ -199,6 +310,7 @@ fn strip_media_from_tool_value_at_depth(
scope: ToolMediaScope,
replacement_block: &Value,
replacement_text: &str,
clamp_parsed_strings: bool,
depth: usize,
) -> usize {
if depth > MAX_MEDIA_TRAVERSAL_DEPTH {
@@ -228,9 +340,13 @@ fn strip_media_from_tool_value_at_depth(
scope,
replacement_block,
replacement_text,
clamp_parsed_strings,
depth + 1,
);
if replaced > 0 {
if clamp_parsed_strings {
clamp_base64ish_strings(&mut parsed);
}
*text = canonical_json_string(&parsed);
}
replaced
@@ -244,6 +360,7 @@ fn strip_media_from_tool_value_at_depth(
scope,
replacement_block,
replacement_text,
clamp_parsed_strings,
depth + 1,
)
})
@@ -266,6 +383,7 @@ fn strip_media_from_tool_value_at_depth(
scope,
replacement_block,
replacement_text,
clamp_parsed_strings,
depth + 1,
)
})
@@ -297,13 +415,9 @@ fn tool_media_kind(part: &Value) -> Option<ToolMediaKind> {
fn chat_image_part(part: &Value) -> Option<Value> {
match part.get("type").and_then(Value::as_str) {
Some("input_image" | "image_url") => {
normalized_image_url(part).map(|image_url| image_url_content_part(part, image_url))
}
Some("image") => {
typed_image_url(part).map(|image_url| image_url_content_part(part, image_url))
}
None => loose_data_image_url(part).map(|image_url| image_url_content_part(part, image_url)),
Some("input_image" | "image_url") => normalized_image_url(part).map(image_url_content_part),
Some("image") => typed_image_url(part).map(image_url_content_part),
None => loose_data_image_url(part).map(image_url_content_part),
_ => None,
}
}
@@ -440,19 +554,25 @@ fn typed_image_url(part: &Value) -> Option<Value> {
Some(Value::Object(image_url))
}
fn image_url_content_part(source_part: &Value, image_url: Value) -> Value {
fn image_url_content_part(image_url: Value) -> Value {
let mut content_part = Map::new();
content_part.insert("type".to_string(), Value::String("image_url".to_string()));
content_part.insert("image_url".to_string(), image_url);
for key in ["cache_control", "prompt_cache_breakpoint"] {
if let Some(value) = source_part.get(key) {
content_part.insert(key.to_string(), value.clone());
}
}
Value::Object(content_part)
}
fn chat_image_part_has_inline_data(part: &Value) -> bool {
part.pointer("/image_url/url")
.and_then(Value::as_str)
.is_some_and(|url| {
let trimmed = url.trim();
let Some(comma_index) = trimmed.find(',') else {
return false;
};
comma_index + 1 < trimmed.len() && is_image_base64_data_url(trimmed)
})
}
fn merge_top_level_detail(part: &Value, image_url: &mut Map<String, Value>) {
if image_url.get("detail").is_none() {
if let Some(detail) = part.get("detail") {
@@ -529,12 +649,23 @@ mod tests {
"image_url": {
"url": "https://example.com/image.png",
"detail": "low"
}
},
"cache_control": {"type": "ephemeral"},
"prompt_cache_breakpoint": true
});
let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap();
assert_eq!(mapped, part);
assert_eq!(
mapped,
json!({
"type": "image_url",
"image_url": {
"url": "https://example.com/image.png",
"detail": "low"
}
})
);
}
#[test]
@@ -625,6 +756,43 @@ mod tests {
));
}
#[test]
fn inline_image_scope_rejects_remote_and_malformed_data_urls() {
let inline = json!({
"type": "image_url",
"image_url": {"url": "data:image/png;base64,YWJj"}
});
let remote = json!({
"type": "image_url",
"image_url": {"url": "https://example.com/image.png"}
});
let missing_base64 = json!({
"type": "image_url",
"image_url": {"url": "data:image/png,YWJj"}
});
let empty_data = json!({
"type": "image_url",
"image_url": {"url": "data:image/png;base64,"}
});
assert!(tool_output_contains_media(
&inline,
ToolMediaScope::InlineImagesOnly
));
assert!(!tool_output_contains_media(
&remote,
ToolMediaScope::InlineImagesOnly
));
assert!(!tool_output_contains_media(
&missing_base64,
ToolMediaScope::InlineImagesOnly
));
assert!(!tool_output_contains_media(
&empty_data,
ToolMediaScope::InlineImagesOnly
));
}
#[test]
fn does_not_scan_embedded_data_urls_inside_plain_text() {
let data_url = large_image_data_url();
@@ -693,6 +861,55 @@ mod tests {
assert!(!serialized.contains("iVBORw0KGgo"));
}
#[test]
fn chat_plan_keeps_scalar_tool_strings_unquoted() {
let raw_data_url = large_image_data_url();
let raw_plan = plan_chat_tool_output_media(Value::String(raw_data_url.clone())).unwrap();
assert_eq!(raw_plan.tool_content, TOOL_RESULT_MEDIA_MOVED_MARKER);
assert_eq!(raw_plan.media_parts[0]["image_url"]["url"], raw_data_url);
let encoded = json!({
"content": [{
"type": "input_image",
"image_url": raw_data_url
}]
})
.to_string();
let encoded_plan = plan_chat_tool_output_media(Value::String(encoded)).unwrap();
assert!(encoded_plan.tool_content.starts_with('{'));
assert!(encoded_plan
.tool_content
.contains(TOOL_RESULT_MEDIA_MOVED_MARKER));
assert!(!encoded_plan.tool_content.starts_with('"'));
}
#[test]
fn chat_plan_clamps_residual_base64_inside_json_string_before_serializing() {
let residual_base64 = "A".repeat(20_000);
let encoded = json!({
"content": [
{
"type": "input_image",
"image_url": "data:image/png;base64,IMAGE_SENTINEL"
},
{
"type": "video",
"data": residual_base64
}
]
})
.to_string();
let plan = plan_chat_tool_output_media(Value::String(encoded)).unwrap();
assert!(plan
.tool_content
.contains("[cc-switch: omitted 20000 bytes]"));
assert!(!plan.tool_content.contains(&"A".repeat(64)));
assert!(!plan.tool_content.contains("IMAGE_SENTINEL"));
assert_eq!(plan.media_parts.len(), 1);
}
#[test]
fn image_only_scope_ignores_file_and_audio() {
let file = json!({"type": "input_file", "file_id": "file_1"});