admin/CC-Switch
1
0
Fork 0
mirror of https://github.com/farion1231/cc-switch.git synced 2026-07-29 09:37:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(proxy): extend tool-result media handling to all conversion bridges

Browse Source 
Generalize the Codex Responses-to-Chat tool media mechanism from 6c9d444c
to the remaining protocol bridges, so image-bearing tool results are never
tokenized as base64 text on any conversion path:

- Claude-to-Chat: tool_result images become native image_url parts,
  batched into one synthetic user message after the tool message batch.
- Claude-to-Responses: JSON-string, MCP, and nested variants are restored
  as native input_image parts inside function_call_output.
- Codex/GrokBuild-to-Anthropic: non-standard tool images are restored as
  Anthropic image blocks (case-insensitive data:/http prefix parsing).
- Claude-to-Gemini: Gemini 3 uses multimodal functionResponse.parts;
  older models get inlineData parts in the same user turn. The new
  InlineImagesOnly scope keeps remote URLs and malformed data URLs in the
  legacy text form instead of emitting fileData the API would reject.

Shared changes in tool_media:

- Centralize the plan/queue/flush helpers used by both Chat bridges.
- strip_and_clamp_media_from_tool_value clamps residual base64 inside
  parsed JSON strings at any nesting depth before re-serialization.
- Emitted Chat parts no longer carry cache_control or
  prompt_cache_breakpoint, preserving the strip-all-cache_control
  contract that strict upstreams (GLM/Qwen) depend on.
- Media detection now requires full convertibility, keeping detection
  and extraction symmetric by construction.

The media sanitizer detects and strips the new shapes symmetrically
(Chat string tool results, Anthropic string tool results, Gemini
inlineData/fileData and functionResponse.parts); the legacy typed-block
replacement runs first so replacement stays a superset of detection and
cache_control survives on replaced Anthropic blocks.

No-media tool outputs keep byte-identical legacy representations on all
bridges to protect prompt-cache prefixes.
This commit is contained in:
Jason
2026-07-24 12:12:22 +08:00
parent 6c9d444c8a
commit 878c26f31e
8 changed files with 1666 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src-tauri/src/proxy/providers/transform.rs
+192 -15
View File
@@ -3,7 +3,14 @@
//! 实现 Anthropic ↔ OpenAI 格式转换，用于 OpenRouter 支持
//! 参考: anthropic-proxy-rs
use crate::proxy::{error::ProxyError, json_canonical::canonical_json_string};
use crate::proxy::{
error::ProxyError,
json_canonical::canonical_json_string,
tool_media::{
chat_media_part_from_tool_part, flush_pending_chat_tool_media, plan_chat_tool_output_media,
queue_chat_tool_output_media, ToolMediaScope,
},
};
use serde_json::{json, Value};
const ANTHROPIC_BILLING_HEADER_PREFIX: &str = "x-anthropic-billing-header:";
@@ -375,6 +382,7 @@ fn convert_message_to_openai(
if let Some(blocks) = content.as_array() {
let mut content_parts = Vec::new();
let mut tool_calls = Vec::new();
let mut pending_tool_media = Vec::new();
// reasoning_parts: 仅在兼容 Moonshot/Kimi/DeepSeek thinking tool-call 路径时
// 生成 reasoning_content，通用 OpenAI-compatible 路径不发送该非标准字段。
let mut reasoning_parts = Vec::new();
@@ -389,16 +397,10 @@ fn convert_message_to_openai(
}
}
"image" => {
if let Some(source) = block.get("source") {
let media_type = source
.get("media_type")
.and_then(|m| m.as_str())
.unwrap_or("image/png");
let data = source.get("data").and_then(|d| d.as_str()).unwrap_or("");
content_parts.push(json!({
"type": "image_url",
"image_url": {"url": format!("data:{};base64,{}", media_type, data)}
}));
if let Some(image) =
chat_media_part_from_tool_part(block, ToolMediaScope::ImagesOnly)
{
content_parts.push(image);
}
}
"tool_use" => {
@@ -421,10 +423,22 @@ fn convert_message_to_openai(
.and_then(|i| i.as_str())
.unwrap_or("");
let content_val = block.get("content");
let content_str = match content_val {
Some(Value::String(s)) => s.clone(),
Some(v) => canonical_json_string(v),
None => String::new(),
let media_plan = content_val.cloned().and_then(plan_chat_tool_output_media);
let content_str = if let Some(media_plan) = media_plan {
queue_chat_tool_output_media(
&mut pending_tool_media,
tool_use_id,
media_plan.media_parts,
);
media_plan.tool_content
} else {
// Keep the no-media representation exactly equal to
// the legacy converter for prompt-cache stability.
match content_val {
Some(Value::String(s)) => s.clone(),
Some(v) => canonical_json_string(v),
None => String::new(),
}
};
result.push(json!({
"role": "tool",
@@ -452,6 +466,11 @@ fn convert_message_to_openai(
}
}
// Chat tool messages cannot carry image parts. Keep parallel tool
// results adjacent, then present all extracted media in one user turn
// before any ordinary message content from the same Anthropic turn.
flush_pending_chat_tool_media(&mut result, &mut pending_tool_media);
// 添加带内容和/或工具调用的消息
if !content_parts.is_empty() || !tool_calls.is_empty() {
let mut msg = json!({"role": role});
@@ -1133,6 +1152,164 @@ mod tests {
assert_eq!(msg["content"], "Sunny, 25°C");
}
#[test]
fn test_anthropic_to_openai_no_media_tool_results_keep_legacy_representation() {
let raw_json_string = "{ \"status\": \"ok\", \"count\": 2 }";
let input = json!({
"model": "claude-3-opus",
"messages": [{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "call_string",
"content": raw_json_string
},
{
"type": "tool_result",
"tool_use_id": "call_array",
"content": [{"type": "text", "text": "plain"}]
}
]
}]
});
let result = anthropic_to_openai(input).unwrap();
let messages = result["messages"].as_array().unwrap();
assert_eq!(messages.len(), 2);
assert_eq!(messages[0]["content"], raw_json_string);
assert_eq!(
messages[1]["content"],
canonical_json_string(&json!([{"type": "text", "text": "plain"}]))
);
}
#[test]
fn test_anthropic_to_openai_moves_tool_result_image_to_user_message() {
let input = json!({
"model": "claude-3-opus",
"max_tokens": 1024,
"messages": [{
"role": "user",
"content": [{
"type": "tool_result",
"tool_use_id": "call_image",
"content": [
{"type": "text", "text": "caption"},
{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/png",
"data": "CLAUDE_CHAT_IMAGE_SENTINEL"
},
"cache_control": {"type": "ephemeral"},
"prompt_cache_breakpoint": true
}
]
}]
}]
});
let result = anthropic_to_openai(input).unwrap();
let messages = result["messages"].as_array().unwrap();
assert_eq!(messages.len(), 2);
assert_eq!(messages[0]["role"], "tool");
assert_eq!(messages[0]["tool_call_id"], "call_image");
assert!(messages[0]["content"]
.as_str()
.unwrap()
.contains("tool result media moved"));
assert!(!messages[0]["content"]
.as_str()
.unwrap()
.contains("CLAUDE_CHAT_IMAGE_SENTINEL"));
assert_eq!(messages[1]["role"], "user");
assert_eq!(
messages[1]["content"][0]["text"],
"[cc-switch: media output of tool call call_image]"
);
assert_eq!(messages[1]["content"][1]["type"], "image_url");
assert!(messages[1]["content"][1].get("cache_control").is_none());
assert!(messages[1]["content"][1]
.get("prompt_cache_breakpoint")
.is_none());
assert_eq!(
messages[1]["content"][1]["image_url"]["url"],
"data:image/png;base64,CLAUDE_CHAT_IMAGE_SENTINEL"
);
}
#[test]
fn test_anthropic_to_openai_batches_parallel_tool_result_media() {
let input = json!({
"model": "claude-3-opus",
"messages": [{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "call_1",
"content": [{
"type": "image",
"source": {"type": "base64", "media_type": "image/png", "data": "ONE"}
}]
},
{
"type": "tool_result",
"tool_use_id": "call_2",
"content": [{
"type": "image",
"source": {"type": "base64", "media_type": "image/jpeg", "data": "TWO"}
}]
}
]
}]
});
let result = anthropic_to_openai(input).unwrap();
let messages = result["messages"].as_array().unwrap();
assert_eq!(messages.len(), 3);
assert_eq!(messages[0]["role"], "tool");
assert_eq!(messages[1]["role"], "tool");
assert_eq!(messages[2]["role"], "user");
assert_eq!(messages[2]["content"].as_array().unwrap().len(), 4);
}
#[test]
fn test_anthropic_to_openai_maps_remote_image_source() {
let input = json!({
"model": "claude-3-opus",
"messages": [{
"role": "user",
"content": [{
"type": "image",
"source": {
"type": "url",
"url": "https://example.com/image.png"
},
"cache_control": {"type": "ephemeral"},
"prompt_cache_breakpoint": true
}]
}]
});
let result = anthropic_to_openai(input).unwrap();
assert_eq!(
result["messages"][0]["content"][0]["image_url"]["url"],
"https://example.com/image.png"
);
assert!(result["messages"][0]["content"][0]
.get("cache_control")
.is_none());
assert!(result["messages"][0]["content"][0]
.get("prompt_cache_breakpoint")
.is_none());
}
#[test]
fn test_openai_to_anthropic_simple() {
let input = json!({