mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-07-25 13:45:03 +08:00
fix(proxy): move Codex tool-result media out of stringified tool text
The Responses->Chat conversion serialized image-bearing *_output items into role:"tool" text via canonical_json_string, so view_image results were tokenized as base64 text (~9000x inflation). Codex replays full history every turn, so sessions hit context-limit 400s and wedged (#4465, #5663). - add proxy/tool_media: shared detection/strip/clamp walker for tool output media (typed input_image / image_url / input_file / input_audio, Anthropic source and MCP data+mimeType image shapes, untyped data: image_url, whole-string bare data URLs) - transform_codex_chat: replace media blocks in place with marker text so tool content stays a plain string, and flush the extracted media as one synthetic role:"user" message after each consecutive tool batch; media-free traffic stays byte-identical to keep prompt-cache prefixes stable - media_sanitizer: detect and strip tool-output media symmetrically (including JSON-string outputs) so reactive image stripping can heal upstream modality rejections - forwarder: regression tests pinning the reactive trigger and the context-limit-400 non-trigger E2E against Kimi K3 through the proxy: the replayed turn stays ~12k input tokens with 99% cache hit, versus ~85k+ of base64 text per replay before.
This commit is contained in:
@@ -4755,6 +4755,27 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
fn body_with_codex_tool_output_image(stringified: bool) -> Value {
|
||||
let output = json!({
|
||||
"content": [{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,TOOL_OUTPUT_SENTINEL"
|
||||
}]
|
||||
});
|
||||
json!({
|
||||
"model": "any-model",
|
||||
"input": [{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_1",
|
||||
"output": if stringified {
|
||||
Value::String(output.to_string())
|
||||
} else {
|
||||
output
|
||||
}
|
||||
}]
|
||||
})
|
||||
}
|
||||
|
||||
fn image_unsupported_error() -> ProxyError {
|
||||
ProxyError::UpstreamError {
|
||||
status: 400,
|
||||
@@ -4857,6 +4878,31 @@ mod tests {
|
||||
assert!(fwd.media_retry_should_trigger("Codex", false, &body, &error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reactive_triggers_for_structured_and_stringified_codex_tool_images() {
|
||||
let fwd = forwarder_with_rectifier(RectifierConfig::default());
|
||||
|
||||
for stringified in [false, true] {
|
||||
let body = body_with_codex_tool_output_image(stringified);
|
||||
assert!(
|
||||
fwd.media_retry_should_trigger("Codex", false, &body, &image_unsupported_error()),
|
||||
"tool-output image should trigger retry (stringified={stringified})"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reactive_does_not_treat_context_limit_as_image_rejection() {
|
||||
let fwd = forwarder_with_rectifier(RectifierConfig::default());
|
||||
let body = body_with_codex_tool_output_image(false);
|
||||
let context_error = ProxyError::UpstreamError {
|
||||
status: 400,
|
||||
body: Some(r#"{"error":{"message":"maximum context length exceeded"}}"#.to_string()),
|
||||
};
|
||||
|
||||
assert!(!fwd.media_retry_should_trigger("Codex", false, &body, &context_error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reactive_skipped_when_media_fallback_off() {
|
||||
// 关闭 request_media_fallback:上游报图片错误也不触发兜底重试。
|
||||
|
||||
@@ -3,6 +3,9 @@ use crate::model_capabilities::is_confirmed_text_only_model as confirmed_text_on
|
||||
use crate::model_capabilities::{image_input_capability_from_settings, ImageInputCapability};
|
||||
use crate::provider::Provider;
|
||||
use crate::proxy::error::ProxyError;
|
||||
use crate::proxy::tool_media::{
|
||||
strip_media_from_tool_value, tool_output_contains_media, ToolMediaScope,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub const UNSUPPORTED_IMAGE_MARKER: &str = "[Unsupported Image]";
|
||||
@@ -193,6 +196,9 @@ fn responses_input_item_has_image_blocks(item: &Value) -> bool {
|
||||
}
|
||||
|
||||
item.get("content").is_some_and(content_has_image_blocks)
|
||||
|| item
|
||||
.get("output")
|
||||
.is_some_and(|output| tool_output_contains_media(output, ToolMediaScope::ImagesOnly))
|
||||
}
|
||||
|
||||
fn replace_images_in_responses_input(input: &mut Value) -> usize {
|
||||
@@ -218,6 +224,23 @@ fn replace_images_in_responses_input_item(item: &mut Value) -> usize {
|
||||
replaced += replace_images_in_content_with_text_type(content, "input_text");
|
||||
}
|
||||
|
||||
if let Some(output) = item.get_mut("output") {
|
||||
// The image-capability fallback deliberately strips images only.
|
||||
// Tool-output files/audio remain a known unsupported-modality gap.
|
||||
let replacement_block = json!({
|
||||
"type": "input_text",
|
||||
"text": UNSUPPORTED_IMAGE_MARKER
|
||||
});
|
||||
let mut discarded_media = Vec::new();
|
||||
replaced += strip_media_from_tool_value(
|
||||
output,
|
||||
&mut discarded_media,
|
||||
ToolMediaScope::ImagesOnly,
|
||||
&replacement_block,
|
||||
UNSUPPORTED_IMAGE_MARKER,
|
||||
);
|
||||
}
|
||||
|
||||
replaced
|
||||
}
|
||||
|
||||
@@ -283,6 +306,13 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn large_tool_data_url() -> String {
|
||||
format!(
|
||||
"data:image/png;base64,{}",
|
||||
"SANITIZER_TOOL_MEDIA_SENTINEL".repeat(400)
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_images_when_model_capability_is_unknown() {
|
||||
let provider = provider(json!({}));
|
||||
@@ -603,6 +633,219 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_and_replaces_responses_function_output_images() {
|
||||
let data_url = large_tool_data_url();
|
||||
let mut body = json!({
|
||||
"model": "text-only",
|
||||
"input": [{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_1",
|
||||
"output": {
|
||||
"content": [
|
||||
{"type": "input_text", "text": "caption"},
|
||||
{"type": "input_image", "image_url": data_url.clone()},
|
||||
{"type": "image", "mimeType": "image/webp", "data": "MCP_SENTINEL"}
|
||||
]
|
||||
}
|
||||
}]
|
||||
});
|
||||
|
||||
assert!(contains_image_blocks(&body));
|
||||
let replaced = replace_image_blocks_with_marker(&mut body);
|
||||
|
||||
assert_eq!(replaced, 2);
|
||||
assert_eq!(
|
||||
body["input"][0]["output"]["content"][1],
|
||||
json!({"type": "input_text", "text": UNSUPPORTED_IMAGE_MARKER})
|
||||
);
|
||||
assert_eq!(
|
||||
body["input"][0]["output"]["content"][2],
|
||||
json!({"type": "input_text", "text": UNSUPPORTED_IMAGE_MARKER})
|
||||
);
|
||||
assert!(!body.to_string().contains(&data_url));
|
||||
assert!(!body.to_string().contains("MCP_SENTINEL"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proactive_text_only_sanitizer_covers_responses_tool_outputs() {
|
||||
let provider = provider(json!({
|
||||
"models": [{"id": "text-model", "input": ["text"]}]
|
||||
}));
|
||||
let mut body = json!({
|
||||
"model": "text-model",
|
||||
"input": [{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_1",
|
||||
"output": [{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,PROACTIVE_SENTINEL"
|
||||
}]
|
||||
}]
|
||||
});
|
||||
|
||||
let replaced = replace_images_for_text_only_model(&mut body, &provider, true);
|
||||
|
||||
assert_eq!(replaced, 1);
|
||||
assert_eq!(body["input"][0]["output"][0]["type"], "input_text");
|
||||
assert!(!body.to_string().contains("PROACTIVE_SENTINEL"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_and_replaces_json_string_tool_output_symmetrically() {
|
||||
let data_url = large_tool_data_url();
|
||||
let output = json!({
|
||||
"content": [{
|
||||
"type": "input_image",
|
||||
"image_url": data_url.clone()
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
let mut body = json!({
|
||||
"input": [{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_string",
|
||||
"output": output
|
||||
}]
|
||||
});
|
||||
|
||||
assert!(contains_image_blocks(&body));
|
||||
let replaced = replace_image_blocks_with_marker(&mut body);
|
||||
|
||||
assert_eq!(replaced, 1);
|
||||
let rewritten = body["input"][0]["output"].as_str().unwrap();
|
||||
assert!(rewritten.contains(UNSUPPORTED_IMAGE_MARKER));
|
||||
assert!(!rewritten.contains(&data_url));
|
||||
let parsed: Value = serde_json::from_str(rewritten).unwrap();
|
||||
assert_eq!(parsed["content"][0]["type"], "input_text");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_and_replaces_whole_string_tool_image_data_url() {
|
||||
let data_url = large_tool_data_url();
|
||||
let mut body = json!({
|
||||
"input": [{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_raw",
|
||||
"output": data_url.clone()
|
||||
}]
|
||||
});
|
||||
|
||||
assert!(contains_image_blocks(&body));
|
||||
let replaced = replace_image_blocks_with_marker(&mut body);
|
||||
|
||||
assert_eq!(replaced, 1);
|
||||
assert_eq!(
|
||||
body["input"][0]["output"],
|
||||
Value::String(UNSUPPORTED_IMAGE_MARKER.to_string())
|
||||
);
|
||||
assert!(!body.to_string().contains(&data_url));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_and_replaces_custom_tool_output_images() {
|
||||
let mut body = json!({
|
||||
"input": [{
|
||||
"type": "custom_tool_call_output",
|
||||
"call_id": "call_custom",
|
||||
"status": "completed",
|
||||
"output": [{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.com/render.png"}
|
||||
}]
|
||||
}]
|
||||
});
|
||||
|
||||
assert!(contains_image_blocks(&body));
|
||||
let replaced = replace_image_blocks_with_marker(&mut body);
|
||||
|
||||
assert_eq!(replaced, 1);
|
||||
assert_eq!(body["input"][0]["status"], "completed");
|
||||
assert_eq!(body["input"][0]["output"][0]["type"], "input_text");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_no_media_and_untyped_remote_tool_outputs() {
|
||||
let mut body = json!({
|
||||
"input": [
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_text",
|
||||
"output": {"content": [{"type": "text", "text": "ordinary result"}]}
|
||||
},
|
||||
{
|
||||
"type": "tool_search_output",
|
||||
"call_id": "call_search",
|
||||
"output": {
|
||||
"image_url": {"url": "https://example.com/search-thumbnail.png"}
|
||||
}
|
||||
}
|
||||
]
|
||||
});
|
||||
let original = body.clone();
|
||||
|
||||
assert!(!contains_image_blocks(&body));
|
||||
assert_eq!(replace_image_blocks_with_marker(&mut body), 0);
|
||||
assert_eq!(body, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_retry_scope_intentionally_ignores_tool_files_and_audio() {
|
||||
let mut body = json!({
|
||||
"input": [{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_modalities",
|
||||
"output": {
|
||||
"content": [
|
||||
{"type": "input_file", "file_id": "file_1"},
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "AUDIO", "format": "wav"}
|
||||
}
|
||||
]
|
||||
}
|
||||
}]
|
||||
});
|
||||
let original = body.clone();
|
||||
|
||||
assert!(!contains_image_blocks(&body));
|
||||
assert_eq!(replace_image_blocks_with_marker(&mut body), 0);
|
||||
assert_eq!(body, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replaces_synthetic_user_and_tool_role_chat_image_parts() {
|
||||
let mut body = json!({
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "tool media"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64,USER_SENTINEL"}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": [{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.com/tool.png"}
|
||||
}]
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
assert!(contains_image_blocks(&body));
|
||||
let replaced = replace_image_blocks_with_marker(&mut body);
|
||||
|
||||
assert_eq!(replaced, 2);
|
||||
assert_eq!(body["messages"][0]["content"][1]["type"], "text");
|
||||
assert_eq!(body["messages"][1]["content"][0]["type"], "text");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_unsupported_image_errors() {
|
||||
let error = ProxyError::UpstreamError {
|
||||
|
||||
@@ -33,6 +33,7 @@ pub(crate) mod switch_lock;
|
||||
pub mod thinking_budget_rectifier;
|
||||
pub mod thinking_optimizer;
|
||||
pub mod thinking_rectifier;
|
||||
pub(crate) mod tool_media;
|
||||
pub(crate) mod types;
|
||||
pub mod usage;
|
||||
|
||||
|
||||
@@ -16,6 +16,10 @@ use crate::proxy::{
|
||||
canonical_json_string, canonicalize_json_string_if_parseable, canonicalize_tool_arguments,
|
||||
short_sha256_hex,
|
||||
},
|
||||
tool_media::{
|
||||
chat_file_from_input_file, clamp_base64ish_strings, strip_media_from_tool_value,
|
||||
whole_string_image_data_url, ToolMediaScope,
|
||||
},
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
@@ -42,6 +46,8 @@ const CUSTOM_TOOL_INPUT_FIELD: &str = "input";
|
||||
const CHAT_TOOL_NAME_MAX_LEN: usize = 64;
|
||||
const CUSTOM_TOOL_INPUT_DESCRIPTION: &str = "Raw string input for the original custom tool. Preserve formatting exactly and follow the original tool definition embedded in the description.";
|
||||
const CUSTOM_TOOL_PRESERVED_METADATA_HEADING: &str = "Original tool definition:";
|
||||
const TOOL_RESULT_MEDIA_MOVED_MARKER: &str =
|
||||
"[cc-switch: tool result media moved to the following user message]";
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) enum CodexToolKind {
|
||||
@@ -539,12 +545,77 @@ fn instruction_text(value: &Value) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
struct ToolOutputMediaPlan {
|
||||
tool_content: String,
|
||||
media_parts: Vec<Value>,
|
||||
}
|
||||
|
||||
fn plan_tool_output_media(mut output: Value) -> Option<ToolOutputMediaPlan> {
|
||||
let replacement_block = json!({
|
||||
"type": "text",
|
||||
"text": TOOL_RESULT_MEDIA_MOVED_MARKER
|
||||
});
|
||||
let mut media_parts = Vec::new();
|
||||
let replaced = strip_media_from_tool_value(
|
||||
&mut output,
|
||||
&mut media_parts,
|
||||
ToolMediaScope::AllSupported,
|
||||
&replacement_block,
|
||||
TOOL_RESULT_MEDIA_MOVED_MARKER,
|
||||
);
|
||||
if replaced == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Only media-bearing outputs enter this path. Remove any residual large
|
||||
// data/base64 scalar that was not itself a recognized content block while
|
||||
// preserving ordinary long OCR/log/source text.
|
||||
clamp_base64ish_strings(&mut output);
|
||||
Some(ToolOutputMediaPlan {
|
||||
tool_content: canonical_json_string(&output),
|
||||
media_parts,
|
||||
})
|
||||
}
|
||||
|
||||
fn plan_raw_tool_output_data_url(output: &str) -> Option<ToolOutputMediaPlan> {
|
||||
whole_string_image_data_url(output).map(|media_part| ToolOutputMediaPlan {
|
||||
// A raw tool-output string remains a raw tool-output string. Do not add
|
||||
// JSON string-literal quotes around the replacement marker.
|
||||
tool_content: TOOL_RESULT_MEDIA_MOVED_MARKER.to_string(),
|
||||
media_parts: vec![media_part],
|
||||
})
|
||||
}
|
||||
|
||||
fn queue_tool_output_media(pending_media: &mut Vec<Value>, call_id: &str, media_parts: Vec<Value>) {
|
||||
if media_parts.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
pending_media.push(json!({
|
||||
"type": "text",
|
||||
"text": format!("[cc-switch: media output of tool call {call_id}]")
|
||||
}));
|
||||
pending_media.extend(media_parts);
|
||||
}
|
||||
|
||||
fn flush_pending_media(messages: &mut Vec<Value>, pending_media: &mut Vec<Value>) {
|
||||
if pending_media.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
messages.push(json!({
|
||||
"role": "user",
|
||||
"content": std::mem::take(pending_media)
|
||||
}));
|
||||
}
|
||||
|
||||
fn append_responses_input_as_chat_messages(
|
||||
input: &Value,
|
||||
messages: &mut Vec<Value>,
|
||||
tool_context: &CodexToolContext,
|
||||
) -> Result<(), ProxyError> {
|
||||
let mut pending_tool_calls = Vec::new();
|
||||
let mut pending_media = Vec::new();
|
||||
let mut pending_reasoning: Option<String> = None;
|
||||
let mut last_assistant_index: Option<usize> = None;
|
||||
|
||||
@@ -561,6 +632,7 @@ fn append_responses_input_as_chat_messages(
|
||||
item,
|
||||
messages,
|
||||
&mut pending_tool_calls,
|
||||
&mut pending_media,
|
||||
&mut pending_reasoning,
|
||||
&mut last_assistant_index,
|
||||
tool_context,
|
||||
@@ -572,6 +644,7 @@ fn append_responses_input_as_chat_messages(
|
||||
input,
|
||||
messages,
|
||||
&mut pending_tool_calls,
|
||||
&mut pending_media,
|
||||
&mut pending_reasoning,
|
||||
&mut last_assistant_index,
|
||||
tool_context,
|
||||
@@ -580,9 +653,14 @@ fn append_responses_input_as_chat_messages(
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// If a later assistant tool-call batch was accumulated after an earlier
|
||||
// media-bearing result, the synthetic user media belongs before that next
|
||||
// assistant turn.
|
||||
flush_pending_media(messages, &mut pending_media);
|
||||
flush_pending_tool_calls(
|
||||
messages,
|
||||
&mut pending_tool_calls,
|
||||
&mut pending_media,
|
||||
&mut pending_reasoning,
|
||||
&mut last_assistant_index,
|
||||
);
|
||||
@@ -603,6 +681,7 @@ fn append_responses_item_as_chat_message(
|
||||
item: &Value,
|
||||
messages: &mut Vec<Value>,
|
||||
pending_tool_calls: &mut Vec<Value>,
|
||||
pending_media: &mut Vec<Value>,
|
||||
pending_reasoning: &mut Option<String>,
|
||||
last_assistant_index: &mut Option<usize>,
|
||||
tool_context: &CodexToolContext,
|
||||
@@ -628,14 +707,35 @@ fn append_responses_item_as_chat_message(
|
||||
flush_pending_tool_calls(
|
||||
messages,
|
||||
pending_tool_calls,
|
||||
pending_media,
|
||||
pending_reasoning,
|
||||
last_assistant_index,
|
||||
);
|
||||
let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let output = match item.get("output") {
|
||||
Some(Value::String(s)) => canonicalize_json_string_if_parseable(s),
|
||||
Some(v) => canonical_json_string(v),
|
||||
None => String::new(),
|
||||
let media_plan = match item.get("output") {
|
||||
Some(Value::String(output)) => {
|
||||
plan_raw_tool_output_data_url(output).or_else(|| {
|
||||
serde_json::from_str::<Value>(output.trim())
|
||||
.ok()
|
||||
.and_then(plan_tool_output_media)
|
||||
})
|
||||
}
|
||||
Some(output @ (Value::Array(_) | Value::Object(_))) => {
|
||||
plan_tool_output_media(output.clone())
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let output = if let Some(media_plan) = media_plan {
|
||||
queue_tool_output_media(pending_media, call_id, media_plan.media_parts);
|
||||
media_plan.tool_content
|
||||
} else {
|
||||
// Cache-sensitive no-media fallback: keep these expressions
|
||||
// byte-for-byte equivalent to the pre-fix conversion.
|
||||
match item.get("output") {
|
||||
Some(Value::String(s)) => canonicalize_json_string_if_parseable(s),
|
||||
Some(v) => canonical_json_string(v),
|
||||
None => String::new(),
|
||||
}
|
||||
};
|
||||
messages.push(json!({
|
||||
"role": "tool",
|
||||
@@ -647,11 +747,39 @@ fn append_responses_item_as_chat_message(
|
||||
flush_pending_tool_calls(
|
||||
messages,
|
||||
pending_tool_calls,
|
||||
pending_media,
|
||||
pending_reasoning,
|
||||
last_assistant_index,
|
||||
);
|
||||
let call_id = item.get("call_id").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let output = canonical_json_string(item);
|
||||
let mut transformed_item = item.clone();
|
||||
let replacement_block = json!({
|
||||
"type": "text",
|
||||
"text": TOOL_RESULT_MEDIA_MOVED_MARKER
|
||||
});
|
||||
let mut media_parts = Vec::new();
|
||||
let replaced = transformed_item
|
||||
.get_mut("output")
|
||||
.map(|output| {
|
||||
strip_media_from_tool_value(
|
||||
output,
|
||||
&mut media_parts,
|
||||
ToolMediaScope::AllSupported,
|
||||
&replacement_block,
|
||||
TOOL_RESULT_MEDIA_MOVED_MARKER,
|
||||
)
|
||||
})
|
||||
.unwrap_or(0);
|
||||
let output = if replaced > 0 {
|
||||
if let Some(output) = transformed_item.get_mut("output") {
|
||||
clamp_base64ish_strings(output);
|
||||
}
|
||||
queue_tool_output_media(pending_media, call_id, media_parts);
|
||||
canonical_json_string(&transformed_item)
|
||||
} else {
|
||||
// Preserve the legacy whole-item representation exactly.
|
||||
canonical_json_string(item)
|
||||
};
|
||||
messages.push(json!({
|
||||
"role": "tool",
|
||||
"tool_call_id": call_id,
|
||||
@@ -672,9 +800,14 @@ fn append_responses_item_as_chat_message(
|
||||
flush_pending_tool_calls(
|
||||
messages,
|
||||
pending_tool_calls,
|
||||
pending_media,
|
||||
pending_reasoning,
|
||||
last_assistant_index,
|
||||
);
|
||||
// `flush_pending_tool_calls` intentionally returns early when
|
||||
// there is no new assistant batch. A previous tool result may
|
||||
// still have media waiting, so flush it before this new message.
|
||||
flush_pending_media(messages, pending_media);
|
||||
let role = item
|
||||
.get("role")
|
||||
.and_then(|v| v.as_str())
|
||||
@@ -705,13 +838,15 @@ fn append_responses_item_as_chat_message(
|
||||
messages.push(message);
|
||||
}
|
||||
Some("message") | None => {
|
||||
flush_pending_tool_calls(
|
||||
messages,
|
||||
pending_tool_calls,
|
||||
pending_reasoning,
|
||||
last_assistant_index,
|
||||
);
|
||||
if item.get("role").is_some() || item.get("content").is_some() {
|
||||
flush_pending_tool_calls(
|
||||
messages,
|
||||
pending_tool_calls,
|
||||
pending_media,
|
||||
pending_reasoning,
|
||||
last_assistant_index,
|
||||
);
|
||||
flush_pending_media(messages, pending_media);
|
||||
let message = responses_message_item_to_chat_message(
|
||||
item,
|
||||
pending_reasoning,
|
||||
@@ -720,16 +855,28 @@ fn append_responses_item_as_chat_message(
|
||||
);
|
||||
update_last_assistant_index(messages, &message, last_assistant_index);
|
||||
messages.push(message);
|
||||
} else if pending_media.is_empty() {
|
||||
// Preserve legacy no-media ordering: inert message-like items
|
||||
// used to close a pending tool-call batch.
|
||||
flush_pending_tool_calls(
|
||||
messages,
|
||||
pending_tool_calls,
|
||||
pending_media,
|
||||
pending_reasoning,
|
||||
last_assistant_index,
|
||||
);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
flush_pending_tool_calls(
|
||||
messages,
|
||||
pending_tool_calls,
|
||||
pending_reasoning,
|
||||
last_assistant_index,
|
||||
);
|
||||
if item.get("role").is_some() || item.get("content").is_some() {
|
||||
flush_pending_tool_calls(
|
||||
messages,
|
||||
pending_tool_calls,
|
||||
pending_media,
|
||||
pending_reasoning,
|
||||
last_assistant_index,
|
||||
);
|
||||
flush_pending_media(messages, pending_media);
|
||||
let message = responses_message_item_to_chat_message(
|
||||
item,
|
||||
pending_reasoning,
|
||||
@@ -738,6 +885,16 @@ fn append_responses_item_as_chat_message(
|
||||
);
|
||||
update_last_assistant_index(messages, &message, last_assistant_index);
|
||||
messages.push(message);
|
||||
} else if pending_media.is_empty() {
|
||||
// Preserve legacy no-media ordering without letting an inert
|
||||
// unknown item flush a media-bearing result batch.
|
||||
flush_pending_tool_calls(
|
||||
messages,
|
||||
pending_tool_calls,
|
||||
pending_media,
|
||||
pending_reasoning,
|
||||
last_assistant_index,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -748,6 +905,7 @@ fn append_responses_item_as_chat_message(
|
||||
fn flush_pending_tool_calls(
|
||||
messages: &mut Vec<Value>,
|
||||
pending_tool_calls: &mut Vec<Value>,
|
||||
pending_media: &mut Vec<Value>,
|
||||
pending_reasoning: &mut Option<String>,
|
||||
last_assistant_index: &mut Option<usize>,
|
||||
) {
|
||||
@@ -755,6 +913,10 @@ fn flush_pending_tool_calls(
|
||||
return;
|
||||
}
|
||||
|
||||
// Media from the preceding tool-result batch must be presented before a
|
||||
// new assistant tool-call turn. Consecutive outputs do not enter here
|
||||
// because `pending_tool_calls` is empty after the first output.
|
||||
flush_pending_media(messages, pending_media);
|
||||
let mut message = json!({
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
@@ -1057,18 +1219,7 @@ fn responses_content_to_chat_content(_role: &str, content: &Value) -> Value {
|
||||
}
|
||||
|
||||
fn responses_input_file_to_chat_file(part: &Value) -> Option<Value> {
|
||||
let mut file = serde_json::Map::new();
|
||||
let has_supported_file_ref = part.get("file_id").is_some() || part.get("file_data").is_some();
|
||||
if !has_supported_file_ref {
|
||||
return None;
|
||||
}
|
||||
|
||||
for key in ["file_id", "file_data", "filename"] {
|
||||
if let Some(value) = part.get(key) {
|
||||
file.insert(key.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
Some(Value::Object(file))
|
||||
chat_file_from_input_file(part)
|
||||
}
|
||||
|
||||
fn collect_tool_search_output_tools(value: &Value, context: &mut CodexToolContext) {
|
||||
@@ -1834,6 +1985,50 @@ pub fn chat_error_to_response_error(body: Option<&Value>) -> Value {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use base64::{engine::general_purpose::STANDARD, Engine as _};
|
||||
|
||||
fn large_test_image_data_url() -> String {
|
||||
let bytes = b"CC_SWITCH_TOOL_MEDIA_SENTINEL".repeat(400);
|
||||
format!("data:image/png;base64,{}", STANDARD.encode(bytes))
|
||||
}
|
||||
|
||||
fn message_roles(result: &Value) -> Vec<&str> {
|
||||
result["messages"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter_map(|message| message.get("role").and_then(Value::as_str))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn test_function_call(call_id: &str) -> Value {
|
||||
json!({
|
||||
"type": "function_call",
|
||||
"call_id": call_id,
|
||||
"name": "view_image",
|
||||
"arguments": "{}"
|
||||
})
|
||||
}
|
||||
|
||||
fn test_function_output(call_id: &str, output: Value) -> Value {
|
||||
json!({
|
||||
"type": "function_call_output",
|
||||
"call_id": call_id,
|
||||
"output": output
|
||||
})
|
||||
}
|
||||
|
||||
fn convert_test_input(items: Vec<Value>) -> Value {
|
||||
responses_to_chat_completions(json!({
|
||||
"model": "kimi-k3",
|
||||
"input": items
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn result_messages(result: &Value) -> &[Value] {
|
||||
result["messages"].as_array().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_with_stream_injects_include_usage() {
|
||||
@@ -3061,6 +3256,538 @@ mod tests {
|
||||
assert_eq!(messages[1]["content"], "plain text result");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_moves_tool_image_to_synthetic_user_message() {
|
||||
let data_url = large_test_image_data_url();
|
||||
let result = convert_test_input(vec![
|
||||
test_function_call("call_image"),
|
||||
test_function_output(
|
||||
"call_image",
|
||||
json!([
|
||||
{"type": "input_text", "text": "screenshot follows"},
|
||||
{"type": "input_image", "image_url": data_url.clone()}
|
||||
]),
|
||||
),
|
||||
]);
|
||||
let messages = result_messages(&result);
|
||||
|
||||
assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]);
|
||||
assert!(messages[1]["content"].is_string());
|
||||
let tool_content: Value =
|
||||
serde_json::from_str(messages[1]["content"].as_str().unwrap()).unwrap();
|
||||
assert_eq!(tool_content[0]["text"], "screenshot follows");
|
||||
assert_eq!(tool_content[1]["type"], "text");
|
||||
assert_eq!(tool_content[1]["text"], TOOL_RESULT_MEDIA_MOVED_MARKER);
|
||||
assert!(!messages[1]["content"].as_str().unwrap().contains(&data_url));
|
||||
|
||||
assert_eq!(
|
||||
messages[2]["content"][0]["text"],
|
||||
"[cc-switch: media output of tool call call_image]"
|
||||
);
|
||||
assert_eq!(messages[2]["content"][1]["type"], "image_url");
|
||||
assert_eq!(messages[2]["content"][1]["image_url"]["url"], data_url);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_groups_parallel_media_after_all_tool_outputs() {
|
||||
let first_url = large_test_image_data_url();
|
||||
let second_payload = "MCP_TOOL_MEDIA_SENTINEL";
|
||||
let result = convert_test_input(vec![
|
||||
test_function_call("call_1"),
|
||||
test_function_call("call_2"),
|
||||
test_function_output(
|
||||
"call_1",
|
||||
json!({"type": "input_image", "image_url": first_url.clone()}),
|
||||
),
|
||||
json!({
|
||||
"type": "reasoning",
|
||||
"summary": [{"type": "summary_text", "text": "keep outputs adjacent"}]
|
||||
}),
|
||||
test_function_output(
|
||||
"call_2",
|
||||
json!({
|
||||
"type": "image",
|
||||
"mimeType": "image/webp",
|
||||
"data": second_payload
|
||||
}),
|
||||
),
|
||||
]);
|
||||
let messages = result_messages(&result);
|
||||
|
||||
assert_eq!(
|
||||
message_roles(&result),
|
||||
vec!["assistant", "tool", "tool", "user"]
|
||||
);
|
||||
assert_eq!(messages[0]["reasoning_content"], "keep outputs adjacent");
|
||||
assert_ne!(messages[0]["reasoning_content"], "tool call");
|
||||
assert_eq!(messages[1]["tool_call_id"], "call_1");
|
||||
assert_eq!(messages[2]["tool_call_id"], "call_2");
|
||||
assert!(messages[1]["content"].is_string());
|
||||
assert!(messages[2]["content"].is_string());
|
||||
assert!(!messages[1]["content"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains(&first_url));
|
||||
assert!(!messages[2]["content"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains(second_payload));
|
||||
assert_eq!(messages[3]["content"].as_array().unwrap().len(), 4);
|
||||
assert_eq!(
|
||||
messages[3]["content"][3]["image_url"]["url"],
|
||||
format!("data:image/webp;base64,{second_payload}")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_flushes_media_before_next_tool_call_batch() {
|
||||
let result = convert_test_input(vec![
|
||||
test_function_call("call_1"),
|
||||
test_function_output(
|
||||
"call_1",
|
||||
json!({
|
||||
"type": "input_image",
|
||||
"image_url": large_test_image_data_url()
|
||||
}),
|
||||
),
|
||||
test_function_call("call_2"),
|
||||
test_function_output("call_2", json!("second result")),
|
||||
]);
|
||||
let messages = result_messages(&result);
|
||||
|
||||
assert_eq!(
|
||||
message_roles(&result),
|
||||
vec!["assistant", "tool", "user", "assistant", "tool"]
|
||||
);
|
||||
assert_eq!(messages[0]["tool_calls"][0]["id"], "call_1");
|
||||
assert_eq!(messages[3]["tool_calls"][0]["id"], "call_2");
|
||||
assert_eq!(messages[4]["tool_call_id"], "call_2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_flushes_media_before_real_user_messages() {
|
||||
let boundaries = [
|
||||
json!({"type": "input_text", "text": "continue"}),
|
||||
json!({"type": "future_message", "role": "user", "content": "continue"}),
|
||||
];
|
||||
|
||||
for boundary in boundaries {
|
||||
let result = convert_test_input(vec![
|
||||
test_function_call("call_1"),
|
||||
test_function_output(
|
||||
"call_1",
|
||||
json!({
|
||||
"type": "input_image",
|
||||
"image_url": large_test_image_data_url()
|
||||
}),
|
||||
),
|
||||
boundary,
|
||||
]);
|
||||
let messages = result_messages(&result);
|
||||
|
||||
assert_eq!(
|
||||
message_roles(&result),
|
||||
vec!["assistant", "tool", "user", "user"]
|
||||
);
|
||||
assert!(messages[2]["content"].is_array());
|
||||
assert_eq!(messages[3]["role"], "user");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_handles_raw_data_url_thresholds() {
|
||||
let large = large_test_image_data_url();
|
||||
let large_result = convert_test_input(vec![
|
||||
test_function_call("call_large"),
|
||||
test_function_output("call_large", Value::String(large.clone())),
|
||||
]);
|
||||
let large_messages = result_messages(&large_result);
|
||||
|
||||
assert_eq!(
|
||||
message_roles(&large_result),
|
||||
vec!["assistant", "tool", "user"]
|
||||
);
|
||||
assert_eq!(large_messages[1]["content"], TOOL_RESULT_MEDIA_MOVED_MARKER);
|
||||
assert_eq!(large_messages[2]["content"][1]["image_url"]["url"], large);
|
||||
|
||||
let small = "data:image/png;base64,YWJj";
|
||||
let small_result = convert_test_input(vec![
|
||||
test_function_call("call_small"),
|
||||
test_function_output("call_small", json!(small)),
|
||||
]);
|
||||
assert_eq!(message_roles(&small_result), vec!["assistant", "tool"]);
|
||||
assert_eq!(small_result["messages"][1]["content"], small);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_maps_supported_structured_image_shapes() {
|
||||
let cases = vec![
|
||||
(
|
||||
json!({
|
||||
"type": "input_image",
|
||||
"image_url": {"url": "https://example.com/input.png"},
|
||||
"detail": "high"
|
||||
}),
|
||||
"https://example.com/input.png",
|
||||
Some("high"),
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"type": "image_url",
|
||||
"image_url": "https://example.com/chat-string.png"
|
||||
}),
|
||||
"https://example.com/chat-string.png",
|
||||
None,
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://example.com/chat-object.png",
|
||||
"detail": "low"
|
||||
}
|
||||
}),
|
||||
"https://example.com/chat-object.png",
|
||||
Some("low"),
|
||||
),
|
||||
(
|
||||
json!({"image_url": "data:image/gif;base64,LOOSE_SENTINEL"}),
|
||||
"data:image/gif;base64,LOOSE_SENTINEL",
|
||||
None,
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"type": "image",
|
||||
"source": {
|
||||
"media_type": "image/jpeg",
|
||||
"data": "ANTHROPIC_SENTINEL"
|
||||
}
|
||||
}),
|
||||
"data:image/jpeg;base64,ANTHROPIC_SENTINEL",
|
||||
None,
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"type": "image",
|
||||
"mimeType": "image/webp",
|
||||
"data": "MCP_SENTINEL"
|
||||
}),
|
||||
"data:image/webp;base64,MCP_SENTINEL",
|
||||
None,
|
||||
),
|
||||
];
|
||||
|
||||
for (index, (output, expected_url, expected_detail)) in cases.into_iter().enumerate() {
|
||||
let call_id = format!("call_shape_{index}");
|
||||
let result = convert_test_input(vec![
|
||||
test_function_call(&call_id),
|
||||
test_function_output(&call_id, output),
|
||||
]);
|
||||
let image = &result["messages"][2]["content"][1];
|
||||
|
||||
assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]);
|
||||
assert_eq!(image["type"], "image_url");
|
||||
assert_eq!(image["image_url"]["url"], expected_url);
|
||||
match expected_detail {
|
||||
Some(detail) => assert_eq!(image["image_url"]["detail"], detail),
|
||||
None => assert!(image["image_url"].get("detail").is_none()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_extracts_media_from_json_string_and_content_wrapper() {
|
||||
let output = json!({
|
||||
"content": [
|
||||
{"type": "input_text", "text": "MCP response"},
|
||||
{
|
||||
"type": "image",
|
||||
"mimeType": "image/png",
|
||||
"data": "STRING_MCP_SENTINEL"
|
||||
}
|
||||
]
|
||||
})
|
||||
.to_string();
|
||||
let result = convert_test_input(vec![
|
||||
test_function_call("call_string"),
|
||||
test_function_output("call_string", Value::String(output)),
|
||||
]);
|
||||
let messages = result_messages(&result);
|
||||
|
||||
assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]);
|
||||
let tool_content: Value =
|
||||
serde_json::from_str(messages[1]["content"].as_str().unwrap()).unwrap();
|
||||
assert_eq!(tool_content["content"][0]["text"], "MCP response");
|
||||
assert_eq!(
|
||||
tool_content["content"][1]["text"],
|
||||
TOOL_RESULT_MEDIA_MOVED_MARKER
|
||||
);
|
||||
assert!(!messages[1]["content"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("STRING_MCP_SENTINEL"));
|
||||
assert_eq!(
|
||||
messages[2]["content"][1]["image_url"]["url"],
|
||||
"data:image/png;base64,STRING_MCP_SENTINEL"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_extracts_tool_files_and_audio() {
|
||||
let result = convert_test_input(vec![
|
||||
test_function_call("call_media"),
|
||||
test_function_output(
|
||||
"call_media",
|
||||
json!({
|
||||
"content": [
|
||||
{
|
||||
"type": "input_file",
|
||||
"file_id": "file_123",
|
||||
"filename": "report.pdf"
|
||||
},
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "AUDIO_SENTINEL", "format": "wav"}
|
||||
}
|
||||
]
|
||||
}),
|
||||
),
|
||||
]);
|
||||
let messages = result_messages(&result);
|
||||
|
||||
assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]);
|
||||
assert_eq!(messages[2]["content"][1]["type"], "file");
|
||||
assert_eq!(messages[2]["content"][1]["file"]["file_id"], "file_123");
|
||||
assert_eq!(messages[2]["content"][2]["type"], "input_audio");
|
||||
assert_eq!(
|
||||
messages[2]["content"][2]["input_audio"]["data"],
|
||||
"AUDIO_SENTINEL"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_extracts_custom_and_tool_search_output_media() {
|
||||
let cases = [
|
||||
(
|
||||
json!({
|
||||
"type": "custom_tool_call",
|
||||
"call_id": "call_custom",
|
||||
"name": "render",
|
||||
"input": "draw"
|
||||
}),
|
||||
json!({
|
||||
"type": "custom_tool_call_output",
|
||||
"call_id": "call_custom",
|
||||
"status": "completed",
|
||||
"output": {
|
||||
"content": [{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,CUSTOM_SENTINEL"
|
||||
}]
|
||||
}
|
||||
}),
|
||||
),
|
||||
(
|
||||
json!({
|
||||
"type": "tool_search_call",
|
||||
"call_id": "call_search",
|
||||
"arguments": {"query": "image tool"}
|
||||
}),
|
||||
json!({
|
||||
"type": "tool_search_output",
|
||||
"call_id": "call_search",
|
||||
"status": "completed",
|
||||
"output": {
|
||||
"content": [{
|
||||
"type": "image",
|
||||
"mimeType": "image/png",
|
||||
"data": "SEARCH_SENTINEL"
|
||||
}]
|
||||
}
|
||||
}),
|
||||
),
|
||||
];
|
||||
|
||||
for (call, output) in cases {
|
||||
let expected_type = output["type"].as_str().unwrap().to_string();
|
||||
let result = convert_test_input(vec![call, output]);
|
||||
let messages = result_messages(&result);
|
||||
let tool_content: Value =
|
||||
serde_json::from_str(messages[1]["content"].as_str().unwrap()).unwrap();
|
||||
|
||||
assert_eq!(message_roles(&result), vec!["assistant", "tool", "user"]);
|
||||
assert_eq!(tool_content["type"], expected_type);
|
||||
assert_eq!(tool_content["status"], "completed");
|
||||
assert_eq!(
|
||||
tool_content["output"]["content"][0]["text"],
|
||||
TOOL_RESULT_MEDIA_MOVED_MARKER
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_rejects_false_positive_media_shapes() {
|
||||
let outputs = [
|
||||
json!({"type": "image", "name": "business metadata"}),
|
||||
json!({
|
||||
"type": "image",
|
||||
"mimeType": "text/plain",
|
||||
"data": "NOT_AN_IMAGE"
|
||||
}),
|
||||
json!({
|
||||
"image_url": {
|
||||
"url": "https://example.com/search-thumbnail.png"
|
||||
}
|
||||
}),
|
||||
];
|
||||
|
||||
for (index, output) in outputs.into_iter().enumerate() {
|
||||
let call_id = format!("call_false_positive_{index}");
|
||||
let expected = canonical_json_string(&output);
|
||||
let result = convert_test_input(vec![
|
||||
test_function_call(&call_id),
|
||||
test_function_output(&call_id, output),
|
||||
]);
|
||||
|
||||
assert_eq!(message_roles(&result), vec!["assistant", "tool"]);
|
||||
assert_eq!(result["messages"][1]["content"], expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_keeps_no_media_tool_output_bytes_stable() {
|
||||
let cases = [
|
||||
(Some(json!("plain text")), "plain text".to_string()),
|
||||
(
|
||||
Some(json!("{ \"z\": true, \"a\": [2, 1] }")),
|
||||
r#"{"a":[2,1],"z":true}"#.to_string(),
|
||||
),
|
||||
(
|
||||
Some(json!(["main.rs", "lib.rs"])),
|
||||
r#"["main.rs","lib.rs"]"#.to_string(),
|
||||
),
|
||||
(
|
||||
Some(json!({"z": true, "a": [2, 1]})),
|
||||
r#"{"a":[2,1],"z":true}"#.to_string(),
|
||||
),
|
||||
(Some(json!([])), "[]".to_string()),
|
||||
(None, String::new()),
|
||||
];
|
||||
|
||||
for (index, (output, expected)) in cases.into_iter().enumerate() {
|
||||
let call_id = format!("call_stable_{index}");
|
||||
let mut item = json!({
|
||||
"type": "function_call_output",
|
||||
"call_id": call_id
|
||||
});
|
||||
if let Some(output) = output {
|
||||
item["output"] = output;
|
||||
}
|
||||
let result = convert_test_input(vec![test_function_call(&call_id), item]);
|
||||
|
||||
assert_eq!(message_roles(&result), vec!["assistant", "tool"]);
|
||||
assert_eq!(result["messages"][1]["content"], expected);
|
||||
}
|
||||
|
||||
for item in [
|
||||
json!({
|
||||
"type": "custom_tool_call_output",
|
||||
"call_id": "call_custom",
|
||||
"status": "completed",
|
||||
"output": {"text": "unchanged"}
|
||||
}),
|
||||
json!({
|
||||
"type": "tool_search_output",
|
||||
"call_id": "call_search",
|
||||
"status": "completed",
|
||||
"output": []
|
||||
}),
|
||||
] {
|
||||
let expected = canonical_json_string(&item);
|
||||
let result = convert_test_input(vec![item]);
|
||||
assert_eq!(result["messages"][0]["content"], expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_preserves_legacy_unknown_item_batch_boundary_without_media() {
|
||||
let result = convert_test_input(vec![
|
||||
test_function_call("call_1"),
|
||||
json!({"type": "future_metadata", "value": 1}),
|
||||
json!({
|
||||
"type": "reasoning",
|
||||
"summary": [{"type": "summary_text", "text": "second batch reasoning"}]
|
||||
}),
|
||||
test_function_call("call_2"),
|
||||
test_function_output("call_1", json!("first result")),
|
||||
test_function_output("call_2", json!("second result")),
|
||||
]);
|
||||
let messages = result_messages(&result);
|
||||
|
||||
assert_eq!(
|
||||
message_roles(&result),
|
||||
vec!["assistant", "assistant", "tool", "tool"]
|
||||
);
|
||||
assert_eq!(messages[0]["tool_calls"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(messages[0]["tool_calls"][0]["id"], "call_1");
|
||||
assert_eq!(messages[0]["reasoning_content"], "tool call");
|
||||
assert_eq!(messages[1]["tool_calls"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(messages[1]["tool_calls"][0]["id"], "call_2");
|
||||
assert_eq!(messages[1]["reasoning_content"], "second batch reasoning");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_clamps_only_residual_base64ish_strings() {
|
||||
let data_url = large_test_image_data_url();
|
||||
let long_text = format!("{}end", "ordinary OCR text with spaces. ".repeat(3_500));
|
||||
let residual_base64 = "A".repeat(20_000);
|
||||
let result = convert_test_input(vec![
|
||||
test_function_call("call_clamp"),
|
||||
test_function_output(
|
||||
"call_clamp",
|
||||
json!([
|
||||
{"type": "input_image", "image_url": data_url.clone()},
|
||||
{"type": "text", "text": long_text.clone()},
|
||||
{"raw": residual_base64}
|
||||
]),
|
||||
),
|
||||
]);
|
||||
let tool_content_text = result["messages"][1]["content"].as_str().unwrap();
|
||||
let tool_content: Value = serde_json::from_str(tool_content_text).unwrap();
|
||||
|
||||
assert_eq!(tool_content[1]["text"], long_text);
|
||||
assert!(tool_content[2]["raw"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.starts_with("[cc-switch: omitted 20000 bytes]"));
|
||||
assert!(!tool_content_text.contains(&data_url));
|
||||
assert!(!tool_content_text.contains("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn responses_request_to_chat_media_conversion_is_deterministic() {
|
||||
let input = json!({
|
||||
"model": "kimi-k3",
|
||||
"input": [
|
||||
test_function_call("call_repeat"),
|
||||
test_function_output(
|
||||
"call_repeat",
|
||||
json!({
|
||||
"content": [{
|
||||
"type": "input_image",
|
||||
"image_url": large_test_image_data_url()
|
||||
}]
|
||||
})
|
||||
)
|
||||
]
|
||||
});
|
||||
|
||||
let first = responses_to_chat_completions(input.clone()).unwrap();
|
||||
let second = responses_to_chat_completions(input).unwrap();
|
||||
|
||||
assert_eq!(first, second);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_response_to_responses_maps_text_tool_calls_and_usage() {
|
||||
let input = json!({
|
||||
|
||||
@@ -0,0 +1,774 @@
|
||||
//! Shared media handling for Codex tool outputs.
|
||||
//!
|
||||
//! Responses tool outputs may carry structured media blocks. Chat Completions
|
||||
//! tool messages are text-only, so the Responses→Chat bridge extracts those
|
||||
//! blocks and re-emits them in a synthetic user message. The media sanitizer
|
||||
//! reuses the same recognition and traversal rules when it needs to remove
|
||||
//! images for a text-only upstream.
|
||||
|
||||
use crate::proxy::json_canonical::canonical_json_string;
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
pub(crate) const WHOLE_DATA_URL_MIN_BYTES: usize = 8 * 1024;
|
||||
const BASE64ISH_MIN_BYTES: usize = 16 * 1024;
|
||||
const MAX_MEDIA_TRAVERSAL_DEPTH: usize = 32;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum ToolMediaScope {
|
||||
/// Used by the existing image-capability sanitizer and its retry path.
|
||||
ImagesOnly,
|
||||
/// Used by Responses→Chat conversion, where user messages can carry all
|
||||
/// currently mapped Chat input modalities.
|
||||
AllSupported,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ToolMediaKind {
|
||||
Image,
|
||||
File,
|
||||
Audio,
|
||||
}
|
||||
|
||||
impl ToolMediaScope {
|
||||
fn allows(self, kind: ToolMediaKind) -> bool {
|
||||
matches!(kind, ToolMediaKind::Image) || matches!(self, Self::AllSupported)
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert one recognized Responses/tool media block to a Chat user content
|
||||
/// part. This is the single shape-recognition entry point used by extraction.
|
||||
pub(crate) fn chat_media_part_from_tool_part(part: &Value, scope: ToolMediaScope) -> Option<Value> {
|
||||
let kind = tool_media_kind(part)?;
|
||||
if !scope.allows(kind) {
|
||||
return None;
|
||||
}
|
||||
|
||||
match kind {
|
||||
ToolMediaKind::Image => chat_image_part(part),
|
||||
ToolMediaKind::File => chat_file_from_input_file(part).map(|file| {
|
||||
json!({
|
||||
"type": "file",
|
||||
"file": file
|
||||
})
|
||||
}),
|
||||
ToolMediaKind::Audio => part.get("input_audio").map(|input_audio| {
|
||||
json!({
|
||||
"type": "input_audio",
|
||||
"input_audio": input_audio.clone()
|
||||
})
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a Responses `input_file` block to the Chat file payload. Kept here so
|
||||
/// top-level content and tool-output extraction share the exact same rules.
|
||||
pub(crate) fn chat_file_from_input_file(part: &Value) -> Option<Value> {
|
||||
let mut file = Map::new();
|
||||
let has_supported_file_ref = part.get("file_id").is_some() || part.get("file_data").is_some();
|
||||
if !has_supported_file_ref {
|
||||
return None;
|
||||
}
|
||||
|
||||
for key in ["file_id", "file_data", "filename"] {
|
||||
if let Some(value) = part.get(key) {
|
||||
file.insert(key.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
Some(Value::Object(file))
|
||||
}
|
||||
|
||||
/// Recognize a complete image data URL stored as a scalar string.
|
||||
///
|
||||
/// Only whole-string matches are accepted. Embedded data URLs in HTML/CSS/SVG
|
||||
/// source are deliberately left alone. Small values remain text as well, which
|
||||
/// preserves workflows that intentionally inspect tiny inline icons.
|
||||
pub(crate) fn whole_string_image_data_url(value: &str) -> Option<Value> {
|
||||
let trimmed = value.trim();
|
||||
if trimmed.len() < WHOLE_DATA_URL_MIN_BYTES || !is_image_base64_data_url(trimmed) {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(json!({
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": trimmed
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
/// Read-only media detection using the same shape classifier and recursive
|
||||
/// boundaries as [`strip_media_from_tool_value`].
|
||||
pub(crate) fn tool_output_contains_media(value: &Value, scope: ToolMediaScope) -> bool {
|
||||
tool_output_contains_media_at_depth(value, scope, 0)
|
||||
}
|
||||
|
||||
/// Extract recognized media blocks and replace them in-place.
|
||||
///
|
||||
/// `replacement_block` is used for structured array/object parts. A scalar
|
||||
/// string that is itself a complete image data URL uses `replacement_text`, so
|
||||
/// an originally plain string stays a plain string. Parseable JSON strings are
|
||||
/// recursively transformed and canonicalized back into a string only when a
|
||||
/// replacement actually occurred.
|
||||
pub(crate) fn strip_media_from_tool_value(
|
||||
value: &mut Value,
|
||||
media_parts: &mut Vec<Value>,
|
||||
scope: ToolMediaScope,
|
||||
replacement_block: &Value,
|
||||
replacement_text: &str,
|
||||
) -> usize {
|
||||
strip_media_from_tool_value_at_depth(
|
||||
value,
|
||||
media_parts,
|
||||
scope,
|
||||
replacement_block,
|
||||
replacement_text,
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
/// Remove residual data/base64 payloads only after a tool output has already
|
||||
/// been positively identified as media-bearing. Ordinary long text is kept.
|
||||
pub(crate) fn clamp_base64ish_strings(value: &mut Value) {
|
||||
match value {
|
||||
Value::String(text) => {
|
||||
let trimmed = text.trim();
|
||||
let should_omit = (trimmed.len() >= WHOLE_DATA_URL_MIN_BYTES
|
||||
&& trimmed
|
||||
.get(..5)
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")))
|
||||
|| looks_like_base64_payload(trimmed);
|
||||
if should_omit {
|
||||
let byte_len = text.len();
|
||||
*text = format!("[cc-switch: omitted {byte_len} bytes]");
|
||||
}
|
||||
}
|
||||
Value::Array(items) => {
|
||||
for item in items {
|
||||
clamp_base64ish_strings(item);
|
||||
}
|
||||
}
|
||||
Value::Object(object) => {
|
||||
for nested in object.values_mut() {
|
||||
clamp_base64ish_strings(nested);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn tool_output_contains_media_at_depth(value: &Value, scope: ToolMediaScope, depth: usize) -> bool {
|
||||
if depth > MAX_MEDIA_TRAVERSAL_DEPTH {
|
||||
return false;
|
||||
}
|
||||
|
||||
match value {
|
||||
Value::String(text) => {
|
||||
if scope.allows(ToolMediaKind::Image) && whole_string_image_data_url(text).is_some() {
|
||||
return true;
|
||||
}
|
||||
|
||||
let trimmed = text.trim();
|
||||
if trimmed.is_empty() {
|
||||
return false;
|
||||
}
|
||||
serde_json::from_str::<Value>(trimmed)
|
||||
.ok()
|
||||
.is_some_and(|parsed| {
|
||||
tool_output_contains_media_at_depth(&parsed, scope, depth + 1)
|
||||
})
|
||||
}
|
||||
Value::Array(items) => items
|
||||
.iter()
|
||||
.any(|item| tool_output_contains_media_at_depth(item, scope, depth + 1)),
|
||||
Value::Object(object) => {
|
||||
if tool_media_kind(value).is_some_and(|kind| scope.allows(kind)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
object.get("content").is_some_and(|content| {
|
||||
tool_output_contains_media_at_depth(content, scope, depth + 1)
|
||||
})
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn strip_media_from_tool_value_at_depth(
|
||||
value: &mut Value,
|
||||
media_parts: &mut Vec<Value>,
|
||||
scope: ToolMediaScope,
|
||||
replacement_block: &Value,
|
||||
replacement_text: &str,
|
||||
depth: usize,
|
||||
) -> usize {
|
||||
if depth > MAX_MEDIA_TRAVERSAL_DEPTH {
|
||||
return 0;
|
||||
}
|
||||
|
||||
match value {
|
||||
Value::String(text) => {
|
||||
if scope.allows(ToolMediaKind::Image) {
|
||||
if let Some(media_part) = whole_string_image_data_url(text) {
|
||||
media_parts.push(media_part);
|
||||
*text = replacement_text.to_string();
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
let trimmed = text.trim();
|
||||
if trimmed.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
let Ok(mut parsed) = serde_json::from_str::<Value>(trimmed) else {
|
||||
return 0;
|
||||
};
|
||||
let replaced = strip_media_from_tool_value_at_depth(
|
||||
&mut parsed,
|
||||
media_parts,
|
||||
scope,
|
||||
replacement_block,
|
||||
replacement_text,
|
||||
depth + 1,
|
||||
);
|
||||
if replaced > 0 {
|
||||
*text = canonical_json_string(&parsed);
|
||||
}
|
||||
replaced
|
||||
}
|
||||
Value::Array(items) => items
|
||||
.iter_mut()
|
||||
.map(|item| {
|
||||
strip_media_from_tool_value_at_depth(
|
||||
item,
|
||||
media_parts,
|
||||
scope,
|
||||
replacement_block,
|
||||
replacement_text,
|
||||
depth + 1,
|
||||
)
|
||||
})
|
||||
.sum(),
|
||||
Value::Object(_) => {
|
||||
if let Some(media_part) = chat_media_part_from_tool_part(value, scope) {
|
||||
media_parts.push(media_part);
|
||||
*value = replacement_block.clone();
|
||||
return 1;
|
||||
}
|
||||
|
||||
value
|
||||
.as_object_mut()
|
||||
.expect("object match arm must remain an object")
|
||||
.get_mut("content")
|
||||
.map(|content| {
|
||||
strip_media_from_tool_value_at_depth(
|
||||
content,
|
||||
media_parts,
|
||||
scope,
|
||||
replacement_block,
|
||||
replacement_text,
|
||||
depth + 1,
|
||||
)
|
||||
})
|
||||
.unwrap_or(0)
|
||||
}
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn tool_media_kind(part: &Value) -> Option<ToolMediaKind> {
|
||||
let object = part.as_object()?;
|
||||
let part_type = object.get("type").and_then(Value::as_str);
|
||||
|
||||
match part_type {
|
||||
Some("input_image" | "image_url") if normalized_image_url(part).is_some() => {
|
||||
Some(ToolMediaKind::Image)
|
||||
}
|
||||
Some("input_file") if part.get("file_id").is_some() || part.get("file_data").is_some() => {
|
||||
Some(ToolMediaKind::File)
|
||||
}
|
||||
Some("input_audio") if part.get("input_audio").is_some_and(Value::is_object) => {
|
||||
Some(ToolMediaKind::Audio)
|
||||
}
|
||||
Some("image") if typed_image_has_payload(part) => Some(ToolMediaKind::Image),
|
||||
None if loose_data_image_url(part).is_some() => Some(ToolMediaKind::Image),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn chat_image_part(part: &Value) -> Option<Value> {
|
||||
match part.get("type").and_then(Value::as_str) {
|
||||
Some("input_image" | "image_url") => {
|
||||
normalized_image_url(part).map(|image_url| image_url_content_part(part, image_url))
|
||||
}
|
||||
Some("image") => {
|
||||
typed_image_url(part).map(|image_url| image_url_content_part(part, image_url))
|
||||
}
|
||||
None => loose_data_image_url(part).map(|image_url| image_url_content_part(part, image_url)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn normalized_image_url(part: &Value) -> Option<Value> {
|
||||
let image_url = part.get("image_url")?;
|
||||
let mut object = match image_url {
|
||||
Value::String(url) if !url.trim().is_empty() => {
|
||||
let mut object = Map::new();
|
||||
object.insert("url".to_string(), Value::String(url.clone()));
|
||||
object
|
||||
}
|
||||
Value::Object(object)
|
||||
if object
|
||||
.get("url")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|url| !url.trim().is_empty()) =>
|
||||
{
|
||||
object.clone()
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
merge_top_level_detail(part, &mut object);
|
||||
Some(Value::Object(object))
|
||||
}
|
||||
|
||||
fn loose_data_image_url(part: &Value) -> Option<Value> {
|
||||
if part.get("type").is_some() {
|
||||
return None;
|
||||
}
|
||||
let normalized = normalized_image_url(part)?;
|
||||
let url = normalized.get("url").and_then(Value::as_str)?;
|
||||
if !url
|
||||
.get(..5)
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(normalized)
|
||||
}
|
||||
|
||||
fn typed_image_has_payload(part: &Value) -> bool {
|
||||
let Some(object) = part.as_object() else {
|
||||
return false;
|
||||
};
|
||||
|
||||
if let Some(source) = object.get("source").and_then(Value::as_object) {
|
||||
if source_media_type_is_image(source) {
|
||||
let has_url = source
|
||||
.get("url")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|url| !url.trim().is_empty());
|
||||
let has_data = source
|
||||
.get("data")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|data| !data.is_empty());
|
||||
if has_url || has_data {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object
|
||||
.get("data")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|data| !data.is_empty())
|
||||
&& object
|
||||
.get("mimeType")
|
||||
.or_else(|| object.get("mime_type"))
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(is_image_mime_type)
|
||||
}
|
||||
|
||||
fn typed_image_url(part: &Value) -> Option<Value> {
|
||||
let object = part.as_object()?;
|
||||
|
||||
if let Some(source) = object.get("source").and_then(Value::as_object) {
|
||||
if !source_media_type_is_image(source) {
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(url) = source
|
||||
.get("url")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|url| !url.trim().is_empty())
|
||||
{
|
||||
let mut image_url = Map::new();
|
||||
image_url.insert("url".to_string(), Value::String(url.to_string()));
|
||||
merge_top_level_detail(part, &mut image_url);
|
||||
return Some(Value::Object(image_url));
|
||||
}
|
||||
|
||||
if let Some(data) = source
|
||||
.get("data")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|data| !data.is_empty())
|
||||
{
|
||||
let media_type = source
|
||||
.get("media_type")
|
||||
.or_else(|| source.get("mime_type"))
|
||||
.or_else(|| source.get("mimeType"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("image/png");
|
||||
let url = if data
|
||||
.get(..11)
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:image/"))
|
||||
{
|
||||
data.to_string()
|
||||
} else {
|
||||
format!("data:{media_type};base64,{data}")
|
||||
};
|
||||
let mut image_url = Map::new();
|
||||
image_url.insert("url".to_string(), Value::String(url));
|
||||
merge_top_level_detail(part, &mut image_url);
|
||||
return Some(Value::Object(image_url));
|
||||
}
|
||||
}
|
||||
|
||||
let data = object
|
||||
.get("data")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|data| !data.is_empty())?;
|
||||
let media_type = object
|
||||
.get("mimeType")
|
||||
.or_else(|| object.get("mime_type"))
|
||||
.and_then(Value::as_str)
|
||||
.filter(|media_type| is_image_mime_type(media_type))?;
|
||||
let mut image_url = Map::new();
|
||||
image_url.insert(
|
||||
"url".to_string(),
|
||||
Value::String(format!("data:{media_type};base64,{data}")),
|
||||
);
|
||||
merge_top_level_detail(part, &mut image_url);
|
||||
Some(Value::Object(image_url))
|
||||
}
|
||||
|
||||
fn image_url_content_part(source_part: &Value, image_url: Value) -> Value {
|
||||
let mut content_part = Map::new();
|
||||
content_part.insert("type".to_string(), Value::String("image_url".to_string()));
|
||||
content_part.insert("image_url".to_string(), image_url);
|
||||
|
||||
for key in ["cache_control", "prompt_cache_breakpoint"] {
|
||||
if let Some(value) = source_part.get(key) {
|
||||
content_part.insert(key.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
Value::Object(content_part)
|
||||
}
|
||||
|
||||
fn merge_top_level_detail(part: &Value, image_url: &mut Map<String, Value>) {
|
||||
if image_url.get("detail").is_none() {
|
||||
if let Some(detail) = part.get("detail") {
|
||||
image_url.insert("detail".to_string(), detail.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn source_media_type_is_image(source: &Map<String, Value>) -> bool {
|
||||
source
|
||||
.get("media_type")
|
||||
.or_else(|| source.get("mime_type"))
|
||||
.or_else(|| source.get("mimeType"))
|
||||
.and_then(Value::as_str)
|
||||
.is_none_or(is_image_mime_type)
|
||||
}
|
||||
|
||||
fn is_image_mime_type(value: &str) -> bool {
|
||||
value
|
||||
.get(..6)
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/"))
|
||||
}
|
||||
|
||||
fn is_image_base64_data_url(value: &str) -> bool {
|
||||
let Some(comma_index) = value.find(',') else {
|
||||
return false;
|
||||
};
|
||||
let header = &value[..comma_index];
|
||||
let header = header.to_ascii_lowercase();
|
||||
header.starts_with("data:image/") && header.ends_with(";base64")
|
||||
}
|
||||
|
||||
fn looks_like_base64_payload(value: &str) -> bool {
|
||||
if value.len() < BASE64ISH_MIN_BYTES {
|
||||
return false;
|
||||
}
|
||||
|
||||
value
|
||||
.bytes()
|
||||
.all(|byte| matches!(byte, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'+' | b'/' | b'='))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use base64::{engine::general_purpose::STANDARD, Engine as _};
|
||||
|
||||
fn large_image_data_url() -> String {
|
||||
format!(
|
||||
"data:image/png;base64,{}",
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAE".repeat(400)
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_input_image_and_merges_top_level_detail() {
|
||||
let part = json!({
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
"detail": "high"
|
||||
});
|
||||
|
||||
let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap();
|
||||
|
||||
assert_eq!(mapped["type"], "image_url");
|
||||
assert_eq!(mapped["image_url"]["url"], "https://example.com/image.png");
|
||||
assert_eq!(mapped["image_url"]["detail"], "high");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_already_chat_shaped_image_url() {
|
||||
let part = json!({
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://example.com/image.png",
|
||||
"detail": "low"
|
||||
}
|
||||
});
|
||||
|
||||
let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap();
|
||||
|
||||
assert_eq!(mapped, part);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_anthropic_and_mcp_image_shapes() {
|
||||
let anthropic = json!({
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/jpeg",
|
||||
"data": "YWJj"
|
||||
}
|
||||
});
|
||||
let mcp = json!({
|
||||
"type": "image",
|
||||
"mimeType": "image/webp",
|
||||
"data": "ZGVm"
|
||||
});
|
||||
let anthropic_url = json!({
|
||||
"type": "image",
|
||||
"source": {
|
||||
"url": "https://example.com/anthropic.png"
|
||||
}
|
||||
});
|
||||
|
||||
let anthropic =
|
||||
chat_media_part_from_tool_part(&anthropic, ToolMediaScope::AllSupported).unwrap();
|
||||
let mcp = chat_media_part_from_tool_part(&mcp, ToolMediaScope::AllSupported).unwrap();
|
||||
let anthropic_url =
|
||||
chat_media_part_from_tool_part(&anthropic_url, ToolMediaScope::AllSupported).unwrap();
|
||||
|
||||
assert_eq!(anthropic["image_url"]["url"], "data:image/jpeg;base64,YWJj");
|
||||
assert_eq!(mcp["image_url"]["url"], "data:image/webp;base64,ZGVm");
|
||||
assert_eq!(
|
||||
anthropic_url["image_url"]["url"],
|
||||
"https://example.com/anthropic.png"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_anthropic_source_data_when_optional_url_is_empty() {
|
||||
let part = json!({
|
||||
"type": "image",
|
||||
"source": {
|
||||
"url": "",
|
||||
"media_type": "image/png",
|
||||
"data": "YWJj"
|
||||
}
|
||||
});
|
||||
|
||||
let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap();
|
||||
|
||||
assert_eq!(mapped["image_url"]["url"], "data:image/png;base64,YWJj");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_image_metadata_and_non_image_mcp_payloads() {
|
||||
let metadata = json!({"type": "image", "name": "cover"});
|
||||
let non_image = json!({
|
||||
"type": "image",
|
||||
"mimeType": "text/plain",
|
||||
"data": "aGVsbG8="
|
||||
});
|
||||
|
||||
assert!(chat_media_part_from_tool_part(&metadata, ToolMediaScope::AllSupported).is_none());
|
||||
assert!(chat_media_part_from_tool_part(&non_image, ToolMediaScope::AllSupported).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loose_data_image_url_is_media_but_loose_remote_url_is_not() {
|
||||
let data = json!({
|
||||
"image_url": {
|
||||
"url": "data:application/octet-stream;base64,YWJj"
|
||||
}
|
||||
});
|
||||
let remote = json!({
|
||||
"image_url": {
|
||||
"url": "https://example.com/search-thumbnail.png"
|
||||
}
|
||||
});
|
||||
|
||||
assert!(tool_output_contains_media(
|
||||
&data,
|
||||
ToolMediaScope::ImagesOnly
|
||||
));
|
||||
assert!(!tool_output_contains_media(
|
||||
&remote,
|
||||
ToolMediaScope::ImagesOnly
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_scan_embedded_data_urls_inside_plain_text() {
|
||||
let data_url = large_image_data_url();
|
||||
let mut value = json!(format!("<html><img src=\"{data_url}\"></html>"));
|
||||
let original = value.clone();
|
||||
let replacement = json!({"type": "text", "text": "moved"});
|
||||
let mut media = Vec::new();
|
||||
|
||||
assert!(!tool_output_contains_media(
|
||||
&value,
|
||||
ToolMediaScope::AllSupported
|
||||
));
|
||||
assert_eq!(
|
||||
strip_media_from_tool_value(
|
||||
&mut value,
|
||||
&mut media,
|
||||
ToolMediaScope::AllSupported,
|
||||
&replacement,
|
||||
"moved",
|
||||
),
|
||||
0
|
||||
);
|
||||
assert!(media.is_empty());
|
||||
assert_eq!(value, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whole_string_data_url_respects_threshold() {
|
||||
let large = large_image_data_url();
|
||||
let small = "data:image/png;base64,YWJj";
|
||||
|
||||
assert!(whole_string_image_data_url(&large).is_some());
|
||||
assert!(whole_string_image_data_url(small).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_media_from_json_string_and_nested_content() {
|
||||
let data_url = large_image_data_url();
|
||||
let mut value = Value::String(
|
||||
json!({
|
||||
"content": [
|
||||
{"type": "input_text", "text": "caption"},
|
||||
{"type": "input_image", "image_url": data_url}
|
||||
]
|
||||
})
|
||||
.to_string(),
|
||||
);
|
||||
let replacement = json!({
|
||||
"type": "text",
|
||||
"text": "moved"
|
||||
});
|
||||
let mut media = Vec::new();
|
||||
|
||||
let replaced = strip_media_from_tool_value(
|
||||
&mut value,
|
||||
&mut media,
|
||||
ToolMediaScope::AllSupported,
|
||||
&replacement,
|
||||
"moved",
|
||||
);
|
||||
|
||||
assert_eq!(replaced, 1);
|
||||
assert_eq!(media.len(), 1);
|
||||
let serialized = value.as_str().unwrap();
|
||||
assert!(serialized.contains("\"text\":\"moved\""));
|
||||
assert!(!serialized.contains("iVBORw0KGgo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_only_scope_ignores_file_and_audio() {
|
||||
let file = json!({"type": "input_file", "file_id": "file_1"});
|
||||
let audio = json!({
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "YWJj", "format": "wav"}
|
||||
});
|
||||
|
||||
assert!(!tool_output_contains_media(
|
||||
&file,
|
||||
ToolMediaScope::ImagesOnly
|
||||
));
|
||||
assert!(!tool_output_contains_media(
|
||||
&audio,
|
||||
ToolMediaScope::ImagesOnly
|
||||
));
|
||||
assert!(tool_output_contains_media(
|
||||
&file,
|
||||
ToolMediaScope::AllSupported
|
||||
));
|
||||
assert!(tool_output_contains_media(
|
||||
&audio,
|
||||
ToolMediaScope::AllSupported
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clamp_preserves_long_text_but_removes_data_and_base64_payloads() {
|
||||
let long_text = format!(
|
||||
"{} with spaces and punctuation!",
|
||||
"ordinary text ".repeat(9000)
|
||||
);
|
||||
let data_url = large_image_data_url();
|
||||
let bytes = (0_u8..=255).cycle().take(18_000).collect::<Vec<_>>();
|
||||
let base64 = STANDARD.encode(bytes);
|
||||
let mut value = json!({
|
||||
"text": long_text,
|
||||
"data_url": data_url,
|
||||
"raw": base64
|
||||
});
|
||||
|
||||
clamp_base64ish_strings(&mut value);
|
||||
|
||||
assert_eq!(value["text"], long_text);
|
||||
assert!(value["data_url"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.starts_with("[cc-switch: omitted "));
|
||||
assert!(value["raw"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.starts_with("[cc-switch: omitted "));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_media_strip_is_byte_stable() {
|
||||
let mut value = json!({
|
||||
"content": [
|
||||
{"type": "text", "text": "hello"},
|
||||
{"type": "image", "name": "business metadata"}
|
||||
]
|
||||
});
|
||||
let before = canonical_json_string(&value);
|
||||
let replacement = json!({"type": "text", "text": "moved"});
|
||||
let mut media = Vec::new();
|
||||
|
||||
let replaced = strip_media_from_tool_value(
|
||||
&mut value,
|
||||
&mut media,
|
||||
ToolMediaScope::AllSupported,
|
||||
&replacement,
|
||||
"moved",
|
||||
);
|
||||
|
||||
assert_eq!(replaced, 0);
|
||||
assert!(media.is_empty());
|
||||
assert_eq!(canonical_json_string(&value), before);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user