mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-08-02 18:41:35 +08:00
fix(proxy): move Codex tool-result media out of stringified tool text
The Responses->Chat conversion serialized image-bearing *_output items into role:"tool" text via canonical_json_string, so view_image results were tokenized as base64 text (~9000x inflation). Codex replays full history every turn, so sessions hit context-limit 400s and wedged (#4465, #5663). - add proxy/tool_media: shared detection/strip/clamp walker for tool output media (typed input_image / image_url / input_file / input_audio, Anthropic source and MCP data+mimeType image shapes, untyped data: image_url, whole-string bare data URLs) - transform_codex_chat: replace media blocks in place with marker text so tool content stays a plain string, and flush the extracted media as one synthetic role:"user" message after each consecutive tool batch; media-free traffic stays byte-identical to keep prompt-cache prefixes stable - media_sanitizer: detect and strip tool-output media symmetrically (including JSON-string outputs) so reactive image stripping can heal upstream modality rejections - forwarder: regression tests pinning the reactive trigger and the context-limit-400 non-trigger E2E against Kimi K3 through the proxy: the replayed turn stays ~12k input tokens with 99% cache hit, versus ~85k+ of base64 text per replay before.
This commit is contained in:
Jason
@@ -0,0 +1,774 @@
|
||||
//! Shared media handling for Codex tool outputs.
|
||||
//!
|
||||
//! Responses tool outputs may carry structured media blocks. Chat Completions
|
||||
//! tool messages are text-only, so the Responses→Chat bridge extracts those
|
||||
//! blocks and re-emits them in a synthetic user message. The media sanitizer
|
||||
//! reuses the same recognition and traversal rules when it needs to remove
|
||||
//! images for a text-only upstream.
|
||||
|
||||
use crate::proxy::json_canonical::canonical_json_string;
|
||||
use serde_json::{json, Map, Value};
|
||||
|
||||
pub(crate) const WHOLE_DATA_URL_MIN_BYTES: usize = 8 * 1024;
|
||||
const BASE64ISH_MIN_BYTES: usize = 16 * 1024;
|
||||
const MAX_MEDIA_TRAVERSAL_DEPTH: usize = 32;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum ToolMediaScope {
|
||||
/// Used by the existing image-capability sanitizer and its retry path.
|
||||
ImagesOnly,
|
||||
/// Used by Responses→Chat conversion, where user messages can carry all
|
||||
/// currently mapped Chat input modalities.
|
||||
AllSupported,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ToolMediaKind {
|
||||
Image,
|
||||
File,
|
||||
Audio,
|
||||
}
|
||||
|
||||
impl ToolMediaScope {
|
||||
fn allows(self, kind: ToolMediaKind) -> bool {
|
||||
matches!(kind, ToolMediaKind::Image) || matches!(self, Self::AllSupported)
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert one recognized Responses/tool media block to a Chat user content
|
||||
/// part. This is the single shape-recognition entry point used by extraction.
|
||||
pub(crate) fn chat_media_part_from_tool_part(part: &Value, scope: ToolMediaScope) -> Option<Value> {
|
||||
let kind = tool_media_kind(part)?;
|
||||
if !scope.allows(kind) {
|
||||
return None;
|
||||
}
|
||||
|
||||
match kind {
|
||||
ToolMediaKind::Image => chat_image_part(part),
|
||||
ToolMediaKind::File => chat_file_from_input_file(part).map(|file| {
|
||||
json!({
|
||||
"type": "file",
|
||||
"file": file
|
||||
})
|
||||
}),
|
||||
ToolMediaKind::Audio => part.get("input_audio").map(|input_audio| {
|
||||
json!({
|
||||
"type": "input_audio",
|
||||
"input_audio": input_audio.clone()
|
||||
})
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Map a Responses `input_file` block to the Chat file payload. Kept here so
|
||||
/// top-level content and tool-output extraction share the exact same rules.
|
||||
pub(crate) fn chat_file_from_input_file(part: &Value) -> Option<Value> {
|
||||
let mut file = Map::new();
|
||||
let has_supported_file_ref = part.get("file_id").is_some() || part.get("file_data").is_some();
|
||||
if !has_supported_file_ref {
|
||||
return None;
|
||||
}
|
||||
|
||||
for key in ["file_id", "file_data", "filename"] {
|
||||
if let Some(value) = part.get(key) {
|
||||
file.insert(key.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
Some(Value::Object(file))
|
||||
}
|
||||
|
||||
/// Recognize a complete image data URL stored as a scalar string.
|
||||
///
|
||||
/// Only whole-string matches are accepted. Embedded data URLs in HTML/CSS/SVG
|
||||
/// source are deliberately left alone. Small values remain text as well, which
|
||||
/// preserves workflows that intentionally inspect tiny inline icons.
|
||||
pub(crate) fn whole_string_image_data_url(value: &str) -> Option<Value> {
|
||||
let trimmed = value.trim();
|
||||
if trimmed.len() < WHOLE_DATA_URL_MIN_BYTES || !is_image_base64_data_url(trimmed) {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(json!({
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": trimmed
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
/// Read-only media detection using the same shape classifier and recursive
|
||||
/// boundaries as [`strip_media_from_tool_value`].
|
||||
pub(crate) fn tool_output_contains_media(value: &Value, scope: ToolMediaScope) -> bool {
|
||||
tool_output_contains_media_at_depth(value, scope, 0)
|
||||
}
|
||||
|
||||
/// Extract recognized media blocks and replace them in-place.
|
||||
///
|
||||
/// `replacement_block` is used for structured array/object parts. A scalar
|
||||
/// string that is itself a complete image data URL uses `replacement_text`, so
|
||||
/// an originally plain string stays a plain string. Parseable JSON strings are
|
||||
/// recursively transformed and canonicalized back into a string only when a
|
||||
/// replacement actually occurred.
|
||||
pub(crate) fn strip_media_from_tool_value(
|
||||
value: &mut Value,
|
||||
media_parts: &mut Vec<Value>,
|
||||
scope: ToolMediaScope,
|
||||
replacement_block: &Value,
|
||||
replacement_text: &str,
|
||||
) -> usize {
|
||||
strip_media_from_tool_value_at_depth(
|
||||
value,
|
||||
media_parts,
|
||||
scope,
|
||||
replacement_block,
|
||||
replacement_text,
|
||||
0,
|
||||
)
|
||||
}
|
||||
|
||||
/// Remove residual data/base64 payloads only after a tool output has already
|
||||
/// been positively identified as media-bearing. Ordinary long text is kept.
|
||||
pub(crate) fn clamp_base64ish_strings(value: &mut Value) {
|
||||
match value {
|
||||
Value::String(text) => {
|
||||
let trimmed = text.trim();
|
||||
let should_omit = (trimmed.len() >= WHOLE_DATA_URL_MIN_BYTES
|
||||
&& trimmed
|
||||
.get(..5)
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:")))
|
||||
|| looks_like_base64_payload(trimmed);
|
||||
if should_omit {
|
||||
let byte_len = text.len();
|
||||
*text = format!("[cc-switch: omitted {byte_len} bytes]");
|
||||
}
|
||||
}
|
||||
Value::Array(items) => {
|
||||
for item in items {
|
||||
clamp_base64ish_strings(item);
|
||||
}
|
||||
}
|
||||
Value::Object(object) => {
|
||||
for nested in object.values_mut() {
|
||||
clamp_base64ish_strings(nested);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn tool_output_contains_media_at_depth(value: &Value, scope: ToolMediaScope, depth: usize) -> bool {
|
||||
if depth > MAX_MEDIA_TRAVERSAL_DEPTH {
|
||||
return false;
|
||||
}
|
||||
|
||||
match value {
|
||||
Value::String(text) => {
|
||||
if scope.allows(ToolMediaKind::Image) && whole_string_image_data_url(text).is_some() {
|
||||
return true;
|
||||
}
|
||||
|
||||
let trimmed = text.trim();
|
||||
if trimmed.is_empty() {
|
||||
return false;
|
||||
}
|
||||
serde_json::from_str::<Value>(trimmed)
|
||||
.ok()
|
||||
.is_some_and(|parsed| {
|
||||
tool_output_contains_media_at_depth(&parsed, scope, depth + 1)
|
||||
})
|
||||
}
|
||||
Value::Array(items) => items
|
||||
.iter()
|
||||
.any(|item| tool_output_contains_media_at_depth(item, scope, depth + 1)),
|
||||
Value::Object(object) => {
|
||||
if tool_media_kind(value).is_some_and(|kind| scope.allows(kind)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
object.get("content").is_some_and(|content| {
|
||||
tool_output_contains_media_at_depth(content, scope, depth + 1)
|
||||
})
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn strip_media_from_tool_value_at_depth(
|
||||
value: &mut Value,
|
||||
media_parts: &mut Vec<Value>,
|
||||
scope: ToolMediaScope,
|
||||
replacement_block: &Value,
|
||||
replacement_text: &str,
|
||||
depth: usize,
|
||||
) -> usize {
|
||||
if depth > MAX_MEDIA_TRAVERSAL_DEPTH {
|
||||
return 0;
|
||||
}
|
||||
|
||||
match value {
|
||||
Value::String(text) => {
|
||||
if scope.allows(ToolMediaKind::Image) {
|
||||
if let Some(media_part) = whole_string_image_data_url(text) {
|
||||
media_parts.push(media_part);
|
||||
*text = replacement_text.to_string();
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
let trimmed = text.trim();
|
||||
if trimmed.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
let Ok(mut parsed) = serde_json::from_str::<Value>(trimmed) else {
|
||||
return 0;
|
||||
};
|
||||
let replaced = strip_media_from_tool_value_at_depth(
|
||||
&mut parsed,
|
||||
media_parts,
|
||||
scope,
|
||||
replacement_block,
|
||||
replacement_text,
|
||||
depth + 1,
|
||||
);
|
||||
if replaced > 0 {
|
||||
*text = canonical_json_string(&parsed);
|
||||
}
|
||||
replaced
|
||||
}
|
||||
Value::Array(items) => items
|
||||
.iter_mut()
|
||||
.map(|item| {
|
||||
strip_media_from_tool_value_at_depth(
|
||||
item,
|
||||
media_parts,
|
||||
scope,
|
||||
replacement_block,
|
||||
replacement_text,
|
||||
depth + 1,
|
||||
)
|
||||
})
|
||||
.sum(),
|
||||
Value::Object(_) => {
|
||||
if let Some(media_part) = chat_media_part_from_tool_part(value, scope) {
|
||||
media_parts.push(media_part);
|
||||
*value = replacement_block.clone();
|
||||
return 1;
|
||||
}
|
||||
|
||||
value
|
||||
.as_object_mut()
|
||||
.expect("object match arm must remain an object")
|
||||
.get_mut("content")
|
||||
.map(|content| {
|
||||
strip_media_from_tool_value_at_depth(
|
||||
content,
|
||||
media_parts,
|
||||
scope,
|
||||
replacement_block,
|
||||
replacement_text,
|
||||
depth + 1,
|
||||
)
|
||||
})
|
||||
.unwrap_or(0)
|
||||
}
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn tool_media_kind(part: &Value) -> Option<ToolMediaKind> {
|
||||
let object = part.as_object()?;
|
||||
let part_type = object.get("type").and_then(Value::as_str);
|
||||
|
||||
match part_type {
|
||||
Some("input_image" | "image_url") if normalized_image_url(part).is_some() => {
|
||||
Some(ToolMediaKind::Image)
|
||||
}
|
||||
Some("input_file") if part.get("file_id").is_some() || part.get("file_data").is_some() => {
|
||||
Some(ToolMediaKind::File)
|
||||
}
|
||||
Some("input_audio") if part.get("input_audio").is_some_and(Value::is_object) => {
|
||||
Some(ToolMediaKind::Audio)
|
||||
}
|
||||
Some("image") if typed_image_has_payload(part) => Some(ToolMediaKind::Image),
|
||||
None if loose_data_image_url(part).is_some() => Some(ToolMediaKind::Image),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn chat_image_part(part: &Value) -> Option<Value> {
|
||||
match part.get("type").and_then(Value::as_str) {
|
||||
Some("input_image" | "image_url") => {
|
||||
normalized_image_url(part).map(|image_url| image_url_content_part(part, image_url))
|
||||
}
|
||||
Some("image") => {
|
||||
typed_image_url(part).map(|image_url| image_url_content_part(part, image_url))
|
||||
}
|
||||
None => loose_data_image_url(part).map(|image_url| image_url_content_part(part, image_url)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn normalized_image_url(part: &Value) -> Option<Value> {
|
||||
let image_url = part.get("image_url")?;
|
||||
let mut object = match image_url {
|
||||
Value::String(url) if !url.trim().is_empty() => {
|
||||
let mut object = Map::new();
|
||||
object.insert("url".to_string(), Value::String(url.clone()));
|
||||
object
|
||||
}
|
||||
Value::Object(object)
|
||||
if object
|
||||
.get("url")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|url| !url.trim().is_empty()) =>
|
||||
{
|
||||
object.clone()
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
merge_top_level_detail(part, &mut object);
|
||||
Some(Value::Object(object))
|
||||
}
|
||||
|
||||
fn loose_data_image_url(part: &Value) -> Option<Value> {
|
||||
if part.get("type").is_some() {
|
||||
return None;
|
||||
}
|
||||
let normalized = normalized_image_url(part)?;
|
||||
let url = normalized.get("url").and_then(Value::as_str)?;
|
||||
if !url
|
||||
.get(..5)
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:"))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(normalized)
|
||||
}
|
||||
|
||||
fn typed_image_has_payload(part: &Value) -> bool {
|
||||
let Some(object) = part.as_object() else {
|
||||
return false;
|
||||
};
|
||||
|
||||
if let Some(source) = object.get("source").and_then(Value::as_object) {
|
||||
if source_media_type_is_image(source) {
|
||||
let has_url = source
|
||||
.get("url")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|url| !url.trim().is_empty());
|
||||
let has_data = source
|
||||
.get("data")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|data| !data.is_empty());
|
||||
if has_url || has_data {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
object
|
||||
.get("data")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|data| !data.is_empty())
|
||||
&& object
|
||||
.get("mimeType")
|
||||
.or_else(|| object.get("mime_type"))
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(is_image_mime_type)
|
||||
}
|
||||
|
||||
fn typed_image_url(part: &Value) -> Option<Value> {
|
||||
let object = part.as_object()?;
|
||||
|
||||
if let Some(source) = object.get("source").and_then(Value::as_object) {
|
||||
if !source_media_type_is_image(source) {
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(url) = source
|
||||
.get("url")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|url| !url.trim().is_empty())
|
||||
{
|
||||
let mut image_url = Map::new();
|
||||
image_url.insert("url".to_string(), Value::String(url.to_string()));
|
||||
merge_top_level_detail(part, &mut image_url);
|
||||
return Some(Value::Object(image_url));
|
||||
}
|
||||
|
||||
if let Some(data) = source
|
||||
.get("data")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|data| !data.is_empty())
|
||||
{
|
||||
let media_type = source
|
||||
.get("media_type")
|
||||
.or_else(|| source.get("mime_type"))
|
||||
.or_else(|| source.get("mimeType"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("image/png");
|
||||
let url = if data
|
||||
.get(..11)
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("data:image/"))
|
||||
{
|
||||
data.to_string()
|
||||
} else {
|
||||
format!("data:{media_type};base64,{data}")
|
||||
};
|
||||
let mut image_url = Map::new();
|
||||
image_url.insert("url".to_string(), Value::String(url));
|
||||
merge_top_level_detail(part, &mut image_url);
|
||||
return Some(Value::Object(image_url));
|
||||
}
|
||||
}
|
||||
|
||||
let data = object
|
||||
.get("data")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|data| !data.is_empty())?;
|
||||
let media_type = object
|
||||
.get("mimeType")
|
||||
.or_else(|| object.get("mime_type"))
|
||||
.and_then(Value::as_str)
|
||||
.filter(|media_type| is_image_mime_type(media_type))?;
|
||||
let mut image_url = Map::new();
|
||||
image_url.insert(
|
||||
"url".to_string(),
|
||||
Value::String(format!("data:{media_type};base64,{data}")),
|
||||
);
|
||||
merge_top_level_detail(part, &mut image_url);
|
||||
Some(Value::Object(image_url))
|
||||
}
|
||||
|
||||
fn image_url_content_part(source_part: &Value, image_url: Value) -> Value {
|
||||
let mut content_part = Map::new();
|
||||
content_part.insert("type".to_string(), Value::String("image_url".to_string()));
|
||||
content_part.insert("image_url".to_string(), image_url);
|
||||
|
||||
for key in ["cache_control", "prompt_cache_breakpoint"] {
|
||||
if let Some(value) = source_part.get(key) {
|
||||
content_part.insert(key.to_string(), value.clone());
|
||||
}
|
||||
}
|
||||
Value::Object(content_part)
|
||||
}
|
||||
|
||||
fn merge_top_level_detail(part: &Value, image_url: &mut Map<String, Value>) {
|
||||
if image_url.get("detail").is_none() {
|
||||
if let Some(detail) = part.get("detail") {
|
||||
image_url.insert("detail".to_string(), detail.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn source_media_type_is_image(source: &Map<String, Value>) -> bool {
|
||||
source
|
||||
.get("media_type")
|
||||
.or_else(|| source.get("mime_type"))
|
||||
.or_else(|| source.get("mimeType"))
|
||||
.and_then(Value::as_str)
|
||||
.is_none_or(is_image_mime_type)
|
||||
}
|
||||
|
||||
fn is_image_mime_type(value: &str) -> bool {
|
||||
value
|
||||
.get(..6)
|
||||
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("image/"))
|
||||
}
|
||||
|
||||
fn is_image_base64_data_url(value: &str) -> bool {
|
||||
let Some(comma_index) = value.find(',') else {
|
||||
return false;
|
||||
};
|
||||
let header = &value[..comma_index];
|
||||
let header = header.to_ascii_lowercase();
|
||||
header.starts_with("data:image/") && header.ends_with(";base64")
|
||||
}
|
||||
|
||||
fn looks_like_base64_payload(value: &str) -> bool {
|
||||
if value.len() < BASE64ISH_MIN_BYTES {
|
||||
return false;
|
||||
}
|
||||
|
||||
value
|
||||
.bytes()
|
||||
.all(|byte| matches!(byte, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'+' | b'/' | b'='))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use base64::{engine::general_purpose::STANDARD, Engine as _};
|
||||
|
||||
fn large_image_data_url() -> String {
|
||||
format!(
|
||||
"data:image/png;base64,{}",
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAE".repeat(400)
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_input_image_and_merges_top_level_detail() {
|
||||
let part = json!({
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
"detail": "high"
|
||||
});
|
||||
|
||||
let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap();
|
||||
|
||||
assert_eq!(mapped["type"], "image_url");
|
||||
assert_eq!(mapped["image_url"]["url"], "https://example.com/image.png");
|
||||
assert_eq!(mapped["image_url"]["detail"], "high");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_already_chat_shaped_image_url() {
|
||||
let part = json!({
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://example.com/image.png",
|
||||
"detail": "low"
|
||||
}
|
||||
});
|
||||
|
||||
let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap();
|
||||
|
||||
assert_eq!(mapped, part);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_anthropic_and_mcp_image_shapes() {
|
||||
let anthropic = json!({
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/jpeg",
|
||||
"data": "YWJj"
|
||||
}
|
||||
});
|
||||
let mcp = json!({
|
||||
"type": "image",
|
||||
"mimeType": "image/webp",
|
||||
"data": "ZGVm"
|
||||
});
|
||||
let anthropic_url = json!({
|
||||
"type": "image",
|
||||
"source": {
|
||||
"url": "https://example.com/anthropic.png"
|
||||
}
|
||||
});
|
||||
|
||||
let anthropic =
|
||||
chat_media_part_from_tool_part(&anthropic, ToolMediaScope::AllSupported).unwrap();
|
||||
let mcp = chat_media_part_from_tool_part(&mcp, ToolMediaScope::AllSupported).unwrap();
|
||||
let anthropic_url =
|
||||
chat_media_part_from_tool_part(&anthropic_url, ToolMediaScope::AllSupported).unwrap();
|
||||
|
||||
assert_eq!(anthropic["image_url"]["url"], "data:image/jpeg;base64,YWJj");
|
||||
assert_eq!(mcp["image_url"]["url"], "data:image/webp;base64,ZGVm");
|
||||
assert_eq!(
|
||||
anthropic_url["image_url"]["url"],
|
||||
"https://example.com/anthropic.png"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maps_anthropic_source_data_when_optional_url_is_empty() {
|
||||
let part = json!({
|
||||
"type": "image",
|
||||
"source": {
|
||||
"url": "",
|
||||
"media_type": "image/png",
|
||||
"data": "YWJj"
|
||||
}
|
||||
});
|
||||
|
||||
let mapped = chat_media_part_from_tool_part(&part, ToolMediaScope::AllSupported).unwrap();
|
||||
|
||||
assert_eq!(mapped["image_url"]["url"], "data:image/png;base64,YWJj");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_image_metadata_and_non_image_mcp_payloads() {
|
||||
let metadata = json!({"type": "image", "name": "cover"});
|
||||
let non_image = json!({
|
||||
"type": "image",
|
||||
"mimeType": "text/plain",
|
||||
"data": "aGVsbG8="
|
||||
});
|
||||
|
||||
assert!(chat_media_part_from_tool_part(&metadata, ToolMediaScope::AllSupported).is_none());
|
||||
assert!(chat_media_part_from_tool_part(&non_image, ToolMediaScope::AllSupported).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loose_data_image_url_is_media_but_loose_remote_url_is_not() {
|
||||
let data = json!({
|
||||
"image_url": {
|
||||
"url": "data:application/octet-stream;base64,YWJj"
|
||||
}
|
||||
});
|
||||
let remote = json!({
|
||||
"image_url": {
|
||||
"url": "https://example.com/search-thumbnail.png"
|
||||
}
|
||||
});
|
||||
|
||||
assert!(tool_output_contains_media(
|
||||
&data,
|
||||
ToolMediaScope::ImagesOnly
|
||||
));
|
||||
assert!(!tool_output_contains_media(
|
||||
&remote,
|
||||
ToolMediaScope::ImagesOnly
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_scan_embedded_data_urls_inside_plain_text() {
|
||||
let data_url = large_image_data_url();
|
||||
let mut value = json!(format!("<html><img src=\"{data_url}\"></html>"));
|
||||
let original = value.clone();
|
||||
let replacement = json!({"type": "text", "text": "moved"});
|
||||
let mut media = Vec::new();
|
||||
|
||||
assert!(!tool_output_contains_media(
|
||||
&value,
|
||||
ToolMediaScope::AllSupported
|
||||
));
|
||||
assert_eq!(
|
||||
strip_media_from_tool_value(
|
||||
&mut value,
|
||||
&mut media,
|
||||
ToolMediaScope::AllSupported,
|
||||
&replacement,
|
||||
"moved",
|
||||
),
|
||||
0
|
||||
);
|
||||
assert!(media.is_empty());
|
||||
assert_eq!(value, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whole_string_data_url_respects_threshold() {
|
||||
let large = large_image_data_url();
|
||||
let small = "data:image/png;base64,YWJj";
|
||||
|
||||
assert!(whole_string_image_data_url(&large).is_some());
|
||||
assert!(whole_string_image_data_url(small).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_media_from_json_string_and_nested_content() {
|
||||
let data_url = large_image_data_url();
|
||||
let mut value = Value::String(
|
||||
json!({
|
||||
"content": [
|
||||
{"type": "input_text", "text": "caption"},
|
||||
{"type": "input_image", "image_url": data_url}
|
||||
]
|
||||
})
|
||||
.to_string(),
|
||||
);
|
||||
let replacement = json!({
|
||||
"type": "text",
|
||||
"text": "moved"
|
||||
});
|
||||
let mut media = Vec::new();
|
||||
|
||||
let replaced = strip_media_from_tool_value(
|
||||
&mut value,
|
||||
&mut media,
|
||||
ToolMediaScope::AllSupported,
|
||||
&replacement,
|
||||
"moved",
|
||||
);
|
||||
|
||||
assert_eq!(replaced, 1);
|
||||
assert_eq!(media.len(), 1);
|
||||
let serialized = value.as_str().unwrap();
|
||||
assert!(serialized.contains("\"text\":\"moved\""));
|
||||
assert!(!serialized.contains("iVBORw0KGgo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_only_scope_ignores_file_and_audio() {
|
||||
let file = json!({"type": "input_file", "file_id": "file_1"});
|
||||
let audio = json!({
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "YWJj", "format": "wav"}
|
||||
});
|
||||
|
||||
assert!(!tool_output_contains_media(
|
||||
&file,
|
||||
ToolMediaScope::ImagesOnly
|
||||
));
|
||||
assert!(!tool_output_contains_media(
|
||||
&audio,
|
||||
ToolMediaScope::ImagesOnly
|
||||
));
|
||||
assert!(tool_output_contains_media(
|
||||
&file,
|
||||
ToolMediaScope::AllSupported
|
||||
));
|
||||
assert!(tool_output_contains_media(
|
||||
&audio,
|
||||
ToolMediaScope::AllSupported
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clamp_preserves_long_text_but_removes_data_and_base64_payloads() {
|
||||
let long_text = format!(
|
||||
"{} with spaces and punctuation!",
|
||||
"ordinary text ".repeat(9000)
|
||||
);
|
||||
let data_url = large_image_data_url();
|
||||
let bytes = (0_u8..=255).cycle().take(18_000).collect::<Vec<_>>();
|
||||
let base64 = STANDARD.encode(bytes);
|
||||
let mut value = json!({
|
||||
"text": long_text,
|
||||
"data_url": data_url,
|
||||
"raw": base64
|
||||
});
|
||||
|
||||
clamp_base64ish_strings(&mut value);
|
||||
|
||||
assert_eq!(value["text"], long_text);
|
||||
assert!(value["data_url"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.starts_with("[cc-switch: omitted "));
|
||||
assert!(value["raw"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.starts_with("[cc-switch: omitted "));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_media_strip_is_byte_stable() {
|
||||
let mut value = json!({
|
||||
"content": [
|
||||
{"type": "text", "text": "hello"},
|
||||
{"type": "image", "name": "business metadata"}
|
||||
]
|
||||
});
|
||||
let before = canonical_json_string(&value);
|
||||
let replacement = json!({"type": "text", "text": "moved"});
|
||||
let mut media = Vec::new();
|
||||
|
||||
let replaced = strip_media_from_tool_value(
|
||||
&mut value,
|
||||
&mut media,
|
||||
ToolMediaScope::AllSupported,
|
||||
&replacement,
|
||||
"moved",
|
||||
);
|
||||
|
||||
assert_eq!(replaced, 0);
|
||||
assert!(media.is_empty());
|
||||
assert_eq!(canonical_json_string(&value), before);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user