fix(proxy): extend image rectifier to Codex /responses text-only path

Codex /responses requests routed to text-only OpenAI-chat upstreams
(e.g. DeepSeek deepseek-v4-flash) failed with HTTP 400 "unknown variant
image_url" when images were sent: the responses->chat conversion turns
input_image items into image_url blocks the model rejects. The media
rectifier previously covered only the Claude adapter, so neither the
proactive strip nor the reactive retry fired for Codex.

- media_retry_should_trigger: accept "Codex" adapter, not just "Claude"
- contains_image_blocks / replace_images: also scan responses `input`
  (input_image) in addition to chat `messages`
- is_image_block_type: match image | image_url | input_image
- is_unsupported_image_error: add "unknown variant" hint for the
  deserialize error
- forward(): proactively run apply_media_prevention for Codex after the
  responses->chat conversion

Proactively strips images for known text-only models (heuristic on by
default) and reactively retries with images replaced on upstream
image-unsupported errors. Adds tests for chat image_url, codex
input_image, the reactive trigger, and the deserialize error match.
This commit is contained in:
Jason
2026-06-09 22:11:39 +08:00
parent cb01593f7d
commit 3390fe7ea0
2 changed files with 188 additions and 29 deletions
+33 -2
View File
@@ -159,7 +159,7 @@ impl RequestForwarder {
provider_body: &Value,
error: &ProxyError,
) -> bool {
adapter_name == "Claude"
matches!(adapter_name, "Claude" | "Codex")
&& self.rectifier_config.enabled
&& self.rectifier_config.request_media_fallback
&& !already_retried
@@ -1321,7 +1321,7 @@ impl RequestForwarder {
};
// 转换请求体(如果需要)
let request_body = if codex_responses_to_chat {
let mut request_body = if codex_responses_to_chat {
let mut mapped_body = mapped_body;
let restored = self
.codex_chat_history
@@ -1359,6 +1359,10 @@ impl RequestForwarder {
mapped_body
};
if matches!(app_type, AppType::Codex) {
self.apply_media_prevention(&mut request_body, provider);
}
// 过滤私有参数(以 `_` 开头的字段),防止内部信息泄露到上游
// 默认使用空白名单,过滤所有 _ 前缀字段
let filtered_body = prepare_upstream_request_body(request_body);
@@ -3298,6 +3302,18 @@ mod tests {
})
}
fn body_with_codex_input_image(model: &str) -> Value {
json!({
"model": model,
"input": [{
"role": "user",
"content": [
{ "type": "input_image", "image_url": "data:image/png;base64,abc" }
]
}]
})
}
fn image_unsupported_error() -> ProxyError {
ProxyError::UpstreamError {
status: 400,
@@ -3385,6 +3401,21 @@ mod tests {
assert!(fwd.media_retry_should_trigger("Claude", false, &body, &image_unsupported_error()));
}
#[test]
fn reactive_triggers_for_codex_image_url_deserialize_errors() {
let fwd = forwarder_with_rectifier(RectifierConfig::default());
let body = body_with_codex_input_image("deepseek-v4-flash");
let error = ProxyError::UpstreamError {
status: 400,
body: Some(
r#"{"error":{"message":"Failed to deserialize the JSON body into the target type: messages[11]: unknown variant image_url, expected text"}}"#
.to_string(),
),
};
assert!(fwd.media_retry_should_trigger("Codex", false, &body, &error));
}
#[test]
fn reactive_skipped_when_media_fallback_off() {
// 关闭 request_media_fallback:上游报图片错误也不触发兜底重试。
+155 -27
View File
@@ -41,14 +41,7 @@ pub fn replace_images_for_text_only_model(
}
pub fn contains_image_blocks(body: &Value) -> bool {
body.get("messages")
.and_then(Value::as_array)
.is_some_and(|messages| {
messages
.iter()
.filter_map(|message| message.get("content"))
.any(content_has_image_blocks)
})
messages_have_image_blocks(body) || responses_input_has_image_blocks(body.get("input"))
}
pub fn replace_image_blocks_with_marker(body: &mut Value) -> usize {
@@ -95,6 +88,7 @@ pub fn is_unsupported_image_error(error: &ProxyError) -> bool {
"text-only",
"invalid content type",
"invalid message content",
"unknown variant",
"unknown content type",
"unrecognized content type",
"cannot process",
@@ -113,51 +107,124 @@ fn content_has_image_blocks(content: &Value) -> bool {
};
blocks.iter().any(|block| {
block.get("type").and_then(Value::as_str) == Some("image")
is_image_block_type(block.get("type").and_then(Value::as_str))
|| block.get("content").is_some_and(content_has_image_blocks)
})
}
fn replace_images_in_body(body: &mut Value) -> usize {
let Some(messages) = body.get_mut("messages").and_then(Value::as_array_mut) else {
return 0;
};
let message_replacements = body
.get_mut("messages")
.and_then(Value::as_array_mut)
.map(|messages| {
messages
.iter_mut()
.filter_map(|message| message.get_mut("content"))
.map(replace_images_in_content)
.sum()
})
.unwrap_or(0);
messages
.iter_mut()
.filter_map(|message| message.get_mut("content"))
.map(replace_images_in_content)
.sum()
message_replacements
+ body
.get_mut("input")
.map(replace_images_in_responses_input)
.unwrap_or(0)
}
fn replace_images_in_content(content: &mut Value) -> usize {
replace_images_in_content_with_text_type(content, "text")
}
fn replace_images_in_content_with_text_type(content: &mut Value, text_type: &str) -> usize {
let Some(blocks) = content.as_array_mut() else {
return 0;
};
let mut replaced = 0usize;
for block in blocks {
if block.get("type").and_then(Value::as_str) == Some("image") {
let cache_control = block.get("cache_control").cloned();
*block = json!({
"type": "text",
"text": UNSUPPORTED_IMAGE_MARKER
});
if let (Some(cache_control), Some(object)) = (cache_control, block.as_object_mut()) {
object.insert("cache_control".to_string(), cache_control);
}
if is_image_block_type(block.get("type").and_then(Value::as_str)) {
replace_image_block_with_text_marker(block, text_type);
replaced += 1;
continue;
}
if let Some(nested_content) = block.get_mut("content") {
replaced += replace_images_in_content(nested_content);
replaced += replace_images_in_content_with_text_type(nested_content, text_type);
}
}
replaced
}
fn messages_have_image_blocks(body: &Value) -> bool {
body.get("messages")
.and_then(Value::as_array)
.is_some_and(|messages| {
messages
.iter()
.filter_map(|message| message.get("content"))
.any(content_has_image_blocks)
})
}
fn responses_input_has_image_blocks(input: Option<&Value>) -> bool {
match input {
Some(Value::Array(items)) => items.iter().any(responses_input_item_has_image_blocks),
Some(item @ Value::Object(_)) => responses_input_item_has_image_blocks(item),
_ => false,
}
}
fn responses_input_item_has_image_blocks(item: &Value) -> bool {
if item.get("type").and_then(Value::as_str) == Some("input_image") {
return true;
}
item.get("content").is_some_and(content_has_image_blocks)
}
fn replace_images_in_responses_input(input: &mut Value) -> usize {
match input {
Value::Array(items) => items
.iter_mut()
.map(replace_images_in_responses_input_item)
.sum(),
Value::Object(_) => replace_images_in_responses_input_item(input),
_ => 0,
}
}
fn replace_images_in_responses_input_item(item: &mut Value) -> usize {
let mut replaced = 0usize;
if item.get("type").and_then(Value::as_str) == Some("input_image") {
replace_image_block_with_text_marker(item, "input_text");
replaced += 1;
}
if let Some(content) = item.get_mut("content") {
replaced += replace_images_in_content_with_text_type(content, "input_text");
}
replaced
}
fn is_image_block_type(block_type: Option<&str>) -> bool {
matches!(block_type, Some("image" | "image_url" | "input_image"))
}
fn replace_image_block_with_text_marker(block: &mut Value, text_type: &str) {
let cache_control = block.get("cache_control").cloned();
*block = json!({
"type": text_type,
"text": UNSUPPORTED_IMAGE_MARKER
});
if let (Some(cache_control), Some(object)) = (cache_control, block.as_object_mut()) {
object.insert("cache_control".to_string(), cache_control);
}
}
fn explicit_model_image_support(provider: &Provider, model: &str) -> Option<bool> {
let settings = &provider.settings_config;
[
@@ -369,6 +436,54 @@ mod tests {
);
}
#[test]
fn known_text_only_models_replace_chat_image_url_before_send() {
let provider = provider(json!({}));
let mut body = json!({
"model": "deepseek-v4-flash",
"messages": [{
"role": "user",
"content": [
{ "type": "text", "text": "look" },
{ "type": "image_url", "image_url": { "url": "data:image/png;base64,abc" } }
]
}]
});
let count = replace_images_for_text_only_model(&mut body, &provider, true);
assert_eq!(count, 1);
assert_eq!(body["messages"][0]["content"][1]["type"], "text");
assert_eq!(
body["messages"][0]["content"][1]["text"],
UNSUPPORTED_IMAGE_MARKER
);
}
#[test]
fn known_text_only_models_replace_codex_input_image_before_send() {
let provider = provider(json!({}));
let mut body = json!({
"model": "deepseek-v4-flash",
"input": [{
"role": "user",
"content": [
{ "type": "input_text", "text": "look" },
{ "type": "input_image", "image_url": "data:image/png;base64,abc" }
]
}]
});
let count = replace_images_for_text_only_model(&mut body, &provider, true);
assert_eq!(count, 1);
assert_eq!(body["input"][0]["content"][1]["type"], "input_text");
assert_eq!(
body["input"][0]["content"][1]["text"],
UNSUPPORTED_IMAGE_MARKER
);
}
#[test]
fn explicit_text_modalities_replace_images_before_send() {
let provider = provider(json!({
@@ -654,6 +769,19 @@ mod tests {
assert!(is_unsupported_image_error(&attachment_error));
}
#[test]
fn detects_chat_content_unknown_variant_image_url_errors() {
let error = ProxyError::UpstreamError {
status: 400,
body: Some(
r#"{"error":{"message":"Failed to deserialize the JSON body into the target type: messages[11]: unknown variant image_url, expected text"}}"#
.to_string(),
),
};
assert!(is_unsupported_image_error(&error));
}
#[test]
fn heuristic_disabled_keeps_images_for_listed_text_only_models() {
// allow_heuristic = false:内置列表不再预测性剥图,避免误判多模态模型时静默丢图。