mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-07-24 12:44:18 +08:00
fix(proxy): extend image rectifier to Codex /responses text-only path
Codex /responses requests routed to text-only OpenAI-chat upstreams (e.g. DeepSeek deepseek-v4-flash) failed with HTTP 400 "unknown variant image_url" when images were sent: the responses->chat conversion turns input_image items into image_url blocks the model rejects. The media rectifier previously covered only the Claude adapter, so neither the proactive strip nor the reactive retry fired for Codex. - media_retry_should_trigger: accept "Codex" adapter, not just "Claude" - contains_image_blocks / replace_images: also scan responses `input` (input_image) in addition to chat `messages` - is_image_block_type: match image | image_url | input_image - is_unsupported_image_error: add "unknown variant" hint for the deserialize error - forward(): proactively run apply_media_prevention for Codex after the responses->chat conversion Proactively strips images for known text-only models (heuristic on by default) and reactively retries with images replaced on upstream image-unsupported errors. Adds tests for chat image_url, codex input_image, the reactive trigger, and the deserialize error match.
This commit is contained in:
@@ -159,7 +159,7 @@ impl RequestForwarder {
|
||||
provider_body: &Value,
|
||||
error: &ProxyError,
|
||||
) -> bool {
|
||||
adapter_name == "Claude"
|
||||
matches!(adapter_name, "Claude" | "Codex")
|
||||
&& self.rectifier_config.enabled
|
||||
&& self.rectifier_config.request_media_fallback
|
||||
&& !already_retried
|
||||
@@ -1321,7 +1321,7 @@ impl RequestForwarder {
|
||||
};
|
||||
|
||||
// 转换请求体(如果需要)
|
||||
let request_body = if codex_responses_to_chat {
|
||||
let mut request_body = if codex_responses_to_chat {
|
||||
let mut mapped_body = mapped_body;
|
||||
let restored = self
|
||||
.codex_chat_history
|
||||
@@ -1359,6 +1359,10 @@ impl RequestForwarder {
|
||||
mapped_body
|
||||
};
|
||||
|
||||
if matches!(app_type, AppType::Codex) {
|
||||
self.apply_media_prevention(&mut request_body, provider);
|
||||
}
|
||||
|
||||
// 过滤私有参数(以 `_` 开头的字段),防止内部信息泄露到上游
|
||||
// 默认使用空白名单,过滤所有 _ 前缀字段
|
||||
let filtered_body = prepare_upstream_request_body(request_body);
|
||||
@@ -3298,6 +3302,18 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
fn body_with_codex_input_image(model: &str) -> Value {
|
||||
json!({
|
||||
"model": model,
|
||||
"input": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{ "type": "input_image", "image_url": "data:image/png;base64,abc" }
|
||||
]
|
||||
}]
|
||||
})
|
||||
}
|
||||
|
||||
fn image_unsupported_error() -> ProxyError {
|
||||
ProxyError::UpstreamError {
|
||||
status: 400,
|
||||
@@ -3385,6 +3401,21 @@ mod tests {
|
||||
assert!(fwd.media_retry_should_trigger("Claude", false, &body, &image_unsupported_error()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reactive_triggers_for_codex_image_url_deserialize_errors() {
|
||||
let fwd = forwarder_with_rectifier(RectifierConfig::default());
|
||||
let body = body_with_codex_input_image("deepseek-v4-flash");
|
||||
let error = ProxyError::UpstreamError {
|
||||
status: 400,
|
||||
body: Some(
|
||||
r#"{"error":{"message":"Failed to deserialize the JSON body into the target type: messages[11]: unknown variant image_url, expected text"}}"#
|
||||
.to_string(),
|
||||
),
|
||||
};
|
||||
|
||||
assert!(fwd.media_retry_should_trigger("Codex", false, &body, &error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reactive_skipped_when_media_fallback_off() {
|
||||
// 关闭 request_media_fallback:上游报图片错误也不触发兜底重试。
|
||||
|
||||
@@ -41,14 +41,7 @@ pub fn replace_images_for_text_only_model(
|
||||
}
|
||||
|
||||
pub fn contains_image_blocks(body: &Value) -> bool {
|
||||
body.get("messages")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|messages| {
|
||||
messages
|
||||
.iter()
|
||||
.filter_map(|message| message.get("content"))
|
||||
.any(content_has_image_blocks)
|
||||
})
|
||||
messages_have_image_blocks(body) || responses_input_has_image_blocks(body.get("input"))
|
||||
}
|
||||
|
||||
pub fn replace_image_blocks_with_marker(body: &mut Value) -> usize {
|
||||
@@ -95,6 +88,7 @@ pub fn is_unsupported_image_error(error: &ProxyError) -> bool {
|
||||
"text-only",
|
||||
"invalid content type",
|
||||
"invalid message content",
|
||||
"unknown variant",
|
||||
"unknown content type",
|
||||
"unrecognized content type",
|
||||
"cannot process",
|
||||
@@ -113,51 +107,124 @@ fn content_has_image_blocks(content: &Value) -> bool {
|
||||
};
|
||||
|
||||
blocks.iter().any(|block| {
|
||||
block.get("type").and_then(Value::as_str) == Some("image")
|
||||
is_image_block_type(block.get("type").and_then(Value::as_str))
|
||||
|| block.get("content").is_some_and(content_has_image_blocks)
|
||||
})
|
||||
}
|
||||
|
||||
fn replace_images_in_body(body: &mut Value) -> usize {
|
||||
let Some(messages) = body.get_mut("messages").and_then(Value::as_array_mut) else {
|
||||
return 0;
|
||||
};
|
||||
let message_replacements = body
|
||||
.get_mut("messages")
|
||||
.and_then(Value::as_array_mut)
|
||||
.map(|messages| {
|
||||
messages
|
||||
.iter_mut()
|
||||
.filter_map(|message| message.get_mut("content"))
|
||||
.map(replace_images_in_content)
|
||||
.sum()
|
||||
})
|
||||
.unwrap_or(0);
|
||||
|
||||
messages
|
||||
.iter_mut()
|
||||
.filter_map(|message| message.get_mut("content"))
|
||||
.map(replace_images_in_content)
|
||||
.sum()
|
||||
message_replacements
|
||||
+ body
|
||||
.get_mut("input")
|
||||
.map(replace_images_in_responses_input)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn replace_images_in_content(content: &mut Value) -> usize {
|
||||
replace_images_in_content_with_text_type(content, "text")
|
||||
}
|
||||
|
||||
fn replace_images_in_content_with_text_type(content: &mut Value, text_type: &str) -> usize {
|
||||
let Some(blocks) = content.as_array_mut() else {
|
||||
return 0;
|
||||
};
|
||||
|
||||
let mut replaced = 0usize;
|
||||
for block in blocks {
|
||||
if block.get("type").and_then(Value::as_str) == Some("image") {
|
||||
let cache_control = block.get("cache_control").cloned();
|
||||
*block = json!({
|
||||
"type": "text",
|
||||
"text": UNSUPPORTED_IMAGE_MARKER
|
||||
});
|
||||
if let (Some(cache_control), Some(object)) = (cache_control, block.as_object_mut()) {
|
||||
object.insert("cache_control".to_string(), cache_control);
|
||||
}
|
||||
if is_image_block_type(block.get("type").and_then(Value::as_str)) {
|
||||
replace_image_block_with_text_marker(block, text_type);
|
||||
replaced += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(nested_content) = block.get_mut("content") {
|
||||
replaced += replace_images_in_content(nested_content);
|
||||
replaced += replace_images_in_content_with_text_type(nested_content, text_type);
|
||||
}
|
||||
}
|
||||
|
||||
replaced
|
||||
}
|
||||
|
||||
fn messages_have_image_blocks(body: &Value) -> bool {
|
||||
body.get("messages")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|messages| {
|
||||
messages
|
||||
.iter()
|
||||
.filter_map(|message| message.get("content"))
|
||||
.any(content_has_image_blocks)
|
||||
})
|
||||
}
|
||||
|
||||
fn responses_input_has_image_blocks(input: Option<&Value>) -> bool {
|
||||
match input {
|
||||
Some(Value::Array(items)) => items.iter().any(responses_input_item_has_image_blocks),
|
||||
Some(item @ Value::Object(_)) => responses_input_item_has_image_blocks(item),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn responses_input_item_has_image_blocks(item: &Value) -> bool {
|
||||
if item.get("type").and_then(Value::as_str) == Some("input_image") {
|
||||
return true;
|
||||
}
|
||||
|
||||
item.get("content").is_some_and(content_has_image_blocks)
|
||||
}
|
||||
|
||||
fn replace_images_in_responses_input(input: &mut Value) -> usize {
|
||||
match input {
|
||||
Value::Array(items) => items
|
||||
.iter_mut()
|
||||
.map(replace_images_in_responses_input_item)
|
||||
.sum(),
|
||||
Value::Object(_) => replace_images_in_responses_input_item(input),
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn replace_images_in_responses_input_item(item: &mut Value) -> usize {
|
||||
let mut replaced = 0usize;
|
||||
|
||||
if item.get("type").and_then(Value::as_str) == Some("input_image") {
|
||||
replace_image_block_with_text_marker(item, "input_text");
|
||||
replaced += 1;
|
||||
}
|
||||
|
||||
if let Some(content) = item.get_mut("content") {
|
||||
replaced += replace_images_in_content_with_text_type(content, "input_text");
|
||||
}
|
||||
|
||||
replaced
|
||||
}
|
||||
|
||||
fn is_image_block_type(block_type: Option<&str>) -> bool {
|
||||
matches!(block_type, Some("image" | "image_url" | "input_image"))
|
||||
}
|
||||
|
||||
fn replace_image_block_with_text_marker(block: &mut Value, text_type: &str) {
|
||||
let cache_control = block.get("cache_control").cloned();
|
||||
*block = json!({
|
||||
"type": text_type,
|
||||
"text": UNSUPPORTED_IMAGE_MARKER
|
||||
});
|
||||
if let (Some(cache_control), Some(object)) = (cache_control, block.as_object_mut()) {
|
||||
object.insert("cache_control".to_string(), cache_control);
|
||||
}
|
||||
}
|
||||
|
||||
fn explicit_model_image_support(provider: &Provider, model: &str) -> Option<bool> {
|
||||
let settings = &provider.settings_config;
|
||||
[
|
||||
@@ -369,6 +436,54 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn known_text_only_models_replace_chat_image_url_before_send() {
|
||||
let provider = provider(json!({}));
|
||||
let mut body = json!({
|
||||
"model": "deepseek-v4-flash",
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{ "type": "text", "text": "look" },
|
||||
{ "type": "image_url", "image_url": { "url": "data:image/png;base64,abc" } }
|
||||
]
|
||||
}]
|
||||
});
|
||||
|
||||
let count = replace_images_for_text_only_model(&mut body, &provider, true);
|
||||
|
||||
assert_eq!(count, 1);
|
||||
assert_eq!(body["messages"][0]["content"][1]["type"], "text");
|
||||
assert_eq!(
|
||||
body["messages"][0]["content"][1]["text"],
|
||||
UNSUPPORTED_IMAGE_MARKER
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn known_text_only_models_replace_codex_input_image_before_send() {
|
||||
let provider = provider(json!({}));
|
||||
let mut body = json!({
|
||||
"model": "deepseek-v4-flash",
|
||||
"input": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{ "type": "input_text", "text": "look" },
|
||||
{ "type": "input_image", "image_url": "data:image/png;base64,abc" }
|
||||
]
|
||||
}]
|
||||
});
|
||||
|
||||
let count = replace_images_for_text_only_model(&mut body, &provider, true);
|
||||
|
||||
assert_eq!(count, 1);
|
||||
assert_eq!(body["input"][0]["content"][1]["type"], "input_text");
|
||||
assert_eq!(
|
||||
body["input"][0]["content"][1]["text"],
|
||||
UNSUPPORTED_IMAGE_MARKER
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_text_modalities_replace_images_before_send() {
|
||||
let provider = provider(json!({
|
||||
@@ -654,6 +769,19 @@ mod tests {
|
||||
assert!(is_unsupported_image_error(&attachment_error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_chat_content_unknown_variant_image_url_errors() {
|
||||
let error = ProxyError::UpstreamError {
|
||||
status: 400,
|
||||
body: Some(
|
||||
r#"{"error":{"message":"Failed to deserialize the JSON body into the target type: messages[11]: unknown variant image_url, expected text"}}"#
|
||||
.to_string(),
|
||||
),
|
||||
};
|
||||
|
||||
assert!(is_unsupported_image_error(&error));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heuristic_disabled_keeps_images_for_listed_text_only_models() {
|
||||
// allow_heuristic = false:内置列表不再预测性剥图,避免误判多模态模型时静默丢图。
|
||||
|
||||
Reference in New Issue
Block a user