mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-08-02 18:41:35 +08:00
fix(proxy): extend image rectifier to Codex /responses text-only path
Codex /responses requests routed to text-only OpenAI-chat upstreams (e.g. DeepSeek deepseek-v4-flash) failed with HTTP 400 "unknown variant image_url" when images were sent: the responses->chat conversion turns input_image items into image_url blocks the model rejects. The media rectifier previously covered only the Claude adapter, so neither the proactive strip nor the reactive retry fired for Codex. - media_retry_should_trigger: accept "Codex" adapter, not just "Claude" - contains_image_blocks / replace_images: also scan responses `input` (input_image) in addition to chat `messages` - is_image_block_type: match image | image_url | input_image - is_unsupported_image_error: add "unknown variant" hint for the deserialize error - forward(): proactively run apply_media_prevention for Codex after the responses->chat conversion Proactively strips images for known text-only models (heuristic on by default) and reactively retries with images replaced on upstream image-unsupported errors. Adds tests for chat image_url, codex input_image, the reactive trigger, and the deserialize error match.
This commit is contained in:
@@ -159,7 +159,7 @@ impl RequestForwarder {
|
|||||||
provider_body: &Value,
|
provider_body: &Value,
|
||||||
error: &ProxyError,
|
error: &ProxyError,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
adapter_name == "Claude"
|
matches!(adapter_name, "Claude" | "Codex")
|
||||||
&& self.rectifier_config.enabled
|
&& self.rectifier_config.enabled
|
||||||
&& self.rectifier_config.request_media_fallback
|
&& self.rectifier_config.request_media_fallback
|
||||||
&& !already_retried
|
&& !already_retried
|
||||||
@@ -1321,7 +1321,7 @@ impl RequestForwarder {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// 转换请求体(如果需要)
|
// 转换请求体(如果需要)
|
||||||
let request_body = if codex_responses_to_chat {
|
let mut request_body = if codex_responses_to_chat {
|
||||||
let mut mapped_body = mapped_body;
|
let mut mapped_body = mapped_body;
|
||||||
let restored = self
|
let restored = self
|
||||||
.codex_chat_history
|
.codex_chat_history
|
||||||
@@ -1359,6 +1359,10 @@ impl RequestForwarder {
|
|||||||
mapped_body
|
mapped_body
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if matches!(app_type, AppType::Codex) {
|
||||||
|
self.apply_media_prevention(&mut request_body, provider);
|
||||||
|
}
|
||||||
|
|
||||||
// 过滤私有参数(以 `_` 开头的字段),防止内部信息泄露到上游
|
// 过滤私有参数(以 `_` 开头的字段),防止内部信息泄露到上游
|
||||||
// 默认使用空白名单,过滤所有 _ 前缀字段
|
// 默认使用空白名单,过滤所有 _ 前缀字段
|
||||||
let filtered_body = prepare_upstream_request_body(request_body);
|
let filtered_body = prepare_upstream_request_body(request_body);
|
||||||
@@ -3298,6 +3302,18 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn body_with_codex_input_image(model: &str) -> Value {
|
||||||
|
json!({
|
||||||
|
"model": model,
|
||||||
|
"input": [{
|
||||||
|
"role": "user",
|
||||||
|
"content": [
|
||||||
|
{ "type": "input_image", "image_url": "data:image/png;base64,abc" }
|
||||||
|
]
|
||||||
|
}]
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn image_unsupported_error() -> ProxyError {
|
fn image_unsupported_error() -> ProxyError {
|
||||||
ProxyError::UpstreamError {
|
ProxyError::UpstreamError {
|
||||||
status: 400,
|
status: 400,
|
||||||
@@ -3385,6 +3401,21 @@ mod tests {
|
|||||||
assert!(fwd.media_retry_should_trigger("Claude", false, &body, &image_unsupported_error()));
|
assert!(fwd.media_retry_should_trigger("Claude", false, &body, &image_unsupported_error()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reactive_triggers_for_codex_image_url_deserialize_errors() {
|
||||||
|
let fwd = forwarder_with_rectifier(RectifierConfig::default());
|
||||||
|
let body = body_with_codex_input_image("deepseek-v4-flash");
|
||||||
|
let error = ProxyError::UpstreamError {
|
||||||
|
status: 400,
|
||||||
|
body: Some(
|
||||||
|
r#"{"error":{"message":"Failed to deserialize the JSON body into the target type: messages[11]: unknown variant image_url, expected text"}}"#
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(fwd.media_retry_should_trigger("Codex", false, &body, &error));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reactive_skipped_when_media_fallback_off() {
|
fn reactive_skipped_when_media_fallback_off() {
|
||||||
// 关闭 request_media_fallback:上游报图片错误也不触发兜底重试。
|
// 关闭 request_media_fallback:上游报图片错误也不触发兜底重试。
|
||||||
|
|||||||
@@ -41,14 +41,7 @@ pub fn replace_images_for_text_only_model(
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn contains_image_blocks(body: &Value) -> bool {
|
pub fn contains_image_blocks(body: &Value) -> bool {
|
||||||
body.get("messages")
|
messages_have_image_blocks(body) || responses_input_has_image_blocks(body.get("input"))
|
||||||
.and_then(Value::as_array)
|
|
||||||
.is_some_and(|messages| {
|
|
||||||
messages
|
|
||||||
.iter()
|
|
||||||
.filter_map(|message| message.get("content"))
|
|
||||||
.any(content_has_image_blocks)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn replace_image_blocks_with_marker(body: &mut Value) -> usize {
|
pub fn replace_image_blocks_with_marker(body: &mut Value) -> usize {
|
||||||
@@ -95,6 +88,7 @@ pub fn is_unsupported_image_error(error: &ProxyError) -> bool {
|
|||||||
"text-only",
|
"text-only",
|
||||||
"invalid content type",
|
"invalid content type",
|
||||||
"invalid message content",
|
"invalid message content",
|
||||||
|
"unknown variant",
|
||||||
"unknown content type",
|
"unknown content type",
|
||||||
"unrecognized content type",
|
"unrecognized content type",
|
||||||
"cannot process",
|
"cannot process",
|
||||||
@@ -113,51 +107,124 @@ fn content_has_image_blocks(content: &Value) -> bool {
|
|||||||
};
|
};
|
||||||
|
|
||||||
blocks.iter().any(|block| {
|
blocks.iter().any(|block| {
|
||||||
block.get("type").and_then(Value::as_str) == Some("image")
|
is_image_block_type(block.get("type").and_then(Value::as_str))
|
||||||
|| block.get("content").is_some_and(content_has_image_blocks)
|
|| block.get("content").is_some_and(content_has_image_blocks)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn replace_images_in_body(body: &mut Value) -> usize {
|
fn replace_images_in_body(body: &mut Value) -> usize {
|
||||||
let Some(messages) = body.get_mut("messages").and_then(Value::as_array_mut) else {
|
let message_replacements = body
|
||||||
return 0;
|
.get_mut("messages")
|
||||||
};
|
.and_then(Value::as_array_mut)
|
||||||
|
.map(|messages| {
|
||||||
|
messages
|
||||||
|
.iter_mut()
|
||||||
|
.filter_map(|message| message.get_mut("content"))
|
||||||
|
.map(replace_images_in_content)
|
||||||
|
.sum()
|
||||||
|
})
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
messages
|
message_replacements
|
||||||
.iter_mut()
|
+ body
|
||||||
.filter_map(|message| message.get_mut("content"))
|
.get_mut("input")
|
||||||
.map(replace_images_in_content)
|
.map(replace_images_in_responses_input)
|
||||||
.sum()
|
.unwrap_or(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn replace_images_in_content(content: &mut Value) -> usize {
|
fn replace_images_in_content(content: &mut Value) -> usize {
|
||||||
|
replace_images_in_content_with_text_type(content, "text")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn replace_images_in_content_with_text_type(content: &mut Value, text_type: &str) -> usize {
|
||||||
let Some(blocks) = content.as_array_mut() else {
|
let Some(blocks) = content.as_array_mut() else {
|
||||||
return 0;
|
return 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut replaced = 0usize;
|
let mut replaced = 0usize;
|
||||||
for block in blocks {
|
for block in blocks {
|
||||||
if block.get("type").and_then(Value::as_str) == Some("image") {
|
if is_image_block_type(block.get("type").and_then(Value::as_str)) {
|
||||||
let cache_control = block.get("cache_control").cloned();
|
replace_image_block_with_text_marker(block, text_type);
|
||||||
*block = json!({
|
|
||||||
"type": "text",
|
|
||||||
"text": UNSUPPORTED_IMAGE_MARKER
|
|
||||||
});
|
|
||||||
if let (Some(cache_control), Some(object)) = (cache_control, block.as_object_mut()) {
|
|
||||||
object.insert("cache_control".to_string(), cache_control);
|
|
||||||
}
|
|
||||||
replaced += 1;
|
replaced += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(nested_content) = block.get_mut("content") {
|
if let Some(nested_content) = block.get_mut("content") {
|
||||||
replaced += replace_images_in_content(nested_content);
|
replaced += replace_images_in_content_with_text_type(nested_content, text_type);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
replaced
|
replaced
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn messages_have_image_blocks(body: &Value) -> bool {
|
||||||
|
body.get("messages")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.is_some_and(|messages| {
|
||||||
|
messages
|
||||||
|
.iter()
|
||||||
|
.filter_map(|message| message.get("content"))
|
||||||
|
.any(content_has_image_blocks)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn responses_input_has_image_blocks(input: Option<&Value>) -> bool {
|
||||||
|
match input {
|
||||||
|
Some(Value::Array(items)) => items.iter().any(responses_input_item_has_image_blocks),
|
||||||
|
Some(item @ Value::Object(_)) => responses_input_item_has_image_blocks(item),
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn responses_input_item_has_image_blocks(item: &Value) -> bool {
|
||||||
|
if item.get("type").and_then(Value::as_str) == Some("input_image") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
item.get("content").is_some_and(content_has_image_blocks)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn replace_images_in_responses_input(input: &mut Value) -> usize {
|
||||||
|
match input {
|
||||||
|
Value::Array(items) => items
|
||||||
|
.iter_mut()
|
||||||
|
.map(replace_images_in_responses_input_item)
|
||||||
|
.sum(),
|
||||||
|
Value::Object(_) => replace_images_in_responses_input_item(input),
|
||||||
|
_ => 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn replace_images_in_responses_input_item(item: &mut Value) -> usize {
|
||||||
|
let mut replaced = 0usize;
|
||||||
|
|
||||||
|
if item.get("type").and_then(Value::as_str) == Some("input_image") {
|
||||||
|
replace_image_block_with_text_marker(item, "input_text");
|
||||||
|
replaced += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(content) = item.get_mut("content") {
|
||||||
|
replaced += replace_images_in_content_with_text_type(content, "input_text");
|
||||||
|
}
|
||||||
|
|
||||||
|
replaced
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_image_block_type(block_type: Option<&str>) -> bool {
|
||||||
|
matches!(block_type, Some("image" | "image_url" | "input_image"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn replace_image_block_with_text_marker(block: &mut Value, text_type: &str) {
|
||||||
|
let cache_control = block.get("cache_control").cloned();
|
||||||
|
*block = json!({
|
||||||
|
"type": text_type,
|
||||||
|
"text": UNSUPPORTED_IMAGE_MARKER
|
||||||
|
});
|
||||||
|
if let (Some(cache_control), Some(object)) = (cache_control, block.as_object_mut()) {
|
||||||
|
object.insert("cache_control".to_string(), cache_control);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn explicit_model_image_support(provider: &Provider, model: &str) -> Option<bool> {
|
fn explicit_model_image_support(provider: &Provider, model: &str) -> Option<bool> {
|
||||||
let settings = &provider.settings_config;
|
let settings = &provider.settings_config;
|
||||||
[
|
[
|
||||||
@@ -369,6 +436,54 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn known_text_only_models_replace_chat_image_url_before_send() {
|
||||||
|
let provider = provider(json!({}));
|
||||||
|
let mut body = json!({
|
||||||
|
"model": "deepseek-v4-flash",
|
||||||
|
"messages": [{
|
||||||
|
"role": "user",
|
||||||
|
"content": [
|
||||||
|
{ "type": "text", "text": "look" },
|
||||||
|
{ "type": "image_url", "image_url": { "url": "data:image/png;base64,abc" } }
|
||||||
|
]
|
||||||
|
}]
|
||||||
|
});
|
||||||
|
|
||||||
|
let count = replace_images_for_text_only_model(&mut body, &provider, true);
|
||||||
|
|
||||||
|
assert_eq!(count, 1);
|
||||||
|
assert_eq!(body["messages"][0]["content"][1]["type"], "text");
|
||||||
|
assert_eq!(
|
||||||
|
body["messages"][0]["content"][1]["text"],
|
||||||
|
UNSUPPORTED_IMAGE_MARKER
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn known_text_only_models_replace_codex_input_image_before_send() {
|
||||||
|
let provider = provider(json!({}));
|
||||||
|
let mut body = json!({
|
||||||
|
"model": "deepseek-v4-flash",
|
||||||
|
"input": [{
|
||||||
|
"role": "user",
|
||||||
|
"content": [
|
||||||
|
{ "type": "input_text", "text": "look" },
|
||||||
|
{ "type": "input_image", "image_url": "data:image/png;base64,abc" }
|
||||||
|
]
|
||||||
|
}]
|
||||||
|
});
|
||||||
|
|
||||||
|
let count = replace_images_for_text_only_model(&mut body, &provider, true);
|
||||||
|
|
||||||
|
assert_eq!(count, 1);
|
||||||
|
assert_eq!(body["input"][0]["content"][1]["type"], "input_text");
|
||||||
|
assert_eq!(
|
||||||
|
body["input"][0]["content"][1]["text"],
|
||||||
|
UNSUPPORTED_IMAGE_MARKER
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn explicit_text_modalities_replace_images_before_send() {
|
fn explicit_text_modalities_replace_images_before_send() {
|
||||||
let provider = provider(json!({
|
let provider = provider(json!({
|
||||||
@@ -654,6 +769,19 @@ mod tests {
|
|||||||
assert!(is_unsupported_image_error(&attachment_error));
|
assert!(is_unsupported_image_error(&attachment_error));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detects_chat_content_unknown_variant_image_url_errors() {
|
||||||
|
let error = ProxyError::UpstreamError {
|
||||||
|
status: 400,
|
||||||
|
body: Some(
|
||||||
|
r#"{"error":{"message":"Failed to deserialize the JSON body into the target type: messages[11]: unknown variant image_url, expected text"}}"#
|
||||||
|
.to_string(),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(is_unsupported_image_error(&error));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn heuristic_disabled_keeps_images_for_listed_text_only_models() {
|
fn heuristic_disabled_keeps_images_for_listed_text_only_models() {
|
||||||
// allow_heuristic = false:内置列表不再预测性剥图,避免误判多模态模型时静默丢图。
|
// allow_heuristic = false:内置列表不再预测性剥图,避免误判多模态模型时静默丢图。
|
||||||
|
|||||||
Reference in New Issue
Block a user