Add media fallback rectifier for text-only models

Replace unsupported Anthropic image blocks with an [Unsupported Image] marker when routed models are text-only or upstream rejects image input.

Add rectifier settings for media fallback and heuristic model detection, wire the controls into the settings UI, and cover the sanitizer and forwarder gates with regression tests.
This commit is contained in:
Jason
2026-06-02 23:35:30 +08:00
parent 33eafbad51
commit 6692343d1e
12 changed files with 1134 additions and 4 deletions
+320
View File
@@ -122,6 +122,51 @@ pub struct RequestForwarder {
}
impl RequestForwarder {
/// 预防式 media 降级:发送前对 text-only 模型把图片块替换为标记。
///
/// 受 `enabled && request_media_fallback` 管辖;其中"启发式模型名单预测"
/// 再受 `request_media_heuristic` 单独管辖(显式声明 text-only 始终生效)。
/// 返回被替换的图片块数量(0 = 未触发或开关关闭)。
fn apply_media_prevention(&self, body: &mut Value, provider: &Provider) -> usize {
if !(self.rectifier_config.enabled && self.rectifier_config.request_media_fallback) {
return 0;
}
let replaced_images = super::media_sanitizer::replace_images_for_text_only_model(
body,
provider,
self.rectifier_config.request_media_heuristic,
);
if replaced_images > 0 {
let model = body.get("model").and_then(Value::as_str).unwrap_or("");
log::info!(
"[Media] Replaced {replaced_images} image block(s) with {} for text-only provider={}, model={}",
super::media_sanitizer::UNSUPPORTED_IMAGE_MARKER,
provider.id,
model
);
}
replaced_images
}
/// 反应式 media 重试判定:上游因图片输入报错后,是否应替换图片块并对同一供应商重试一次。
///
/// 受 `enabled && request_media_fallback` 管辖;不涉及 `request_media_heuristic`——
/// 这里是上游"实测"错误后的纯恢复,不是预测,故启发式开关与它无关。
fn media_retry_should_trigger(
&self,
adapter_name: &str,
already_retried: bool,
provider_body: &Value,
error: &ProxyError,
) -> bool {
adapter_name == "Claude"
&& self.rectifier_config.enabled
&& self.rectifier_config.request_media_fallback
&& !already_retried
&& super::media_sanitizer::contains_image_blocks(provider_body)
&& super::media_sanitizer::is_unsupported_image_error(error)
}
#[allow(clippy::too_many_arguments)]
pub fn new(
router: Arc<ProviderRouter>,
@@ -346,6 +391,7 @@ impl RequestForwarder {
// —— 首家 provider 整流后被 5xx/timeout 击落时,下家仍能用整流后的请求体走整流流程
let mut rectifier_retried = false;
let mut budget_rectifier_retried = false;
let mut media_rectifier_retried = false;
// 上限检查:尊重用户在 AppProxyConfig.max_retries 上配置的「重试次数」。
// 放在熔断器 allow 检查之前,避免在已经超限时还占用 HalfOpen 探测名额。
@@ -477,6 +523,123 @@ impl RequestForwarder {
);
let mut signature_rectifier_non_retryable_client_error = false;
if self.media_retry_should_trigger(
adapter.name(),
media_rectifier_retried,
&provider_body,
&e,
) {
let mut media_body = provider_body.clone();
let replaced_images =
super::media_sanitizer::replace_image_blocks_with_marker(
&mut media_body,
);
if replaced_images > 0 {
let _ = std::mem::replace(&mut media_rectifier_retried, true);
let model = media_body
.get("model")
.and_then(Value::as_str)
.unwrap_or("");
log::info!(
"[{app_type_str}] [Media] Upstream rejected image input; retrying provider={} model={} with {replaced_images} image block(s) replaced by {}",
provider.id,
model,
super::media_sanitizer::UNSUPPORTED_IMAGE_MARKER
);
match self
.forward(
app_type,
&method,
provider,
endpoint,
&media_body,
&headers,
&extensions,
adapter.as_ref(),
)
.await
{
Ok((response, claude_api_format)) => {
log::info!(
"[{app_type_str}] [Media] Unsupported-image retry succeeded"
);
self.record_success_result(
&provider.id,
app_type_str,
used_half_open_permit,
)
.await;
{
let mut current_providers =
self.current_providers.write().await;
current_providers.insert(
app_type_str.to_string(),
(provider.id.clone(), provider.name.clone()),
);
}
{
let mut status = self.status.write().await;
status.success_requests += 1;
status.last_error = None;
let should_switch =
self.current_provider_id_at_start.as_str()
!= provider.id.as_str();
if should_switch {
status.failover_count += 1;
let fm = self.failover_manager.clone();
let ah = self.app_handle.clone();
let pid = provider.id.clone();
let pname = provider.name.clone();
let at = app_type_str.to_string();
tokio::spawn(async move {
let _ = fm
.try_switch(ah.as_ref(), &at, &pid, &pname)
.await;
});
}
if status.total_requests > 0 {
status.success_rate = (status.success_requests as f32
/ status.total_requests as f32)
* 100.0;
}
}
return Ok(ForwardResult {
response,
provider: provider.clone(),
claude_api_format,
connection_guard: None,
});
}
Err(retry_err) => {
log::warn!(
"[{app_type_str}] [Media] Unsupported-image retry still failed: {retry_err}"
);
if let Some(err) = self
.handle_rectifier_retry_failure(
retry_err,
provider,
app_type_str,
used_half_open_permit,
"media 降级",
&mut last_error,
&mut last_provider,
)
.await
{
return Err(err);
}
continue;
}
}
}
}
if is_anthropic_provider {
let error_message = extract_error_message(&e);
if should_rectify_thinking_signature(
@@ -1114,6 +1277,7 @@ impl RequestForwarder {
provider,
api_format,
);
self.apply_media_prevention(&mut mapped_body, provider);
}
}
let needs_transform = match resolved_claude_api_format.as_deref() {
@@ -3106,4 +3270,160 @@ mod tests {
assert_eq!(will_replace, should_replace, "{desc}");
}
}
// ===== P3: forwarder 层 media 开关回归测试 =====
// 验证 gate 在 forwarder 这一层的"接线",而非 media_sanitizer 纯函数本身。
fn forwarder_with_rectifier(config: RectifierConfig) -> RequestForwarder {
let mut fwd = test_forwarder(Duration::from_secs(1), Duration::from_secs(1));
fwd.rectifier_config = config;
fwd
}
fn provider_with_settings(settings_config: Value) -> Provider {
let mut p = test_provider_with_type(Some("anthropic"));
p.settings_config = settings_config;
p
}
fn body_with_image(model: &str) -> Value {
json!({
"model": model,
"messages": [{
"role": "user",
"content": [
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
})
}
fn image_unsupported_error() -> ProxyError {
ProxyError::UpstreamError {
status: 400,
body: Some(
r#"{"error":{"message":"This model does not support image input"}}"#.to_string(),
),
}
}
#[test]
fn prevention_replaces_when_all_switches_on_and_model_in_heuristic_list() {
let fwd = forwarder_with_rectifier(RectifierConfig::default());
let provider = provider_with_settings(json!({}));
let mut body = body_with_image("deepseek-v4-pro");
let replaced = fwd.apply_media_prevention(&mut body, &provider);
assert_eq!(replaced, 1, "默认全开 + 名单内模型应预替换");
assert_eq!(body["messages"][0]["content"][0]["type"], "text");
}
#[test]
fn prevention_skipped_when_media_fallback_off() {
// 关闭 request_media_fallback:即使名单命中也不预替换。
let fwd = forwarder_with_rectifier(RectifierConfig {
request_media_fallback: false,
..RectifierConfig::default()
});
let provider = provider_with_settings(json!({}));
let mut body = body_with_image("deepseek-v4-pro");
let replaced = fwd.apply_media_prevention(&mut body, &provider);
assert_eq!(replaced, 0);
assert_eq!(body["messages"][0]["content"][0]["type"], "image");
}
#[test]
fn prevention_skipped_when_master_switch_off() {
let fwd = forwarder_with_rectifier(RectifierConfig {
enabled: false,
..RectifierConfig::default()
});
let provider = provider_with_settings(json!({}));
let mut body = body_with_image("deepseek-v4-pro");
assert_eq!(fwd.apply_media_prevention(&mut body, &provider), 0);
assert_eq!(body["messages"][0]["content"][0]["type"], "image");
}
#[test]
fn prevention_heuristic_off_skips_list_but_keeps_explicit_text_only() {
// 关闭 request_media_heuristic:名单预测失效,但显式声明 text-only 仍预替换。
let fwd = forwarder_with_rectifier(RectifierConfig {
request_media_heuristic: false,
..RectifierConfig::default()
});
// (a) 名单内模型、无显式声明 → 不再预替换
let bare_provider = provider_with_settings(json!({}));
let mut list_body = body_with_image("deepseek-v4-pro");
assert_eq!(
fwd.apply_media_prevention(&mut list_body, &bare_provider),
0,
"heuristic 关闭后名单模型不应被预替换"
);
assert_eq!(list_body["messages"][0]["content"][0]["type"], "image");
// (b) 显式声明 text-only → 仍预替换(声明驱动,不受 heuristic 开关影响)
let declared_provider = provider_with_settings(json!({
"models": [ { "id": "some-text-model", "input": ["text"] } ]
}));
let mut declared_body = body_with_image("some-text-model");
assert_eq!(
fwd.apply_media_prevention(&mut declared_body, &declared_provider),
1,
"显式 text-only 即使关闭 heuristic 也应预替换"
);
assert_eq!(declared_body["messages"][0]["content"][0]["type"], "text");
}
#[test]
fn reactive_triggers_when_all_switches_on() {
let fwd = forwarder_with_rectifier(RectifierConfig::default());
let body = body_with_image("any-model");
assert!(fwd.media_retry_should_trigger("Claude", false, &body, &image_unsupported_error()));
}
#[test]
fn reactive_skipped_when_media_fallback_off() {
// 关闭 request_media_fallback:上游报图片错误也不触发兜底重试。
let fwd = forwarder_with_rectifier(RectifierConfig {
request_media_fallback: false,
..RectifierConfig::default()
});
let body = body_with_image("any-model");
assert!(!fwd.media_retry_should_trigger(
"Claude",
false,
&body,
&image_unsupported_error()
));
}
#[test]
fn reactive_skipped_when_master_switch_off() {
let fwd = forwarder_with_rectifier(RectifierConfig {
enabled: false,
..RectifierConfig::default()
});
let body = body_with_image("any-model");
assert!(!fwd.media_retry_should_trigger(
"Claude",
false,
&body,
&image_unsupported_error()
));
}
#[test]
fn reactive_unaffected_by_heuristic_switch() {
// 关闭 request_media_heuristic 不影响反应式兜底——它是上游实测错误后的恢复,不是预测。
let fwd = forwarder_with_rectifier(RectifierConfig {
request_media_heuristic: false,
..RectifierConfig::default()
});
let body = body_with_image("any-model");
assert!(fwd.media_retry_should_trigger("Claude", false, &body, &image_unsupported_error()));
}
}
+703
View File
@@ -0,0 +1,703 @@
use crate::provider::Provider;
use crate::proxy::error::ProxyError;
use serde_json::{json, Value};
pub const UNSUPPORTED_IMAGE_MARKER: &str = "[Unsupported Image]";
/// Replace image blocks before sending when the routed model is text-only.
///
/// Two paths, both reached only when the caller's media-fallback switch is on:
/// - explicit capability from the provider config (modelCatalog / modalities) is
/// always trusted — it is declaration-driven, never a guess;
/// - the curated `known_text_only_model` list is a heuristic *prediction* and only
/// runs when `allow_heuristic` is true, so a mislabeled multimodal model cannot
/// have its images silently stripped when the user opts out.
pub fn replace_images_for_text_only_model(
body: &mut Value,
provider: &Provider,
allow_heuristic: bool,
) -> usize {
if !contains_image_blocks(body) {
return 0;
}
let model = body
.get("model")
.and_then(Value::as_str)
.map(str::trim)
.unwrap_or("");
match explicit_model_image_support(provider, model) {
Some(true) => return 0,
Some(false) => return replace_images_in_body(body),
None => {}
}
if !allow_heuristic || !known_text_only_model(model) {
return 0;
}
replace_images_in_body(body)
}
pub fn contains_image_blocks(body: &Value) -> bool {
body.get("messages")
.and_then(Value::as_array)
.is_some_and(|messages| {
messages
.iter()
.filter_map(|message| message.get("content"))
.any(content_has_image_blocks)
})
}
pub fn replace_image_blocks_with_marker(body: &mut Value) -> usize {
replace_images_in_body(body)
}
pub fn is_unsupported_image_error(error: &ProxyError) -> bool {
let ProxyError::UpstreamError { status, body } = error else {
return false;
};
if !matches!(*status, 400 | 415 | 422 | 501) {
return false;
}
let Some(body) = body.as_deref() else {
return false;
};
let message = extract_error_text(body);
let message = message.to_ascii_lowercase();
let mentions_image = message.contains("image")
|| message.contains("vision")
|| message.contains("multimodal")
|| message.contains("multi-modal")
|| message.contains("modality")
|| message.contains("modalities")
|| message.contains("media")
|| message.contains("attachment");
if !mentions_image {
return false;
}
const UNSUPPORTED_HINTS: &[&str] = &[
"unsupported",
"not supported",
"does not support",
"doesn't support",
"do not support",
"don't support",
"only supports text",
"text only",
"text-only",
"invalid content type",
"invalid message content",
"unknown content type",
"unrecognized content type",
"cannot process",
"cannot handle",
"can't process",
"can't handle",
"unable to process",
];
UNSUPPORTED_HINTS.iter().any(|hint| message.contains(hint))
}
fn content_has_image_blocks(content: &Value) -> bool {
let Some(blocks) = content.as_array() else {
return false;
};
blocks.iter().any(|block| {
block.get("type").and_then(Value::as_str) == Some("image")
|| block.get("content").is_some_and(content_has_image_blocks)
})
}
fn replace_images_in_body(body: &mut Value) -> usize {
let Some(messages) = body.get_mut("messages").and_then(Value::as_array_mut) else {
return 0;
};
messages
.iter_mut()
.filter_map(|message| message.get_mut("content"))
.map(replace_images_in_content)
.sum()
}
fn replace_images_in_content(content: &mut Value) -> usize {
let Some(blocks) = content.as_array_mut() else {
return 0;
};
let mut replaced = 0usize;
for block in blocks {
if block.get("type").and_then(Value::as_str) == Some("image") {
let cache_control = block.get("cache_control").cloned();
*block = json!({
"type": "text",
"text": UNSUPPORTED_IMAGE_MARKER
});
if let (Some(cache_control), Some(object)) = (cache_control, block.as_object_mut()) {
object.insert("cache_control".to_string(), cache_control);
}
replaced += 1;
continue;
}
if let Some(nested_content) = block.get_mut("content") {
replaced += replace_images_in_content(nested_content);
}
}
replaced
}
fn explicit_model_image_support(provider: &Provider, model: &str) -> Option<bool> {
let settings = &provider.settings_config;
[
settings
.get("modelCatalog")
.and_then(|catalog| catalog.get("models")),
settings.get("modelCatalog"),
settings.get("models"),
]
.into_iter()
.flatten()
.find_map(|value| explicit_model_image_support_in_value(value, model))
}
fn known_text_only_model(model: &str) -> bool {
let normalized = normalize_model_id(model);
let tail = normalized.rsplit('/').next().unwrap_or(normalized.as_str());
const EXACT_TAILS: &[&str] = &[
"ark-code-latest",
"deepseek-chat",
"deepseek-reasoner",
"deepseek-v4-flash",
"deepseek-v4-pro",
"glm-5.1",
"kat-coder",
"kat-coder-pro",
"kat-coder-pro v1",
"kat-coder-pro v2",
"kat-coder-pro-v1",
"kat-coder-pro-v2",
"ling-2.5-1t",
"longcat-flash-chat",
"mimo-v2.5-pro",
"us.deepseek.r1-v1",
];
const TAIL_PREFIXES: &[&str] = &["minimax-m2.7", "qwen3-coder", "step-3.5-flash"];
EXACT_TAILS.contains(&tail) || TAIL_PREFIXES.iter().any(|prefix| tail.starts_with(prefix))
}
fn explicit_model_image_support_in_value(value: &Value, model: &str) -> Option<bool> {
if let Some(models) = value.as_array() {
return models.iter().find_map(|entry| {
model_entry_matches(entry, None, model).then(|| explicit_image_support(entry))?
});
}
let object = value.as_object()?;
object.iter().find_map(|(key, entry)| {
model_entry_matches(entry, Some(key), model).then(|| explicit_image_support(entry))?
})
}
fn explicit_image_support(entry: &Value) -> Option<bool> {
if let Some(value) = entry
.get("supportsImage")
.or_else(|| entry.get("supports_image"))
.or_else(|| entry.get("vision"))
.and_then(Value::as_bool)
{
return Some(value);
}
[
entry.get("input"),
entry.pointer("/modalities/input"),
entry.get("input_modalities"),
entry.get("inputModalities"),
]
.into_iter()
.flatten()
.find_map(input_modalities_support_image)
}
fn input_modalities_support_image(value: &Value) -> Option<bool> {
let modalities = value.as_array()?;
Some(modalities.iter().any(|item| {
item.as_str()
.map(str::trim)
.is_some_and(|item| item.eq_ignore_ascii_case("image"))
}))
}
fn extract_error_text(body: &str) -> String {
if let Ok(value) = serde_json::from_str::<Value>(body) {
let candidates = [
value.pointer("/error/message"),
value.pointer("/message"),
value.pointer("/detail"),
value.pointer("/error"),
];
if let Some(message) = candidates
.into_iter()
.flatten()
.find_map(|value| value.as_str())
{
return message.to_string();
}
if let Ok(compact) = serde_json::to_string(&value) {
return compact;
}
}
body.to_string()
}
fn model_entry_matches(entry: &Value, key: Option<&str>, model: &str) -> bool {
key.is_some_and(|key| model_ids_match(key, model))
|| ["model", "id", "name"]
.into_iter()
.filter_map(|field| entry.get(field).and_then(Value::as_str))
.any(|candidate| model_ids_match(candidate, model))
}
fn model_ids_match(candidate: &str, model: &str) -> bool {
let candidate = normalize_model_id(candidate);
let model = normalize_model_id(model);
if candidate.is_empty() || model.is_empty() {
return false;
}
if candidate == model {
return true;
}
let candidate_tail = candidate.rsplit('/').next().unwrap_or(candidate.as_str());
let model_tail = model.rsplit('/').next().unwrap_or(model.as_str());
candidate_tail == model_tail || candidate == model_tail || candidate_tail == model
}
fn normalize_model_id(value: &str) -> String {
let mut normalized = value
.trim()
.trim_start_matches("models/")
.trim()
.to_ascii_lowercase();
if let Some(stripped) =
normalized.strip_suffix(crate::claude_desktop_config::ONE_M_CONTEXT_MARKER)
{
normalized = stripped.trim().to_string();
}
normalized
}
#[cfg(test)]
mod tests {
use super::*;
use crate::provider::Provider;
use serde_json::json;
fn provider(settings_config: Value) -> Provider {
Provider {
id: "test".to_string(),
name: "Test".to_string(),
settings_config,
website_url: None,
category: None,
created_at: None,
sort_index: None,
notes: None,
meta: None,
icon: None,
icon_color: None,
in_failover_queue: false,
}
}
#[test]
fn keeps_images_when_model_capability_is_unknown() {
let provider = provider(json!({}));
let mut body = json!({
"model": "unknown-model",
"messages": [{
"role": "user",
"content": [
{ "type": "text", "text": "look" },
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
});
let count = replace_images_for_text_only_model(&mut body, &provider, true);
assert_eq!(count, 0);
assert_eq!(body["messages"][0]["content"][1]["type"], "image");
}
#[test]
fn known_text_only_models_replace_images_before_send() {
let provider = provider(json!({}));
let mut body = json!({
"model": "deepseek/deepseek-v4-pro",
"messages": [{
"role": "user",
"content": [
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
});
let count = replace_images_for_text_only_model(&mut body, &provider, true);
assert_eq!(count, 1);
assert_eq!(
body["messages"][0]["content"][0]["text"],
UNSUPPORTED_IMAGE_MARKER
);
}
#[test]
fn explicit_text_modalities_replace_images_before_send() {
let provider = provider(json!({
"models": [
{ "id": "deepseek-v4-pro", "input": ["text"] }
]
}));
let mut body = json!({
"model": "deepseek-v4-pro",
"messages": [{
"role": "user",
"content": [
{ "type": "text", "text": "look" },
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
});
let count = replace_images_for_text_only_model(&mut body, &provider, true);
assert_eq!(count, 1);
assert_eq!(body["messages"][0]["content"][0]["text"], "look");
assert_eq!(body["messages"][0]["content"][1]["type"], "text");
assert_eq!(
body["messages"][0]["content"][1]["text"],
UNSUPPORTED_IMAGE_MARKER
);
}
#[test]
fn preserves_images_without_explicit_capability_even_for_unknown_models() {
let provider = provider(json!({}));
let mut body = json!({
"model": "unknown-model",
"messages": [{
"role": "user",
"content": [
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
});
let count = replace_images_for_text_only_model(&mut body, &provider, true);
assert_eq!(count, 0);
assert_eq!(body["messages"][0]["content"][0]["type"], "image");
}
#[test]
fn explicit_text_modalities_can_override_visual_model_ids() {
let provider = provider(json!({
"models": [
{ "id": "gpt-4o", "input": ["text"] }
]
}));
let mut body = json!({
"model": "gpt-4o",
"messages": [{
"role": "user",
"content": [
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
});
let count = replace_images_for_text_only_model(&mut body, &provider, true);
assert_eq!(count, 1);
assert_eq!(
body["messages"][0]["content"][0]["text"],
UNSUPPORTED_IMAGE_MARKER
);
}
#[test]
fn explicit_image_modalities_preserve_model_images() {
let provider = provider(json!({
"modelCatalog": {
"models": [
{ "model": "deepseek-v4-pro", "modalities": { "input": ["text", "image"] } }
]
}
}));
let mut body = json!({
"model": "deepseek-v4-pro",
"messages": [{
"role": "user",
"content": [
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
});
let count = replace_images_for_text_only_model(&mut body, &provider, true);
assert_eq!(count, 0);
assert_eq!(body["messages"][0]["content"][0]["type"], "image");
}
#[test]
fn known_mimo_pro_replaces_but_mimo_multimodal_preserves() {
let provider = provider(json!({}));
let mut pro_body = json!({
"model": "xiaomi-mimo-token-plan/mimo-v2.5-pro",
"messages": [{
"role": "user",
"content": [
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
});
let mut multimodal_body = json!({
"model": "xiaomi-mimo-token-plan/mimo-v2.5",
"messages": [{
"role": "user",
"content": [
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
});
let pro_count = replace_images_for_text_only_model(&mut pro_body, &provider, true);
let multimodal_count =
replace_images_for_text_only_model(&mut multimodal_body, &provider, true);
assert_eq!(pro_count, 1);
assert_eq!(multimodal_count, 0);
assert_eq!(
multimodal_body["messages"][0]["content"][0]["type"],
"image"
);
}
#[test]
fn multimodal_kimi_model_is_not_on_text_only_list() {
let provider = provider(json!({}));
let mut body = json!({
"model": "kimi/kimi-k2.6",
"messages": [{
"role": "user",
"content": [
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
});
let count = replace_images_for_text_only_model(&mut body, &provider, true);
assert_eq!(count, 0);
assert_eq!(body["messages"][0]["content"][0]["type"], "image");
}
#[test]
fn known_text_only_prefixes_replace_images_before_send() {
let provider = provider(json!({}));
let mut body = json!({
"model": "therouter/qwen/qwen3-coder-480b",
"messages": [{
"role": "user",
"content": [
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
});
let count = replace_images_for_text_only_model(&mut body, &provider, true);
assert_eq!(count, 1);
assert_eq!(
body["messages"][0]["content"][0]["text"],
UNSUPPORTED_IMAGE_MARKER
);
}
#[test]
fn unconditional_marker_replacement_handles_retry_path() {
let mut body = json!({
"model": "xiaomi-mimo-token-plan/mimo-v2.5-pro",
"messages": [{
"role": "user",
"content": [
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
});
assert!(contains_image_blocks(&body));
let count = replace_image_blocks_with_marker(&mut body);
assert_eq!(count, 1);
assert_eq!(
body["messages"][0]["content"][0]["text"],
UNSUPPORTED_IMAGE_MARKER
);
}
#[test]
fn replaces_nested_tool_result_image_blocks() {
let mut body = json!({
"model": "deepseek-v4-pro",
"messages": [{
"role": "user",
"content": [{
"type": "tool_result",
"tool_use_id": "toolu_1",
"content": [
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
}]
});
let count = replace_image_blocks_with_marker(&mut body);
assert_eq!(count, 1);
assert_eq!(
body["messages"][0]["content"][0]["content"][0]["text"],
UNSUPPORTED_IMAGE_MARKER
);
}
#[test]
fn detects_unsupported_image_errors() {
let error = ProxyError::UpstreamError {
status: 400,
body: Some(
r#"{"error":{"message":"This model does not support image input"}}"#.to_string(),
),
};
assert!(is_unsupported_image_error(&error));
}
#[test]
fn ignores_non_image_errors() {
let error = ProxyError::UpstreamError {
status: 400,
body: Some(r#"{"error":{"message":"Invalid API key"}}"#.to_string()),
};
assert!(!is_unsupported_image_error(&error));
}
#[test]
fn preserves_cache_control_when_replacing_image() {
// image block 可能承载 prompt cache 断点;替换成标记时必须把
// cache_control 迁移到新的 text block,否则会断掉缓存命中。
let mut body = json!({
"model": "deepseek-v4-pro",
"messages": [{
"role": "user",
"content": [{
"type": "image",
"source": { "type": "base64", "media_type": "image/png", "data": "abc" },
"cache_control": { "type": "ephemeral" }
}]
}]
});
let count = replace_image_blocks_with_marker(&mut body);
assert_eq!(count, 1);
let block = &body["messages"][0]["content"][0];
assert_eq!(block["type"], "text");
assert_eq!(block["text"], UNSUPPORTED_IMAGE_MARKER);
assert_eq!(block["cache_control"]["type"], "ephemeral");
}
#[test]
fn detects_media_and_attachment_error_phrasings() {
let media_error = ProxyError::UpstreamError {
status: 400,
body: Some(
r#"{"error":{"message":"This model cannot process media inputs"}}"#.to_string(),
),
};
assert!(is_unsupported_image_error(&media_error));
let attachment_error = ProxyError::UpstreamError {
status: 422,
body: Some(r#"{"message":"attachments are not supported by this model"}"#.to_string()),
};
assert!(is_unsupported_image_error(&attachment_error));
}
#[test]
fn heuristic_disabled_keeps_images_for_listed_text_only_models() {
// allow_heuristic = false:内置列表不再预测性剥图,避免误判多模态模型时静默丢图。
let provider = provider(json!({}));
let mut body = json!({
"model": "deepseek/deepseek-v4-pro",
"messages": [{
"role": "user",
"content": [
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
});
let count = replace_images_for_text_only_model(&mut body, &provider, false);
assert_eq!(count, 0);
assert_eq!(body["messages"][0]["content"][0]["type"], "image");
}
#[test]
fn explicit_text_capability_replaces_even_when_heuristic_disabled() {
// 显式声明 text-only 是声明驱动、零误判,即使关掉启发式也应生效。
let provider = provider(json!({
"models": [
{ "id": "deepseek-v4-pro", "input": ["text"] }
]
}));
let mut body = json!({
"model": "deepseek-v4-pro",
"messages": [{
"role": "user",
"content": [
{ "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
]
}]
});
let count = replace_images_for_text_only_model(&mut body, &provider, false);
assert_eq!(count, 1);
assert_eq!(
body["messages"][0]["content"][0]["text"],
UNSUPPORTED_IMAGE_MARKER
);
}
}
+1
View File
@@ -19,6 +19,7 @@ pub mod http_client;
pub mod hyper_client;
pub(crate) mod json_canonical;
pub mod log_codes;
pub mod media_sanitizer;
pub mod model_mapper;
pub mod provider_router;
pub mod providers;
@@ -148,6 +148,8 @@ mod tests {
enabled: true,
request_thinking_signature: true,
request_thinking_budget: true,
request_media_fallback: true,
request_media_heuristic: true,
}
}
@@ -156,6 +158,8 @@ mod tests {
enabled: true,
request_thinking_signature: true,
request_thinking_budget: false,
request_media_fallback: true,
request_media_heuristic: true,
}
}
@@ -164,6 +168,8 @@ mod tests {
enabled: false,
request_thinking_signature: true,
request_thinking_budget: true,
request_media_fallback: true,
request_media_heuristic: true,
}
}
@@ -251,6 +251,8 @@ mod tests {
enabled: true,
request_thinking_signature: true,
request_thinking_budget: true,
request_media_fallback: true,
request_media_heuristic: true,
}
}
@@ -259,6 +261,8 @@ mod tests {
enabled: true,
request_thinking_signature: false,
request_thinking_budget: false,
request_media_fallback: true,
request_media_heuristic: true,
}
}
@@ -267,6 +271,8 @@ mod tests {
enabled: false,
request_thinking_signature: true,
request_thinking_budget: true,
request_media_fallback: true,
request_media_heuristic: true,
}
}
+44
View File
@@ -213,6 +213,19 @@ pub struct RectifierConfig {
/// 处理错误:budget_tokens + thinking 相关约束
#[serde(default = "default_true")]
pub request_thinking_budget: bool,
/// 请求整流:不支持的图片降级(默认开启)
///
/// 上游拒绝图片输入时,把图片块替换为 [Unsupported Image] 标记,
/// 让对话不中断。总开关,管辖「显式声明 text-only」与「上游报错后兜底」两条事实驱动路径。
#[serde(default = "default_true")]
pub request_media_fallback: bool,
/// 请求整流:启发式 text-only 模型名匹配(默认开启)
///
/// 在模型未声明能力时,按内置模型名列表预测性地剥离图片(发送前)。
/// 受 request_media_fallback 管辖;单独关闭后仅保留「显式声明」与「上游兜底」,
/// 避免内置列表把多模态模型误判成 text-only 而静默剥图。
#[serde(default = "default_true")]
pub request_media_heuristic: bool,
}
fn default_true() -> bool {
@@ -229,6 +242,8 @@ impl Default for RectifierConfig {
enabled: true,
request_thinking_signature: true,
request_thinking_budget: true,
request_media_fallback: true,
request_media_heuristic: true,
}
}
}
@@ -386,6 +401,14 @@ mod tests {
config.request_thinking_budget,
"thinking budget 整流器默认应为 true"
);
assert!(
config.request_media_fallback,
"media 降级总开关默认应为 true"
);
assert!(
config.request_media_heuristic,
"启发式 text-only 模型识别默认应为 true"
);
}
#[test]
@@ -396,6 +419,14 @@ mod tests {
assert!(config.enabled);
assert!(config.request_thinking_signature);
assert!(config.request_thinking_budget);
assert!(
config.request_media_fallback,
"缺 requestMediaFallback 时应回退默认值 true"
);
assert!(
config.request_media_heuristic,
"缺 requestMediaHeuristic 时应回退默认值 true"
);
}
#[test]
@@ -419,6 +450,19 @@ mod tests {
assert!(config.request_thinking_budget);
}
#[test]
fn test_rectifier_config_serde_media_explicit_false() {
// 验证 media 两字段显式 false 时被如实反序列化(用户主动关闭须生效,不能被默认值覆盖)
let json = r#"{"requestMediaFallback": false, "requestMediaHeuristic": false}"#;
let config: RectifierConfig = serde_json::from_str(json).unwrap();
assert!(!config.request_media_fallback);
assert!(!config.request_media_heuristic);
// 其余字段仍走默认 true
assert!(config.enabled);
assert!(config.request_thinking_signature);
assert!(config.request_thinking_budget);
}
#[test]
fn test_log_config_default() {
let config = LogConfig::default();