Add media fallback rectifier for text-only models

Replace unsupported Anthropic image blocks with an [Unsupported Image] marker when routed models are text-only or upstream rejects image input. Add rectifier settings for media fallback and heuristic model detection, wire the controls into the settings UI, and cover the sanitizer and forwarder gates with regression tests.
2026-08-04 19:45:34 +08:00 · 2026-06-02 23:35:30 +08:00
parent 33eafbad51
commit 6692343d1e
12 changed files with 1134 additions and 4 deletions
@@ -122,6 +122,51 @@ pub struct RequestForwarder {
 }

 impl RequestForwarder {
+    /// 预防式 media 降级：发送前对 text-only 模型把图片块替换为标记。
+    ///
+    /// 受 `enabled && request_media_fallback` 管辖；其中"启发式模型名单预测"
+    /// 再受 `request_media_heuristic` 单独管辖（显式声明 text-only 始终生效）。
+    /// 返回被替换的图片块数量（0 = 未触发或开关关闭）。
+    fn apply_media_prevention(&self, body: &mut Value, provider: &Provider) -> usize {
+        if !(self.rectifier_config.enabled && self.rectifier_config.request_media_fallback) {
+            return 0;
+        }
+        let replaced_images = super::media_sanitizer::replace_images_for_text_only_model(
+            body,
+            provider,
+            self.rectifier_config.request_media_heuristic,
+        );
+        if replaced_images > 0 {
+            let model = body.get("model").and_then(Value::as_str).unwrap_or("");
+            log::info!(
+                "[Media] Replaced {replaced_images} image block(s) with {} for text-only provider={}, model={}",
+                super::media_sanitizer::UNSUPPORTED_IMAGE_MARKER,
+                provider.id,
+                model
+            );
+        }
+        replaced_images
+    }
+
+    /// 反应式 media 重试判定：上游因图片输入报错后，是否应替换图片块并对同一供应商重试一次。
+    ///
+    /// 受 `enabled && request_media_fallback` 管辖；不涉及 `request_media_heuristic`——
+    /// 这里是上游"实测"错误后的纯恢复，不是预测，故启发式开关与它无关。
+    fn media_retry_should_trigger(
+        &self,
+        adapter_name: &str,
+        already_retried: bool,
+        provider_body: &Value,
+        error: &ProxyError,
+    ) -> bool {
+        adapter_name == "Claude"
+            && self.rectifier_config.enabled
+            && self.rectifier_config.request_media_fallback
+            && !already_retried
+            && super::media_sanitizer::contains_image_blocks(provider_body)
+            && super::media_sanitizer::is_unsupported_image_error(error)
+    }
+
    #[allow(clippy::too_many_arguments)]
    pub fn new(
        router: Arc<ProviderRouter>,
@@ -346,6 +391,7 @@ impl RequestForwarder {
            // —— 首家 provider 整流后被 5xx/timeout 击落时，下家仍能用整流后的请求体走整流流程
            let mut rectifier_retried = false;
            let mut budget_rectifier_retried = false;
+            let mut media_rectifier_retried = false;

            // 上限检查：尊重用户在 AppProxyConfig.max_retries 上配置的「重试次数」。
            // 放在熔断器 allow 检查之前，避免在已经超限时还占用 HalfOpen 探测名额。
@@ -477,6 +523,123 @@ impl RequestForwarder {
                    );
                    let mut signature_rectifier_non_retryable_client_error = false;

+                    if self.media_retry_should_trigger(
+                        adapter.name(),
+                        media_rectifier_retried,
+                        &provider_body,
+                        &e,
+                    ) {
+                        let mut media_body = provider_body.clone();
+                        let replaced_images =
+                            super::media_sanitizer::replace_image_blocks_with_marker(
+                                &mut media_body,
+                            );
+
+                        if replaced_images > 0 {
+                            let _ = std::mem::replace(&mut media_rectifier_retried, true);
+                            let model = media_body
+                                .get("model")
+                                .and_then(Value::as_str)
+                                .unwrap_or("");
+                            log::info!(
+                                "[{app_type_str}] [Media] Upstream rejected image input; retrying provider={} model={} with {replaced_images} image block(s) replaced by {}",
+                                provider.id,
+                                model,
+                                super::media_sanitizer::UNSUPPORTED_IMAGE_MARKER
+                            );
+
+                            match self
+                                .forward(
+                                    app_type,
+                                    &method,
+                                    provider,
+                                    endpoint,
+                                    &media_body,
+                                    &headers,
+                                    &extensions,
+                                    adapter.as_ref(),
+                                )
+                                .await
+                            {
+                                Ok((response, claude_api_format)) => {
+                                    log::info!(
+                                        "[{app_type_str}] [Media] Unsupported-image retry succeeded"
+                                    );
+                                    self.record_success_result(
+                                        &provider.id,
+                                        app_type_str,
+                                        used_half_open_permit,
+                                    )
+                                    .await;
+
+                                    {
+                                        let mut current_providers =
+                                            self.current_providers.write().await;
+                                        current_providers.insert(
+                                            app_type_str.to_string(),
+                                            (provider.id.clone(), provider.name.clone()),
+                                        );
+                                    }
+
+                                    {
+                                        let mut status = self.status.write().await;
+                                        status.success_requests += 1;
+                                        status.last_error = None;
+                                        let should_switch =
+                                            self.current_provider_id_at_start.as_str()
+                                                != provider.id.as_str();
+                                        if should_switch {
+                                            status.failover_count += 1;
+                                            let fm = self.failover_manager.clone();
+                                            let ah = self.app_handle.clone();
+                                            let pid = provider.id.clone();
+                                            let pname = provider.name.clone();
+                                            let at = app_type_str.to_string();
+
+                                            tokio::spawn(async move {
+                                                let _ = fm
+                                                    .try_switch(ah.as_ref(), &at, &pid, &pname)
+                                                    .await;
+                                            });
+                                        }
+                                        if status.total_requests > 0 {
+                                            status.success_rate = (status.success_requests as f32
+                                                / status.total_requests as f32)
+                                                * 100.0;
+                                        }
+                                    }
+
+                                    return Ok(ForwardResult {
+                                        response,
+                                        provider: provider.clone(),
+                                        claude_api_format,
+                                        connection_guard: None,
+                                    });
+                                }
+                                Err(retry_err) => {
+                                    log::warn!(
+                                        "[{app_type_str}] [Media] Unsupported-image retry still failed: {retry_err}"
+                                    );
+                                    if let Some(err) = self
+                                        .handle_rectifier_retry_failure(
+                                            retry_err,
+                                            provider,
+                                            app_type_str,
+                                            used_half_open_permit,
+                                            "media 降级",
+                                            &mut last_error,
+                                            &mut last_provider,
+                                        )
+                                        .await
+                                    {
+                                        return Err(err);
+                                    }
+                                    continue;
+                                }
+                            }
+                        }
+                    }
+
                    if is_anthropic_provider {
                        let error_message = extract_error_message(&e);
                        if should_rectify_thinking_signature(
@@ -1114,6 +1277,7 @@ impl RequestForwarder {
                    provider,
                    api_format,
                );
+                self.apply_media_prevention(&mut mapped_body, provider);
            }
        }
        let needs_transform = match resolved_claude_api_format.as_deref() {
@@ -3106,4 +3270,160 @@ mod tests {
            assert_eq!(will_replace, should_replace, "{desc}");
        }
    }
+
+    // ===== P3: forwarder 层 media 开关回归测试 =====
+    // 验证 gate 在 forwarder 这一层的"接线"，而非 media_sanitizer 纯函数本身。
+
+    fn forwarder_with_rectifier(config: RectifierConfig) -> RequestForwarder {
+        let mut fwd = test_forwarder(Duration::from_secs(1), Duration::from_secs(1));
+        fwd.rectifier_config = config;
+        fwd
+    }
+
+    fn provider_with_settings(settings_config: Value) -> Provider {
+        let mut p = test_provider_with_type(Some("anthropic"));
+        p.settings_config = settings_config;
+        p
+    }
+
+    fn body_with_image(model: &str) -> Value {
+        json!({
+            "model": model,
+            "messages": [{
+                "role": "user",
+                "content": [
+                    { "type": "image", "source": { "type": "base64", "media_type": "image/png", "data": "abc" } }
+                ]
+            }]
+        })
+    }
+
+    fn image_unsupported_error() -> ProxyError {
+        ProxyError::UpstreamError {
+            status: 400,
+            body: Some(
+                r#"{"error":{"message":"This model does not support image input"}}"#.to_string(),
+            ),
+        }
+    }
+    #[test]
+    fn prevention_replaces_when_all_switches_on_and_model_in_heuristic_list() {
+        let fwd = forwarder_with_rectifier(RectifierConfig::default());
+        let provider = provider_with_settings(json!({}));
+        let mut body = body_with_image("deepseek-v4-pro");
+
+        let replaced = fwd.apply_media_prevention(&mut body, &provider);
+
+        assert_eq!(replaced, 1, "默认全开 + 名单内模型应预替换");
+        assert_eq!(body["messages"][0]["content"][0]["type"], "text");
+    }
+
+    #[test]
+    fn prevention_skipped_when_media_fallback_off() {
+        // 关闭 request_media_fallback：即使名单命中也不预替换。
+        let fwd = forwarder_with_rectifier(RectifierConfig {
+            request_media_fallback: false,
+            ..RectifierConfig::default()
+        });
+        let provider = provider_with_settings(json!({}));
+        let mut body = body_with_image("deepseek-v4-pro");
+
+        let replaced = fwd.apply_media_prevention(&mut body, &provider);
+
+        assert_eq!(replaced, 0);
+        assert_eq!(body["messages"][0]["content"][0]["type"], "image");
+    }
+
+    #[test]
+    fn prevention_skipped_when_master_switch_off() {
+        let fwd = forwarder_with_rectifier(RectifierConfig {
+            enabled: false,
+            ..RectifierConfig::default()
+        });
+        let provider = provider_with_settings(json!({}));
+        let mut body = body_with_image("deepseek-v4-pro");
+
+        assert_eq!(fwd.apply_media_prevention(&mut body, &provider), 0);
+        assert_eq!(body["messages"][0]["content"][0]["type"], "image");
+    }
+
+    #[test]
+    fn prevention_heuristic_off_skips_list_but_keeps_explicit_text_only() {
+        // 关闭 request_media_heuristic：名单预测失效，但显式声明 text-only 仍预替换。
+        let fwd = forwarder_with_rectifier(RectifierConfig {
+            request_media_heuristic: false,
+            ..RectifierConfig::default()
+        });
+
+        // (a) 名单内模型、无显式声明 → 不再预替换
+        let bare_provider = provider_with_settings(json!({}));
+        let mut list_body = body_with_image("deepseek-v4-pro");
+        assert_eq!(
+            fwd.apply_media_prevention(&mut list_body, &bare_provider),
+            0,
+            "heuristic 关闭后名单模型不应被预替换"
+        );
+        assert_eq!(list_body["messages"][0]["content"][0]["type"], "image");
+
+        // (b) 显式声明 text-only → 仍预替换（声明驱动，不受 heuristic 开关影响）
+        let declared_provider = provider_with_settings(json!({
+            "models": [ { "id": "some-text-model", "input": ["text"] } ]
+        }));
+        let mut declared_body = body_with_image("some-text-model");
+        assert_eq!(
+            fwd.apply_media_prevention(&mut declared_body, &declared_provider),
+            1,
+            "显式 text-only 即使关闭 heuristic 也应预替换"
+        );
+        assert_eq!(declared_body["messages"][0]["content"][0]["type"], "text");
+    }
+
+    #[test]
+    fn reactive_triggers_when_all_switches_on() {
+        let fwd = forwarder_with_rectifier(RectifierConfig::default());
+        let body = body_with_image("any-model");
+        assert!(fwd.media_retry_should_trigger("Claude", false, &body, &image_unsupported_error()));
+    }
+
+    #[test]
+    fn reactive_skipped_when_media_fallback_off() {
+        // 关闭 request_media_fallback：上游报图片错误也不触发兜底重试。
+        let fwd = forwarder_with_rectifier(RectifierConfig {
+            request_media_fallback: false,
+            ..RectifierConfig::default()
+        });
+        let body = body_with_image("any-model");
+        assert!(!fwd.media_retry_should_trigger(
+            "Claude",
+            false,
+            &body,
+            &image_unsupported_error()
+        ));
+    }
+
+    #[test]
+    fn reactive_skipped_when_master_switch_off() {
+        let fwd = forwarder_with_rectifier(RectifierConfig {
+            enabled: false,
+            ..RectifierConfig::default()
+        });
+        let body = body_with_image("any-model");
+        assert!(!fwd.media_retry_should_trigger(
+            "Claude",
+            false,
+            &body,
+            &image_unsupported_error()
+        ));
+    }
+
+    #[test]
+    fn reactive_unaffected_by_heuristic_switch() {
+        // 关闭 request_media_heuristic 不影响反应式兜底——它是上游实测错误后的恢复，不是预测。
+        let fwd = forwarder_with_rectifier(RectifierConfig {
+            request_media_heuristic: false,
+            ..RectifierConfig::default()
+        });
+        let body = body_with_image("any-model");
+        assert!(fwd.media_retry_should_trigger("Claude", false, &body, &image_unsupported_error()));
+    }
 }