mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-07-29 18:05:37 +08:00
fix(failover): patch P1-P3 reliability gaps surfaced by team review
- Forwarder buffers non-streaming bodies and primes streaming first chunk before signaling success, so body timeouts and SSE first-chunk failures route through the circuit breaker instead of being recorded as success on response-header arrival - Atomic enable-failover: switch to P1 before persisting the flag, and roll back auto-added queue entries when the switch is rejected (e.g. official providers) - Hot-reload circuit breaker config on per-app proxy config change instead of waiting for a proxy restart - FailoverToggle / FailoverQueueManager / AutoFailoverConfigPanel require proxy takeover for the active app; the backend command also rejects enabling when takeover is off - ProviderHealthBadge consumes the backend is_healthy flag instead of hardcoding the 5-failure threshold Cleanup: - impl From<&AppProxyConfig> for CircuitBreakerConfig and use it from the command layer - Collapse three identical TabsContent blocks into a single map
This commit is contained in:
@@ -25,6 +25,7 @@ use crate::commands::{CodexOAuthState, CopilotAuthState};
|
||||
use crate::proxy::providers::codex_oauth_auth::CodexOAuthManager;
|
||||
use crate::proxy::providers::copilot_auth::CopilotAuthManager;
|
||||
use crate::{app_config::AppType, provider::Provider};
|
||||
use futures::StreamExt;
|
||||
use http::Extensions;
|
||||
use serde_json::Value;
|
||||
use std::sync::Arc;
|
||||
@@ -1164,8 +1165,9 @@ impl RequestForwarder {
|
||||
&filtered_body,
|
||||
self.session_client_provided,
|
||||
);
|
||||
let force_identity_encoding = needs_transform
|
||||
|| should_force_identity_encoding(&effective_endpoint, &filtered_body, headers);
|
||||
let request_is_streaming =
|
||||
is_streaming_request(&effective_endpoint, &filtered_body, headers);
|
||||
let force_identity_encoding = needs_transform || request_is_streaming;
|
||||
|
||||
// Codex OAuth 需要注入的 ChatGPT-Account-Id(在动态 token 获取期间填充)
|
||||
let mut codex_oauth_account_id: Option<String> = None;
|
||||
@@ -1609,8 +1611,6 @@ impl RequestForwarder {
|
||||
);
|
||||
let client = super::http_client::get();
|
||||
let mut request = client.request(method.clone(), &url);
|
||||
let request_is_streaming =
|
||||
is_streaming_request(&effective_endpoint, &filtered_body, headers);
|
||||
if request_is_streaming {
|
||||
// reqwest 的 timeout 是整请求超时;流式请求交给 response_processor
|
||||
// 的首包/静默期超时控制,避免长流被总时长误杀。
|
||||
@@ -1663,6 +1663,9 @@ impl RequestForwarder {
|
||||
let status = response.status();
|
||||
|
||||
if status.is_success() {
|
||||
let response = self
|
||||
.prepare_success_response_for_failover(response, request_is_streaming)
|
||||
.await?;
|
||||
Ok((response, resolved_claude_api_format))
|
||||
} else {
|
||||
let status_code = status.as_u16();
|
||||
@@ -1675,6 +1678,73 @@ impl RequestForwarder {
|
||||
}
|
||||
}
|
||||
|
||||
/// 故障转移开启时,成功不能只看上游响应头。
|
||||
///
|
||||
/// - 非流式:先把完整 body 读到内存,读超时/连接中断会回到 retry loop 尝试下一家。
|
||||
/// - 流式:至少等首个 chunk 到达,避免上游返回 200 后一直不吐 SSE 时被误记成功。
|
||||
async fn prepare_success_response_for_failover(
|
||||
&self,
|
||||
response: ProxyResponse,
|
||||
request_is_streaming: bool,
|
||||
) -> Result<ProxyResponse, ProxyError> {
|
||||
if request_is_streaming {
|
||||
return self.prime_streaming_response(response).await;
|
||||
}
|
||||
|
||||
if self.non_streaming_timeout.is_zero() {
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
let status = response.status();
|
||||
let headers = response.headers().clone();
|
||||
let body_timeout = self.non_streaming_timeout;
|
||||
let body = tokio::time::timeout(body_timeout, response.bytes())
|
||||
.await
|
||||
.map_err(|_| {
|
||||
ProxyError::Timeout(format!(
|
||||
"响应体读取超时: {}s(上游发完响应头后 body 未到达)",
|
||||
body_timeout.as_secs()
|
||||
))
|
||||
})??;
|
||||
|
||||
Ok(ProxyResponse::buffered(status, headers, body))
|
||||
}
|
||||
|
||||
async fn prime_streaming_response(
|
||||
&self,
|
||||
response: ProxyResponse,
|
||||
) -> Result<ProxyResponse, ProxyError> {
|
||||
if self.streaming_first_byte_timeout.is_zero() {
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
let status = response.status();
|
||||
let headers = response.headers().clone();
|
||||
let timeout = self.streaming_first_byte_timeout;
|
||||
let mut stream = Box::pin(response.bytes_stream());
|
||||
|
||||
let first = tokio::time::timeout(timeout, stream.next())
|
||||
.await
|
||||
.map_err(|_| {
|
||||
ProxyError::Timeout(format!(
|
||||
"流式响应首包超时: {}s(上游已返回响应头但未返回数据)",
|
||||
timeout.as_secs()
|
||||
))
|
||||
})?;
|
||||
|
||||
let Some(first) = first else {
|
||||
return Err(ProxyError::ForwardFailed(
|
||||
"流式响应在首包到达前结束".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
let first =
|
||||
first.map_err(|e| ProxyError::ForwardFailed(format!("读取流式响应首包失败: {e}")))?;
|
||||
|
||||
let replay = futures::stream::once(async move { Ok(first) }).chain(stream);
|
||||
Ok(ProxyResponse::streamed(status, headers, replay))
|
||||
}
|
||||
|
||||
async fn resolve_claude_api_format(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
@@ -2213,9 +2283,14 @@ fn value_for_log(value: &Value) -> String {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::database::Database;
|
||||
use axum::http::header::{HeaderValue, ACCEPT};
|
||||
use axum::http::HeaderMap;
|
||||
use bytes::Bytes;
|
||||
use http::StatusCode;
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
fn test_provider_with_type(provider_type: Option<&str>) -> Provider {
|
||||
Provider {
|
||||
@@ -2237,6 +2312,31 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn test_forwarder(
|
||||
non_streaming_timeout: Duration,
|
||||
streaming_first_byte_timeout: Duration,
|
||||
) -> RequestForwarder {
|
||||
let db = Arc::new(Database::memory().expect("memory db"));
|
||||
|
||||
RequestForwarder {
|
||||
router: Arc::new(ProviderRouter::new(db.clone())),
|
||||
status: Arc::new(RwLock::new(ProxyStatus::default())),
|
||||
current_providers: Arc::new(RwLock::new(HashMap::new())),
|
||||
gemini_shadow: Arc::new(GeminiShadowStore::new()),
|
||||
failover_manager: Arc::new(FailoverSwitchManager::new(db)),
|
||||
app_handle: None,
|
||||
current_provider_id_at_start: String::new(),
|
||||
session_id: String::new(),
|
||||
session_client_provided: false,
|
||||
rectifier_config: RectifierConfig::default(),
|
||||
optimizer_config: OptimizerConfig::default(),
|
||||
copilot_optimizer_config: CopilotOptimizerConfig::default(),
|
||||
non_streaming_timeout,
|
||||
streaming_first_byte_timeout,
|
||||
max_attempts: 1,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_provider_retryable_log_uses_single_provider_code() {
|
||||
let error = ProxyError::UpstreamError {
|
||||
@@ -2382,6 +2482,96 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn non_streaming_success_is_buffered_before_marking_provider_successful() {
|
||||
let forwarder = test_forwarder(Duration::from_secs(1), Duration::from_secs(1));
|
||||
let response = ProxyResponse::streamed(
|
||||
StatusCode::OK,
|
||||
HeaderMap::new(),
|
||||
futures::stream::once(async {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
Ok::<Bytes, std::io::Error>(Bytes::from_static(b"{\"ok\":true}"))
|
||||
}),
|
||||
);
|
||||
|
||||
let prepared = forwarder
|
||||
.prepare_success_response_for_failover(response, false)
|
||||
.await
|
||||
.expect("response should be buffered");
|
||||
|
||||
assert_eq!(
|
||||
prepared.bytes().await.unwrap(),
|
||||
Bytes::from_static(b"{\"ok\":true}")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn non_streaming_body_read_error_is_retryable_before_success_record() {
|
||||
let forwarder = test_forwarder(Duration::from_secs(1), Duration::from_secs(1));
|
||||
let response = ProxyResponse::streamed(
|
||||
StatusCode::OK,
|
||||
HeaderMap::new(),
|
||||
futures::stream::once(async {
|
||||
Err::<Bytes, std::io::Error>(std::io::Error::other("body boom"))
|
||||
}),
|
||||
);
|
||||
|
||||
let err = match forwarder
|
||||
.prepare_success_response_for_failover(response, false)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("body read errors should fail the attempt"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert!(matches!(err, ProxyError::ForwardFailed(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_success_primes_first_chunk_and_replays_it() {
|
||||
let forwarder = test_forwarder(Duration::from_secs(1), Duration::from_secs(1));
|
||||
let response = ProxyResponse::streamed(
|
||||
StatusCode::OK,
|
||||
HeaderMap::new(),
|
||||
futures::stream::iter(vec![
|
||||
Ok::<Bytes, std::io::Error>(Bytes::from_static(b"first")),
|
||||
Ok::<Bytes, std::io::Error>(Bytes::from_static(b"second")),
|
||||
]),
|
||||
);
|
||||
|
||||
let prepared = forwarder
|
||||
.prepare_success_response_for_failover(response, true)
|
||||
.await
|
||||
.expect("stream should be primed");
|
||||
|
||||
assert_eq!(
|
||||
prepared.bytes().await.unwrap(),
|
||||
Bytes::from_static(b"firstsecond")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn streaming_first_chunk_error_is_retryable_before_success_record() {
|
||||
let forwarder = test_forwarder(Duration::from_secs(1), Duration::from_secs(1));
|
||||
let response = ProxyResponse::streamed(
|
||||
StatusCode::OK,
|
||||
HeaderMap::new(),
|
||||
futures::stream::once(async {
|
||||
Err::<Bytes, std::io::Error>(std::io::Error::other("first chunk boom"))
|
||||
}),
|
||||
);
|
||||
|
||||
let err = match forwarder
|
||||
.prepare_success_response_for_failover(response, true)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("first chunk errors should fail the attempt"),
|
||||
Err(err) => err,
|
||||
};
|
||||
|
||||
assert!(matches!(err, ProxyError::ForwardFailed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codex_oauth_session_headers_match_codex_cache_identity() {
|
||||
let headers = build_codex_oauth_session_headers("session-123");
|
||||
|
||||
Reference in New Issue
Block a user