fix(proxy): resolve circuit breaker state persistence and HalfOpen deadlock

This commit addresses several critical issues in the failover system:

**Circuit breaker state persistence (previous fix)**
- Promote ProviderRouter to ProxyState for cross-request state sharing
- Remove redundant router.rs module
- Fix 429 errors to be retryable (rate limiting should try other providers)

**Hot-update circuit breaker config**
- Add update_circuit_breaker_configs() to ProxyServer and ProxyService
- Connect update_circuit_breaker_config command to running circuit breakers
- Add reset_provider_circuit_breaker() for manual breaker reset

**Fix HalfOpen deadlock bug**
- Change half_open_requests from cumulative count to in-flight count
- Release quota in record_success()/record_failure() when in HalfOpen state
- Prevents permanent deadlock when success_threshold > 1

**Fix duplicate select_providers() call**
- Store providers list in RequestContext, pass to forward_with_retry()
- Avoid consuming HalfOpen quota twice per request
- Single call to select_providers() per request lifecycle

**Add per-provider retry with exponential backoff**
- Implement forward_with_provider_retry() with configurable max_retries
- Backoff delays: 100ms, 200ms, 400ms, etc.
This commit is contained in:
Jason
2025-12-13 22:47:49 +08:00
parent 5d424b1383
commit 5a5ca2a989
11 changed files with 347 additions and 141 deletions
+31 -10
View File
@@ -5,8 +5,7 @@
use crate::app_config::AppType;
use crate::provider::Provider;
use crate::proxy::{
forwarder::RequestForwarder, router::ProviderRouter, server::ProxyState, types::ProxyConfig,
ProxyError,
forwarder::RequestForwarder, server::ProxyState, types::ProxyConfig, ProxyError,
};
use std::time::Instant;
@@ -15,7 +14,7 @@ use std::time::Instant;
/// 贯穿整个请求生命周期,包含:
/// - 计时信息
/// - 代理配置
/// - 选中的 Provider
/// - 选中的 Provider 列表(用于故障转移)
/// - 请求模型名称
/// - 日志标签
pub struct RequestContext {
@@ -23,8 +22,10 @@ pub struct RequestContext {
pub start_time: Instant,
/// 代理配置快照
pub config: ProxyConfig,
/// 选中的 Provider
/// 选中的 Provider(故障转移链的第一个)
pub provider: Provider,
/// 完整的 Provider 列表(用于故障转移)
providers: Vec<Provider>,
/// 请求中的模型名称
pub request_model: String,
/// 日志标签(如 "Claude"、"Codex"、"Gemini"
@@ -65,21 +66,32 @@ impl RequestContext {
.unwrap_or("unknown")
.to_string();
// Provider 选择
let router = ProviderRouter::new(state.db.clone());
let provider = router.select_provider(&app_type, &[]).await?;
// 使用共享的 ProviderRouter 选择 Provider(熔断器状态跨请求保持)
// 注意:只在这里调用一次,结果传递给 forwarder,避免重复消耗 HalfOpen 名额
let providers = state
.provider_router
.select_providers(app_type_str)
.await
.map_err(|e| ProxyError::DatabaseError(e.to_string()))?;
let provider = providers
.first()
.cloned()
.ok_or(ProxyError::NoAvailableProvider)?;
log::info!(
"[{}] Provider: {}, model: {}",
"[{}] Provider: {}, model: {}, failover chain: {} providers",
tag,
provider.name,
request_model
request_model,
providers.len()
);
Ok(Self {
start_time,
config,
provider,
providers,
request_model,
tag,
app_type_str,
@@ -110,9 +122,11 @@ impl RequestContext {
}
/// 创建 RequestForwarder
///
/// 使用共享的 ProviderRouter,确保熔断器状态跨请求保持
pub fn create_forwarder(&self, state: &ProxyState) -> RequestForwarder {
RequestForwarder::new(
state.db.clone(),
state.provider_router.clone(),
self.config.request_timeout,
self.config.max_retries,
state.status.clone(),
@@ -120,6 +134,13 @@ impl RequestContext {
)
}
/// 获取 Provider 列表(用于故障转移)
///
/// 返回在创建上下文时已选择的 providers,避免重复调用 select_providers()
pub fn get_providers(&self) -> Vec<Provider> {
self.providers.clone()
}
/// 计算请求延迟(毫秒)
#[inline]
pub fn latency_ms(&self) -> u64 {