mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-08-02 02:05:57 +08:00
fix(proxy): resolve circuit breaker state persistence and HalfOpen deadlock
This commit addresses several critical issues in the failover system: **Circuit breaker state persistence (previous fix)** - Promote ProviderRouter to ProxyState for cross-request state sharing - Remove redundant router.rs module - Fix 429 errors to be retryable (rate limiting should try other providers) **Hot-update circuit breaker config** - Add update_circuit_breaker_configs() to ProxyServer and ProxyService - Connect update_circuit_breaker_config command to running circuit breakers - Add reset_provider_circuit_breaker() for manual breaker reset **Fix HalfOpen deadlock bug** - Change half_open_requests from cumulative count to in-flight count - Release quota in record_success()/record_failure() when in HalfOpen state - Prevents permanent deadlock when success_threshold > 1 **Fix duplicate select_providers() call** - Store providers list in RequestContext, pass to forward_with_retry() - Avoid consuming HalfOpen quota twice per request - Single call to select_providers() per request lifecycle **Add per-provider retry with exponential backoff** - Implement forward_with_provider_retry() with configurable max_retries - Backoff delays: 100ms, 200ms, 400ms, etc.
This commit is contained in:
@@ -130,15 +130,17 @@ pub async fn reset_circuit_breaker(
|
||||
provider_id: String,
|
||||
app_type: String,
|
||||
) -> Result<(), String> {
|
||||
// 重置数据库健康状态
|
||||
// 1. 重置数据库健康状态
|
||||
let db = &state.db;
|
||||
db.update_provider_health(&provider_id, &app_type, true, None)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
// 注意:熔断器状态在内存中,重启代理服务器后会重置
|
||||
// 如果代理服务器正在运行,需要通知它重置熔断器
|
||||
// 目前先通过数据库重置健康状态,熔断器会在下次超时后自动尝试半开
|
||||
// 2. 如果代理正在运行,重置内存中的熔断器状态
|
||||
state
|
||||
.proxy_service
|
||||
.reset_provider_circuit_breaker(&provider_id, &app_type)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -161,9 +163,19 @@ pub async fn update_circuit_breaker_config(
|
||||
config: CircuitBreakerConfig,
|
||||
) -> Result<(), String> {
|
||||
let db = &state.db;
|
||||
|
||||
// 1. 更新数据库配置
|
||||
db.update_circuit_breaker_config(&config)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
// 2. 如果代理正在运行,热更新内存中的熔断器配置
|
||||
state
|
||||
.proxy_service
|
||||
.update_circuit_breaker_configs(config)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 获取熔断器统计信息(仅当代理服务器运行时)
|
||||
|
||||
Reference in New Issue
Block a user