fix(proxy): resolve circuit breaker race condition and error classification

This commit addresses two critical issues in the proxy failover logic:

1. Circuit Breaker HalfOpen Concurrency Bug:
   - Introduced `AllowResult` struct to track half-open permit usage
   - Added state guard in `transition_to_half_open()` to prevent duplicate resets
   - Replaced `fetch_sub` with CAS loop in `release_half_open_permit()` to prevent underflow
   - Separated `is_available()` (routing) from `allow_request()` (permit acquisition)

2. Error Classification Conflation:
   - Split retry logic into `should_retry_same_provider()` and `categorize_proxy_error()`
   - Same-provider retry: only for transient errors (timeout, 429, 5xx)
   - Cross-provider failover: now includes ConfigError, TransformError, AuthError
   - 4xx errors (401/403) no longer waste retries on the same provider
This commit is contained in:
Jason
2025-12-17 08:49:47 +08:00
parent 3d514c8250
commit 1b73b26c0e
3 changed files with 183 additions and 63 deletions
+8 -7
View File
@@ -5,7 +5,7 @@
use crate::database::Database;
use crate::error::AppError;
use crate::provider::Provider;
use crate::proxy::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig};
use crate::proxy::circuit_breaker::{AllowResult, CircuitBreaker, CircuitBreakerConfig};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
@@ -123,7 +123,7 @@ impl ProviderRouter {
///
/// 注意:调用方必须在请求结束后通过 `record_result()` 释放 HalfOpen 名额,
/// 否则会导致该 Provider 长时间无法进入探测状态。
pub async fn allow_provider_request(&self, provider_id: &str, app_type: &str) -> bool {
pub async fn allow_provider_request(&self, provider_id: &str, app_type: &str) -> AllowResult {
let circuit_key = format!("{app_type}:{provider_id}");
let breaker = self.get_or_create_circuit_breaker(&circuit_key).await;
breaker.allow_request().await
@@ -134,6 +134,7 @@ impl ProviderRouter {
&self,
provider_id: &str,
app_type: &str,
used_half_open_permit: bool,
success: bool,
error_msg: Option<String>,
) -> Result<(), AppError> {
@@ -146,10 +147,10 @@ impl ProviderRouter {
let breaker = self.get_or_create_circuit_breaker(&circuit_key).await;
if success {
breaker.record_success().await;
breaker.record_success(used_half_open_permit).await;
log::debug!("Provider {provider_id} request succeeded");
} else {
breaker.record_failure().await;
breaker.record_failure(used_half_open_permit).await;
log::warn!(
"Provider {} request failed: {}",
provider_id,
@@ -265,7 +266,7 @@ mod tests {
// 测试创建熔断器
let breaker = router.get_or_create_circuit_breaker("claude:test").await;
assert!(breaker.allow_request().await);
assert!(breaker.allow_request().await.allowed);
}
#[tokio::test]
@@ -296,7 +297,7 @@ mod tests {
// 让 B 进入 Open 状态(failure_threshold=1
router
.record_result("b", "claude", false, Some("fail".to_string()))
.record_result("b", "claude", false, false, Some("fail".to_string()))
.await
.unwrap();
@@ -305,6 +306,6 @@ mod tests {
assert_eq!(providers.len(), 2);
// 如果 select_providers 错误地消耗了 HalfOpen 名额,这里会返回 false(被限流拒绝)
assert!(router.allow_provider_request("b", "claude").await);
assert!(router.allow_provider_request("b", "claude").await.allowed);
}
}