feat(failover): auto-switch to higher priority provider on recovery

- After circuit breaker reset, check if recovered provider has higher priority
- Automatically switch back if queue_order is lower (higher priority)
- Stream health check now resets circuit breaker on success/degraded
This commit is contained in:
YoVinchen
2025-12-22 00:35:36 +08:00
parent 59096d9c15
commit deea8455a9
2 changed files with 83 additions and 9 deletions
+74 -8
View File
@@ -14,14 +14,6 @@ pub async fn start_proxy_server(
state.proxy_service.start().await
}
/// 启动代理服务器(带 Live 配置接管)
#[tauri::command]
pub async fn start_proxy_with_takeover(
state: tauri::State<'_, AppState>,
) -> Result<ProxyServerInfo, String> {
state.proxy_service.start_with_takeover().await
}
/// 停止代理服务器(恢复 Live 配置)
#[tauri::command]
pub async fn stop_proxy_with_restore(state: tauri::State<'_, AppState>) -> Result<(), String> {
@@ -111,8 +103,13 @@ pub async fn get_provider_health(
}
/// 重置熔断器
///
/// 重置后会检查是否应该切回队列中优先级更高的供应商:
/// 1. 检查自动故障转移是否开启
/// 2. 如果恢复的供应商在队列中优先级更高(queue_order 更小),则自动切换
#[tauri::command]
pub async fn reset_circuit_breaker(
app_handle: tauri::AppHandle,
state: tauri::State<'_, AppState>,
provider_id: String,
app_type: String,
@@ -129,6 +126,75 @@ pub async fn reset_circuit_breaker(
.reset_provider_circuit_breaker(&provider_id, &app_type)
.await?;
// 3. 检查是否应该切回优先级更高的供应商
let failover_key = format!("auto_failover_enabled_{app_type}");
let auto_failover_enabled = match db.get_setting(&failover_key) {
Ok(Some(value)) => value == "true",
Ok(None) => {
log::debug!(
"[{app_type}] Failover setting '{failover_key}' not found, defaulting to disabled"
);
false
}
Err(e) => {
log::error!(
"[{app_type}] Failed to read failover setting '{failover_key}': {e}, defaulting to disabled"
);
false
}
};
if auto_failover_enabled && state.proxy_service.is_running().await {
// 获取当前供应商 ID
let current_id = db
.get_current_provider(&app_type)
.map_err(|e| e.to_string())?;
if let Some(current_id) = current_id {
// 获取故障转移队列
let queue = db
.get_failover_queue(&app_type)
.map_err(|e| e.to_string())?;
// 找到恢复的供应商和当前供应商在队列中的位置
let restored_order = queue
.iter()
.find(|item| item.provider_id == provider_id && item.enabled)
.map(|item| item.queue_order);
let current_order = queue
.iter()
.find(|item| item.provider_id == current_id && item.enabled)
.map(|item| item.queue_order);
// 如果恢复的供应商优先级更高(queue_order 更小),则切换
if let (Some(restored), Some(current)) = (restored_order, current_order) {
if restored < current {
log::info!(
"[Recovery] 供应商 {provider_id} 已恢复且优先级更高 (P{restored} vs P{current}),自动切换"
);
// 获取供应商名称用于日志和事件
let provider_name = db
.get_all_providers(&app_type)
.ok()
.and_then(|providers| providers.get(&provider_id).map(|p| p.name.clone()))
.unwrap_or_else(|| provider_id.clone());
// 创建故障转移切换管理器并执行切换
let switch_manager =
crate::proxy::failover_switch::FailoverSwitchManager::new(db.clone());
if let Err(e) = switch_manager
.try_switch(Some(&app_handle), &app_type, &provider_id, &provider_name)
.await
{
log::error!("[Recovery] 自动切换失败: {e}");
}
}
}
}
}
Ok(())
}
+9 -1
View File
@@ -6,10 +6,12 @@ import {
type StreamCheckResult,
} from "@/lib/api/model-test";
import type { AppId } from "@/lib/api";
import { useResetCircuitBreaker } from "@/lib/query/failover";
export function useStreamCheck(appId: AppId) {
const { t } = useTranslation();
const [checkingIds, setCheckingIds] = useState<Set<string>>(new Set());
const resetCircuitBreaker = useResetCircuitBreaker();
const checkProvider = useCallback(
async (
@@ -30,6 +32,9 @@ export function useStreamCheck(appId: AppId) {
}),
{ closeButton: true },
);
// 测试通过后重置熔断器状态
resetCircuitBreaker.mutate({ providerId, appType: appId });
} else if (result.status === "degraded") {
toast.warning(
t("streamCheck.degraded", {
@@ -38,6 +43,9 @@ export function useStreamCheck(appId: AppId) {
defaultValue: `${providerName} 响应较慢 (${result.responseTimeMs}ms)`,
}),
);
// 降级状态也重置熔断器,因为至少能通信
resetCircuitBreaker.mutate({ providerId, appType: appId });
} else {
toast.error(
t("streamCheck.failed", {
@@ -66,7 +74,7 @@ export function useStreamCheck(appId: AppId) {
});
}
},
[appId, t],
[appId, t, resetCircuitBreaker],
);
const isChecking = useCallback(