mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-07-25 13:45:03 +08:00
Feat/auto failover switch (#440)
* feat(failover): add auto-failover master switch with proxy integration
- Add persistent auto_failover_enabled setting in database
- Add get/set_auto_failover_enabled commands
- Provider router respects master switch state
- Proxy shutdown automatically disables failover
- Enabling failover auto-starts proxy server
- Optimistic updates for failover queue toggle
* feat(proxy): persist proxy takeover state across app restarts
- Add proxy_takeover_{app_type} settings for per-app state tracking
- Restore proxy takeover state automatically on app startup
- Preserve state on normal exit, clear on manual stop
- Add stop_with_restore_keep_state method for graceful shutdown
* fix(proxy): set takeover state for all apps in start_with_takeover
* fix(windows): hide console window when checking CLI versions
Add CREATE_NO_WINDOW flag to prevent command prompt from flashing
when detecting claude/codex/gemini CLI versions on Windows.
* refactor(failover): make auto-failover toggle per-app independent
- Change setting key from 'auto_failover_enabled' to 'auto_failover_enabled_{app_type}'
- Update provider_router to check per-app failover setting
- When failover disabled, use current provider only; when enabled, use queue order
- Add unit tests for failover enabled/disabled behavior
* feat(failover): auto-switch to higher priority provider on recovery
- After circuit breaker reset, check if recovered provider has higher priority
- Automatically switch back if queue_order is lower (higher priority)
- Stream health check now resets circuit breaker on success/degraded
* chore: remove unused start_proxy_with_takeover command
- Remove command registration from lib.rs
- Add comment clarifying failover queue is preserved on proxy stop
* feat(ui): integrate failover controls into provider cards
- Add failover toggle button to provider card actions
- Show priority badge (P1, P2, ...) for queued providers
- Highlight active provider with green border in failover mode
- Sync drag-drop order with failover queue
- Move per-app failover toggle to FailoverQueueManager
- Simplify SettingsPage failover section
* test(providers): add mocks for failover hooks in ProviderList tests
* refactor(failover): merge failover_queue table into providers
- Add in_failover_queue field to providers table
- Remove standalone failover_queue table and related indexes
- Simplify queue ordering by reusing sort_index field
- Remove reorder_failover_queue and set_failover_item_enabled commands
- Update frontend to use simplified FailoverQueueItem type
* fix(database): ensure in_failover_queue column exists for v2 databases
Add column check in create_tables to handle existing v2 databases
that were created before the failover queue refactor.
* fix(ui): differentiate active provider border color by proxy mode
- Use green border/gradient when proxy takeover is active
- Use blue border/gradient in normal mode (no proxy)
- Improves visual distinction between proxy and non-proxy states
* fix(database): clear provider health record when removing from failover queue
When a provider is removed from the failover queue, its health monitoring
is no longer needed. This change ensures the health record is also deleted
from the database to prevent stale data.
* fix(failover): improve cache cleanup for provider health and circuit breaker
- Use removeQueries instead of invalidateQueries when stopping proxy to
completely clear health and circuit breaker caches
- Clear provider health and circuit breaker caches when removing from
failover queue
- Refresh failover queue after drag-sort since queue order depends on
sort_index
- Only show health badge when provider is in failover queue
* style: apply prettier formatting to App.tsx and ProviderList.tsx
* fix(proxy): handle missing health records and clear health on proxy stop
- Return default healthy state when provider health record not found
- Add clear_provider_health_for_app to clear health for specific app
- Clear app health records when stopping proxy takeover
* fix(proxy): track actual provider used in forwarding for accurate logging
Introduce ForwardResult and ForwardError structs to return the actual
provider that handled the request. This ensures usage statistics and
error logs reflect the correct provider after failover.
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
//! 故障转移队列命令
|
||||
//!
|
||||
//! 管理代理模式下的故障转移队列
|
||||
//! 管理代理模式下的故障转移队列(基于 providers 表的 in_failover_queue 字段)
|
||||
|
||||
use crate::database::FailoverQueueItem;
|
||||
use crate::provider::Provider;
|
||||
@@ -56,54 +56,35 @@ pub async fn remove_from_failover_queue(
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// 重新排序故障转移队列
|
||||
/// 获取指定应用的自动故障转移开关状态
|
||||
#[tauri::command]
|
||||
pub async fn reorder_failover_queue(
|
||||
pub async fn get_auto_failover_enabled(
|
||||
state: tauri::State<'_, AppState>,
|
||||
app_type: String,
|
||||
provider_ids: Vec<String>,
|
||||
) -> Result<(), String> {
|
||||
) -> Result<bool, String> {
|
||||
let key = format!("auto_failover_enabled_{app_type}");
|
||||
state
|
||||
.db
|
||||
.reorder_failover_queue(&app_type, &provider_ids)
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// 设置故障转移队列项的启用状态
|
||||
#[tauri::command]
|
||||
pub async fn set_failover_item_enabled(
|
||||
state: tauri::State<'_, AppState>,
|
||||
app_type: String,
|
||||
provider_id: String,
|
||||
enabled: bool,
|
||||
) -> Result<(), String> {
|
||||
state
|
||||
.db
|
||||
.set_failover_item_enabled(&app_type, &provider_id, enabled)
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// 获取自动故障转移总开关状态
|
||||
#[tauri::command]
|
||||
pub async fn get_auto_failover_enabled(state: tauri::State<'_, AppState>) -> Result<bool, String> {
|
||||
state
|
||||
.db
|
||||
.get_setting("auto_failover_enabled")
|
||||
.get_setting(&key)
|
||||
.map(|v| v.map(|s| s == "true").unwrap_or(false)) // 默认关闭
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// 设置自动故障转移总开关状态
|
||||
/// 设置指定应用的自动故障转移开关状态
|
||||
///
|
||||
/// 注意:关闭故障转移时不会清除队列,队列内容会保留供下次开启时使用
|
||||
#[tauri::command]
|
||||
pub async fn set_auto_failover_enabled(
|
||||
state: tauri::State<'_, AppState>,
|
||||
app_type: String,
|
||||
enabled: bool,
|
||||
) -> Result<(), String> {
|
||||
state
|
||||
.db
|
||||
.set_setting(
|
||||
"auto_failover_enabled",
|
||||
if enabled { "true" } else { "false" },
|
||||
)
|
||||
.map_err(|e| e.to_string())
|
||||
let key = format!("auto_failover_enabled_{app_type}");
|
||||
let value = if enabled { "true" } else { "false" };
|
||||
|
||||
log::info!(
|
||||
"[Failover] Setting auto_failover_enabled: key='{key}', value='{value}', app_type='{app_type}'"
|
||||
);
|
||||
|
||||
state.db.set_setting(&key, value).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
@@ -4,6 +4,12 @@ use crate::init_status::InitErrorPayload;
|
||||
use tauri::AppHandle;
|
||||
use tauri_plugin_opener::OpenerExt;
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
use std::os::windows::process::CommandExt;
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
const CREATE_NO_WINDOW: u32 = 0x08000000;
|
||||
|
||||
/// 打开外部链接
|
||||
#[tauri::command]
|
||||
pub async fn open_external(app: AppHandle, url: String) -> Result<bool, String> {
|
||||
@@ -142,11 +148,16 @@ fn extract_version(raw: &str) -> String {
|
||||
fn try_get_version(tool: &str) -> (Option<String>, Option<String>) {
|
||||
use std::process::Command;
|
||||
|
||||
let output = if cfg!(target_os = "windows") {
|
||||
#[cfg(target_os = "windows")]
|
||||
let output = {
|
||||
Command::new("cmd")
|
||||
.args(["/C", &format!("{tool} --version")])
|
||||
.creation_flags(CREATE_NO_WINDOW)
|
||||
.output()
|
||||
} else {
|
||||
};
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
let output = {
|
||||
Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(format!("{tool} --version"))
|
||||
@@ -239,10 +250,22 @@ fn scan_cli_version(tool: &str) -> (Option<String>, Option<String>) {
|
||||
let current_path = std::env::var("PATH").unwrap_or_default();
|
||||
let new_path = format!("{}:{}", path.display(), current_path);
|
||||
|
||||
let output = Command::new(&tool_path)
|
||||
.arg("--version")
|
||||
.env("PATH", &new_path)
|
||||
.output();
|
||||
#[cfg(target_os = "windows")]
|
||||
let output = {
|
||||
Command::new(&tool_path)
|
||||
.arg("--version")
|
||||
.env("PATH", &new_path)
|
||||
.creation_flags(CREATE_NO_WINDOW)
|
||||
.output()
|
||||
};
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
let output = {
|
||||
Command::new(&tool_path)
|
||||
.arg("--version")
|
||||
.env("PATH", &new_path)
|
||||
.output()
|
||||
};
|
||||
|
||||
if let Ok(out) = output {
|
||||
let stdout = String::from_utf8_lossy(&out.stdout).trim().to_string();
|
||||
|
||||
@@ -14,14 +14,6 @@ pub async fn start_proxy_server(
|
||||
state.proxy_service.start().await
|
||||
}
|
||||
|
||||
/// 启动代理服务器(带 Live 配置接管)
|
||||
#[tauri::command]
|
||||
pub async fn start_proxy_with_takeover(
|
||||
state: tauri::State<'_, AppState>,
|
||||
) -> Result<ProxyServerInfo, String> {
|
||||
state.proxy_service.start_with_takeover().await
|
||||
}
|
||||
|
||||
/// 停止代理服务器(恢复 Live 配置)
|
||||
#[tauri::command]
|
||||
pub async fn stop_proxy_with_restore(state: tauri::State<'_, AppState>) -> Result<(), String> {
|
||||
@@ -111,8 +103,13 @@ pub async fn get_provider_health(
|
||||
}
|
||||
|
||||
/// 重置熔断器
|
||||
///
|
||||
/// 重置后会检查是否应该切回队列中优先级更高的供应商:
|
||||
/// 1. 检查自动故障转移是否开启
|
||||
/// 2. 如果恢复的供应商在队列中优先级更高(queue_order 更小),则自动切换
|
||||
#[tauri::command]
|
||||
pub async fn reset_circuit_breaker(
|
||||
app_handle: tauri::AppHandle,
|
||||
state: tauri::State<'_, AppState>,
|
||||
provider_id: String,
|
||||
app_type: String,
|
||||
@@ -129,6 +126,75 @@ pub async fn reset_circuit_breaker(
|
||||
.reset_provider_circuit_breaker(&provider_id, &app_type)
|
||||
.await?;
|
||||
|
||||
// 3. 检查是否应该切回优先级更高的供应商
|
||||
let failover_key = format!("auto_failover_enabled_{app_type}");
|
||||
let auto_failover_enabled = match db.get_setting(&failover_key) {
|
||||
Ok(Some(value)) => value == "true",
|
||||
Ok(None) => {
|
||||
log::debug!(
|
||||
"[{app_type}] Failover setting '{failover_key}' not found, defaulting to disabled"
|
||||
);
|
||||
false
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!(
|
||||
"[{app_type}] Failed to read failover setting '{failover_key}': {e}, defaulting to disabled"
|
||||
);
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
if auto_failover_enabled && state.proxy_service.is_running().await {
|
||||
// 获取当前供应商 ID
|
||||
let current_id = db
|
||||
.get_current_provider(&app_type)
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
if let Some(current_id) = current_id {
|
||||
// 获取故障转移队列
|
||||
let queue = db
|
||||
.get_failover_queue(&app_type)
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
// 找到恢复的供应商和当前供应商在队列中的位置(使用 sort_index)
|
||||
let restored_order = queue
|
||||
.iter()
|
||||
.find(|item| item.provider_id == provider_id)
|
||||
.and_then(|item| item.sort_index);
|
||||
|
||||
let current_order = queue
|
||||
.iter()
|
||||
.find(|item| item.provider_id == current_id)
|
||||
.and_then(|item| item.sort_index);
|
||||
|
||||
// 如果恢复的供应商优先级更高(sort_index 更小),则切换
|
||||
if let (Some(restored), Some(current)) = (restored_order, current_order) {
|
||||
if restored < current {
|
||||
log::info!(
|
||||
"[Recovery] 供应商 {provider_id} 已恢复且优先级更高 (P{restored} vs P{current}),自动切换"
|
||||
);
|
||||
|
||||
// 获取供应商名称用于日志和事件
|
||||
let provider_name = db
|
||||
.get_all_providers(&app_type)
|
||||
.ok()
|
||||
.and_then(|providers| providers.get(&provider_id).map(|p| p.name.clone()))
|
||||
.unwrap_or_else(|| provider_id.clone());
|
||||
|
||||
// 创建故障转移切换管理器并执行切换
|
||||
let switch_manager =
|
||||
crate::proxy::failover_switch::FailoverSwitchManager::new(db.clone());
|
||||
if let Err(e) = switch_manager
|
||||
.try_switch(Some(&app_handle), &app_type, &provider_id, &provider_name)
|
||||
.await
|
||||
{
|
||||
log::error!("[Recovery] 自动切换失败: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -52,9 +52,7 @@ pub async fn stream_check_all_providers(
|
||||
}
|
||||
if let Ok(queue) = state.db.get_failover_queue(app_type.as_str()) {
|
||||
for item in queue {
|
||||
if item.enabled {
|
||||
ids.insert(item.provider_id);
|
||||
}
|
||||
ids.insert(item.provider_id);
|
||||
}
|
||||
}
|
||||
Some(ids)
|
||||
|
||||
@@ -1,36 +1,32 @@
|
||||
//! 故障转移队列 DAO
|
||||
//!
|
||||
//! 管理代理模式下的故障转移队列
|
||||
//! 管理代理模式下的故障转移队列(基于 providers 表的 in_failover_queue 字段)
|
||||
|
||||
use crate::database::{lock_conn, Database};
|
||||
use crate::error::AppError;
|
||||
use crate::provider::Provider;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// 故障转移队列条目
|
||||
/// 故障转移队列条目(简化版,用于前端展示)
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct FailoverQueueItem {
|
||||
pub provider_id: String,
|
||||
pub provider_name: String,
|
||||
pub queue_order: i32,
|
||||
pub enabled: bool,
|
||||
pub created_at: i64,
|
||||
pub sort_index: Option<usize>,
|
||||
}
|
||||
|
||||
impl Database {
|
||||
/// 获取故障转移队列(按 queue_order 排序)
|
||||
/// 获取故障转移队列(按 sort_index 排序)
|
||||
pub fn get_failover_queue(&self, app_type: &str) -> Result<Vec<FailoverQueueItem>, AppError> {
|
||||
let conn = lock_conn!(self.conn);
|
||||
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
"SELECT fq.provider_id, p.name, fq.queue_order, fq.enabled, fq.created_at
|
||||
FROM failover_queue fq
|
||||
JOIN providers p ON fq.provider_id = p.id AND fq.app_type = p.app_type
|
||||
WHERE fq.app_type = ?1
|
||||
ORDER BY fq.queue_order ASC",
|
||||
"SELECT id, name, sort_index
|
||||
FROM providers
|
||||
WHERE app_type = ?1 AND in_failover_queue = 1
|
||||
ORDER BY COALESCE(sort_index, 999999), id ASC",
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
|
||||
@@ -39,9 +35,7 @@ impl Database {
|
||||
Ok(FailoverQueueItem {
|
||||
provider_id: row.get(0)?,
|
||||
provider_name: row.get(1)?,
|
||||
queue_order: row.get(2)?,
|
||||
enabled: row.get(3)?,
|
||||
created_at: row.get(4)?,
|
||||
sort_index: row.get(2)?,
|
||||
})
|
||||
})
|
||||
.map_err(|e| AppError::Database(e.to_string()))?
|
||||
@@ -53,43 +47,23 @@ impl Database {
|
||||
|
||||
/// 获取故障转移队列中的供应商(完整 Provider 信息,按顺序)
|
||||
pub fn get_failover_providers(&self, app_type: &str) -> Result<Vec<Provider>, AppError> {
|
||||
let queue = self.get_failover_queue(app_type)?;
|
||||
let all_providers = self.get_all_providers(app_type)?;
|
||||
|
||||
let mut result = Vec::new();
|
||||
for item in queue {
|
||||
if item.enabled {
|
||||
if let Some(provider) = all_providers.get(&item.provider_id) {
|
||||
result.push(provider.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
let result: Vec<Provider> = all_providers
|
||||
.into_values()
|
||||
.filter(|p| p.in_failover_queue)
|
||||
.collect();
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// 添加供应商到故障转移队列末尾
|
||||
/// 添加供应商到故障转移队列
|
||||
pub fn add_to_failover_queue(&self, app_type: &str, provider_id: &str) -> Result<(), AppError> {
|
||||
let conn = lock_conn!(self.conn);
|
||||
|
||||
// 获取当前最大 queue_order
|
||||
let max_order: i32 = conn
|
||||
.query_row(
|
||||
"SELECT COALESCE(MAX(queue_order), 0) FROM failover_queue WHERE app_type = ?1",
|
||||
[app_type],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs() as i64;
|
||||
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO failover_queue (app_type, provider_id, queue_order, enabled, created_at)
|
||||
VALUES (?1, ?2, ?3, 1, ?4)",
|
||||
rusqlite::params![app_type, provider_id, max_order + 1, now],
|
||||
"UPDATE providers SET in_failover_queue = 1 WHERE id = ?1 AND app_type = ?2",
|
||||
rusqlite::params![provider_id, app_type],
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
|
||||
@@ -104,103 +78,21 @@ impl Database {
|
||||
) -> Result<(), AppError> {
|
||||
let conn = lock_conn!(self.conn);
|
||||
|
||||
// 获取被删除项的 queue_order
|
||||
let removed_order: Option<i32> = conn
|
||||
.query_row(
|
||||
"SELECT queue_order FROM failover_queue WHERE app_type = ?1 AND provider_id = ?2",
|
||||
[app_type, provider_id],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.ok();
|
||||
|
||||
// 删除该项
|
||||
// 1. 从队列中移除
|
||||
conn.execute(
|
||||
"DELETE FROM failover_queue WHERE app_type = ?1 AND provider_id = ?2",
|
||||
[app_type, provider_id],
|
||||
"UPDATE providers SET in_failover_queue = 0 WHERE id = ?1 AND app_type = ?2",
|
||||
rusqlite::params![provider_id, app_type],
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
|
||||
// 重新排序后面的项(填补空隙)
|
||||
if let Some(order) = removed_order {
|
||||
conn.execute(
|
||||
"UPDATE failover_queue
|
||||
SET queue_order = queue_order - 1
|
||||
WHERE app_type = ?1 AND queue_order > ?2",
|
||||
rusqlite::params![app_type, order],
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
}
|
||||
// 2. 清除该供应商的健康状态(退出队列后不再需要健康监控)
|
||||
conn.execute(
|
||||
"DELETE FROM provider_health WHERE provider_id = ?1 AND app_type = ?2",
|
||||
rusqlite::params![provider_id, app_type],
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 重新排序故障转移队列
|
||||
/// provider_ids: 按新顺序排列的 provider_id 列表
|
||||
pub fn reorder_failover_queue(
|
||||
&self,
|
||||
app_type: &str,
|
||||
provider_ids: &[String],
|
||||
) -> Result<(), AppError> {
|
||||
let conn = lock_conn!(self.conn);
|
||||
|
||||
// 使用事务确保原子性
|
||||
conn.execute("BEGIN TRANSACTION", [])
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
|
||||
let result = (|| {
|
||||
for (index, provider_id) in provider_ids.iter().enumerate() {
|
||||
conn.execute(
|
||||
"UPDATE failover_queue
|
||||
SET queue_order = ?3
|
||||
WHERE app_type = ?1 AND provider_id = ?2",
|
||||
rusqlite::params![app_type, provider_id, (index + 1) as i32],
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
|
||||
match result {
|
||||
Ok(_) => {
|
||||
conn.execute("COMMIT", [])
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
conn.execute("ROLLBACK", []).ok();
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 设置故障转移队列中供应商的启用状态
|
||||
pub fn set_failover_item_enabled(
|
||||
&self,
|
||||
app_type: &str,
|
||||
provider_id: &str,
|
||||
enabled: bool,
|
||||
) -> Result<(), AppError> {
|
||||
let conn = lock_conn!(self.conn);
|
||||
|
||||
let rows_affected = conn
|
||||
.execute(
|
||||
"UPDATE failover_queue SET enabled = ?3 WHERE app_type = ?1 AND provider_id = ?2",
|
||||
rusqlite::params![app_type, provider_id, enabled],
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
|
||||
if rows_affected == 0 {
|
||||
log::warn!(
|
||||
"set_failover_item_enabled: 未找到匹配记录 app_type={app_type}, provider_id={provider_id}"
|
||||
);
|
||||
return Err(AppError::Database(format!(
|
||||
"未找到故障转移队列项: app_type={app_type}, provider_id={provider_id}"
|
||||
)));
|
||||
}
|
||||
|
||||
log::info!(
|
||||
"set_failover_item_enabled: 已更新 app_type={app_type}, provider_id={provider_id}, enabled={enabled}"
|
||||
);
|
||||
log::info!("已从故障转移队列移除供应商 {provider_id} ({app_type}), 并清除其健康状态");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -209,8 +101,11 @@ impl Database {
|
||||
pub fn clear_failover_queue(&self, app_type: &str) -> Result<(), AppError> {
|
||||
let conn = lock_conn!(self.conn);
|
||||
|
||||
conn.execute("DELETE FROM failover_queue WHERE app_type = ?1", [app_type])
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
conn.execute(
|
||||
"UPDATE providers SET in_failover_queue = 0 WHERE app_type = ?1",
|
||||
[app_type],
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -223,15 +118,15 @@ impl Database {
|
||||
) -> Result<bool, AppError> {
|
||||
let conn = lock_conn!(self.conn);
|
||||
|
||||
let count: i32 = conn
|
||||
let in_queue: bool = conn
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM failover_queue WHERE app_type = ?1 AND provider_id = ?2",
|
||||
[app_type, provider_id],
|
||||
"SELECT in_failover_queue FROM providers WHERE id = ?1 AND app_type = ?2",
|
||||
rusqlite::params![provider_id, app_type],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
.unwrap_or(false);
|
||||
|
||||
Ok(count > 0)
|
||||
Ok(in_queue)
|
||||
}
|
||||
|
||||
/// 获取可添加到故障转移队列的供应商(不在队列中的)
|
||||
@@ -240,14 +135,10 @@ impl Database {
|
||||
app_type: &str,
|
||||
) -> Result<Vec<Provider>, AppError> {
|
||||
let all_providers = self.get_all_providers(app_type)?;
|
||||
let queue = self.get_failover_queue(app_type)?;
|
||||
|
||||
let queue_ids: std::collections::HashSet<_> =
|
||||
queue.iter().map(|item| &item.provider_id).collect();
|
||||
|
||||
let available: Vec<Provider> = all_providers
|
||||
.into_values()
|
||||
.filter(|p| !queue_ids.contains(&p.id))
|
||||
.filter(|p| !p.in_failover_queue)
|
||||
.collect();
|
||||
|
||||
Ok(available)
|
||||
|
||||
@@ -17,7 +17,7 @@ impl Database {
|
||||
) -> Result<IndexMap<String, Provider>, AppError> {
|
||||
let conn = lock_conn!(self.conn);
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, name, settings_config, website_url, category, created_at, sort_index, notes, icon, icon_color, meta
|
||||
"SELECT id, name, settings_config, website_url, category, created_at, sort_index, notes, icon, icon_color, meta, in_failover_queue
|
||||
FROM providers WHERE app_type = ?1
|
||||
ORDER BY COALESCE(sort_index, 999999), created_at ASC, id ASC"
|
||||
).map_err(|e| AppError::Database(e.to_string()))?;
|
||||
@@ -35,6 +35,7 @@ impl Database {
|
||||
let icon: Option<String> = row.get(8)?;
|
||||
let icon_color: Option<String> = row.get(9)?;
|
||||
let meta_str: String = row.get(10)?;
|
||||
let in_failover_queue: bool = row.get(11)?;
|
||||
|
||||
let settings_config =
|
||||
serde_json::from_str(&settings_config_str).unwrap_or(serde_json::Value::Null);
|
||||
@@ -54,6 +55,7 @@ impl Database {
|
||||
meta: Some(meta),
|
||||
icon,
|
||||
icon_color,
|
||||
in_failover_queue,
|
||||
},
|
||||
))
|
||||
})
|
||||
@@ -129,7 +131,7 @@ impl Database {
|
||||
) -> Result<Option<Provider>, AppError> {
|
||||
let conn = lock_conn!(self.conn);
|
||||
let result = conn.query_row(
|
||||
"SELECT name, settings_config, website_url, category, created_at, sort_index, notes, icon, icon_color, meta
|
||||
"SELECT name, settings_config, website_url, category, created_at, sort_index, notes, icon, icon_color, meta, in_failover_queue
|
||||
FROM providers WHERE id = ?1 AND app_type = ?2",
|
||||
params![id, app_type],
|
||||
|row| {
|
||||
@@ -143,6 +145,7 @@ impl Database {
|
||||
let icon: Option<String> = row.get(7)?;
|
||||
let icon_color: Option<String> = row.get(8)?;
|
||||
let meta_str: String = row.get(9)?;
|
||||
let in_failover_queue: bool = row.get(10)?;
|
||||
|
||||
let settings_config = serde_json::from_str(&settings_config_str).unwrap_or(serde_json::Value::Null);
|
||||
let meta: ProviderMeta = serde_json::from_str(&meta_str).unwrap_or_default();
|
||||
@@ -159,6 +162,7 @@ impl Database {
|
||||
meta: Some(meta),
|
||||
icon,
|
||||
icon_color,
|
||||
in_failover_queue,
|
||||
})
|
||||
},
|
||||
);
|
||||
@@ -184,17 +188,18 @@ impl Database {
|
||||
let mut meta_clone = provider.meta.clone().unwrap_or_default();
|
||||
let endpoints = std::mem::take(&mut meta_clone.custom_endpoints);
|
||||
|
||||
// 检查是否存在(用于判断新增/更新,以及保留 is_current)
|
||||
let existing: Option<bool> = tx
|
||||
// 检查是否存在(用于判断新增/更新,以及保留 is_current 和 in_failover_queue)
|
||||
let existing: Option<(bool, bool)> = tx
|
||||
.query_row(
|
||||
"SELECT is_current FROM providers WHERE id = ?1 AND app_type = ?2",
|
||||
"SELECT is_current, in_failover_queue FROM providers WHERE id = ?1 AND app_type = ?2",
|
||||
params![provider.id, app_type],
|
||||
|row| row.get(0),
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.ok();
|
||||
|
||||
let is_update = existing.is_some();
|
||||
let is_current = existing.unwrap_or(false);
|
||||
let (is_current, in_failover_queue) =
|
||||
existing.unwrap_or((false, provider.in_failover_queue));
|
||||
|
||||
if is_update {
|
||||
// 更新模式:使用 UPDATE 避免触发 ON DELETE CASCADE
|
||||
@@ -210,8 +215,9 @@ impl Database {
|
||||
icon = ?8,
|
||||
icon_color = ?9,
|
||||
meta = ?10,
|
||||
is_current = ?11
|
||||
WHERE id = ?12 AND app_type = ?13",
|
||||
is_current = ?11,
|
||||
in_failover_queue = ?12
|
||||
WHERE id = ?13 AND app_type = ?14",
|
||||
params![
|
||||
provider.name,
|
||||
serde_json::to_string(&provider.settings_config).unwrap(),
|
||||
@@ -224,6 +230,7 @@ impl Database {
|
||||
provider.icon_color,
|
||||
serde_json::to_string(&meta_clone).unwrap(),
|
||||
is_current,
|
||||
in_failover_queue,
|
||||
provider.id,
|
||||
app_type,
|
||||
],
|
||||
@@ -234,8 +241,8 @@ impl Database {
|
||||
tx.execute(
|
||||
"INSERT INTO providers (
|
||||
id, app_type, name, settings_config, website_url, category,
|
||||
created_at, sort_index, notes, icon, icon_color, meta, is_current
|
||||
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)",
|
||||
created_at, sort_index, notes, icon, icon_color, meta, is_current, in_failover_queue
|
||||
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)",
|
||||
params![
|
||||
provider.id,
|
||||
app_type,
|
||||
@@ -250,6 +257,7 @@ impl Database {
|
||||
provider.icon_color,
|
||||
serde_json::to_string(&meta_clone).unwrap(),
|
||||
is_current,
|
||||
in_failover_queue,
|
||||
],
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
|
||||
@@ -102,28 +102,45 @@ impl Database {
|
||||
provider_id: &str,
|
||||
app_type: &str,
|
||||
) -> Result<ProviderHealth, AppError> {
|
||||
let conn = lock_conn!(self.conn);
|
||||
let result = {
|
||||
let conn = lock_conn!(self.conn);
|
||||
|
||||
conn.query_row(
|
||||
"SELECT provider_id, app_type, is_healthy, consecutive_failures,
|
||||
last_success_at, last_failure_at, last_error, updated_at
|
||||
FROM provider_health
|
||||
WHERE provider_id = ?1 AND app_type = ?2",
|
||||
rusqlite::params![provider_id, app_type],
|
||||
|row| {
|
||||
Ok(ProviderHealth {
|
||||
provider_id: row.get(0)?,
|
||||
app_type: row.get(1)?,
|
||||
is_healthy: row.get::<_, i64>(2)? != 0,
|
||||
consecutive_failures: row.get::<_, i64>(3)? as u32,
|
||||
last_success_at: row.get(4)?,
|
||||
last_failure_at: row.get(5)?,
|
||||
last_error: row.get(6)?,
|
||||
updated_at: row.get(7)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))
|
||||
conn.query_row(
|
||||
"SELECT provider_id, app_type, is_healthy, consecutive_failures,
|
||||
last_success_at, last_failure_at, last_error, updated_at
|
||||
FROM provider_health
|
||||
WHERE provider_id = ?1 AND app_type = ?2",
|
||||
rusqlite::params![provider_id, app_type],
|
||||
|row| {
|
||||
Ok(ProviderHealth {
|
||||
provider_id: row.get(0)?,
|
||||
app_type: row.get(1)?,
|
||||
is_healthy: row.get::<_, i64>(2)? != 0,
|
||||
consecutive_failures: row.get::<_, i64>(3)? as u32,
|
||||
last_success_at: row.get(4)?,
|
||||
last_failure_at: row.get(5)?,
|
||||
last_error: row.get(6)?,
|
||||
updated_at: row.get(7)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(health) => Ok(health),
|
||||
// 缺少记录时视为健康(关闭后清空状态,再次打开时默认正常)
|
||||
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(ProviderHealth {
|
||||
provider_id: provider_id.to_string(),
|
||||
app_type: app_type.to_string(),
|
||||
is_healthy: true,
|
||||
consecutive_failures: 0,
|
||||
last_success_at: None,
|
||||
last_failure_at: None,
|
||||
last_error: None,
|
||||
updated_at: chrono::Utc::now().to_rfc3339(),
|
||||
}),
|
||||
Err(e) => Err(AppError::Database(e.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
/// 更新Provider健康状态
|
||||
@@ -228,6 +245,20 @@ impl Database {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 清空指定应用的健康状态(关闭单个代理时使用)
|
||||
pub async fn clear_provider_health_for_app(&self, app_type: &str) -> Result<(), AppError> {
|
||||
let conn = lock_conn!(self.conn);
|
||||
|
||||
conn.execute(
|
||||
"DELETE FROM provider_health WHERE app_type = ?1",
|
||||
[app_type],
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
|
||||
log::debug!("Cleared provider health records for app {app_type}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 清空所有Provider健康状态(代理停止时调用)
|
||||
pub async fn clear_all_provider_health(&self) -> Result<(), AppError> {
|
||||
let conn = lock_conn!(self.conn);
|
||||
|
||||
@@ -31,6 +31,7 @@ impl Database {
|
||||
icon_color TEXT,
|
||||
meta TEXT NOT NULL DEFAULT '{}',
|
||||
is_current BOOLEAN NOT NULL DEFAULT 0,
|
||||
in_failover_queue BOOLEAN NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (id, app_type)
|
||||
)",
|
||||
[],
|
||||
@@ -312,29 +313,24 @@ impl Database {
|
||||
[],
|
||||
);
|
||||
|
||||
// 14. Failover Queue 表 (故障转移队列)
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS failover_queue (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
app_type TEXT NOT NULL,
|
||||
provider_id TEXT NOT NULL,
|
||||
queue_order INTEGER NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
created_at INTEGER NOT NULL,
|
||||
UNIQUE (app_type, provider_id),
|
||||
FOREIGN KEY (provider_id, app_type) REFERENCES providers(id, app_type) ON DELETE CASCADE
|
||||
)",
|
||||
[],
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
// 确保 in_failover_queue 列存在(对于已存在的 v2 数据库)
|
||||
Self::add_column_if_missing(
|
||||
conn,
|
||||
"providers",
|
||||
"in_failover_queue",
|
||||
"BOOLEAN NOT NULL DEFAULT 0",
|
||||
)?;
|
||||
|
||||
// 为故障转移队列创建索引
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_failover_queue_order
|
||||
ON failover_queue(app_type, queue_order)",
|
||||
// 删除旧的 failover_queue 表(如果存在)
|
||||
let _ = conn.execute("DROP INDEX IF EXISTS idx_failover_queue_order", []);
|
||||
let _ = conn.execute("DROP TABLE IF EXISTS failover_queue", []);
|
||||
|
||||
// 为故障转移队列创建索引(基于 providers 表)
|
||||
let _ = conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_providers_failover
|
||||
ON providers(app_type, in_failover_queue, sort_index)",
|
||||
[],
|
||||
)
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -472,6 +468,26 @@ impl Database {
|
||||
Self::add_column_if_missing(conn, "providers", "limit_daily_usd", "TEXT")?;
|
||||
Self::add_column_if_missing(conn, "providers", "limit_monthly_usd", "TEXT")?;
|
||||
Self::add_column_if_missing(conn, "providers", "provider_type", "TEXT")?;
|
||||
Self::add_column_if_missing(
|
||||
conn,
|
||||
"providers",
|
||||
"in_failover_queue",
|
||||
"BOOLEAN NOT NULL DEFAULT 0",
|
||||
)?;
|
||||
|
||||
// 删除旧的 failover_queue 表(如果存在)
|
||||
conn.execute("DROP INDEX IF EXISTS idx_failover_queue_order", [])
|
||||
.map_err(|e| AppError::Database(format!("删除 failover_queue 索引失败: {e}")))?;
|
||||
conn.execute("DROP TABLE IF EXISTS failover_queue", [])
|
||||
.map_err(|e| AppError::Database(format!("删除 failover_queue 表失败: {e}")))?;
|
||||
|
||||
// 创建 failover 索引
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_providers_failover
|
||||
ON providers(app_type, in_failover_queue, sort_index)",
|
||||
[],
|
||||
)
|
||||
.map_err(|e| AppError::Database(format!("创建 failover 索引失败: {e}")))?;
|
||||
|
||||
// proxy_request_logs 表(包含所有字段)
|
||||
conn.execute(
|
||||
|
||||
@@ -245,6 +245,7 @@ fn dry_run_validates_schema_compatibility() {
|
||||
meta: None,
|
||||
icon: None,
|
||||
icon_color: None,
|
||||
in_failover_queue: false,
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -132,6 +132,7 @@ pub(crate) fn build_provider_from_request(
|
||||
meta,
|
||||
icon: request.icon.clone(),
|
||||
icon_color: None,
|
||||
in_failover_queue: false,
|
||||
};
|
||||
|
||||
Ok(provider)
|
||||
|
||||
@@ -650,7 +650,6 @@ pub fn run() {
|
||||
commands::get_auto_launch_status,
|
||||
// Proxy server management
|
||||
commands::start_proxy_server,
|
||||
commands::start_proxy_with_takeover,
|
||||
commands::stop_proxy_with_restore,
|
||||
commands::get_proxy_takeover_status,
|
||||
commands::set_proxy_takeover_for_app,
|
||||
@@ -671,8 +670,6 @@ pub fn run() {
|
||||
commands::get_available_providers_for_failover,
|
||||
commands::add_to_failover_queue,
|
||||
commands::remove_from_failover_queue,
|
||||
commands::reorder_failover_queue,
|
||||
commands::set_failover_item_enabled,
|
||||
commands::get_auto_failover_enabled,
|
||||
commands::set_auto_failover_enabled,
|
||||
// Usage statistics
|
||||
|
||||
@@ -36,6 +36,10 @@ pub struct Provider {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[serde(rename = "iconColor")]
|
||||
pub icon_color: Option<String>,
|
||||
/// 是否加入故障转移队列
|
||||
#[serde(default)]
|
||||
#[serde(rename = "inFailoverQueue")]
|
||||
pub in_failover_queue: bool,
|
||||
}
|
||||
|
||||
impl Provider {
|
||||
@@ -58,6 +62,7 @@ impl Provider {
|
||||
meta: None,
|
||||
icon: None,
|
||||
icon_color: None,
|
||||
in_failover_queue: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,16 @@ use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
pub struct ForwardResult {
|
||||
pub response: Response,
|
||||
pub provider: Provider,
|
||||
}
|
||||
|
||||
pub struct ForwardError {
|
||||
pub error: ProxyError,
|
||||
pub provider: Option<Provider>,
|
||||
}
|
||||
|
||||
pub struct RequestForwarder {
|
||||
client: Client,
|
||||
/// 共享的 ProviderRouter(持有熔断器状态)
|
||||
@@ -134,13 +144,16 @@ impl RequestForwarder {
|
||||
body: Value,
|
||||
headers: axum::http::HeaderMap,
|
||||
providers: Vec<Provider>,
|
||||
) -> Result<Response, ProxyError> {
|
||||
) -> Result<ForwardResult, ForwardError> {
|
||||
// 获取适配器
|
||||
let adapter = get_adapter(app_type);
|
||||
let app_type_str = app_type.as_str();
|
||||
|
||||
if providers.is_empty() {
|
||||
return Err(ProxyError::NoAvailableProvider);
|
||||
return Err(ForwardError {
|
||||
error: ProxyError::NoAvailableProvider,
|
||||
provider: None,
|
||||
});
|
||||
}
|
||||
|
||||
log::info!(
|
||||
@@ -150,6 +163,7 @@ impl RequestForwarder {
|
||||
);
|
||||
|
||||
let mut last_error = None;
|
||||
let mut last_provider = None;
|
||||
let mut attempted_providers = 0usize;
|
||||
|
||||
// 依次尝试每个供应商
|
||||
@@ -269,7 +283,10 @@ impl RequestForwarder {
|
||||
latency
|
||||
);
|
||||
|
||||
return Ok(response);
|
||||
return Ok(ForwardResult {
|
||||
response,
|
||||
provider: provider.clone(),
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
let latency = start.elapsed().as_millis() as u64;
|
||||
@@ -310,6 +327,7 @@ impl RequestForwarder {
|
||||
);
|
||||
|
||||
last_error = Some(e);
|
||||
last_provider = Some(provider.clone());
|
||||
// 继续尝试下一个供应商
|
||||
continue;
|
||||
}
|
||||
@@ -331,7 +349,10 @@ impl RequestForwarder {
|
||||
provider.name,
|
||||
e
|
||||
);
|
||||
return Err(e);
|
||||
return Err(ForwardError {
|
||||
error: e,
|
||||
provider: Some(provider.clone()),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -349,7 +370,10 @@ impl RequestForwarder {
|
||||
(status.success_requests as f32 / status.total_requests as f32) * 100.0;
|
||||
}
|
||||
}
|
||||
return Err(ProxyError::NoAvailableProvider);
|
||||
return Err(ForwardError {
|
||||
error: ProxyError::NoAvailableProvider,
|
||||
provider: None,
|
||||
});
|
||||
}
|
||||
|
||||
// 所有供应商都失败了
|
||||
@@ -369,7 +393,10 @@ impl RequestForwarder {
|
||||
providers.len()
|
||||
);
|
||||
|
||||
Err(last_error.unwrap_or(ProxyError::MaxRetriesExceeded))
|
||||
Err(ForwardError {
|
||||
error: last_error.unwrap_or(ProxyError::MaxRetriesExceeded),
|
||||
provider: last_provider,
|
||||
})
|
||||
}
|
||||
|
||||
/// 转发单个请求(使用适配器)
|
||||
|
||||
@@ -61,27 +61,16 @@ pub async fn handle_messages(
|
||||
headers: axum::http::HeaderMap,
|
||||
Json(body): Json<Value>,
|
||||
) -> Result<axum::response::Response, ProxyError> {
|
||||
let ctx = RequestContext::new(&state, &body, AppType::Claude, "Claude", "claude").await?;
|
||||
|
||||
// 检查是否需要格式转换(OpenRouter 等中转服务)
|
||||
let adapter = get_adapter(&AppType::Claude);
|
||||
let needs_transform = adapter.needs_transform(&ctx.provider);
|
||||
let mut ctx = RequestContext::new(&state, &body, AppType::Claude, "Claude", "claude").await?;
|
||||
|
||||
let is_stream = body
|
||||
.get("stream")
|
||||
.and_then(|s| s.as_bool())
|
||||
.unwrap_or(false);
|
||||
|
||||
log::info!(
|
||||
"[Claude] Provider: {}, needs_transform: {}, is_stream: {}",
|
||||
ctx.provider.name,
|
||||
needs_transform,
|
||||
is_stream
|
||||
);
|
||||
|
||||
// 转发请求
|
||||
let forwarder = ctx.create_forwarder(&state);
|
||||
let response = match forwarder
|
||||
let result = match forwarder
|
||||
.forward_with_retry(
|
||||
&AppType::Claude,
|
||||
"/v1/messages",
|
||||
@@ -91,13 +80,30 @@ pub async fn handle_messages(
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(resp) => resp,
|
||||
Err(e) => {
|
||||
log_forward_error(&state, &ctx, is_stream, &e);
|
||||
return Err(e);
|
||||
Ok(result) => result,
|
||||
Err(mut err) => {
|
||||
if let Some(provider) = err.provider.take() {
|
||||
ctx.provider = provider;
|
||||
}
|
||||
log_forward_error(&state, &ctx, is_stream, &err.error);
|
||||
return Err(err.error);
|
||||
}
|
||||
};
|
||||
|
||||
ctx.provider = result.provider;
|
||||
let response = result.response;
|
||||
|
||||
// 检查是否需要格式转换(OpenRouter 等中转服务)
|
||||
let adapter = get_adapter(&AppType::Claude);
|
||||
let needs_transform = adapter.needs_transform(&ctx.provider);
|
||||
|
||||
log::info!(
|
||||
"[Claude] Provider: {}, needs_transform: {}, is_stream: {}",
|
||||
ctx.provider.name,
|
||||
needs_transform,
|
||||
is_stream
|
||||
);
|
||||
|
||||
let status = response.status();
|
||||
log::info!("[Claude] 上游响应状态: {status}");
|
||||
|
||||
@@ -295,7 +301,7 @@ pub async fn handle_chat_completions(
|
||||
) -> Result<axum::response::Response, ProxyError> {
|
||||
log::info!("[Codex] ====== /v1/chat/completions 请求开始 ======");
|
||||
|
||||
let ctx = RequestContext::new(&state, &body, AppType::Codex, "Codex", "codex").await?;
|
||||
let mut ctx = RequestContext::new(&state, &body, AppType::Codex, "Codex", "codex").await?;
|
||||
|
||||
let is_stream = body
|
||||
.get("stream")
|
||||
@@ -309,7 +315,7 @@ pub async fn handle_chat_completions(
|
||||
);
|
||||
|
||||
let forwarder = ctx.create_forwarder(&state);
|
||||
let response = match forwarder
|
||||
let result = match forwarder
|
||||
.forward_with_retry(
|
||||
&AppType::Codex,
|
||||
"/v1/chat/completions",
|
||||
@@ -319,13 +325,19 @@ pub async fn handle_chat_completions(
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(resp) => resp,
|
||||
Err(e) => {
|
||||
log_forward_error(&state, &ctx, is_stream, &e);
|
||||
return Err(e);
|
||||
Ok(result) => result,
|
||||
Err(mut err) => {
|
||||
if let Some(provider) = err.provider.take() {
|
||||
ctx.provider = provider;
|
||||
}
|
||||
log_forward_error(&state, &ctx, is_stream, &err.error);
|
||||
return Err(err.error);
|
||||
}
|
||||
};
|
||||
|
||||
ctx.provider = result.provider;
|
||||
let response = result.response;
|
||||
|
||||
log::info!("[Codex] 上游响应状态: {}", response.status());
|
||||
|
||||
process_response(response, &ctx, &state, &OPENAI_PARSER_CONFIG).await
|
||||
@@ -337,7 +349,7 @@ pub async fn handle_responses(
|
||||
headers: axum::http::HeaderMap,
|
||||
Json(body): Json<Value>,
|
||||
) -> Result<axum::response::Response, ProxyError> {
|
||||
let ctx = RequestContext::new(&state, &body, AppType::Codex, "Codex", "codex").await?;
|
||||
let mut ctx = RequestContext::new(&state, &body, AppType::Codex, "Codex", "codex").await?;
|
||||
|
||||
let is_stream = body
|
||||
.get("stream")
|
||||
@@ -345,7 +357,7 @@ pub async fn handle_responses(
|
||||
.unwrap_or(false);
|
||||
|
||||
let forwarder = ctx.create_forwarder(&state);
|
||||
let response = match forwarder
|
||||
let result = match forwarder
|
||||
.forward_with_retry(
|
||||
&AppType::Codex,
|
||||
"/v1/responses",
|
||||
@@ -355,13 +367,19 @@ pub async fn handle_responses(
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(resp) => resp,
|
||||
Err(e) => {
|
||||
log_forward_error(&state, &ctx, is_stream, &e);
|
||||
return Err(e);
|
||||
Ok(result) => result,
|
||||
Err(mut err) => {
|
||||
if let Some(provider) = err.provider.take() {
|
||||
ctx.provider = provider;
|
||||
}
|
||||
log_forward_error(&state, &ctx, is_stream, &err.error);
|
||||
return Err(err.error);
|
||||
}
|
||||
};
|
||||
|
||||
ctx.provider = result.provider;
|
||||
let response = result.response;
|
||||
|
||||
log::info!("[Codex] 上游响应状态: {}", response.status());
|
||||
|
||||
process_response(response, &ctx, &state, &CODEX_PARSER_CONFIG).await
|
||||
@@ -379,7 +397,7 @@ pub async fn handle_gemini(
|
||||
Json(body): Json<Value>,
|
||||
) -> Result<axum::response::Response, ProxyError> {
|
||||
// Gemini 的模型名称在 URI 中
|
||||
let ctx = RequestContext::new(&state, &body, AppType::Gemini, "Gemini", "gemini")
|
||||
let mut ctx = RequestContext::new(&state, &body, AppType::Gemini, "Gemini", "gemini")
|
||||
.await?
|
||||
.with_model_from_uri(&uri);
|
||||
|
||||
@@ -397,7 +415,7 @@ pub async fn handle_gemini(
|
||||
.unwrap_or(false);
|
||||
|
||||
let forwarder = ctx.create_forwarder(&state);
|
||||
let response = match forwarder
|
||||
let result = match forwarder
|
||||
.forward_with_retry(
|
||||
&AppType::Gemini,
|
||||
endpoint,
|
||||
@@ -407,13 +425,19 @@ pub async fn handle_gemini(
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(resp) => resp,
|
||||
Err(e) => {
|
||||
log_forward_error(&state, &ctx, is_stream, &e);
|
||||
return Err(e);
|
||||
Ok(result) => result,
|
||||
Err(mut err) => {
|
||||
if let Some(provider) = err.provider.take() {
|
||||
ctx.provider = provider;
|
||||
}
|
||||
log_forward_error(&state, &ctx, is_stream, &err.error);
|
||||
return Err(err.error);
|
||||
}
|
||||
};
|
||||
|
||||
ctx.provider = result.provider;
|
||||
let response = result.response;
|
||||
|
||||
log::info!("[Gemini] 上游响应状态: {}", response.status());
|
||||
|
||||
process_response(response, &ctx, &state, &GEMINI_PARSER_CONFIG).await
|
||||
|
||||
@@ -30,85 +30,92 @@ impl ProviderRouter {
|
||||
/// 选择可用的供应商(支持故障转移)
|
||||
///
|
||||
/// 返回按优先级排序的可用供应商列表:
|
||||
/// 1. 当前供应商(is_current=true)始终第一位
|
||||
/// 2. 故障转移队列中的其他供应商(按 queue_order 排序)- 仅当自动故障转移开关开启时
|
||||
/// 3. 只返回熔断器未打开的供应商
|
||||
/// - 故障转移关闭时:仅返回当前供应商
|
||||
/// - 故障转移开启时:完全按照故障转移队列顺序返回,忽略当前供应商设置
|
||||
pub async fn select_providers(&self, app_type: &str) -> Result<Vec<Provider>, AppError> {
|
||||
let mut result = Vec::new();
|
||||
let all_providers = self.db.get_all_providers(app_type)?;
|
||||
|
||||
// 检查自动故障转移总开关是否开启
|
||||
let auto_failover_enabled = self
|
||||
.db
|
||||
.get_setting("auto_failover_enabled")
|
||||
.map(|v| v.map(|s| s == "true").unwrap_or(false)) // 默认关闭
|
||||
.unwrap_or(false);
|
||||
// 检查该应用的自动故障转移开关是否开启
|
||||
let failover_key = format!("auto_failover_enabled_{app_type}");
|
||||
let auto_failover_enabled = match self.db.get_setting(&failover_key) {
|
||||
Ok(Some(value)) => {
|
||||
let enabled = value == "true";
|
||||
log::info!(
|
||||
"[{app_type}] Failover setting '{failover_key}' = '{value}', enabled: {enabled}"
|
||||
);
|
||||
enabled
|
||||
}
|
||||
Ok(None) => {
|
||||
log::warn!(
|
||||
"[{app_type}] Failover setting '{failover_key}' not found in database, defaulting to disabled"
|
||||
);
|
||||
false
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!(
|
||||
"[{app_type}] Failed to read failover setting '{failover_key}': {e}, defaulting to disabled"
|
||||
);
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
// 1. 当前供应商始终第一位
|
||||
if let Some(current_id) = self.db.get_current_provider(app_type)? {
|
||||
if let Some(current) = all_providers.get(¤t_id) {
|
||||
let circuit_key = format!("{}:{}", app_type, current.id);
|
||||
if auto_failover_enabled {
|
||||
// 故障转移开启:使用 in_failover_queue 标记的供应商,按 sort_index 排序
|
||||
let failover_providers = self.db.get_failover_providers(app_type)?;
|
||||
log::info!(
|
||||
"[{}] Failover enabled, using queue order ({} items)",
|
||||
app_type,
|
||||
failover_providers.len()
|
||||
);
|
||||
|
||||
for provider in failover_providers {
|
||||
// 检查熔断器状态
|
||||
let circuit_key = format!("{}:{}", app_type, provider.id);
|
||||
let breaker = self.get_or_create_circuit_breaker(&circuit_key).await;
|
||||
|
||||
if breaker.is_available().await {
|
||||
log::info!(
|
||||
"[{}] Current provider available: {} ({})",
|
||||
"[{}] Queue provider available: {} ({}) at sort_index {:?}",
|
||||
app_type,
|
||||
current.name,
|
||||
current.id
|
||||
provider.name,
|
||||
provider.id,
|
||||
provider.sort_index
|
||||
);
|
||||
result.push(current.clone());
|
||||
result.push(provider);
|
||||
} else {
|
||||
log::warn!(
|
||||
"[{}] Current provider {} circuit breaker open, checking failover queue",
|
||||
log::debug!(
|
||||
"[{}] Queue provider {} circuit breaker open, skipping",
|
||||
app_type,
|
||||
current.name
|
||||
provider.name
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 故障转移关闭:仅使用当前供应商
|
||||
log::info!("[{app_type}] Failover disabled, using current provider only");
|
||||
|
||||
// 2. 获取故障转移队列中的供应商(仅当自动故障转移开关开启时)
|
||||
if auto_failover_enabled {
|
||||
let queue = self.db.get_failover_queue(app_type)?;
|
||||
|
||||
for item in queue {
|
||||
// 跳过已添加的当前供应商
|
||||
if result.iter().any(|p| p.id == item.provider_id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 跳过禁用的队列项
|
||||
if !item.enabled {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 获取供应商信息
|
||||
if let Some(provider) = all_providers.get(&item.provider_id) {
|
||||
// 检查熔断器状态
|
||||
let circuit_key = format!("{}:{}", app_type, provider.id);
|
||||
if let Some(current_id) = self.db.get_current_provider(app_type)? {
|
||||
if let Some(current) = self.db.get_provider_by_id(¤t_id, app_type)? {
|
||||
let circuit_key = format!("{}:{}", app_type, current.id);
|
||||
let breaker = self.get_or_create_circuit_breaker(&circuit_key).await;
|
||||
|
||||
if breaker.is_available().await {
|
||||
log::info!(
|
||||
"[{}] Failover provider available: {} ({}) at queue position {}",
|
||||
"[{}] Current provider available: {} ({})",
|
||||
app_type,
|
||||
provider.name,
|
||||
provider.id,
|
||||
item.queue_order
|
||||
current.name,
|
||||
current.id
|
||||
);
|
||||
result.push(provider.clone());
|
||||
result.push(current);
|
||||
} else {
|
||||
log::debug!(
|
||||
"[{}] Failover provider {} circuit breaker open, skipping",
|
||||
log::warn!(
|
||||
"[{}] Current provider {} circuit breaker open",
|
||||
app_type,
|
||||
provider.name
|
||||
current.name
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log::info!("[{app_type}] Auto-failover disabled, using only current provider");
|
||||
}
|
||||
|
||||
if result.is_empty() {
|
||||
@@ -118,7 +125,7 @@ impl ProviderRouter {
|
||||
}
|
||||
|
||||
log::info!(
|
||||
"[{}] Failover chain: {} provider(s) available",
|
||||
"[{}] Provider chain: {} provider(s) available",
|
||||
app_type,
|
||||
result.len()
|
||||
);
|
||||
@@ -275,25 +282,14 @@ mod tests {
|
||||
let db = Arc::new(Database::memory().unwrap());
|
||||
let router = ProviderRouter::new(db);
|
||||
|
||||
// 测试创建熔断器
|
||||
let breaker = router.get_or_create_circuit_breaker("claude:test").await;
|
||||
assert!(breaker.allow_request().await.allowed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn select_providers_does_not_consume_half_open_permit() {
|
||||
async fn test_failover_disabled_uses_current_provider() {
|
||||
let db = Arc::new(Database::memory().unwrap());
|
||||
|
||||
// 配置:让熔断器 Open 后立刻进入 HalfOpen(timeout_seconds=0),并用 1 次失败就打开熔断器
|
||||
db.update_circuit_breaker_config(&CircuitBreakerConfig {
|
||||
failure_threshold: 1,
|
||||
timeout_seconds: 0,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// 准备 2 个 Provider:A(当前)+ B(队列)
|
||||
let provider_a =
|
||||
Provider::with_id("a".to_string(), "Provider A".to_string(), json!({}), None);
|
||||
let provider_b =
|
||||
@@ -304,19 +300,78 @@ mod tests {
|
||||
db.set_current_provider("claude", "a").unwrap();
|
||||
db.add_to_failover_queue("claude", "b").unwrap();
|
||||
|
||||
let router = ProviderRouter::new(db.clone());
|
||||
let providers = router.select_providers("claude").await.unwrap();
|
||||
|
||||
assert_eq!(providers.len(), 1);
|
||||
assert_eq!(providers[0].id, "a");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_failover_enabled_uses_queue_order() {
|
||||
let db = Arc::new(Database::memory().unwrap());
|
||||
|
||||
// 设置 sort_index 来控制顺序:b=1, a=2
|
||||
let mut provider_a =
|
||||
Provider::with_id("a".to_string(), "Provider A".to_string(), json!({}), None);
|
||||
provider_a.sort_index = Some(2);
|
||||
let mut provider_b =
|
||||
Provider::with_id("b".to_string(), "Provider B".to_string(), json!({}), None);
|
||||
provider_b.sort_index = Some(1);
|
||||
|
||||
db.save_provider("claude", &provider_a).unwrap();
|
||||
db.save_provider("claude", &provider_b).unwrap();
|
||||
db.set_current_provider("claude", "a").unwrap();
|
||||
|
||||
db.add_to_failover_queue("claude", "b").unwrap();
|
||||
db.add_to_failover_queue("claude", "a").unwrap();
|
||||
db.set_setting("auto_failover_enabled_claude", "true")
|
||||
.unwrap();
|
||||
|
||||
let router = ProviderRouter::new(db.clone());
|
||||
let providers = router.select_providers("claude").await.unwrap();
|
||||
|
||||
assert_eq!(providers.len(), 2);
|
||||
// 按 sort_index 排序:b(1) 在前,a(2) 在后
|
||||
assert_eq!(providers[0].id, "b");
|
||||
assert_eq!(providers[1].id, "a");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_select_providers_does_not_consume_half_open_permit() {
|
||||
let db = Arc::new(Database::memory().unwrap());
|
||||
|
||||
db.update_circuit_breaker_config(&CircuitBreakerConfig {
|
||||
failure_threshold: 1,
|
||||
timeout_seconds: 0,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let provider_a =
|
||||
Provider::with_id("a".to_string(), "Provider A".to_string(), json!({}), None);
|
||||
let provider_b =
|
||||
Provider::with_id("b".to_string(), "Provider B".to_string(), json!({}), None);
|
||||
|
||||
db.save_provider("claude", &provider_a).unwrap();
|
||||
db.save_provider("claude", &provider_b).unwrap();
|
||||
|
||||
db.add_to_failover_queue("claude", "a").unwrap();
|
||||
db.add_to_failover_queue("claude", "b").unwrap();
|
||||
db.set_setting("auto_failover_enabled_claude", "true")
|
||||
.unwrap();
|
||||
|
||||
let router = ProviderRouter::new(db.clone());
|
||||
|
||||
// 让 B 进入 Open 状态(failure_threshold=1)
|
||||
router
|
||||
.record_result("b", "claude", false, false, Some("fail".to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// select_providers 只做“可用性判断”,不应占用 HalfOpen 探测名额
|
||||
let providers = router.select_providers("claude").await.unwrap();
|
||||
assert_eq!(providers.len(), 2);
|
||||
|
||||
// 如果 select_providers 错误地消耗了 HalfOpen 名额,这里会返回 false(被限流拒绝)
|
||||
assert!(router.allow_provider_request("b", "claude").await.allowed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -270,6 +270,7 @@ mod tests {
|
||||
meta: None,
|
||||
icon: None,
|
||||
icon_color: None,
|
||||
in_failover_queue: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -174,6 +174,7 @@ mod tests {
|
||||
meta: None,
|
||||
icon: None,
|
||||
icon_color: None,
|
||||
in_failover_queue: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -250,6 +250,7 @@ mod tests {
|
||||
meta: None,
|
||||
icon: None,
|
||||
icon_color: None,
|
||||
in_failover_queue: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -209,6 +209,7 @@ mod tests {
|
||||
meta: None,
|
||||
icon: None,
|
||||
icon_color: None,
|
||||
in_failover_queue: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -394,6 +394,7 @@ mod tests {
|
||||
meta: None,
|
||||
icon: None,
|
||||
icon_color: None,
|
||||
in_failover_queue: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -258,7 +258,13 @@ impl ProxyService {
|
||||
.set_proxy_takeover_enabled(app_type_str, false)
|
||||
.map_err(|e| format!("清除 {app_type_str} 接管状态失败: {e}"))?;
|
||||
|
||||
// 4) 若无其它接管,更新旧标志,并停止代理服务
|
||||
// 4) 清除该应用的健康状态(关闭代理时重置队列状态)
|
||||
self.db
|
||||
.clear_provider_health_for_app(app_type_str)
|
||||
.await
|
||||
.map_err(|e| format!("清除 {app_type_str} 健康状态失败: {e}"))?;
|
||||
|
||||
// 5) 若无其它接管,更新旧标志,并停止代理服务
|
||||
let has_any_backup = self
|
||||
.db
|
||||
.has_any_live_backup()
|
||||
@@ -538,6 +544,7 @@ impl ProxyService {
|
||||
.await
|
||||
.map_err(|e| format!("重置健康状态失败: {e}"))?;
|
||||
|
||||
// 注意:不清除故障转移队列和开关状态,保留供下次开启代理时使用
|
||||
log::info!("代理已停止,Live 配置已恢复");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user