refactor(db): clean up unused database tables and optimize schema

Remove deprecated and unused database tables:
- Remove proxy_usage table (replaced by proxy_request_logs)
- Remove usage_daily_stats table (aggregation done on-the-fly)
- Rename model_test_logs to stream_check_logs with updated schema
- Remove related DAO methods for proxy_usage
- Update usage_stats service to use proxy_request_logs only
- Refactor usage_script to work with new schema

This simplifies the database schema and removes redundant data storage.
This commit is contained in:
YoVinchen
2025-12-10 15:54:21 +08:00
parent 454c3ed111
commit ed5ad7ad3d
6 changed files with 66 additions and 254 deletions
+1
View File
@@ -8,5 +8,6 @@ pub mod providers;
pub mod proxy;
pub mod settings;
pub mod skills;
pub mod stream_check;
// 所有 DAO 方法都通过 Database impl 提供,无需单独导出
-77
View File
@@ -184,83 +184,6 @@ impl Database {
Ok(())
}
// ==================== Proxy Usage (可选) ====================
/// 记录代理使用统计
#[allow(dead_code)]
pub async fn record_proxy_usage(&self, record: &ProxyUsageRecord) -> Result<(), AppError> {
let conn = lock_conn!(self.conn);
conn.execute(
"INSERT INTO proxy_usage
(provider_id, app_type, endpoint, request_tokens, response_tokens,
status_code, latency_ms, error, timestamp)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
rusqlite::params![
&record.provider_id,
&record.app_type,
&record.endpoint,
record.request_tokens,
record.response_tokens,
record.status_code as i64,
record.latency_ms as i64,
&record.error,
&record.timestamp,
],
)
.map_err(|e| AppError::Database(e.to_string()))?;
Ok(())
}
/// 查询最近的使用统计
#[allow(dead_code)]
pub async fn get_recent_usage(
&self,
provider_id: &str,
app_type: &str,
limit: usize,
) -> Result<Vec<ProxyUsageRecord>, AppError> {
let conn = lock_conn!(self.conn);
let mut stmt = conn
.prepare(
"SELECT provider_id, app_type, endpoint, request_tokens, response_tokens,
status_code, latency_ms, error, timestamp
FROM proxy_usage
WHERE provider_id = ?1 AND app_type = ?2
ORDER BY timestamp DESC
LIMIT ?3",
)
.map_err(|e| AppError::Database(e.to_string()))?;
let rows = stmt
.query_map(
rusqlite::params![provider_id, app_type, limit as i64],
|row| {
Ok(ProxyUsageRecord {
provider_id: row.get(0)?,
app_type: row.get(1)?,
endpoint: row.get(2)?,
request_tokens: row.get(3)?,
response_tokens: row.get(4)?,
status_code: row.get::<_, i64>(5)? as u16,
latency_ms: row.get::<_, i64>(6)? as u64,
error: row.get(7)?,
timestamp: row.get(8)?,
})
},
)
.map_err(|e| AppError::Database(e.to_string()))?;
let mut records = Vec::new();
for row in rows {
records.push(row.map_err(|e| AppError::Database(e.to_string()))?);
}
Ok(records)
}
// ==================== Circuit Breaker Config ====================
/// 获取熔断器配置
+10 -86
View File
@@ -173,40 +173,7 @@ impl Database {
)
.map_err(|e| AppError::Database(e.to_string()))?;
// 10. Proxy Usage 表 (代理使用统计,可选)
conn.execute(
"CREATE TABLE IF NOT EXISTS proxy_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT,
provider_id TEXT NOT NULL,
app_type TEXT NOT NULL,
endpoint TEXT NOT NULL,
request_tokens INTEGER,
response_tokens INTEGER,
status_code INTEGER NOT NULL,
latency_ms INTEGER NOT NULL,
error TEXT,
timestamp TEXT NOT NULL
)",
[],
)
.map_err(|e| AppError::Database(e.to_string()))?;
// 为 proxy_usage 创建索引
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_proxy_usage_timestamp
ON proxy_usage(timestamp)",
[],
)
.map_err(|e| AppError::Database(e.to_string()))?;
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_proxy_usage_provider
ON proxy_usage(provider_id, app_type)",
[],
)
.map_err(|e| AppError::Database(e.to_string()))?;
// 11. Proxy Request Logs 表 (详细请求日志)
// 10. Proxy Request Logs 表 (详细请求日志)
conn.execute(
"CREATE TABLE IF NOT EXISTS proxy_request_logs (
request_id TEXT PRIMARY KEY,
@@ -272,7 +239,7 @@ impl Database {
)
.map_err(|e| AppError::Database(e.to_string()))?;
// 12. Model Pricing 表 (模型定价)
// 11. Model Pricing 表 (模型定价)
conn.execute(
"CREATE TABLE IF NOT EXISTS model_pricing (
model_id TEXT PRIMARY KEY,
@@ -286,38 +253,20 @@ impl Database {
)
.map_err(|e| AppError::Database(e.to_string()))?;
// 13. Usage Daily Stats 表 (每日聚合统计)
// 12. Stream Check Logs 表 (流式健康检查日志)
conn.execute(
"CREATE TABLE IF NOT EXISTS usage_daily_stats (
date TEXT NOT NULL,
provider_id TEXT NOT NULL,
app_type TEXT NOT NULL,
model TEXT NOT NULL,
request_count INTEGER NOT NULL DEFAULT 0,
total_input_tokens INTEGER NOT NULL DEFAULT 0,
total_output_tokens INTEGER NOT NULL DEFAULT 0,
total_cost_usd TEXT NOT NULL DEFAULT '0',
success_count INTEGER NOT NULL DEFAULT 0,
error_count INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (date, provider_id, app_type, model)
)",
[],
)
.map_err(|e| AppError::Database(e.to_string()))?;
// 14. Model Test Logs 表 (模型测试日志,独立于代理使用统计)
conn.execute(
"CREATE TABLE IF NOT EXISTS model_test_logs (
"CREATE TABLE IF NOT EXISTS stream_check_logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
provider_id TEXT NOT NULL,
provider_name TEXT NOT NULL,
app_type TEXT NOT NULL,
model TEXT NOT NULL,
prompt TEXT NOT NULL,
status TEXT NOT NULL,
success INTEGER NOT NULL,
message TEXT NOT NULL,
response_time_ms INTEGER,
http_status INTEGER,
model_used TEXT,
retry_count INTEGER DEFAULT 0,
tested_at INTEGER NOT NULL
)",
[],
@@ -325,20 +274,13 @@ impl Database {
.map_err(|e| AppError::Database(e.to_string()))?;
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_model_test_logs_provider
ON model_test_logs(provider_id, app_type)",
"CREATE INDEX IF NOT EXISTS idx_stream_check_logs_provider
ON stream_check_logs(app_type, provider_id, tested_at DESC)",
[],
)
.map_err(|e| AppError::Database(e.to_string()))?;
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_model_test_logs_tested_at
ON model_test_logs(tested_at DESC)",
[],
)
.map_err(|e| AppError::Database(e.to_string()))?;
// 15. Circuit Breaker Config 表 (熔断器配置)
// 13. Circuit Breaker Config 表 (熔断器配置)
conn.execute(
"CREATE TABLE IF NOT EXISTS circuit_breaker_config (
id INTEGER PRIMARY KEY CHECK (id = 1),
@@ -557,24 +499,6 @@ impl Database {
[],
)?;
// usage_daily_stats 表
conn.execute(
"CREATE TABLE IF NOT EXISTS usage_daily_stats (
date TEXT NOT NULL,
provider_id TEXT NOT NULL,
app_type TEXT NOT NULL,
model TEXT NOT NULL,
request_count INTEGER NOT NULL DEFAULT 0,
total_input_tokens INTEGER NOT NULL DEFAULT 0,
total_output_tokens INTEGER NOT NULL DEFAULT 0,
total_cost_usd TEXT NOT NULL DEFAULT '0',
success_count INTEGER NOT NULL DEFAULT 0,
error_count INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (date, provider_id, app_type, model)
)",
[],
)?;
// 清空并重新插入模型定价
conn.execute("DELETE FROM model_pricing", [])
.map_err(|e| AppError::Database(format!("清空模型定价失败: {e}")))?;
-14
View File
@@ -103,17 +103,3 @@ pub struct ProviderHealth {
pub last_error: Option<String>,
pub updated_at: String,
}
/// 使用统计记录
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProxyUsageRecord {
pub provider_id: String,
pub app_type: String,
pub endpoint: String,
pub request_tokens: Option<i32>,
pub response_tokens: Option<i32>,
pub status_code: u16,
pub latency_ms: u64,
pub error: Option<String>,
pub timestamp: String,
}
-50
View File
@@ -652,56 +652,6 @@ impl Database {
monthly_exceeded,
})
}
/// 更新每日统计聚合
///
/// 在请求完成后调用,更新 usage_daily_stats 表
#[allow(clippy::too_many_arguments)]
pub fn update_daily_stats(
&self,
provider_id: &str,
app_type: &str,
model: &str,
input_tokens: u32,
output_tokens: u32,
total_cost: &str,
is_success: bool,
) -> Result<(), AppError> {
let conn = lock_conn!(self.conn);
let date = Utc::now().format("%Y-%m-%d").to_string();
// 使用 UPSERT 更新或插入统计
conn.execute(
"INSERT INTO usage_daily_stats (
date, provider_id, app_type, model,
request_count, total_input_tokens, total_output_tokens,
total_cost_usd, success_count, error_count
) VALUES (?1, ?2, ?3, ?4, 1, ?5, ?6, ?7, ?8, ?9)
ON CONFLICT(date, provider_id, app_type, model) DO UPDATE SET
request_count = request_count + 1,
total_input_tokens = total_input_tokens + ?5,
total_output_tokens = total_output_tokens + ?6,
total_cost_usd = CAST(
CAST(total_cost_usd AS REAL) + CAST(?7 AS REAL) AS TEXT
),
success_count = success_count + ?8,
error_count = error_count + ?9",
params![
date,
provider_id,
app_type,
model,
input_tokens,
output_tokens,
total_cost,
if is_success { 1 } else { 0 },
if is_success { 0 } else { 1 },
],
)
.map_err(|e| AppError::Database(format!("更新每日统计失败: {e}")))?;
Ok(())
}
}
/// Provider 限额状态
+55 -27
View File
@@ -525,23 +525,18 @@ fn validate_request_url(request_url: &str, base_url: &str) -> Result<(), AppErro
// 检查端口是否匹配(考虑默认端口)
// 使用 port_or_known_default() 会自动处理默认端口(http->80, https->443
match (parsed_request.port_or_known_default(), parsed_base.port_or_known_default()) {
match (
parsed_request.port_or_known_default(),
parsed_base.port_or_known_default(),
) {
(Some(request_port), Some(base_port)) if request_port == base_port => {
// 端口匹配,继续执行
}
(Some(request_port), Some(base_port)) => {
return Err(AppError::localized(
"usage_script.request_port_mismatch",
format!(
"请求端口 {} 必须与 base_url 端口 {} 匹配",
request_port,
base_port
),
format!(
"Request port {} must match base_url port {}",
request_port,
base_port
),
format!("请求端口 {request_port} 必须与 base_url 端口 {base_port} 匹配"),
format!("Request port {request_port} must match base_url port {base_port}"),
));
}
_ => {
@@ -774,22 +769,25 @@ mod tests {
fn test_https_bypass_prevention() {
// 非本地域名的 HTTP 应该被拒绝
let result = validate_base_url("http://127.0.0.1.evil.com/api");
assert!(result.is_err(), "Should reject HTTP for non-localhost domains");
assert!(
result.is_err(),
"Should reject HTTP for non-localhost domains"
);
}
#[test]
fn test_edge_cases() {
// 边界情况测试
assert!(is_private_ip("172.16.0.0")); // RFC1918起始
assert!(is_private_ip("172.16.0.0")); // RFC1918起始
assert!(is_private_ip("172.31.255.255")); // RFC1918结束
assert!(is_private_ip("10.0.0.0")); // 10.0.0.0/8起始
assert!(is_private_ip("10.0.0.0")); // 10.0.0.0/8起始
assert!(is_private_ip("10.255.255.255")); // 10.0.0.0/8结束
assert!(is_private_ip("192.168.0.0")); // 192.168.0.0/16起始
assert!(is_private_ip("192.168.0.0")); // 192.168.0.0/16起始
assert!(is_private_ip("192.168.255.255")); // 192.168.0.0/16结束
// 紧邻RFC1918的公网地址 - 应该返回false
assert!(!is_private_ip("172.15.255.255")); // 172.16.0.0的前一个
assert!(!is_private_ip("172.32.0.0")); // 172.31.255.255的后一个
assert!(!is_private_ip("172.32.0.0")); // 172.31.255.255的后一个
}
#[test]
@@ -815,27 +813,57 @@ mod tests {
// 测试用例:(base_url, request_url, should_match)
let test_cases = vec![
// HTTPS默认端口测试
("https://api.example.com", "https://api.example.com/v1/test", true),
("https://api.example.com", "https://api.example.com:443/v1/test", true),
("https://api.example.com:443", "https://api.example.com/v1/test", true),
("https://api.example.com:443", "https://api.example.com:443/v1/test", true),
(
"https://api.example.com",
"https://api.example.com/v1/test",
true,
),
(
"https://api.example.com",
"https://api.example.com:443/v1/test",
true,
),
(
"https://api.example.com:443",
"https://api.example.com/v1/test",
true,
),
(
"https://api.example.com:443",
"https://api.example.com:443/v1/test",
true,
),
// 端口不匹配测试
("https://api.example.com", "https://api.example.com:8443/v1/test", false),
("https://api.example.com:443", "https://api.example.com:8443/v1/test", false),
(
"https://api.example.com",
"https://api.example.com:8443/v1/test",
false,
),
(
"https://api.example.com:443",
"https://api.example.com:8443/v1/test",
false,
),
];
for (base_url, request_url, should_match) in test_cases {
let result = validate_request_url(request_url, base_url);
if should_match {
assert!(result.is_ok(),
assert!(
result.is_ok(),
"应该匹配的URL被拒绝: base_url={}, request_url={}, error={}",
base_url, request_url, result.unwrap_err());
base_url,
request_url,
result.unwrap_err()
);
} else {
assert!(result.is_err(),
assert!(
result.is_err(),
"应该不匹配的URL被允许: base_url={}, request_url={}",
base_url, request_url);
base_url,
request_url
);
}
}
}