mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-07-24 12:44:18 +08:00
feat(usage): persist pricing basis and takeover dimensions in storage (schema v11)
- proxy_request_logs: add pricing_model column recording the basis actually used at write time (NULL = pre-v11 rows, '' = unpriced error rows) - cost backfill recomputes strictly by the persisted basis; the request_model fallback now only applies to placeholder models, so real-but-unpriced takeover rows stay at zero cost until pricing is added instead of being permanently frozen at the alias's price - backfill_missing_usage_costs_for_model can locate rows by pricing_model - usage_daily_rollups: rebuild with request_model + pricing_model in the primary key so the alias-to-real-model mapping and the pricing basis survive the 30-day prune; legacy rows migrate with '' - rollup_and_prune backfills costs before pruning: prune is irreversible and used to run before the startup backfill, permanently booking then-unpriced rows as zero - get_model_stats groups by the effective pricing model (COALESCE(NULLIF(pricing_model,''), model)) so costs aggregate under the model whose prices produced them; response-mode behavior unchanged
This commit is contained in:
@@ -75,6 +75,15 @@ impl Database {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
// 剪枝是不可逆的:明细一旦汇总删除,0 成本行就永远失去按 pricing_model
|
||||
// 补价重算的机会(启动序列里 seed 定价先于 rollup、但启动回填在 rollup
|
||||
// 之后;周期任务同理)。所以剪枝前先尽力回填一次。失败仅告警不阻断——
|
||||
// 否则一行损坏的定价数据会永久卡死日志清理。
|
||||
// 注意必须在 SAVEPOINT 之外调用:回填内部自己开顶层事务。
|
||||
if let Err(e) = Self::backfill_missing_usage_costs_on_conn(&conn, None) {
|
||||
log::warn!("Pre-prune cost backfill failed, pruning anyway: {e}");
|
||||
}
|
||||
|
||||
// Use a savepoint for atomicity
|
||||
conn.execute("SAVEPOINT rollup_prune;", [])
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
@@ -106,15 +115,18 @@ impl Database {
|
||||
fn do_rollup_and_prune(conn: &rusqlite::Connection, cutoff: i64) -> Result<u64, AppError> {
|
||||
// Aggregate old logs, merging with any pre-existing rollup rows via LEFT JOIN.
|
||||
let effective_filter = effective_usage_log_filter("l");
|
||||
// request_model 维度保留路由接管的「客户端别名 → 真实模型」映射,
|
||||
// pricing_model 维度保留写入时的计价基准(request 计价模式下与 model 分叉);
|
||||
// 明细行的这两列可能为 NULL(历史/手工数据),归一为 ''。
|
||||
let aggregation_sql = format!(
|
||||
"INSERT OR REPLACE INTO usage_daily_rollups
|
||||
(date, app_type, provider_id, model,
|
||||
(date, app_type, provider_id, model, request_model, pricing_model,
|
||||
request_count, success_count,
|
||||
input_tokens, output_tokens,
|
||||
cache_read_tokens, cache_creation_tokens,
|
||||
total_cost_usd, avg_latency_ms)
|
||||
SELECT
|
||||
d, a, p, m,
|
||||
d, a, p, m, rm, pm,
|
||||
COALESCE(old.request_count, 0) + new_req,
|
||||
COALESCE(old.success_count, 0) + new_succ,
|
||||
COALESCE(old.input_tokens, 0) + new_in,
|
||||
@@ -131,6 +143,8 @@ impl Database {
|
||||
SELECT
|
||||
date(l.created_at, 'unixepoch', 'localtime') as d,
|
||||
l.app_type as a, l.provider_id as p, l.model as m,
|
||||
COALESCE(l.request_model, '') as rm,
|
||||
COALESCE(l.pricing_model, '') as pm,
|
||||
COUNT(*) as new_req,
|
||||
SUM(CASE WHEN l.status_code >= 200 AND l.status_code < 300 THEN 1 ELSE 0 END) as new_succ,
|
||||
COALESCE(SUM(l.input_tokens), 0) as new_in,
|
||||
@@ -141,11 +155,12 @@ impl Database {
|
||||
COALESCE(AVG(l.latency_ms), 0) as new_lat
|
||||
FROM proxy_request_logs l
|
||||
WHERE l.created_at < ?1 AND {effective_filter}
|
||||
GROUP BY d, a, p, m
|
||||
GROUP BY d, a, p, m, rm, pm
|
||||
) agg
|
||||
LEFT JOIN usage_daily_rollups old
|
||||
ON old.date = agg.d AND old.app_type = agg.a
|
||||
AND old.provider_id = agg.p AND old.model = agg.m"
|
||||
AND old.provider_id = agg.p AND old.model = agg.m
|
||||
AND old.request_model = agg.rm AND old.pricing_model = agg.pm"
|
||||
);
|
||||
|
||||
conn.execute(&aggregation_sql, [cutoff])
|
||||
@@ -325,6 +340,144 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rollup_preserves_request_model_dimension() -> Result<(), AppError> {
|
||||
let db = Database::memory()?;
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let old_ts = now - 40 * 86400;
|
||||
|
||||
{
|
||||
let conn = crate::database::lock_conn!(db.conn);
|
||||
// 路由接管行:model 是真实上游模型,request_model 是客户端别名。
|
||||
// 同 model 下两个不同别名必须各自成行,prune 后映射关系仍可审计。
|
||||
for (i, request_model) in [
|
||||
("a", "claude-sonnet-4-6"),
|
||||
("b", "claude-sonnet-4-6"),
|
||||
("c", "claude-haiku-4-5"),
|
||||
] {
|
||||
conn.execute(
|
||||
"INSERT INTO proxy_request_logs (
|
||||
request_id, provider_id, app_type, model, request_model,
|
||||
input_tokens, output_tokens, total_cost_usd,
|
||||
latency_ms, status_code, created_at
|
||||
) VALUES (?1, 'p1', 'claude', 'kimi-k2', ?2, 100, 50, '0.01', 100, 200, ?3)",
|
||||
rusqlite::params![format!("takeover-{i}"), request_model, old_ts],
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
let deleted = db.rollup_and_prune(30)?;
|
||||
assert_eq!(deleted, 3);
|
||||
|
||||
let conn = crate::database::lock_conn!(db.conn);
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT request_model, request_count FROM usage_daily_rollups
|
||||
WHERE model = 'kimi-k2' ORDER BY request_model",
|
||||
)?;
|
||||
let rows = stmt
|
||||
.query_map([], |row| {
|
||||
Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
|
||||
})?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
assert_eq!(
|
||||
rows,
|
||||
vec![
|
||||
("claude-haiku-4-5".to_string(), 1),
|
||||
("claude-sonnet-4-6".to_string(), 2),
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rollup_preserves_pricing_model_dimension() -> Result<(), AppError> {
|
||||
let db = Database::memory()?;
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let old_ts = now - 40 * 86400;
|
||||
|
||||
{
|
||||
let conn = crate::database::lock_conn!(db.conn);
|
||||
// request 计价模式下 pricing_model 与 model 分叉,必须各自成行
|
||||
conn.execute(
|
||||
"INSERT INTO proxy_request_logs (
|
||||
request_id, provider_id, app_type, model, request_model, pricing_model,
|
||||
input_tokens, output_tokens, total_cost_usd,
|
||||
latency_ms, status_code, created_at
|
||||
) VALUES ('pm-a', 'p1', 'claude', 'kimi-k2', 'claude-sonnet-4-6', 'kimi-k2',
|
||||
100, 50, '0.01', 100, 200, ?1)",
|
||||
rusqlite::params![old_ts],
|
||||
)?;
|
||||
conn.execute(
|
||||
"INSERT INTO proxy_request_logs (
|
||||
request_id, provider_id, app_type, model, request_model, pricing_model,
|
||||
input_tokens, output_tokens, total_cost_usd,
|
||||
latency_ms, status_code, created_at
|
||||
) VALUES ('pm-b', 'p1', 'claude', 'kimi-k2', 'claude-sonnet-4-6', 'claude-sonnet-4-6',
|
||||
100, 50, '0.30', 100, 200, ?1)",
|
||||
rusqlite::params![old_ts],
|
||||
)?;
|
||||
}
|
||||
|
||||
let deleted = db.rollup_and_prune(30)?;
|
||||
assert_eq!(deleted, 2);
|
||||
|
||||
let conn = crate::database::lock_conn!(db.conn);
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT pricing_model, total_cost_usd FROM usage_daily_rollups
|
||||
WHERE model = 'kimi-k2' ORDER BY pricing_model",
|
||||
)?;
|
||||
let rows = stmt
|
||||
.query_map([], |row| {
|
||||
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
|
||||
})?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
assert_eq!(rows.len(), 2);
|
||||
assert_eq!(rows[0].0, "claude-sonnet-4-6");
|
||||
assert_eq!(rows[1].0, "kimi-k2");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rollup_backfills_costs_before_pruning() -> Result<(), AppError> {
|
||||
let db = Database::memory()?;
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let old_ts = now - 40 * 86400;
|
||||
|
||||
{
|
||||
let conn = crate::database::lock_conn!(db.conn);
|
||||
// >30 天的 0 成本行:pricing_model(gpt-5.5)在 seed 定价表中有价。
|
||||
// 剪枝是不可逆的,rollup 必须先回填再汇总,否则按 0 永久入账。
|
||||
conn.execute(
|
||||
"INSERT INTO proxy_request_logs (
|
||||
request_id, provider_id, app_type, model, request_model, pricing_model,
|
||||
input_tokens, output_tokens, total_cost_usd,
|
||||
latency_ms, status_code, created_at
|
||||
) VALUES ('prune-backfill', 'p1', 'codex', 'gpt-5.5', 'gpt-5.5', 'gpt-5.5',
|
||||
1000000, 0, '0', 100, 200, ?1)",
|
||||
rusqlite::params![old_ts],
|
||||
)?;
|
||||
}
|
||||
|
||||
let deleted = db.rollup_and_prune(30)?;
|
||||
assert_eq!(deleted, 1);
|
||||
|
||||
let conn = crate::database::lock_conn!(db.conn);
|
||||
let total_cost: f64 = conn.query_row(
|
||||
"SELECT CAST(total_cost_usd AS REAL) FROM usage_daily_rollups
|
||||
WHERE model = 'gpt-5.5'",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
// gpt-5.5 input $5/M × 1M tokens,回填后再汇总
|
||||
assert!(
|
||||
(total_cost - 5.0).abs() < 1e-6,
|
||||
"expected backfilled cost 5.0, got {total_cost}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rollup_noop_when_no_old_data() -> Result<(), AppError> {
|
||||
let db = Database::memory()?;
|
||||
|
||||
@@ -49,7 +49,7 @@ use std::sync::Mutex;
|
||||
|
||||
/// 当前 Schema 版本号
|
||||
/// 每次修改表结构时递增,并在 schema.rs 中添加相应的迁移逻辑
|
||||
pub(crate) const SCHEMA_VERSION: i32 = 10;
|
||||
pub(crate) const SCHEMA_VERSION: i32 = 11;
|
||||
|
||||
/// 安全地序列化 JSON,避免 unwrap panic
|
||||
pub(crate) fn to_json_string<T: Serialize>(value: &T) -> Result<String, AppError> {
|
||||
|
||||
@@ -181,9 +181,12 @@ impl Database {
|
||||
)", []).map_err(|e| AppError::Database(e.to_string()))?;
|
||||
|
||||
// 10. Proxy Request Logs 表
|
||||
// pricing_model = 写入时实际用于计价的模型名(pricing_model_source 解析结果),
|
||||
// 回填按它重算;NULL 表示 v11 之前的历史行,'' 表示未计价的错误行。
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS proxy_request_logs (
|
||||
request_id TEXT PRIMARY KEY, provider_id TEXT NOT NULL, app_type TEXT NOT NULL, model TEXT NOT NULL,
|
||||
request_model TEXT,
|
||||
pricing_model TEXT,
|
||||
input_tokens INTEGER NOT NULL DEFAULT 0, output_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
cache_read_tokens INTEGER NOT NULL DEFAULT 0, cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
input_cost_usd TEXT NOT NULL DEFAULT '0', output_cost_usd TEXT NOT NULL DEFAULT '0',
|
||||
@@ -255,12 +258,17 @@ impl Database {
|
||||
.map_err(|e| AppError::Database(e.to_string()))?;
|
||||
|
||||
// 17. Usage Daily Rollups 表 (日聚合统计)
|
||||
// request_model 保留路由接管的「客户端别名 → 真实模型」映射维度,
|
||||
// pricing_model 保留写入时的计价基准(request 计价模式下与 model 分叉),
|
||||
// 否则明细被 prune 后接管计费不可审计;历史行迁移时填 ''(未知)。
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS usage_daily_rollups (
|
||||
date TEXT NOT NULL,
|
||||
app_type TEXT NOT NULL,
|
||||
provider_id TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
request_model TEXT NOT NULL DEFAULT '',
|
||||
pricing_model TEXT NOT NULL DEFAULT '',
|
||||
request_count INTEGER NOT NULL DEFAULT 0,
|
||||
success_count INTEGER NOT NULL DEFAULT 0,
|
||||
input_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
@@ -269,7 +277,7 @@ impl Database {
|
||||
cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
total_cost_usd TEXT NOT NULL DEFAULT '0',
|
||||
avg_latency_ms INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (date, app_type, provider_id, model)
|
||||
PRIMARY KEY (date, app_type, provider_id, model, request_model, pricing_model)
|
||||
)",
|
||||
[],
|
||||
)
|
||||
@@ -431,6 +439,11 @@ impl Database {
|
||||
Self::migrate_v9_to_v10(conn)?;
|
||||
Self::set_user_version(conn, 10)?;
|
||||
}
|
||||
10 => {
|
||||
log::info!("迁移数据库从 v10 到 v11(usage_daily_rollups 保留 request_model 维度)");
|
||||
Self::migrate_v10_to_v11(conn)?;
|
||||
Self::set_user_version(conn, 11)?;
|
||||
}
|
||||
_ => {
|
||||
return Err(AppError::Database(format!(
|
||||
"未知的数据库版本 {version},无法迁移到 {SCHEMA_VERSION}"
|
||||
@@ -1200,6 +1213,63 @@ impl Database {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// v10 -> v11:usage_daily_rollups 增加 request_model 维度(进入主键),
|
||||
/// proxy_request_logs 增加 pricing_model 列(写入时的计价基准,回填依据)。
|
||||
///
|
||||
/// 路由接管下 model(真实上游模型)≠ request_model(客户端别名),
|
||||
/// 旧 rollup 只按 model 聚合,明细 prune 后映射关系永久丢失、计费不可审计。
|
||||
/// SQLite 改主键必须重建表;历史行的 request_model 已不可知,填 ''。
|
||||
fn migrate_v10_to_v11(conn: &Connection) -> Result<(), AppError> {
|
||||
// proxy_request_logs.pricing_model:NULL = v11 前的历史行(回填走
|
||||
// model → 占位符回退 request_model 的旧逻辑),'' = 未计价的错误行
|
||||
if Self::table_exists(conn, "proxy_request_logs")? {
|
||||
Self::add_column_if_missing(conn, "proxy_request_logs", "pricing_model", "TEXT")?;
|
||||
}
|
||||
|
||||
if !Self::table_exists(conn, "usage_daily_rollups")? {
|
||||
log::info!("v10 -> v11:usage_daily_rollups 不存在,跳过重建");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
conn.execute_batch(
|
||||
"ALTER TABLE usage_daily_rollups RENAME TO usage_daily_rollups_v10;
|
||||
CREATE TABLE usage_daily_rollups (
|
||||
date TEXT NOT NULL,
|
||||
app_type TEXT NOT NULL,
|
||||
provider_id TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
request_model TEXT NOT NULL DEFAULT '',
|
||||
pricing_model TEXT NOT NULL DEFAULT '',
|
||||
request_count INTEGER NOT NULL DEFAULT 0,
|
||||
success_count INTEGER NOT NULL DEFAULT 0,
|
||||
input_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
output_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
total_cost_usd TEXT NOT NULL DEFAULT '0',
|
||||
avg_latency_ms INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (date, app_type, provider_id, model, request_model, pricing_model)
|
||||
);
|
||||
INSERT INTO usage_daily_rollups
|
||||
(date, app_type, provider_id, model, request_model, pricing_model,
|
||||
request_count, success_count, input_tokens, output_tokens,
|
||||
cache_read_tokens, cache_creation_tokens, total_cost_usd, avg_latency_ms)
|
||||
SELECT date, app_type, provider_id, model, '', '',
|
||||
request_count, success_count, input_tokens, output_tokens,
|
||||
cache_read_tokens, cache_creation_tokens, total_cost_usd, avg_latency_ms
|
||||
FROM usage_daily_rollups_v10;
|
||||
DROP TABLE usage_daily_rollups_v10;",
|
||||
)
|
||||
.map_err(|e| {
|
||||
AppError::Database(format!("v10 -> v11 重建 usage_daily_rollups 失败: {e}"))
|
||||
})?;
|
||||
|
||||
log::info!(
|
||||
"v10 -> v11 迁移完成:usage_daily_rollups 已保留 request_model/pricing_model 维度"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 插入默认模型定价数据
|
||||
/// 格式: (model_id, display_name, input, output, cache_read, cache_creation)
|
||||
/// 注意: model_id 使用短横线格式(如 claude-haiku-4-5),与 API 返回的模型名称标准化后一致
|
||||
|
||||
@@ -345,6 +345,87 @@ fn schema_migration_v4_adds_pricing_model_columns() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_v10_to_v11_rebuilds_rollups_with_request_model_dimension() {
|
||||
let conn = Connection::open_in_memory().expect("open memory db");
|
||||
|
||||
// 模拟 v10 形状的 rollup 表(主键不含 request_model)+ 一行历史聚合数据,
|
||||
// 以及 v10 形状的明细表(无 pricing_model 列)
|
||||
conn.execute_batch(
|
||||
r#"
|
||||
CREATE TABLE proxy_request_logs (
|
||||
request_id TEXT PRIMARY KEY,
|
||||
model TEXT NOT NULL,
|
||||
request_model TEXT
|
||||
);
|
||||
CREATE TABLE usage_daily_rollups (
|
||||
date TEXT NOT NULL,
|
||||
app_type TEXT NOT NULL,
|
||||
provider_id TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
request_count INTEGER NOT NULL DEFAULT 0,
|
||||
success_count INTEGER NOT NULL DEFAULT 0,
|
||||
input_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
output_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
total_cost_usd TEXT NOT NULL DEFAULT '0',
|
||||
avg_latency_ms INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (date, app_type, provider_id, model)
|
||||
);
|
||||
INSERT INTO usage_daily_rollups
|
||||
(date, app_type, provider_id, model, request_count, success_count,
|
||||
input_tokens, output_tokens, total_cost_usd, avg_latency_ms)
|
||||
VALUES ('2026-05-01', 'claude', 'p1', 'kimi-k2', 7, 7, 1000, 500, '0.07', 120);
|
||||
"#,
|
||||
)
|
||||
.expect("seed v10 rollup table");
|
||||
|
||||
Database::set_user_version(&conn, 10).expect("set user_version=10");
|
||||
Database::apply_schema_migrations_on_conn(&conn).expect("apply migrations");
|
||||
|
||||
// 新列存在且 NOT NULL DEFAULT ''
|
||||
let request_model = get_column_info(&conn, "usage_daily_rollups", "request_model");
|
||||
assert_eq!(request_model.r#type, "TEXT");
|
||||
assert_eq!(request_model.notnull, 1);
|
||||
let rollup_pricing_model = get_column_info(&conn, "usage_daily_rollups", "pricing_model");
|
||||
assert_eq!(rollup_pricing_model.r#type, "TEXT");
|
||||
assert_eq!(rollup_pricing_model.notnull, 1);
|
||||
|
||||
// 明细表补上 pricing_model 列(可空,历史行 NULL)
|
||||
let pricing_model = get_column_info(&conn, "proxy_request_logs", "pricing_model");
|
||||
assert_eq!(pricing_model.r#type, "TEXT");
|
||||
assert_eq!(pricing_model.notnull, 0);
|
||||
|
||||
// 历史行保留,request_model 填 ''(未知)
|
||||
let (rm, count, input, cost): (String, i64, i64, String) = conn
|
||||
.query_row(
|
||||
"SELECT request_model, request_count, input_tokens, total_cost_usd
|
||||
FROM usage_daily_rollups WHERE model = 'kimi-k2'",
|
||||
[],
|
||||
|row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)),
|
||||
)
|
||||
.expect("migrated row");
|
||||
assert_eq!(rm, "");
|
||||
assert_eq!(count, 7);
|
||||
assert_eq!(input, 1000);
|
||||
assert_eq!(cost, "0.07");
|
||||
|
||||
// 主键包含 request_model:同 model 不同别名可共存
|
||||
conn.execute(
|
||||
"INSERT INTO usage_daily_rollups
|
||||
(date, app_type, provider_id, model, request_model, request_count)
|
||||
VALUES ('2026-05-01', 'claude', 'p1', 'kimi-k2', 'claude-sonnet-4-6', 1)",
|
||||
[],
|
||||
)
|
||||
.expect("insert row with same model but different request_model");
|
||||
|
||||
assert_eq!(
|
||||
Database::get_user_version(&conn).expect("version after migration"),
|
||||
SCHEMA_VERSION
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_create_tables_repairs_legacy_proxy_config_singleton_to_per_app() {
|
||||
let conn = Connection::open_in_memory().expect("open memory db");
|
||||
|
||||
@@ -149,17 +149,20 @@ pub struct RequestLogDetail {
|
||||
pub created_at: i64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub data_source: Option<String>,
|
||||
/// 写入时实际用于计价的模型名。None = v11 前的历史行,"" = 未计价的错误行。
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub pricing_model: Option<String>,
|
||||
}
|
||||
|
||||
/// 把 24 列的查询结果映射为 `RequestLogDetail`。
|
||||
/// 把 25 列的查询结果映射为 `RequestLogDetail`。
|
||||
///
|
||||
/// 调用方的 SELECT **必须**按以下顺序返回 24 列:
|
||||
/// 调用方的 SELECT **必须**按以下顺序返回 25 列:
|
||||
/// `request_id, provider_id, provider_name, app_type, model, request_model,
|
||||
/// cost_multiplier, input_tokens, output_tokens, cache_read_tokens,
|
||||
/// cache_creation_tokens, input_cost_usd, output_cost_usd, cache_read_cost_usd,
|
||||
/// cache_creation_cost_usd, total_cost_usd, is_streaming, latency_ms,
|
||||
/// first_token_ms, duration_ms, status_code, error_message, created_at,
|
||||
/// data_source`
|
||||
/// data_source, pricing_model`
|
||||
///
|
||||
/// 不需要 provider_name 时(如 backfill)SELECT `NULL AS provider_name` 占位即可。
|
||||
fn row_to_request_log_detail(row: &rusqlite::Row<'_>) -> rusqlite::Result<RequestLogDetail> {
|
||||
@@ -190,6 +193,7 @@ fn row_to_request_log_detail(row: &rusqlite::Row<'_>) -> rusqlite::Result<Reques
|
||||
error_message: row.get(21)?,
|
||||
created_at: row.get(22)?,
|
||||
data_source: row.get(23)?,
|
||||
pricing_model: row.get(24)?,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1154,6 +1158,11 @@ impl Database {
|
||||
};
|
||||
|
||||
// UNION detail logs + rollup data
|
||||
//
|
||||
// 分组键用「有效计价模型」:pricing_model 非空时优先(成本就是按它的
|
||||
// 定价算的,金额与定价表自洽),NULL/'' 回落 model。默认 response 计价
|
||||
// 模式下两者相同,行为不变;request 模式 + 路由接管下,钱挂在实际计价
|
||||
// 基准名下,而不是上游回显/客户端别名名下。
|
||||
let fresh_input_detail = fresh_input_sql("l");
|
||||
let fresh_input_rollup = fresh_input_sql("r");
|
||||
let sql = format!(
|
||||
@@ -1163,21 +1172,21 @@ impl Database {
|
||||
SUM(total_tokens) as total_tokens,
|
||||
SUM(total_cost) as total_cost
|
||||
FROM (
|
||||
SELECT l.model,
|
||||
SELECT COALESCE(NULLIF(l.pricing_model, ''), l.model) as model,
|
||||
COUNT(*) as request_count,
|
||||
COALESCE(SUM({fresh_input_detail} + l.output_tokens), 0) as total_tokens,
|
||||
COALESCE(SUM(CAST(l.total_cost_usd AS REAL)), 0) as total_cost
|
||||
FROM proxy_request_logs l
|
||||
{detail_where}
|
||||
GROUP BY l.model
|
||||
GROUP BY COALESCE(NULLIF(l.pricing_model, ''), l.model)
|
||||
UNION ALL
|
||||
SELECT r.model,
|
||||
SELECT COALESCE(NULLIF(r.pricing_model, ''), r.model),
|
||||
COALESCE(SUM(r.request_count), 0),
|
||||
COALESCE(SUM({fresh_input_rollup} + r.output_tokens), 0),
|
||||
COALESCE(SUM(CAST(r.total_cost_usd AS REAL)), 0)
|
||||
FROM usage_daily_rollups r
|
||||
{rollup_where}
|
||||
GROUP BY r.model
|
||||
GROUP BY COALESCE(NULLIF(r.pricing_model, ''), r.model)
|
||||
)
|
||||
GROUP BY model
|
||||
ORDER BY total_cost DESC"
|
||||
@@ -1281,7 +1290,7 @@ impl Database {
|
||||
l.input_tokens, l.output_tokens, l.cache_read_tokens, l.cache_creation_tokens,
|
||||
l.input_cost_usd, l.output_cost_usd, l.cache_read_cost_usd, l.cache_creation_cost_usd, l.total_cost_usd,
|
||||
l.is_streaming, l.latency_ms, l.first_token_ms, l.duration_ms,
|
||||
l.status_code, l.error_message, l.created_at, l.data_source
|
||||
l.status_code, l.error_message, l.created_at, l.data_source, l.pricing_model
|
||||
FROM proxy_request_logs l
|
||||
LEFT JOIN providers p ON l.provider_id = p.id AND l.app_type = p.app_type
|
||||
{where_clause}
|
||||
@@ -1324,7 +1333,7 @@ impl Database {
|
||||
input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens,
|
||||
input_cost_usd, output_cost_usd, cache_read_cost_usd, cache_creation_cost_usd, total_cost_usd,
|
||||
is_streaming, latency_ms, first_token_ms, duration_ms,
|
||||
status_code, error_message, created_at, l.data_source
|
||||
status_code, error_message, created_at, l.data_source, l.pricing_model
|
||||
FROM proxy_request_logs l
|
||||
LEFT JOIN providers p ON l.provider_id = p.id AND l.app_type = p.app_type
|
||||
WHERE l.request_id = ?"
|
||||
@@ -1469,7 +1478,7 @@ impl Database {
|
||||
Self::backfill_missing_usage_costs_on_conn(&conn, Some(model_id))
|
||||
}
|
||||
|
||||
fn backfill_missing_usage_costs_on_conn(
|
||||
pub(crate) fn backfill_missing_usage_costs_on_conn(
|
||||
conn: &Connection,
|
||||
only_model_id: Option<&str>,
|
||||
) -> Result<u64, AppError> {
|
||||
@@ -1480,7 +1489,7 @@ impl Database {
|
||||
input_cost_usd, output_cost_usd, cache_read_cost_usd,
|
||||
cache_creation_cost_usd, total_cost_usd, is_streaming, latency_ms,
|
||||
first_token_ms, duration_ms, status_code, error_message, created_at,
|
||||
data_source
|
||||
data_source, pricing_model
|
||||
FROM proxy_request_logs
|
||||
WHERE CAST(total_cost_usd AS REAL) <= 0
|
||||
AND (input_tokens > 0 OR output_tokens > 0
|
||||
@@ -1489,7 +1498,9 @@ impl Database {
|
||||
let mut logs = {
|
||||
match only_model_id {
|
||||
Some(model) => {
|
||||
let sql = format!("{BASE_SQL} AND (model = ?1 OR request_model = ?1)");
|
||||
let sql = format!(
|
||||
"{BASE_SQL} AND (model = ?1 OR request_model = ?1 OR pricing_model = ?1)"
|
||||
);
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
let rows = stmt.query_map([model], row_to_request_log_detail)?;
|
||||
rows.collect::<Result<Vec<_>, _>>()?
|
||||
@@ -1647,10 +1658,32 @@ impl Database {
|
||||
cache: &mut HashMap<String, PricingInfo>,
|
||||
log: &RequestLogDetail,
|
||||
) -> Result<Option<PricingInfo>, AppError> {
|
||||
// 写入时的计价基准已落库(v11+):回填只按它重算,找不到就保持 0 成本
|
||||
// 等补价。不能换用 model/request_model 猜——路由接管 + request 计价模式下
|
||||
// 三者可能各不相同(model=上游回显、request_model=客户端别名、
|
||||
// pricing_model=实际出站模型),换基准会按错误价格永久固化。
|
||||
// 占位符("" = 未计价错误行 / "unknown")视同缺失,走历史行逻辑。
|
||||
if let Some(pricing_model) = log
|
||||
.pricing_model
|
||||
.as_deref()
|
||||
.filter(|pm| !is_placeholder_pricing_model(pm))
|
||||
{
|
||||
return Self::get_model_pricing_cached(conn, cache, pricing_model);
|
||||
}
|
||||
|
||||
if let Some(pricing) = Self::get_model_pricing_cached(conn, cache, &log.model)? {
|
||||
return Ok(Some(pricing));
|
||||
}
|
||||
|
||||
// 仅当 model 列是占位符(解析失败留下的 ""/"unknown" 等)时才回退到
|
||||
// request_model 定价。model 是真实模型名但缺定价时必须保持 0 成本等待
|
||||
// 补价:路由接管下 request_model 是客户端别名(如 claude-sonnet-4-6),
|
||||
// 按别名回填会把真实上游模型的 tokens 按错误价格永久固化(行一旦有成本
|
||||
// 就不再进入回填范围)。
|
||||
if !is_placeholder_pricing_model(&log.model) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let Some(request_model) = log.request_model.as_deref() else {
|
||||
return Ok(None);
|
||||
};
|
||||
@@ -2190,6 +2223,123 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backfill_skips_request_model_fallback_for_real_unpriced_model() -> Result<(), AppError>
|
||||
{
|
||||
let db = Database::memory()?;
|
||||
|
||||
{
|
||||
let conn = lock_conn!(db.conn);
|
||||
// 路由接管场景:model 是上游回显的真实模型(缺定价),request_model
|
||||
// 是客户端别名(有定价)。回填不得按别名定价,必须保持 0 成本等待补价。
|
||||
conn.execute(
|
||||
"INSERT INTO proxy_request_logs (
|
||||
request_id, provider_id, app_type, model, request_model,
|
||||
input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens,
|
||||
input_cost_usd, output_cost_usd, cache_read_cost_usd, cache_creation_cost_usd,
|
||||
total_cost_usd, latency_ms, status_code, created_at, data_source
|
||||
) VALUES (
|
||||
'takeover-unpriced-model', 'provider-1', 'claude',
|
||||
'takeover-real-model-unpriced', 'claude-sonnet-4-6',
|
||||
1000000, 0, 0, 0,
|
||||
'0', '0', '0', '0',
|
||||
'0', 100, 200, 1000, 'proxy'
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
}
|
||||
|
||||
// request_model(claude-sonnet-4-6)有定价,但 model 是真实模型名:不得回退
|
||||
assert_eq!(db.backfill_missing_usage_costs()?, 0);
|
||||
|
||||
{
|
||||
let conn = lock_conn!(db.conn);
|
||||
let total_cost: String = conn.query_row(
|
||||
"SELECT total_cost_usd
|
||||
FROM proxy_request_logs WHERE request_id = 'takeover-unpriced-model'",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
assert_eq!(total_cost, "0");
|
||||
|
||||
// 补上真实模型定价后,回填必须按真实模型价格修复(0 成本行未被污染固化)
|
||||
conn.execute(
|
||||
"INSERT INTO model_pricing (model_id, display_name, input_cost_per_million, output_cost_per_million)
|
||||
VALUES ('takeover-real-model-unpriced', 'Takeover Real Model', '0.6', '2.5')",
|
||||
[],
|
||||
)?;
|
||||
}
|
||||
|
||||
assert_eq!(db.backfill_missing_usage_costs()?, 1);
|
||||
|
||||
let conn = lock_conn!(db.conn);
|
||||
let total_cost: String = conn.query_row(
|
||||
"SELECT total_cost_usd
|
||||
FROM proxy_request_logs WHERE request_id = 'takeover-unpriced-model'",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
assert_eq!(total_cost, "0.600000");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backfill_uses_persisted_pricing_model() -> Result<(), AppError> {
|
||||
let db = Database::memory()?;
|
||||
|
||||
{
|
||||
let conn = lock_conn!(db.conn);
|
||||
// request 计价模式 + 接管:写入时锚定出站模型 kimi-k2-novel(当时缺价),
|
||||
// 但上游回显了别名 → model/request_model 都是 claude-sonnet-4-6(有定价)。
|
||||
// 回填必须按落库的 pricing_model 重算,不得换用 model 列的别名价格。
|
||||
conn.execute(
|
||||
"INSERT INTO proxy_request_logs (
|
||||
request_id, provider_id, app_type, model, request_model, pricing_model,
|
||||
input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens,
|
||||
input_cost_usd, output_cost_usd, cache_read_cost_usd, cache_creation_cost_usd,
|
||||
total_cost_usd, latency_ms, status_code, created_at, data_source
|
||||
) VALUES (
|
||||
'persisted-pricing-model', 'provider-1', 'claude',
|
||||
'claude-sonnet-4-6', 'claude-sonnet-4-6', 'kimi-k2-novel',
|
||||
1000000, 0, 0, 0,
|
||||
'0', '0', '0', '0',
|
||||
'0', 100, 200, 1000, 'proxy'
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
}
|
||||
|
||||
// pricing_model(kimi-k2-novel)缺价:不得回退到 model 列的别名价格
|
||||
assert_eq!(db.backfill_missing_usage_costs()?, 0);
|
||||
|
||||
{
|
||||
let conn = lock_conn!(db.conn);
|
||||
conn.execute(
|
||||
"INSERT INTO model_pricing (model_id, display_name, input_cost_per_million, output_cost_per_million)
|
||||
VALUES ('kimi-k2-novel', 'Kimi K2 Novel', '0.6', '2.5')",
|
||||
[],
|
||||
)?;
|
||||
}
|
||||
|
||||
// 按 pricing_model 也能定位到该行(model/request_model 都不是 kimi-k2-novel)
|
||||
assert_eq!(
|
||||
db.backfill_missing_usage_costs_for_model("kimi-k2-novel")?,
|
||||
1
|
||||
);
|
||||
|
||||
let conn = lock_conn!(db.conn);
|
||||
let total_cost: String = conn.query_row(
|
||||
"SELECT total_cost_usd
|
||||
FROM proxy_request_logs WHERE request_id = 'persisted-pricing-model'",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
assert_eq!(total_cost, "0.600000");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_backfill_missing_usage_costs_keeps_claude_fresh_input() -> Result<(), AppError> {
|
||||
let db = Database::memory()?;
|
||||
|
||||
Reference in New Issue
Block a user