fix(usage): account for cache-write tokens across schema versions

Parse cache_write_tokens from OpenAI usage details and preserve cache creation data across Chat, Responses, and Anthropic conversion paths.

Add explicit input-token semantics to request logs and rollups so legacy rows subtract cache reads only while new total-inclusive rows subtract both cache reads and writes. Migrate v12 databases, normalize rollups to fresh input, and cover historical backfill behavior with regression tests.
This commit is contained in:
Jason
2026-07-11 12:27:39 +08:00
parent 06039540ff
commit f991726ff0
13 changed files with 443 additions and 102 deletions
+74
View File
@@ -189,6 +189,7 @@ impl Database {
pricing_model TEXT,
input_tokens INTEGER NOT NULL DEFAULT 0, output_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_tokens INTEGER NOT NULL DEFAULT 0, cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
input_token_semantics INTEGER NOT NULL DEFAULT 0,
input_cost_usd TEXT NOT NULL DEFAULT '0', output_cost_usd TEXT NOT NULL DEFAULT '0',
cache_read_cost_usd TEXT NOT NULL DEFAULT '0', cache_creation_cost_usd TEXT NOT NULL DEFAULT '0',
total_cost_usd TEXT NOT NULL DEFAULT '0', latency_ms INTEGER NOT NULL, first_token_ms INTEGER,
@@ -275,6 +276,7 @@ impl Database {
output_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
input_token_semantics INTEGER NOT NULL DEFAULT 0,
total_cost_usd TEXT NOT NULL DEFAULT '0',
avg_latency_ms INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (date, app_type, provider_id, model, request_model, pricing_model)
@@ -478,6 +480,11 @@ impl Database {
Self::migrate_v11_to_v12(conn)?;
Self::set_user_version(conn, 12)?;
}
12 => {
log::info!("迁移数据库从 v12 到 v13(记录输入 token 缓存语义)");
Self::migrate_v12_to_v13(conn)?;
Self::set_user_version(conn, 13)?;
}
_ => {
return Err(AppError::Database(format!(
"未知的数据库版本 {version},无法迁移到 {SCHEMA_VERSION}"
@@ -650,6 +657,7 @@ impl Database {
request_model TEXT,
input_tokens INTEGER NOT NULL DEFAULT 0, output_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_tokens INTEGER NOT NULL DEFAULT 0, cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
input_token_semantics INTEGER NOT NULL DEFAULT 0,
input_cost_usd TEXT NOT NULL DEFAULT '0', output_cost_usd TEXT NOT NULL DEFAULT '0',
cache_read_cost_usd TEXT NOT NULL DEFAULT '0', cache_creation_cost_usd TEXT NOT NULL DEFAULT '0',
total_cost_usd TEXT NOT NULL DEFAULT '0', latency_ms INTEGER NOT NULL, first_token_ms INTEGER,
@@ -1322,6 +1330,30 @@ impl Database {
Ok(())
}
/// v12 -> v13:记录 input_tokens 是否包含缓存写入。
///
/// 默认 0 表示旧版/未知语义;旧 Codex 行只包含 cache read,不包含
/// cache creation。新代理行会显式写入 1(total-inclusive) 或 2(fresh)。
fn migrate_v12_to_v13(conn: &Connection) -> Result<(), AppError> {
if Self::table_exists(conn, "proxy_request_logs")? {
Self::add_column_if_missing(
conn,
"proxy_request_logs",
"input_token_semantics",
"INTEGER NOT NULL DEFAULT 0",
)?;
}
if Self::table_exists(conn, "usage_daily_rollups")? {
Self::add_column_if_missing(
conn,
"usage_daily_rollups",
"input_token_semantics",
"INTEGER NOT NULL DEFAULT 0",
)?;
}
Ok(())
}
/// 插入默认模型定价数据
/// 格式: (model_id, display_name, input, output, cache_read, cache_creation)
/// 注意: model_id 使用短横线格式(如 claude-haiku-4-5),与 API 返回的模型名称标准化后一致
@@ -2661,3 +2693,45 @@ impl Database {
Ok(true)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn migrate_v12_to_v13_adds_input_token_semantics_columns() -> Result<(), AppError> {
let conn = Connection::open_in_memory()?;
conn.execute(
"CREATE TABLE proxy_request_logs (request_id TEXT PRIMARY KEY)",
[],
)?;
conn.execute(
"CREATE TABLE usage_daily_rollups (date TEXT PRIMARY KEY)",
[],
)?;
Database::set_user_version(&conn, 12)?;
Database::apply_schema_migrations_on_conn(&conn)?;
assert_eq!(Database::get_user_version(&conn)?, 13);
assert!(Database::has_column(
&conn,
"proxy_request_logs",
"input_token_semantics"
)?);
assert!(Database::has_column(
&conn,
"usage_daily_rollups",
"input_token_semantics"
)?);
let log_default: i64 = conn.query_row(
"SELECT dflt_value = '0' FROM pragma_table_info('proxy_request_logs')
WHERE name = 'input_token_semantics'",
[],
|row| row.get(0),
)?;
assert_eq!(log_default, 1);
Ok(())
}
}