Files
CC-Switch/src-tauri/src/services/session_usage.rs
T
Jason 402570ce31 fix(usage): pricing routing, SSE lifecycle, and validation hardening
* model pricing routing: extend prefix-match families (gpt-/o1-o5/
  gemini-/deepseek-/qwen-/glm-/kimi-/minimax-) with per-family dash
  thresholds so short base IDs like gpt-5 no longer mis-match
  gpt-5-mini; strip ISO and 8-digit date suffixes via UTF-8-safe
  byte matching so claude-haiku-4-5-20251001 falls back to
  claude-haiku-4-5 pricing
* SSE collector: SseUsageFinishGuard (RAII) guarantees finish() on
  early return or panic; AtomicBool fast path lets push() skip the
  Mutex once first-event time is recorded
* validation: shared validate_cost_multiplier / validate_pricing_source
  helpers across DAO and service layers; PRICING_SOURCE_RESPONSE /
  PRICING_SOURCE_REQUEST constants replace string literals; price
  fields in update_model_pricing now reject empty / non-decimal /
  negative input before INSERT
* backfill: add backfill_missing_usage_costs_for_model so a single
  price edit only scans matching rows instead of the full log table;
  startup backfill remains full-scan
* session_usage{,_codex,_gemini}: share find_model_pricing helper from
  usage_stats; metadata_modified_nanos centralizes mtime precision
* frontend: NON_NEGATIVE_DECIMAL_REGEX + isNonNegativeDecimalString
  replace three copies of the same multiplier regex; isUnpricedUsage
  surfaces zero-cost rows that have usage tokens (cached per row to
  avoid double evaluation); invalidate usageKeys.all on pricing mutate
  so backfilled rows refresh
2026-05-14 11:53:51 +08:00

641 lines
21 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Claude Code 会话日志使用追踪
//!
//! 从 ~/.claude/projects/ 下的 JSONL 会话文件中提取 token 使用数据,
//! 实现无代理模式下的使用统计。
//!
//! ## 数据流
//! ```text
//! ~/.claude/projects/*/*.jsonl → 增量解析 → 去重 → 费用计算 → proxy_request_logs 表
//! ```
use crate::config::get_claude_config_dir;
use crate::database::{lock_conn, Database};
use crate::error::AppError;
use crate::proxy::usage::calculator::{CostCalculator, ModelPricing};
use crate::proxy::usage::parser::TokenUsage;
use crate::services::usage_stats::{
effective_usage_log_filter, find_model_pricing, should_skip_session_insert, DedupKey,
};
use rust_decimal::Decimal;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fs;
use std::io::{BufRead, BufReader};
use std::path::{Path, PathBuf};
use std::time::SystemTime;
/// 同步结果
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SessionSyncResult {
pub imported: u32,
pub skipped: u32,
pub files_scanned: u32,
pub errors: Vec<String>,
}
/// 数据来源分布
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DataSourceSummary {
pub data_source: String,
pub request_count: u32,
pub total_cost_usd: String,
}
/// 从 JSONL 中解析出的 assistant 消息使用数据
#[derive(Debug)]
struct ParsedAssistantUsage {
message_id: String,
model: String,
input_tokens: u32,
output_tokens: u32,
cache_read_tokens: u32,
cache_creation_tokens: u32,
stop_reason: Option<String>,
timestamp: Option<String>,
session_id: Option<String>,
}
/// 同步 Claude Code 会话日志到使用统计数据库
pub fn sync_claude_session_logs(db: &Database) -> Result<SessionSyncResult, AppError> {
let projects_dir = get_claude_config_dir().join("projects");
if !projects_dir.exists() {
return Ok(SessionSyncResult {
imported: 0,
skipped: 0,
files_scanned: 0,
errors: vec![],
});
}
let mut result = SessionSyncResult {
imported: 0,
skipped: 0,
files_scanned: 0,
errors: vec![],
};
// 收集所有 .jsonl 文件
let jsonl_files = collect_jsonl_files(&projects_dir);
for file_path in &jsonl_files {
result.files_scanned += 1;
match sync_single_file(db, file_path) {
Ok((imported, skipped)) => {
result.imported += imported;
result.skipped += skipped;
}
Err(e) => {
let msg = format!("{}: {e}", file_path.display());
log::warn!("[SESSION-SYNC] 文件解析失败: {msg}");
result.errors.push(msg);
}
}
}
if result.imported > 0 {
log::info!(
"[SESSION-SYNC] 同步完成: 导入 {} 条, 跳过 {} 条, 扫描 {} 个文件",
result.imported,
result.skipped,
result.files_scanned
);
}
Ok(result)
}
/// 收集目录下所有 .jsonl 文件
fn collect_jsonl_files(projects_dir: &Path) -> Vec<PathBuf> {
let mut files = Vec::new();
let entries = match fs::read_dir(projects_dir) {
Ok(e) => e,
Err(_) => return files,
};
for entry in entries.flatten() {
let path = entry.path();
if !path.is_dir() {
continue;
}
// 每个项目目录下的 .jsonl 文件
if let Ok(sub_entries) = fs::read_dir(&path) {
for sub_entry in sub_entries.flatten() {
let sub_path = sub_entry.path();
if sub_path.extension().and_then(|e| e.to_str()) == Some("jsonl") {
files.push(sub_path);
}
}
}
}
files
}
/// 同步单个 JSONL 文件,返回 (imported, skipped)
fn sync_single_file(db: &Database, file_path: &Path) -> Result<(u32, u32), AppError> {
let file_path_str = file_path.to_string_lossy().to_string();
// 获取文件元数据
let metadata = fs::metadata(file_path)
.map_err(|e| AppError::Config(format!("无法读取文件元数据: {e}")))?;
let file_modified = metadata_modified_nanos(&metadata);
// 检查同步状态
let (last_modified, last_offset) = get_sync_state(db, &file_path_str)?;
// 文件未变化则跳过
if file_modified <= last_modified {
return Ok((0, 0));
}
// 从上次偏移位置开始增量解析
let file =
fs::File::open(file_path).map_err(|e| AppError::Config(format!("无法打开文件: {e}")))?;
let reader = BufReader::new(file);
let mut line_offset: i64 = 0;
let mut messages: HashMap<String, ParsedAssistantUsage> = HashMap::new();
let mut current_session_id: Option<String> = None;
for line_result in reader.lines() {
line_offset += 1;
// 跳过已处理的行
if line_offset <= last_offset {
continue;
}
let line = match line_result {
Ok(l) => l,
Err(_) => continue, // 容忍不完整的最后一行
};
if line.trim().is_empty() {
continue;
}
let value: serde_json::Value = match serde_json::from_str(&line) {
Ok(v) => v,
Err(_) => continue,
};
// 提取 session ID (从 system 或首条消息)
if current_session_id.is_none() {
if let Some(sid) = value.get("sessionId").and_then(|v| v.as_str()) {
current_session_id = Some(sid.to_string());
}
}
// 只处理 assistant 类型的消息
if value.get("type").and_then(|t| t.as_str()) != Some("assistant") {
continue;
}
let message = match value.get("message") {
Some(m) => m,
None => continue,
};
let msg_id = match message.get("id").and_then(|v| v.as_str()) {
Some(id) => id.to_string(),
None => continue,
};
let usage = match message.get("usage") {
Some(u) => u,
None => continue,
};
let parsed = ParsedAssistantUsage {
message_id: msg_id.clone(),
model: message
.get("model")
.and_then(|v| v.as_str())
.unwrap_or("unknown")
.to_string(),
input_tokens: usage
.get("input_tokens")
.and_then(|v| v.as_u64())
.unwrap_or(0) as u32,
output_tokens: usage
.get("output_tokens")
.and_then(|v| v.as_u64())
.unwrap_or(0) as u32,
cache_read_tokens: usage
.get("cache_read_input_tokens")
.and_then(|v| v.as_u64())
.unwrap_or(0) as u32,
cache_creation_tokens: usage
.get("cache_creation_input_tokens")
.and_then(|v| v.as_u64())
.unwrap_or(0) as u32,
stop_reason: message
.get("stop_reason")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
timestamp: value
.get("timestamp")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
session_id: current_session_id.clone(),
};
// 按 message.id 去重:优先保留有 stop_reason 的条目,否则保留最新的
let should_replace = match messages.get(&msg_id) {
None => true,
Some(existing) => {
// 新条目有 stop_reason 而旧条目没有 → 替换
if parsed.stop_reason.is_some() && existing.stop_reason.is_none() {
true
}
// 两个都有或都没有 stop_reason → 取 output_tokens 更大的
else if parsed.stop_reason.is_some() == existing.stop_reason.is_some() {
parsed.output_tokens > existing.output_tokens
} else {
false
}
}
};
if should_replace {
messages.insert(msg_id, parsed);
}
}
// 写入数据库
let mut imported: u32 = 0;
let mut skipped: u32 = 0;
for msg in messages.values() {
// 只导入有 stop_reason 的最终条目(完整的 API 调用)
if msg.stop_reason.is_none() {
continue;
}
let request_id = format!(
"{}{}",
crate::proxy::usage::parser::SESSION_REQUEST_ID_PREFIX,
msg.message_id
);
// 跳过 output_tokens 为 0 的无意义条目
if msg.output_tokens == 0 {
continue;
}
match insert_session_log_entry(db, &request_id, msg) {
Ok(true) => imported += 1,
Ok(false) => skipped += 1,
Err(e) => {
log::warn!("[SESSION-SYNC] 插入失败 ({}): {e}", msg.message_id);
skipped += 1;
}
}
}
// 更新同步状态
update_sync_state(db, &file_path_str, file_modified, line_offset)?;
Ok((imported, skipped))
}
/// 获取 session_log_sync 表中某条目的同步进度。
///
/// Shared by all session_usage_* parsers.
pub(crate) fn get_sync_state(db: &Database, file_path: &str) -> Result<(i64, i64), AppError> {
let conn = lock_conn!(db.conn);
let result = conn.query_row(
"SELECT last_modified, last_line_offset FROM session_log_sync WHERE file_path = ?1",
rusqlite::params![file_path],
|row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)),
);
Ok(result.unwrap_or((0, 0)))
}
/// 返回文件 mtime 的纳秒时间戳。
///
/// `session_log_sync.last_modified` 旧数据是秒级时间戳;新写入纳秒值不需要
/// schema 迁移,旧值会自然触发一次增量重扫,并继续依赖行 offset 避免重复导入。
pub(crate) fn metadata_modified_nanos(metadata: &fs::Metadata) -> i64 {
metadata
.modified()
.ok()
.and_then(|t| t.duration_since(SystemTime::UNIX_EPOCH).ok())
.map(|d| d.as_nanos().min(i64::MAX as u128) as i64)
.unwrap_or(0)
}
/// 更新 session_log_sync 表中某条目的同步进度。
///
/// Shared by all session_usage_* parsers.
pub(crate) fn update_sync_state(
db: &Database,
file_path: &str,
last_modified: i64,
last_offset: i64,
) -> Result<(), AppError> {
let now = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0);
let conn = lock_conn!(db.conn);
conn.execute(
"INSERT OR REPLACE INTO session_log_sync (file_path, last_modified, last_line_offset, last_synced_at)
VALUES (?1, ?2, ?3, ?4)",
rusqlite::params![file_path, last_modified, last_offset, now],
)
.map_err(|e| AppError::Database(format!("更新同步状态失败: {e}")))?;
Ok(())
}
/// 插入单条会话日志到 proxy_request_logs,返回是否成功插入 (true=新插入, false=已存在)
fn insert_session_log_entry(
db: &Database,
request_id: &str,
msg: &ParsedAssistantUsage,
) -> Result<bool, AppError> {
let conn = lock_conn!(db.conn);
let created_at = msg
.timestamp
.as_ref()
.and_then(|ts| {
chrono::DateTime::parse_from_rfc3339(ts)
.ok()
.map(|dt| dt.timestamp())
})
.unwrap_or_else(|| {
SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
});
let dedup_key = DedupKey {
app_type: "claude",
model: &msg.model,
input_tokens: msg.input_tokens,
output_tokens: msg.output_tokens,
cache_read_tokens: msg.cache_read_tokens,
cache_creation_tokens: msg.cache_creation_tokens,
created_at,
};
if should_skip_session_insert(&conn, request_id, &dedup_key)? {
return Ok(false);
}
// 计算费用
let usage = TokenUsage {
input_tokens: msg.input_tokens,
output_tokens: msg.output_tokens,
cache_read_tokens: msg.cache_read_tokens,
cache_creation_tokens: msg.cache_creation_tokens,
model: Some(msg.model.clone()),
message_id: None,
};
let pricing = find_model_pricing_for_session(&conn, &msg.model);
let multiplier = Decimal::from(1);
let (input_cost, output_cost, cache_read_cost, cache_creation_cost, total_cost) = match pricing
{
Some(p) => {
let cost = CostCalculator::calculate(&usage, &p, multiplier);
(
cost.input_cost.to_string(),
cost.output_cost.to_string(),
cost.cache_read_cost.to_string(),
cost.cache_creation_cost.to_string(),
cost.total_cost.to_string(),
)
}
None => (
"0".to_string(),
"0".to_string(),
"0".to_string(),
"0".to_string(),
"0".to_string(),
),
};
conn.execute(
"INSERT OR IGNORE INTO proxy_request_logs (
request_id, provider_id, app_type, model, request_model,
input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens,
input_cost_usd, output_cost_usd, cache_read_cost_usd, cache_creation_cost_usd, total_cost_usd,
latency_ms, first_token_ms, status_code, error_message, session_id,
provider_type, is_streaming, cost_multiplier, created_at, data_source
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24)",
rusqlite::params![
request_id,
"_session", // provider_id: 标记为会话来源
"claude", // app_type
msg.model,
msg.model, // request_model = model
msg.input_tokens,
msg.output_tokens,
msg.cache_read_tokens,
msg.cache_creation_tokens,
input_cost,
output_cost,
cache_read_cost,
cache_creation_cost,
total_cost,
0i64, // latency_ms: 会话日志无此数据
Option::<i64>::None, // first_token_ms
200i64, // status_code: 有 stop_reason 说明请求成功
Option::<String>::None, // error_message
msg.session_id,
Some("session_log"), // provider_type
1i64, // is_streaming: Claude Code 通常使用流式
"1.0", // cost_multiplier
created_at,
"session_log", // data_source
],
)
.map_err(|e| AppError::Database(format!("插入会话日志失败: {e}")))?;
Ok(true)
}
/// 从 model_pricing 表查找模型定价(支持模糊匹配)
fn find_model_pricing_for_session(
conn: &rusqlite::Connection,
model_id: &str,
) -> Option<ModelPricing> {
find_model_pricing(conn, model_id)
}
/// 查询数据来源分布统计
pub fn get_data_source_breakdown(db: &Database) -> Result<Vec<DataSourceSummary>, AppError> {
let conn = lock_conn!(db.conn);
let effective_filter = effective_usage_log_filter("l");
let sql = format!(
"SELECT COALESCE(l.data_source, 'proxy') as ds, COUNT(*) as cnt,
COALESCE(SUM(CAST(l.total_cost_usd AS REAL)), 0) as cost
FROM proxy_request_logs l
WHERE {effective_filter}
GROUP BY ds
ORDER BY cnt DESC"
);
let mut stmt = conn.prepare(&sql)?;
let rows = stmt.query_map([], |row| {
Ok(DataSourceSummary {
data_source: row.get(0)?,
request_count: row.get::<_, i64>(1)? as u32,
total_cost_usd: format!("{:.6}", row.get::<_, f64>(2)?),
})
})?;
let mut summaries = Vec::new();
for row in rows {
summaries.push(row.map_err(|e| AppError::Database(e.to_string()))?);
}
Ok(summaries)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_usage_from_jsonl_line() {
let line = r#"{"type":"assistant","message":{"id":"msg_test123","model":"claude-opus-4-6","usage":{"input_tokens":3,"output_tokens":150,"cache_read_input_tokens":5000,"cache_creation_input_tokens":10000},"stop_reason":"end_turn"},"timestamp":"2026-04-05T12:00:00Z","sessionId":"session-abc"}"#;
let value: serde_json::Value = serde_json::from_str(line).unwrap();
assert_eq!(
value.get("type").and_then(|t| t.as_str()),
Some("assistant")
);
let message = value.get("message").unwrap();
let usage = message.get("usage").unwrap();
assert_eq!(usage.get("input_tokens").unwrap().as_u64().unwrap(), 3);
assert_eq!(usage.get("output_tokens").unwrap().as_u64().unwrap(), 150);
assert_eq!(
usage
.get("cache_read_input_tokens")
.unwrap()
.as_u64()
.unwrap(),
5000
);
assert_eq!(
usage
.get("cache_creation_input_tokens")
.unwrap()
.as_u64()
.unwrap(),
10000
);
assert_eq!(
message.get("stop_reason").unwrap().as_str().unwrap(),
"end_turn"
);
}
#[test]
fn test_dedup_by_message_id() {
// 同一个 message.id 有多条,应该取 stop_reason 有值的那条
let mut messages: HashMap<String, ParsedAssistantUsage> = HashMap::new();
// 中间条目(无 stop_reason
let intermediate = ParsedAssistantUsage {
message_id: "msg_1".to_string(),
model: "claude-opus-4-6".to_string(),
input_tokens: 3,
output_tokens: 26,
cache_read_tokens: 5000,
cache_creation_tokens: 10000,
stop_reason: None,
timestamp: Some("2026-04-05T12:00:00Z".to_string()),
session_id: None,
};
messages.insert("msg_1".to_string(), intermediate);
// 最终条目(有 stop_reason
let final_entry = ParsedAssistantUsage {
message_id: "msg_1".to_string(),
model: "claude-opus-4-6".to_string(),
input_tokens: 3,
output_tokens: 1349,
cache_read_tokens: 5000,
cache_creation_tokens: 10000,
stop_reason: Some("end_turn".to_string()),
timestamp: Some("2026-04-05T12:00:00Z".to_string()),
session_id: None,
};
// 应该替换
let should_replace = final_entry.stop_reason.is_some()
&& messages.get("msg_1").unwrap().stop_reason.is_none();
assert!(should_replace);
messages.insert("msg_1".to_string(), final_entry);
assert_eq!(messages.get("msg_1").unwrap().output_tokens, 1349);
}
#[test]
fn test_insert_claude_session_skips_matching_proxy_log() -> Result<(), AppError> {
let db = Database::memory()?;
{
let conn = lock_conn!(db.conn);
conn.execute(
"INSERT INTO proxy_request_logs (
request_id, provider_id, app_type, model, request_model,
input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens,
total_cost_usd, latency_ms, status_code, created_at, data_source
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
rusqlite::params![
"proxy-different-id",
"openai-compatible",
"claude",
"claude-sonnet-4-5",
"claude-sonnet-4-5",
100,
20,
10,
5,
"0.10",
100,
200,
1000,
"proxy"
],
)?;
}
let msg = ParsedAssistantUsage {
message_id: "msg_1".to_string(),
model: "claude-sonnet-4-5".to_string(),
input_tokens: 100,
output_tokens: 20,
cache_read_tokens: 10,
cache_creation_tokens: 5,
stop_reason: Some("end_turn".to_string()),
timestamp: Some("1970-01-01T00:16:45Z".to_string()),
session_id: Some("session-1".to_string()),
};
let inserted = insert_session_log_entry(&db, "session:msg_1", &msg)?;
assert!(!inserted);
let conn = lock_conn!(db.conn);
let count: i64 = conn.query_row("SELECT COUNT(*) FROM proxy_request_logs", [], |row| {
row.get(0)
})?;
assert_eq!(count, 1);
Ok(())
}
}