mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-07-30 02:14:43 +08:00
fix(usage): prevent double-counting between proxy and session-log sources
Proxy writes and session-log sync wrote to proxy_request_logs with
mismatched request_ids: only Claude on a native Anthropic backend used the
shared `session:{message_id}` key. Codex/Gemini and Claude-through-OpenAI
providers always produced distinct ids, so primary-key dedup never fired
and every real request was recorded twice.
Adds a 7-dim fingerprint dedup (app_type, 4 token counts, 2xx status,
model with case-insensitive match, ±10min window) wired into three layers:
- Write path: should_skip_session_insert() blocks duplicate session rows
before INSERT, unifying the previously-divergent Claude/Codex/Gemini
paths through a single DedupKey-based helper.
- Read path: effective_usage_log_filter() excludes already-covered session
rows from every aggregation query.
- Rollup path: same filter applied so usage_daily_rollups never absorbs
duplicates.
Also adds a covering index (idx_request_logs_dedup_lookup) so the EXISTS
subquery stays index-only, and a transform.rs regression test that pins
openai_to_anthropic id preservation - the missing piece that lets
Claude+OpenAI-compatible providers reuse the session: id scheme.
This commit is contained in:
@@ -4,6 +4,7 @@
|
||||
|
||||
use crate::database::{lock_conn, Database};
|
||||
use crate::error::AppError;
|
||||
use crate::services::usage_stats::effective_usage_log_filter;
|
||||
use chrono::{Duration, Local, TimeZone};
|
||||
|
||||
/// Compute the rollup/prune cutoff aligned to a local-day boundary.
|
||||
@@ -101,7 +102,8 @@ impl Database {
|
||||
|
||||
fn do_rollup_and_prune(conn: &rusqlite::Connection, cutoff: i64) -> Result<u64, AppError> {
|
||||
// Aggregate old logs, merging with any pre-existing rollup rows via LEFT JOIN.
|
||||
conn.execute(
|
||||
let effective_filter = effective_usage_log_filter("l");
|
||||
let aggregation_sql = format!(
|
||||
"INSERT OR REPLACE INTO usage_daily_rollups
|
||||
(date, app_type, provider_id, model,
|
||||
request_count, success_count,
|
||||
@@ -124,27 +126,30 @@ impl Database {
|
||||
ELSE 0 END
|
||||
FROM (
|
||||
SELECT
|
||||
date(created_at, 'unixepoch', 'localtime') as d,
|
||||
app_type as a, provider_id as p, model as m,
|
||||
date(l.created_at, 'unixepoch', 'localtime') as d,
|
||||
l.app_type as a, l.provider_id as p, l.model as m,
|
||||
COUNT(*) as new_req,
|
||||
SUM(CASE WHEN status_code >= 200 AND status_code < 300 THEN 1 ELSE 0 END) as new_succ,
|
||||
COALESCE(SUM(input_tokens), 0) as new_in,
|
||||
COALESCE(SUM(output_tokens), 0) as new_out,
|
||||
COALESCE(SUM(cache_read_tokens), 0) as new_cr,
|
||||
COALESCE(SUM(cache_creation_tokens), 0) as new_cc,
|
||||
COALESCE(SUM(CAST(total_cost_usd AS REAL)), 0) as new_cost,
|
||||
COALESCE(AVG(latency_ms), 0) as new_lat
|
||||
FROM proxy_request_logs WHERE created_at < ?1
|
||||
SUM(CASE WHEN l.status_code >= 200 AND l.status_code < 300 THEN 1 ELSE 0 END) as new_succ,
|
||||
COALESCE(SUM(l.input_tokens), 0) as new_in,
|
||||
COALESCE(SUM(l.output_tokens), 0) as new_out,
|
||||
COALESCE(SUM(l.cache_read_tokens), 0) as new_cr,
|
||||
COALESCE(SUM(l.cache_creation_tokens), 0) as new_cc,
|
||||
COALESCE(SUM(CAST(l.total_cost_usd AS REAL)), 0) as new_cost,
|
||||
COALESCE(AVG(l.latency_ms), 0) as new_lat
|
||||
FROM proxy_request_logs l
|
||||
WHERE l.created_at < ?1 AND {effective_filter}
|
||||
GROUP BY d, a, p, m
|
||||
) agg
|
||||
LEFT JOIN usage_daily_rollups old
|
||||
ON old.date = agg.d AND old.app_type = agg.a
|
||||
AND old.provider_id = agg.p AND old.model = agg.m",
|
||||
[cutoff],
|
||||
)
|
||||
.map_err(|e| AppError::Database(format!("Rollup aggregation failed: {e}")))?;
|
||||
AND old.provider_id = agg.p AND old.model = agg.m"
|
||||
);
|
||||
|
||||
// Delete the aggregated detail rows
|
||||
conn.execute(&aggregation_sql, [cutoff])
|
||||
.map_err(|e| AppError::Database(format!("Rollup aggregation failed: {e}")))?;
|
||||
|
||||
// INSERT uses the effective-log filter to exclude duplicate session rows.
|
||||
// DELETE intentionally prunes all old details so those duplicates are discarded.
|
||||
let deleted = conn
|
||||
.execute(
|
||||
"DELETE FROM proxy_request_logs WHERE created_at < ?1",
|
||||
@@ -254,6 +259,69 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rollup_uses_effective_usage_logs() -> Result<(), AppError> {
|
||||
let db = Database::memory()?;
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let old_ts = now - 40 * 86400;
|
||||
|
||||
{
|
||||
let conn = crate::database::lock_conn!(db.conn);
|
||||
conn.execute(
|
||||
"INSERT INTO proxy_request_logs (
|
||||
request_id, provider_id, app_type, model, request_model,
|
||||
input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens,
|
||||
total_cost_usd, latency_ms, status_code, created_at, data_source
|
||||
) VALUES (?1, 'openai', 'codex', 'gpt-5.4', 'gpt-5.4', 100, 20, 10, 0, '0.10', 100, 200, ?2, 'proxy')",
|
||||
rusqlite::params!["codex-proxy-old", old_ts],
|
||||
)?;
|
||||
conn.execute(
|
||||
"INSERT INTO proxy_request_logs (
|
||||
request_id, provider_id, app_type, model, request_model,
|
||||
input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens,
|
||||
total_cost_usd, latency_ms, status_code, created_at, data_source
|
||||
) VALUES (?1, '_codex_session', 'codex', 'gpt-5.4', 'gpt-5.4', 100, 20, 10, 0, '0.10', 0, 200, ?2, 'codex_session')",
|
||||
rusqlite::params!["codex-session-old-dup", old_ts + 60],
|
||||
)?;
|
||||
}
|
||||
|
||||
let deleted = db.rollup_and_prune(30)?;
|
||||
assert_eq!(deleted, 2);
|
||||
|
||||
let conn = crate::database::lock_conn!(db.conn);
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT provider_id, request_count, input_tokens, output_tokens, cache_read_tokens
|
||||
FROM usage_daily_rollups WHERE app_type = 'codex'",
|
||||
)?;
|
||||
let rows = stmt
|
||||
.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, i64>(1)?,
|
||||
row.get::<_, i64>(2)?,
|
||||
row.get::<_, i64>(3)?,
|
||||
row.get::<_, i64>(4)?,
|
||||
))
|
||||
})?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
assert_eq!(rows.len(), 1);
|
||||
let (provider_id, request_count, input_tokens, output_tokens, cache_read_tokens) = &rows[0];
|
||||
assert_eq!(provider_id, "openai");
|
||||
assert_eq!(*request_count, 1);
|
||||
assert_eq!(*input_tokens, 100);
|
||||
assert_eq!(*output_tokens, 20);
|
||||
assert_eq!(*cache_read_tokens, 10);
|
||||
|
||||
let remaining: i64 =
|
||||
conn.query_row("SELECT COUNT(*) FROM proxy_request_logs", [], |row| {
|
||||
row.get(0)
|
||||
})?;
|
||||
assert_eq!(remaining, 0);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rollup_noop_when_no_old_data() -> Result<(), AppError> {
|
||||
let db = Database::memory()?;
|
||||
|
||||
Reference in New Issue
Block a user