feat: add usage daily rollups, incremental auto-vacuum, and sync-aware backup

- Add usage_daily_rollups table (schema v6) to aggregate proxy request
  logs into daily summaries, reducing query overhead for statistics
- Add rollup_and_prune DAO that aggregates old detail logs (>N days)
  into rollup rows and deletes the originals
- Update all usage stats queries to UNION detail logs with rollup data
- Introduce incremental auto-vacuum for SQLite, with startup and
  periodic cleanup of old stream_check_logs and request log rollups
- Split backup export/import into full vs sync variants: WebDAV sync
  now skips local-only table data (proxy_request_logs,
  stream_check_logs, provider_health, proxy_live_backup,
  usage_daily_rollups) while preserving them on import
- Add enable_logging guard to skip request log writes when disabled
- Apply cargo fmt formatting fixes across multiple modules
This commit is contained in:
Jason
2026-03-08 17:18:21 +08:00
parent 3f711d6504
commit bf40b0138c
16 changed files with 910 additions and 97 deletions
+14 -5
View File
@@ -617,7 +617,10 @@ mod tests {
let mut tool_index_by_call: HashMap<String, u64> = HashMap::new();
for event in &events {
if event.get("type").and_then(|v| v.as_str()) == Some("content_block_start")
&& event.pointer("/content_block/type").and_then(|v| v.as_str()) == Some("tool_use")
&& event
.pointer("/content_block/type")
.and_then(|v| v.as_str())
== Some("tool_use")
{
if let (Some(call_id), Some(index)) = (
event.pointer("/content_block/id").and_then(|v| v.as_str()),
@@ -666,8 +669,7 @@ mod tests {
assert!(events.iter().any(|event| {
event.get("type").and_then(|v| v.as_str()) == Some("message_delta")
&& event.pointer("/delta/stop_reason").and_then(|v| v.as_str())
== Some("tool_use")
&& event.pointer("/delta/stop_reason").and_then(|v| v.as_str()) == Some("tool_use")
}));
}
@@ -701,7 +703,10 @@ mod tests {
.iter()
.filter(|event| {
event.get("type").and_then(|v| v.as_str()) == Some("content_block_start")
&& event.pointer("/content_block/type").and_then(|v| v.as_str()) == Some("tool_use")
&& event
.pointer("/content_block/type")
.and_then(|v| v.as_str())
== Some("tool_use")
})
.collect();
assert_eq!(starts.len(), 1);
@@ -727,7 +732,11 @@ mod tests {
&& event.pointer("/delta/type").and_then(|v| v.as_str())
== Some("input_json_delta")
})
.filter_map(|event| event.pointer("/delta/partial_json").and_then(|v| v.as_str()))
.filter_map(|event| {
event
.pointer("/delta/partial_json")
.and_then(|v| v.as_str())
})
.collect();
assert!(deltas.contains(&"{\"a\":"));
assert!(deltas.contains(&"1}"));
+7 -2
View File
@@ -337,7 +337,10 @@ pub fn openai_to_anthropic(body: Value) -> Result<Value, ProxyError> {
// 兼容旧格式(function_call
if !has_tool_use {
if let Some(function_call) = message.get("function_call") {
let id = function_call.get("id").and_then(|i| i.as_str()).unwrap_or("");
let id = function_call
.get("id")
.and_then(|i| i.as_str())
.unwrap_or("");
let name = function_call
.get("name")
.and_then(|n| n.as_str())
@@ -372,7 +375,9 @@ pub fn openai_to_anthropic(body: Value) -> Result<Value, ProxyError> {
"tool_calls" | "function_call" => "tool_use",
"content_filter" => "end_turn",
other => {
log::warn!("[Claude/OpenAI] Unknown finish_reason in non-streaming response: {other}");
log::warn!(
"[Claude/OpenAI] Unknown finish_reason in non-streaming response: {other}"
);
"end_turn"
}
})
+15
View File
@@ -283,6 +283,11 @@ fn create_usage_collector(
status_code: u16,
parser_config: &UsageParserConfig,
) -> SseUsageCollector {
let logging_enabled = state
.config
.try_read()
.map(|c| c.enable_logging)
.unwrap_or(true);
let state = state.clone();
let provider_id = ctx.provider.id.clone();
let request_model = ctx.request_model.clone();
@@ -294,6 +299,9 @@ fn create_usage_collector(
let session_id = ctx.session_id.clone();
SseUsageCollector::new(start_time, move |events, first_token_ms| {
if !logging_enabled {
return;
}
if let Some(usage) = stream_parser(&events) {
let model = model_extractor(&events, &request_model);
let latency_ms = start_time.elapsed().as_millis() as u64;
@@ -358,6 +366,13 @@ fn spawn_log_usage(
status_code: u16,
is_streaming: bool,
) {
// Check enable_logging before spawning the log task
if let Ok(config) = state.config.try_read() {
if !config.enable_logging {
return;
}
}
let state = state.clone();
let provider_id = ctx.provider.id.clone();
let app_type_str = ctx.app_type_str.to_string();