fix(proxy): improve cache hit rate for Codex/Responses requests

prompt_cache_key was falling back to provider.id when the client did not
supply a session, which collapsed every conversation onto a single key
and defeated upstream prefix caching. Only emit the key when a real
client-provided session/thread identity is available; otherwise let the
upstream use its default matching behaviour.

Additional fixes that affect cache stability:
- Canonicalise (sort) JSON keys in outgoing request bodies and in
  tool_call arguments / tool_result content so semantically identical
  requests produce identical byte sequences for upstream prefix caches.
- Exempt JSON Schema property maps (properties, patternProperties,
  definitions, \$defs) from the underscore-prefix filter so user-defined
  schema keys like _id and _meta survive.
- Add a [CacheTrace] debug log with stable hashes for instructions,
  tools, input and include to help diagnose cache misses.
- Thread session_id into the usage logger for request correlation.
This commit is contained in:
Jason
2026-05-11 08:41:32 +08:00
parent aec055a1d1
commit 00a789e7a3
9 changed files with 392 additions and 31 deletions
+149 -1
View File
@@ -7,6 +7,7 @@ use super::{
body_filter::filter_private_params_with_whitelist,
error::*,
failover_switch::FailoverSwitchManager,
json_canonical::{canonicalize_value, short_value_hash},
log_codes::fwd as log_fwd,
provider_router::ProviderRouter,
providers::{
@@ -1008,7 +1009,15 @@ impl RequestForwarder {
// 过滤私有参数(以 `_` 开头的字段),防止内部信息泄露到上游
// 默认使用空白名单,过滤所有 _ 前缀字段
let filtered_body = filter_private_params_with_whitelist(request_body, &[]);
let filtered_body = prepare_upstream_request_body(request_body);
log_prompt_cache_trace(
app_type,
provider,
&effective_endpoint,
resolved_claude_api_format.as_deref(),
&filtered_body,
self.session_client_provided,
);
let force_identity_encoding = needs_transform
|| should_force_identity_encoding(&effective_endpoint, &filtered_body, headers);
@@ -1976,6 +1985,65 @@ fn summarize_text_for_log(text: &str, max_chars: usize) -> String {
format!("{truncated}...")
}
fn prepare_upstream_request_body(request_body: Value) -> Value {
canonicalize_value(filter_private_params_with_whitelist(request_body, &[]))
}
fn log_prompt_cache_trace(
app_type: &AppType,
provider: &Provider,
endpoint: &str,
api_format: Option<&str>,
body: &Value,
session_client_provided: bool,
) {
if !log::log_enabled!(log::Level::Debug) {
return;
}
let prompt_cache_key = body
.get("prompt_cache_key")
.and_then(|value| value.as_str())
.map(|key| format!("present(len={})", key.len()))
.unwrap_or_else(|| "absent".to_string());
let store = body
.get("store")
.map(value_for_log)
.unwrap_or_else(|| "absent".to_string());
let stream = body
.get("stream")
.map(value_for_log)
.unwrap_or_else(|| "absent".to_string());
log::debug!(
"[CacheTrace] app={}, provider={}, endpoint={}, api_format={}, session_client_provided={}, prompt_cache_key={}, store={}, stream={}, instructions_hash={}, tools_hash={}, input_hash={}, include_hash={}, body_hash={}",
app_type.as_str(),
provider.id,
endpoint,
api_format.unwrap_or("native"),
session_client_provided,
prompt_cache_key,
store,
stream,
short_value_hash(body.get("instructions")),
short_value_hash(body.get("tools")),
short_value_hash(body.get("input")),
short_value_hash(body.get("include")),
short_value_hash(Some(body)),
);
}
fn value_for_log(value: &Value) -> String {
match value {
Value::Bool(value) => value.to_string(),
Value::Number(value) => value.to_string(),
Value::String(value) => value.clone(),
Value::Null => "null".to_string(),
Value::Array(values) => format!("array(len={})", values.len()),
Value::Object(values) => format!("object(len={})", values.len()),
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -2068,6 +2136,86 @@ mod tests {
assert_eq!(summary, "line1 line2...");
}
#[test]
fn canonical_json_sorts_object_keys_for_cache_trace_hashes() {
let left = json!({
"tools": [
{
"parameters": {
"properties": {
"b": {"type": "string"},
"a": {"type": "number"}
},
"type": "object"
},
"name": "lookup"
}
]
});
let right = json!({
"tools": [
{
"name": "lookup",
"parameters": {
"type": "object",
"properties": {
"a": {"type": "number"},
"b": {"type": "string"}
}
}
}
]
});
assert_eq!(
crate::proxy::json_canonical::canonical_json_string(&left),
crate::proxy::json_canonical::canonical_json_string(&right)
);
assert_eq!(
short_value_hash(Some(&left)),
short_value_hash(Some(&right))
);
}
#[test]
fn prepare_upstream_request_body_filters_private_fields_and_canonicalizes_order() {
let body = json!({
"z": 1,
"_internal": "drop",
"tools": [
{
"name": "lookup",
"parameters": {
"type": "object",
"properties": {
"_id": {
"_private_note": "drop",
"type": "string"
},
"b": {"type": "number"},
"a": {"type": "string"}
}
}
}
],
"a": 2
});
let prepared = prepare_upstream_request_body(body);
assert!(prepared.get("_internal").is_none());
assert!(prepared["tools"][0]["parameters"]["properties"]
.get("_id")
.is_some());
assert!(prepared["tools"][0]["parameters"]["properties"]["_id"]
.get("_private_note")
.is_none());
assert_eq!(
serde_json::to_string(&prepared).unwrap(),
r#"{"a":2,"tools":[{"name":"lookup","parameters":{"properties":{"_id":{"type":"string"},"a":{"type":"string"},"b":{"type":"number"}},"type":"object"}}],"z":1}"#
);
}
#[test]
fn codex_oauth_session_headers_match_codex_cache_identity() {
let headers = build_codex_oauth_session_headers("session-123");