fix(cache): strengthen prompt cache breakpoint injection

Honor both the optimizer master switch and the cache-injection sub-switch before mutating native optimizer requests.

Use the available four-breakpoint budget across tools, system content, the latest cacheable message, and an older user anchor for long tool-heavy conversations.

Preserve caller-owned breakpoint limits, avoid thinking blocks as cache targets, normalize configured TTLs, and add regression coverage for disabled optimization and long histories.
This commit is contained in:
Jason
2026-07-12 12:19:28 +08:00
parent 650905af2c
commit b9263a8040
+87 -18
View File
@@ -7,7 +7,7 @@ use serde_json::{json, Value};
/// 在请求体关键位置注入 cache_control 断点 /// 在请求体关键位置注入 cache_control 断点
pub fn inject(body: &mut Value, config: &OptimizerConfig) { pub fn inject(body: &mut Value, config: &OptimizerConfig) {
if !config.cache_injection { if !config.enabled || !config.cache_injection {
return; return;
} }
@@ -89,24 +89,29 @@ pub fn inject(body: &mut Value, config: &OptimizerConfig) {
// user/tool_result 结束;只标 assistant 会让最新稳定前缀无法命中缓存。 // user/tool_result 结束;只标 assistant 会让最新稳定前缀无法命中缓存。
if budget > 0 { if budget > 0 {
if let Some(messages) = body.get_mut("messages").and_then(|m| m.as_array_mut()) { if let Some(messages) = body.get_mut("messages").and_then(|m| m.as_array_mut()) {
'messages: for message in messages.iter_mut().rev() { for message in messages.iter_mut().rev() {
if let Some(content) = message.get_mut("content").and_then(|c| c.as_array_mut()) { if inject_message_breakpoint(message, &config.cache_ttl) {
if let Some(block) = content.iter_mut().rev().find(|block| { budget -= 1;
!matches!( injected.push("msgs-latest");
block.get("type").and_then(Value::as_str), break;
Some("thinking" | "redacted_thinking") }
) }
}) {
if block.get("cache_control").is_none() { // (d) A second, older user anchor helps long tool-result turns where
if let Some(object) = block.as_object_mut() { // the stable prefix falls outside Anthropic's 20-block lookback from
object.insert( // the newest breakpoint. Keep this best-effort and inside the 4-BP cap.
"cache_control".to_string(), if budget > 0 && messages.len() >= 4 {
make_cache_control(&config.cache_ttl), let mut user_count = 0;
); for message in messages.iter_mut().rev() {
injected.push("msgs"); if message.get("role").and_then(Value::as_str) != Some("user") {
} continue;
}
user_count += 1;
if user_count == 2 {
if inject_message_breakpoint(message, &config.cache_ttl) {
injected.push("msgs-prior-user");
} }
break 'messages; break;
} }
} }
} }
@@ -121,6 +126,28 @@ pub fn inject(body: &mut Value, config: &OptimizerConfig) {
); );
} }
fn inject_message_breakpoint(message: &mut Value, ttl: &str) -> bool {
let Some(content) = message.get_mut("content").and_then(Value::as_array_mut) else {
return false;
};
let Some(block) = content.iter_mut().rev().find(|block| {
!matches!(
block.get("type").and_then(Value::as_str),
Some("thinking" | "redacted_thinking")
)
}) else {
return false;
};
if block.get("cache_control").is_some() {
return false;
}
let Some(object) = block.as_object_mut() else {
return false;
};
object.insert("cache_control".to_string(), make_cache_control(ttl));
true
}
fn make_cache_control(ttl: &str) -> Value { fn make_cache_control(ttl: &str) -> Value {
if ttl == "5m" { if ttl == "5m" {
json!({"type": "ephemeral"}) json!({"type": "ephemeral"})
@@ -244,6 +271,30 @@ mod tests {
.is_some()); .is_some());
} }
#[test]
fn test_long_history_uses_fourth_prior_user_breakpoint() {
let mut body = json!({
"model":"test",
"tools":[{"name":"tool1"}],
"system":[{"type":"text","text":"sys"}],
"messages":[
{"role":"user","content":[{"type":"text","text":"first"}]},
{"role":"assistant","content":[{"type":"text","text":"answer"}]},
{"role":"user","content":[{"type":"tool_result","tool_use_id":"c1","content":"result"}]},
{"role":"assistant","content":[{"type":"text","text":"latest"}]}
]
});
inject(&mut body, &default_config());
assert_eq!(count_existing(&body), 4);
assert!(body["messages"][0]["content"][0]
.get("cache_control")
.is_some());
assert!(body["messages"][3]["content"][0]
.get("cache_control")
.is_some());
}
#[test] #[test]
fn test_existing_four_breakpoints_only_upgrades_ttl() { fn test_existing_four_breakpoints_only_upgrades_ttl() {
let mut body = json!({ let mut body = json!({
@@ -379,6 +430,24 @@ mod tests {
assert_eq!(body, original); assert_eq!(body, original);
} }
#[test]
fn test_optimizer_disabled_no_change() {
let config = OptimizerConfig {
enabled: false,
cache_injection: true,
..default_config()
};
let mut body = json!({
"model":"test",
"tools":[{"name":"tool1"}],
"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]
});
let original = body.clone();
inject(&mut body, &config);
assert_eq!(body, original);
}
#[test] #[test]
fn test_skip_thinking_blocks_in_assistant() { fn test_skip_thinking_blocks_in_assistant() {
let mut body = json!({ let mut body = json!({