mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-07-24 12:44:18 +08:00
fix(cache): strengthen prompt cache breakpoint injection
Honor both the optimizer master switch and the cache-injection sub-switch before mutating native optimizer requests. Use the available four-breakpoint budget across tools, system content, the latest cacheable message, and an older user anchor for long tool-heavy conversations. Preserve caller-owned breakpoint limits, avoid thinking blocks as cache targets, normalize configured TTLs, and add regression coverage for disabled optimization and long histories.
This commit is contained in:
@@ -7,7 +7,7 @@ use serde_json::{json, Value};
|
||||
|
||||
/// 在请求体关键位置注入 cache_control 断点
|
||||
pub fn inject(body: &mut Value, config: &OptimizerConfig) {
|
||||
if !config.cache_injection {
|
||||
if !config.enabled || !config.cache_injection {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -89,24 +89,29 @@ pub fn inject(body: &mut Value, config: &OptimizerConfig) {
|
||||
// user/tool_result 结束;只标 assistant 会让最新稳定前缀无法命中缓存。
|
||||
if budget > 0 {
|
||||
if let Some(messages) = body.get_mut("messages").and_then(|m| m.as_array_mut()) {
|
||||
'messages: for message in messages.iter_mut().rev() {
|
||||
if let Some(content) = message.get_mut("content").and_then(|c| c.as_array_mut()) {
|
||||
if let Some(block) = content.iter_mut().rev().find(|block| {
|
||||
!matches!(
|
||||
block.get("type").and_then(Value::as_str),
|
||||
Some("thinking" | "redacted_thinking")
|
||||
)
|
||||
}) {
|
||||
if block.get("cache_control").is_none() {
|
||||
if let Some(object) = block.as_object_mut() {
|
||||
object.insert(
|
||||
"cache_control".to_string(),
|
||||
make_cache_control(&config.cache_ttl),
|
||||
);
|
||||
injected.push("msgs");
|
||||
}
|
||||
for message in messages.iter_mut().rev() {
|
||||
if inject_message_breakpoint(message, &config.cache_ttl) {
|
||||
budget -= 1;
|
||||
injected.push("msgs-latest");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// (d) A second, older user anchor helps long tool-result turns where
|
||||
// the stable prefix falls outside Anthropic's 20-block lookback from
|
||||
// the newest breakpoint. Keep this best-effort and inside the 4-BP cap.
|
||||
if budget > 0 && messages.len() >= 4 {
|
||||
let mut user_count = 0;
|
||||
for message in messages.iter_mut().rev() {
|
||||
if message.get("role").and_then(Value::as_str) != Some("user") {
|
||||
continue;
|
||||
}
|
||||
user_count += 1;
|
||||
if user_count == 2 {
|
||||
if inject_message_breakpoint(message, &config.cache_ttl) {
|
||||
injected.push("msgs-prior-user");
|
||||
}
|
||||
break 'messages;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -121,6 +126,28 @@ pub fn inject(body: &mut Value, config: &OptimizerConfig) {
|
||||
);
|
||||
}
|
||||
|
||||
fn inject_message_breakpoint(message: &mut Value, ttl: &str) -> bool {
|
||||
let Some(content) = message.get_mut("content").and_then(Value::as_array_mut) else {
|
||||
return false;
|
||||
};
|
||||
let Some(block) = content.iter_mut().rev().find(|block| {
|
||||
!matches!(
|
||||
block.get("type").and_then(Value::as_str),
|
||||
Some("thinking" | "redacted_thinking")
|
||||
)
|
||||
}) else {
|
||||
return false;
|
||||
};
|
||||
if block.get("cache_control").is_some() {
|
||||
return false;
|
||||
}
|
||||
let Some(object) = block.as_object_mut() else {
|
||||
return false;
|
||||
};
|
||||
object.insert("cache_control".to_string(), make_cache_control(ttl));
|
||||
true
|
||||
}
|
||||
|
||||
fn make_cache_control(ttl: &str) -> Value {
|
||||
if ttl == "5m" {
|
||||
json!({"type": "ephemeral"})
|
||||
@@ -244,6 +271,30 @@ mod tests {
|
||||
.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_long_history_uses_fourth_prior_user_breakpoint() {
|
||||
let mut body = json!({
|
||||
"model":"test",
|
||||
"tools":[{"name":"tool1"}],
|
||||
"system":[{"type":"text","text":"sys"}],
|
||||
"messages":[
|
||||
{"role":"user","content":[{"type":"text","text":"first"}]},
|
||||
{"role":"assistant","content":[{"type":"text","text":"answer"}]},
|
||||
{"role":"user","content":[{"type":"tool_result","tool_use_id":"c1","content":"result"}]},
|
||||
{"role":"assistant","content":[{"type":"text","text":"latest"}]}
|
||||
]
|
||||
});
|
||||
|
||||
inject(&mut body, &default_config());
|
||||
assert_eq!(count_existing(&body), 4);
|
||||
assert!(body["messages"][0]["content"][0]
|
||||
.get("cache_control")
|
||||
.is_some());
|
||||
assert!(body["messages"][3]["content"][0]
|
||||
.get("cache_control")
|
||||
.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_existing_four_breakpoints_only_upgrades_ttl() {
|
||||
let mut body = json!({
|
||||
@@ -379,6 +430,24 @@ mod tests {
|
||||
assert_eq!(body, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_optimizer_disabled_no_change() {
|
||||
let config = OptimizerConfig {
|
||||
enabled: false,
|
||||
cache_injection: true,
|
||||
..default_config()
|
||||
};
|
||||
let mut body = json!({
|
||||
"model":"test",
|
||||
"tools":[{"name":"tool1"}],
|
||||
"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]
|
||||
});
|
||||
let original = body.clone();
|
||||
|
||||
inject(&mut body, &config);
|
||||
assert_eq!(body, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_skip_thinking_blocks_in_assistant() {
|
||||
let mut body = json!({
|
||||
|
||||
Reference in New Issue
Block a user