mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-07-24 12:44:18 +08:00
fix(cache): strengthen prompt cache breakpoint injection
Honor both the optimizer master switch and the cache-injection sub-switch before mutating native optimizer requests. Use the available four-breakpoint budget across tools, system content, the latest cacheable message, and an older user anchor for long tool-heavy conversations. Preserve caller-owned breakpoint limits, avoid thinking blocks as cache targets, normalize configured TTLs, and add regression coverage for disabled optimization and long histories.
This commit is contained in:
@@ -7,7 +7,7 @@ use serde_json::{json, Value};
|
|||||||
|
|
||||||
/// 在请求体关键位置注入 cache_control 断点
|
/// 在请求体关键位置注入 cache_control 断点
|
||||||
pub fn inject(body: &mut Value, config: &OptimizerConfig) {
|
pub fn inject(body: &mut Value, config: &OptimizerConfig) {
|
||||||
if !config.cache_injection {
|
if !config.enabled || !config.cache_injection {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -89,24 +89,29 @@ pub fn inject(body: &mut Value, config: &OptimizerConfig) {
|
|||||||
// user/tool_result 结束;只标 assistant 会让最新稳定前缀无法命中缓存。
|
// user/tool_result 结束;只标 assistant 会让最新稳定前缀无法命中缓存。
|
||||||
if budget > 0 {
|
if budget > 0 {
|
||||||
if let Some(messages) = body.get_mut("messages").and_then(|m| m.as_array_mut()) {
|
if let Some(messages) = body.get_mut("messages").and_then(|m| m.as_array_mut()) {
|
||||||
'messages: for message in messages.iter_mut().rev() {
|
for message in messages.iter_mut().rev() {
|
||||||
if let Some(content) = message.get_mut("content").and_then(|c| c.as_array_mut()) {
|
if inject_message_breakpoint(message, &config.cache_ttl) {
|
||||||
if let Some(block) = content.iter_mut().rev().find(|block| {
|
budget -= 1;
|
||||||
!matches!(
|
injected.push("msgs-latest");
|
||||||
block.get("type").and_then(Value::as_str),
|
break;
|
||||||
Some("thinking" | "redacted_thinking")
|
}
|
||||||
)
|
}
|
||||||
}) {
|
|
||||||
if block.get("cache_control").is_none() {
|
// (d) A second, older user anchor helps long tool-result turns where
|
||||||
if let Some(object) = block.as_object_mut() {
|
// the stable prefix falls outside Anthropic's 20-block lookback from
|
||||||
object.insert(
|
// the newest breakpoint. Keep this best-effort and inside the 4-BP cap.
|
||||||
"cache_control".to_string(),
|
if budget > 0 && messages.len() >= 4 {
|
||||||
make_cache_control(&config.cache_ttl),
|
let mut user_count = 0;
|
||||||
);
|
for message in messages.iter_mut().rev() {
|
||||||
injected.push("msgs");
|
if message.get("role").and_then(Value::as_str) != Some("user") {
|
||||||
}
|
continue;
|
||||||
|
}
|
||||||
|
user_count += 1;
|
||||||
|
if user_count == 2 {
|
||||||
|
if inject_message_breakpoint(message, &config.cache_ttl) {
|
||||||
|
injected.push("msgs-prior-user");
|
||||||
}
|
}
|
||||||
break 'messages;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -121,6 +126,28 @@ pub fn inject(body: &mut Value, config: &OptimizerConfig) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn inject_message_breakpoint(message: &mut Value, ttl: &str) -> bool {
|
||||||
|
let Some(content) = message.get_mut("content").and_then(Value::as_array_mut) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Some(block) = content.iter_mut().rev().find(|block| {
|
||||||
|
!matches!(
|
||||||
|
block.get("type").and_then(Value::as_str),
|
||||||
|
Some("thinking" | "redacted_thinking")
|
||||||
|
)
|
||||||
|
}) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if block.get("cache_control").is_some() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let Some(object) = block.as_object_mut() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
object.insert("cache_control".to_string(), make_cache_control(ttl));
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
fn make_cache_control(ttl: &str) -> Value {
|
fn make_cache_control(ttl: &str) -> Value {
|
||||||
if ttl == "5m" {
|
if ttl == "5m" {
|
||||||
json!({"type": "ephemeral"})
|
json!({"type": "ephemeral"})
|
||||||
@@ -244,6 +271,30 @@ mod tests {
|
|||||||
.is_some());
|
.is_some());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_long_history_uses_fourth_prior_user_breakpoint() {
|
||||||
|
let mut body = json!({
|
||||||
|
"model":"test",
|
||||||
|
"tools":[{"name":"tool1"}],
|
||||||
|
"system":[{"type":"text","text":"sys"}],
|
||||||
|
"messages":[
|
||||||
|
{"role":"user","content":[{"type":"text","text":"first"}]},
|
||||||
|
{"role":"assistant","content":[{"type":"text","text":"answer"}]},
|
||||||
|
{"role":"user","content":[{"type":"tool_result","tool_use_id":"c1","content":"result"}]},
|
||||||
|
{"role":"assistant","content":[{"type":"text","text":"latest"}]}
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
inject(&mut body, &default_config());
|
||||||
|
assert_eq!(count_existing(&body), 4);
|
||||||
|
assert!(body["messages"][0]["content"][0]
|
||||||
|
.get("cache_control")
|
||||||
|
.is_some());
|
||||||
|
assert!(body["messages"][3]["content"][0]
|
||||||
|
.get("cache_control")
|
||||||
|
.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_existing_four_breakpoints_only_upgrades_ttl() {
|
fn test_existing_four_breakpoints_only_upgrades_ttl() {
|
||||||
let mut body = json!({
|
let mut body = json!({
|
||||||
@@ -379,6 +430,24 @@ mod tests {
|
|||||||
assert_eq!(body, original);
|
assert_eq!(body, original);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_optimizer_disabled_no_change() {
|
||||||
|
let config = OptimizerConfig {
|
||||||
|
enabled: false,
|
||||||
|
cache_injection: true,
|
||||||
|
..default_config()
|
||||||
|
};
|
||||||
|
let mut body = json!({
|
||||||
|
"model":"test",
|
||||||
|
"tools":[{"name":"tool1"}],
|
||||||
|
"messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]
|
||||||
|
});
|
||||||
|
let original = body.clone();
|
||||||
|
|
||||||
|
inject(&mut body, &config);
|
||||||
|
assert_eq!(body, original);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_skip_thinking_blocks_in_assistant() {
|
fn test_skip_thinking_blocks_in_assistant() {
|
||||||
let mut body = json!({
|
let mut body = json!({
|
||||||
|
|||||||
Reference in New Issue
Block a user