mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-08-04 11:43:57 +08:00
fix(pi): bind recovery to durable native authority
This commit is contained in:
@@ -629,6 +629,13 @@ impl ProxyService {
|
||||
}
|
||||
if !crate::settings::pi_takeover_enabled() {
|
||||
self.pi_runtime.close(catalog_epoch).await?;
|
||||
if let Some(expected_native) = expected_native {
|
||||
let models_path = crate::pi_config::native::get_pi_models_path()?;
|
||||
crate::pi_config::document::verify_pi_provider_values(
|
||||
&models_path,
|
||||
&expected_native.values,
|
||||
)?;
|
||||
}
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let listener = self
|
||||
@@ -702,6 +709,36 @@ impl ProxyService {
|
||||
Ok(build.direct_only_provider_ids)
|
||||
}
|
||||
|
||||
/// Re-open a fenced runtime only while Pi's live exact-key projection
|
||||
/// still matches the immutable snapshot which produced that runtime.
|
||||
///
|
||||
/// Rebuilding this witness from the mutable database or settings would
|
||||
/// turn a failed compensation into stale admission, so the projection is
|
||||
/// retained inside `PiRuntimeSnapshot`.
|
||||
async fn republish_current_pi_runtime_if_native_matches(
|
||||
&self,
|
||||
catalog_epoch: u64,
|
||||
) -> Result<bool, AppError> {
|
||||
let Some(expected_projection) = self.pi_runtime.retained_native_projection() else {
|
||||
self.pi_runtime.close(catalog_epoch).await?;
|
||||
return Ok(false);
|
||||
};
|
||||
let verification = crate::pi_config::native::get_pi_models_path().and_then(|models_path| {
|
||||
crate::pi_config::document::verify_pi_provider_values(
|
||||
&models_path,
|
||||
&expected_projection,
|
||||
)
|
||||
});
|
||||
if let Err(error) = verification {
|
||||
self.pi_runtime.close(catalog_epoch).await?;
|
||||
return Err(AppError::Conflict(format!(
|
||||
"Pi native projection changed while runtime admission was fenced; admission \
|
||||
remains closed: {error}"
|
||||
)));
|
||||
}
|
||||
self.pi_runtime.republish_current(catalog_epoch).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn fail_next_pi_reconcile_for_test(&self) {
|
||||
self.fail_next_pi_reconcile.store(true, Ordering::Release);
|
||||
@@ -973,7 +1010,9 @@ impl ProxyService {
|
||||
let old_direct_receipt = match self.restore_pi_direct_projection_at(&old_models_path) {
|
||||
Ok(receipt) => receipt,
|
||||
Err(error) => {
|
||||
let _ = self.pi_runtime.republish_current(epoch).await;
|
||||
let _ = self
|
||||
.republish_current_pi_runtime_if_native_matches(epoch)
|
||||
.await;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
@@ -1019,7 +1058,9 @@ impl ProxyService {
|
||||
.await
|
||||
.is_ok()
|
||||
} else {
|
||||
self.pi_runtime.republish_current(epoch).await.is_ok()
|
||||
self.republish_current_pi_runtime_if_native_matches(epoch)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
};
|
||||
return Err(AppError::Config(format!(
|
||||
"failed to reconcile Pi prompts in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, gateway={runtime_restored}"
|
||||
@@ -1048,7 +1089,9 @@ impl ProxyService {
|
||||
.await
|
||||
.is_ok()
|
||||
} else {
|
||||
self.pi_runtime.republish_current(epoch).await.is_ok()
|
||||
self.republish_current_pi_runtime_if_native_matches(epoch)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
};
|
||||
return Err(AppError::Config(format!(
|
||||
"failed to reconcile Pi Skills in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, gateway={runtime_restored}"
|
||||
@@ -1078,7 +1121,9 @@ impl ProxyService {
|
||||
.await
|
||||
.is_ok()
|
||||
} else {
|
||||
self.pi_runtime.republish_current(epoch).await.is_ok()
|
||||
self.republish_current_pi_runtime_if_native_matches(epoch)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
};
|
||||
return Err(AppError::Config(format!(
|
||||
"failed to publish Pi in the new native directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, old_gateway={old_gateway_restored}"
|
||||
@@ -1096,7 +1141,9 @@ impl ProxyService {
|
||||
let direct_receipt = match self.restore_pi_direct_projection() {
|
||||
Ok(receipt) => receipt,
|
||||
Err(error) => {
|
||||
let _ = self.pi_runtime.republish_current(epoch).await;
|
||||
let _ = self
|
||||
.republish_current_pi_runtime_if_native_matches(epoch)
|
||||
.await;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
@@ -1761,8 +1808,10 @@ impl ProxyService {
|
||||
if let Err(error) = self.reconcile_pi_runtime().await {
|
||||
if was_enabled {
|
||||
let epoch = self.pi_runtime.begin_mutation().await;
|
||||
let restored = self.pi_runtime.republish_current(epoch).await;
|
||||
return Err(if restored.is_ok() {
|
||||
let restored = self
|
||||
.republish_current_pi_runtime_if_native_matches(epoch)
|
||||
.await;
|
||||
return Err(if matches!(restored, Ok(true)) {
|
||||
format!(
|
||||
"failed to refresh Pi gateway catalog; the previous runtime remains active: {error}"
|
||||
)
|
||||
@@ -1814,7 +1863,9 @@ impl ProxyService {
|
||||
let direct_receipt = match self.restore_pi_direct_projection() {
|
||||
Ok(receipt) => receipt,
|
||||
Err(error) => {
|
||||
let _ = self.pi_runtime.republish_current(epoch).await;
|
||||
let _ = self
|
||||
.republish_current_pi_runtime_if_native_matches(epoch)
|
||||
.await;
|
||||
return Err(format!(
|
||||
"failed to restore Pi's direct native projection: {error}"
|
||||
));
|
||||
@@ -5234,6 +5285,112 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn fenced_pi_runtime_is_not_republished_after_external_native_drift() {
|
||||
let home = TempHome::new();
|
||||
crate::settings::reload_settings().expect("reload isolated settings");
|
||||
let pi_dir = home.dir.path().join("pi");
|
||||
std::fs::create_dir_all(&pi_dir).expect("Pi directory");
|
||||
let mut settings = crate::settings::get_settings();
|
||||
settings.pi_config_dir = Some(pi_dir.to_string_lossy().into_owned());
|
||||
settings.pi_takeover_enabled = true;
|
||||
crate::settings::update_settings(settings).expect("Pi settings");
|
||||
|
||||
let db = Arc::new(Database::memory().expect("database"));
|
||||
let direct = json!({
|
||||
"name": "Managed Pi",
|
||||
"api": "openai-responses",
|
||||
"baseUrl": "https://managed.example/v1",
|
||||
"apiKey": "managed-key",
|
||||
"models": [{"id": "model-a", "name": "Model A"}]
|
||||
});
|
||||
db.create_pi_catalog_provider(
|
||||
NewProviderAggregate::from_input(
|
||||
AppType::Pi.as_str(),
|
||||
ProviderMutationInput {
|
||||
id: "managed-pi".to_string(),
|
||||
name: "Managed Pi".to_string(),
|
||||
settings_config: direct.clone(),
|
||||
website_url: None,
|
||||
category: None,
|
||||
created_at: None,
|
||||
sort_index: Some(0),
|
||||
notes: None,
|
||||
meta: None,
|
||||
icon: Some("pi".to_string()),
|
||||
icon_color: None,
|
||||
in_failover_queue: false,
|
||||
},
|
||||
)
|
||||
.expect("aggregate"),
|
||||
"managed-pi",
|
||||
)
|
||||
.expect("managed provider");
|
||||
let models_path = pi_dir.join("models.json");
|
||||
std::fs::write(
|
||||
&models_path,
|
||||
serde_json::to_vec_pretty(&json!({"providers": {"managed-pi": direct}}))
|
||||
.expect("direct models"),
|
||||
)
|
||||
.expect("seed direct models");
|
||||
|
||||
let service = ProxyService::new(db);
|
||||
*service
|
||||
.pi_listener
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(PiListenerIdentity {
|
||||
server_generation: 41,
|
||||
gateway_origin: url::Url::parse("http://127.0.0.1:15721/").expect("gateway origin"),
|
||||
});
|
||||
service
|
||||
.reconcile_pi_runtime()
|
||||
.await
|
||||
.expect("publish initial runtime");
|
||||
assert_eq!(
|
||||
service
|
||||
.get_takeover_status()
|
||||
.await
|
||||
.expect("active status")
|
||||
.pi_operational_state,
|
||||
PiTakeoverOperationalState::Active
|
||||
);
|
||||
|
||||
let epoch = service.begin_pi_catalog_mutation().await;
|
||||
let external = serde_json::to_vec_pretty(&json!({
|
||||
"providers": {
|
||||
"managed-pi": {
|
||||
"name": "External",
|
||||
"api": "openai-responses",
|
||||
"baseUrl": "https://external.example/v1",
|
||||
"apiKey": "external-key",
|
||||
"models": [{"id": "external-model"}]
|
||||
}
|
||||
}
|
||||
}))
|
||||
.expect("external models");
|
||||
std::fs::write(&models_path, &external).expect("external edit");
|
||||
|
||||
assert!(matches!(
|
||||
service
|
||||
.republish_current_pi_runtime_if_native_matches(epoch)
|
||||
.await,
|
||||
Err(AppError::Conflict(_))
|
||||
));
|
||||
assert_eq!(
|
||||
service
|
||||
.get_takeover_status()
|
||||
.await
|
||||
.expect("degraded status")
|
||||
.pi_operational_state,
|
||||
PiTakeoverOperationalState::Degraded
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(&models_path).expect("external file remains"),
|
||||
external
|
||||
);
|
||||
}
|
||||
|
||||
fn seed_codex_model_template() {
|
||||
let codex_dir = crate::codex_config::get_codex_config_dir();
|
||||
std::fs::create_dir_all(&codex_dir).expect("create codex dir");
|
||||
|
||||
Reference in New Issue
Block a user