fix(pi): bind recovery to durable native authority

This commit is contained in:
SaladDay
2026-08-03 04:09:18 +00:00
parent 914fc99b4c
commit 692ef41883
5 changed files with 618 additions and 45 deletions
+165 -8
View File
@@ -629,6 +629,13 @@ impl ProxyService {
}
if !crate::settings::pi_takeover_enabled() {
self.pi_runtime.close(catalog_epoch).await?;
if let Some(expected_native) = expected_native {
let models_path = crate::pi_config::native::get_pi_models_path()?;
crate::pi_config::document::verify_pi_provider_values(
&models_path,
&expected_native.values,
)?;
}
return Ok(Vec::new());
}
let listener = self
@@ -702,6 +709,36 @@ impl ProxyService {
Ok(build.direct_only_provider_ids)
}
/// Re-open a fenced runtime only while Pi's live exact-key projection
/// still matches the immutable snapshot which produced that runtime.
///
/// Rebuilding this witness from the mutable database or settings would
/// turn a failed compensation into stale admission, so the projection is
/// retained inside `PiRuntimeSnapshot`.
async fn republish_current_pi_runtime_if_native_matches(
&self,
catalog_epoch: u64,
) -> Result<bool, AppError> {
let Some(expected_projection) = self.pi_runtime.retained_native_projection() else {
self.pi_runtime.close(catalog_epoch).await?;
return Ok(false);
};
let verification = crate::pi_config::native::get_pi_models_path().and_then(|models_path| {
crate::pi_config::document::verify_pi_provider_values(
&models_path,
&expected_projection,
)
});
if let Err(error) = verification {
self.pi_runtime.close(catalog_epoch).await?;
return Err(AppError::Conflict(format!(
"Pi native projection changed while runtime admission was fenced; admission \
remains closed: {error}"
)));
}
self.pi_runtime.republish_current(catalog_epoch).await
}
#[cfg(test)]
pub(crate) fn fail_next_pi_reconcile_for_test(&self) {
self.fail_next_pi_reconcile.store(true, Ordering::Release);
@@ -973,7 +1010,9 @@ impl ProxyService {
let old_direct_receipt = match self.restore_pi_direct_projection_at(&old_models_path) {
Ok(receipt) => receipt,
Err(error) => {
let _ = self.pi_runtime.republish_current(epoch).await;
let _ = self
.republish_current_pi_runtime_if_native_matches(epoch)
.await;
return Err(error);
}
};
@@ -1019,7 +1058,9 @@ impl ProxyService {
.await
.is_ok()
} else {
self.pi_runtime.republish_current(epoch).await.is_ok()
self.republish_current_pi_runtime_if_native_matches(epoch)
.await
.unwrap_or(false)
};
return Err(AppError::Config(format!(
"failed to reconcile Pi prompts in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, gateway={runtime_restored}"
@@ -1048,7 +1089,9 @@ impl ProxyService {
.await
.is_ok()
} else {
self.pi_runtime.republish_current(epoch).await.is_ok()
self.republish_current_pi_runtime_if_native_matches(epoch)
.await
.unwrap_or(false)
};
return Err(AppError::Config(format!(
"failed to reconcile Pi Skills in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, gateway={runtime_restored}"
@@ -1078,7 +1121,9 @@ impl ProxyService {
.await
.is_ok()
} else {
self.pi_runtime.republish_current(epoch).await.is_ok()
self.republish_current_pi_runtime_if_native_matches(epoch)
.await
.unwrap_or(false)
};
return Err(AppError::Config(format!(
"failed to publish Pi in the new native directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, old_gateway={old_gateway_restored}"
@@ -1096,7 +1141,9 @@ impl ProxyService {
let direct_receipt = match self.restore_pi_direct_projection() {
Ok(receipt) => receipt,
Err(error) => {
let _ = self.pi_runtime.republish_current(epoch).await;
let _ = self
.republish_current_pi_runtime_if_native_matches(epoch)
.await;
return Err(error);
}
};
@@ -1761,8 +1808,10 @@ impl ProxyService {
if let Err(error) = self.reconcile_pi_runtime().await {
if was_enabled {
let epoch = self.pi_runtime.begin_mutation().await;
let restored = self.pi_runtime.republish_current(epoch).await;
return Err(if restored.is_ok() {
let restored = self
.republish_current_pi_runtime_if_native_matches(epoch)
.await;
return Err(if matches!(restored, Ok(true)) {
format!(
"failed to refresh Pi gateway catalog; the previous runtime remains active: {error}"
)
@@ -1814,7 +1863,9 @@ impl ProxyService {
let direct_receipt = match self.restore_pi_direct_projection() {
Ok(receipt) => receipt,
Err(error) => {
let _ = self.pi_runtime.republish_current(epoch).await;
let _ = self
.republish_current_pi_runtime_if_native_matches(epoch)
.await;
return Err(format!(
"failed to restore Pi's direct native projection: {error}"
));
@@ -5234,6 +5285,112 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn fenced_pi_runtime_is_not_republished_after_external_native_drift() {
let home = TempHome::new();
crate::settings::reload_settings().expect("reload isolated settings");
let pi_dir = home.dir.path().join("pi");
std::fs::create_dir_all(&pi_dir).expect("Pi directory");
let mut settings = crate::settings::get_settings();
settings.pi_config_dir = Some(pi_dir.to_string_lossy().into_owned());
settings.pi_takeover_enabled = true;
crate::settings::update_settings(settings).expect("Pi settings");
let db = Arc::new(Database::memory().expect("database"));
let direct = json!({
"name": "Managed Pi",
"api": "openai-responses",
"baseUrl": "https://managed.example/v1",
"apiKey": "managed-key",
"models": [{"id": "model-a", "name": "Model A"}]
});
db.create_pi_catalog_provider(
NewProviderAggregate::from_input(
AppType::Pi.as_str(),
ProviderMutationInput {
id: "managed-pi".to_string(),
name: "Managed Pi".to_string(),
settings_config: direct.clone(),
website_url: None,
category: None,
created_at: None,
sort_index: Some(0),
notes: None,
meta: None,
icon: Some("pi".to_string()),
icon_color: None,
in_failover_queue: false,
},
)
.expect("aggregate"),
"managed-pi",
)
.expect("managed provider");
let models_path = pi_dir.join("models.json");
std::fs::write(
&models_path,
serde_json::to_vec_pretty(&json!({"providers": {"managed-pi": direct}}))
.expect("direct models"),
)
.expect("seed direct models");
let service = ProxyService::new(db);
*service
.pi_listener
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(PiListenerIdentity {
server_generation: 41,
gateway_origin: url::Url::parse("http://127.0.0.1:15721/").expect("gateway origin"),
});
service
.reconcile_pi_runtime()
.await
.expect("publish initial runtime");
assert_eq!(
service
.get_takeover_status()
.await
.expect("active status")
.pi_operational_state,
PiTakeoverOperationalState::Active
);
let epoch = service.begin_pi_catalog_mutation().await;
let external = serde_json::to_vec_pretty(&json!({
"providers": {
"managed-pi": {
"name": "External",
"api": "openai-responses",
"baseUrl": "https://external.example/v1",
"apiKey": "external-key",
"models": [{"id": "external-model"}]
}
}
}))
.expect("external models");
std::fs::write(&models_path, &external).expect("external edit");
assert!(matches!(
service
.republish_current_pi_runtime_if_native_matches(epoch)
.await,
Err(AppError::Conflict(_))
));
assert_eq!(
service
.get_takeover_status()
.await
.expect("degraded status")
.pi_operational_state,
PiTakeoverOperationalState::Degraded
);
assert_eq!(
std::fs::read(&models_path).expect("external file remains"),
external
);
}
fn seed_codex_model_template() {
let codex_dir = crate::codex_config::get_codex_config_dir();
std::fs::create_dir_all(&codex_dir).expect("create codex dir");