fix(pi): harden gateway publication and retry responses

This commit is contained in:
SaladDay
2026-08-03 03:23:48 +00:00
parent 1e9e59051a
commit 40e087c072
5 changed files with 1083 additions and 217 deletions
+596 -78
View File
@@ -611,6 +611,15 @@ impl ProxyService {
pub(crate) async fn reconcile_pi_runtime_at_epoch(
&self,
catalog_epoch: u64,
) -> Result<Vec<String>, AppError> {
self.reconcile_pi_runtime_at_epoch_with_native_precondition(catalog_epoch, None)
.await
}
pub(crate) async fn reconcile_pi_runtime_at_epoch_with_native_precondition(
&self,
catalog_epoch: u64,
expected_native: Option<&crate::pi_config::document::PiProviderValuesSnapshot>,
) -> Result<Vec<String>, AppError> {
#[cfg(test)]
if self.fail_next_pi_reconcile.swap(false, Ordering::AcqRel) {
@@ -642,11 +651,54 @@ impl ProxyService {
token,
app_config,
)?;
crate::pi_config::document::apply_pi_provider_patch(
&crate::pi_config::native::get_pi_models_path()?,
&build.projection_patch,
)?;
self.pi_runtime.publish(build.snapshot).await?;
let models_path = crate::pi_config::native::get_pi_models_path()?;
let native_receipt = if build.projection_patch.is_empty() {
None
} else {
let expected = match expected_native {
Some(expected) => {
crate::pi_config::document::PiProviderValuesSnapshot {
file_existed: expected.file_existed,
values: build
.projection_patch
.keys()
.map(|provider_key| {
expected
.values
.get(provider_key)
.cloned()
.map(|value| (provider_key.clone(), value))
.ok_or_else(|| {
AppError::Config(format!(
"Pi runtime provider key '{provider_key}' was not included in the catalog preflight"
))
})
})
.collect::<Result<_, _>>()?,
}
}
None => self.preflight_pi_owned_projection_at(
&models_path,
&build.projection_patch,
)?,
};
Some(
crate::pi_config::document::apply_pi_provider_patch_with_receipt(
&models_path,
&expected,
&build.projection_patch,
)?,
)
};
if let Err(error) = self.pi_runtime.publish(build.snapshot).await {
let rollback = native_receipt.as_ref().map_or(
Ok(()),
crate::pi_config::document::PiProviderPatchReceipt::rollback,
);
return Err(AppError::Config(format!(
"failed to publish Pi runtime after native projection: {error}; native rollback={rollback:?}"
)));
}
Ok(build.direct_only_provider_ids)
}
@@ -692,12 +744,126 @@ impl ProxyService {
fn restore_pi_direct_projection_at(
&self,
models_path: &std::path::Path,
) -> Result<(), AppError> {
let patch = direct_pi_projection_patch(self.db.as_ref())?;
crate::pi_config::document::apply_pi_provider_patch(models_path, &patch)
) -> Result<crate::pi_config::document::PiProviderPatchReceipt, AppError> {
if self
.pi_listener
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_none()
{
return self.confirm_pi_projection_is_already_direct_at(models_path);
}
let gateway = self.current_pi_gateway_projection()?;
self.restore_pi_direct_projection_at_with_expected_gateway(models_path, &gateway)
}
fn restore_pi_direct_projection(&self) -> Result<(), AppError> {
fn confirm_pi_projection_is_already_direct_at(
&self,
models_path: &std::path::Path,
) -> Result<crate::pi_config::document::PiProviderPatchReceipt, AppError> {
let direct = direct_pi_projection_patch(self.db.as_ref())?;
let before = crate::pi_config::document::snapshot_pi_provider_values(
models_path,
direct.keys().cloned(),
)?;
if let Some(provider_key) = direct.iter().find_map(|(provider_key, expected)| {
(before.values.get(provider_key) != Some(expected)).then_some(provider_key)
}) {
return Err(AppError::Conflict(format!(
"Pi listener is unavailable and provider key '{provider_key}' is not already in its direct projection"
)));
}
crate::pi_config::document::apply_pi_provider_patch_with_receipt(
models_path,
&before,
&direct,
)
}
fn current_pi_gateway_projection(
&self,
) -> Result<indexmap::IndexMap<String, Option<Value>>, AppError> {
let listener = self
.pi_listener
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
.ok_or_else(|| {
AppError::Conflict(
"cannot restore Pi direct projection without its active listener identity"
.to_string(),
)
})?;
let gateway_token = crate::settings::get_pi_gateway_token().or_else(|settings_error| {
self.pi_runtime
.retained_gateway_token(listener.server_generation)
.ok_or(settings_error)
})?;
let gateway = build_pi_runtime(
self.db.as_ref(),
listener.server_generation,
0,
&listener.gateway_origin,
gateway_token,
crate::settings::get_pi_app_proxy_config(),
)?
.projection_patch;
Ok(gateway)
}
fn preflight_pi_owned_projection_at(
&self,
models_path: &std::path::Path,
expected_gateway: &indexmap::IndexMap<String, Option<Value>>,
) -> Result<crate::pi_config::document::PiProviderValuesSnapshot, AppError> {
let direct = direct_pi_projection_patch(self.db.as_ref())?;
let direct_keys = direct.keys().collect::<std::collections::BTreeSet<_>>();
let gateway_keys = expected_gateway
.keys()
.collect::<std::collections::BTreeSet<_>>();
if direct_keys != gateway_keys {
return Err(AppError::Conflict(
"Pi direct and gateway projections cover different exact-key ownership".to_string(),
));
}
let before = crate::pi_config::document::snapshot_pi_provider_values(
models_path,
direct.keys().cloned(),
)?;
for (provider_key, direct_value) in &direct {
let observed = before
.values
.get(provider_key)
.expect("every owned key was preflighted");
let gateway_value = expected_gateway
.get(provider_key)
.expect("gateway ownership keys were checked");
if observed != direct_value && observed != gateway_value {
return Err(AppError::Conflict(format!(
"Pi provider key '{provider_key}' changed outside CC Switch"
)));
}
}
Ok(before)
}
fn restore_pi_direct_projection_at_with_expected_gateway(
&self,
models_path: &std::path::Path,
expected_gateway: &indexmap::IndexMap<String, Option<Value>>,
) -> Result<crate::pi_config::document::PiProviderPatchReceipt, AppError> {
let direct = direct_pi_projection_patch(self.db.as_ref())?;
let before = self.preflight_pi_owned_projection_at(models_path, expected_gateway)?;
crate::pi_config::document::apply_pi_provider_patch_with_receipt(
models_path,
&before,
&direct,
)
}
fn restore_pi_direct_projection(
&self,
) -> Result<crate::pi_config::document::PiProviderPatchReceipt, AppError> {
let models_path = crate::pi_config::native::get_pi_models_path()?;
self.restore_pi_direct_projection_at(&models_path)
}
@@ -751,19 +917,21 @@ impl ProxyService {
if !existing.pi_takeover_enabled {
crate::settings::update_settings(next)?;
if let Err(error) =
crate::pi_config::document::apply_pi_provider_patch(&new_models_path, &direct_patch)
{
let settings_restored = crate::settings::update_settings(existing.clone()).is_ok();
let new_native_restored = crate::pi_config::document::restore_pi_provider_values(
let native_receipt =
match crate::pi_config::document::apply_pi_provider_patch_with_receipt(
&new_models_path,
&new_native_before,
)
.is_ok();
return Err(AppError::Config(format!(
"failed to publish managed Pi providers in the new directory: {error}; rollback: settings={settings_restored}, native={new_native_restored}"
&direct_patch,
) {
Ok(receipt) => receipt,
Err(error) => {
let settings_restored =
crate::settings::update_settings(existing.clone()).is_ok();
return Err(AppError::Config(format!(
"failed to publish managed Pi providers in the new directory: {error}; rollback: settings={settings_restored}"
)));
}
}
};
if let Err(error) =
PromptService::reconcile_pi_native_under_guard(self.db.as_ref(), &prompt_guard)
{
@@ -772,11 +940,7 @@ impl ProxyService {
.db
.save_prompt_selection(AppType::Pi.as_str(), &previous_prompts)
.is_ok();
let new_native_restored = crate::pi_config::document::restore_pi_provider_values(
&new_models_path,
&new_native_before,
)
.is_ok();
let new_native_restored = native_receipt.rollback().is_ok();
return Err(AppError::Config(format!(
"failed to reconcile Pi prompts in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, native={new_native_restored}"
)));
@@ -789,11 +953,7 @@ impl ProxyService {
.db
.save_prompt_selection(AppType::Pi.as_str(), &previous_prompts)
.is_ok();
let new_native_restored = crate::pi_config::document::restore_pi_provider_values(
&new_models_path,
&new_native_before,
)
.is_ok();
let new_native_restored = native_receipt.rollback().is_ok();
let skills_restored = settings_restored
&& crate::services::skill_deployment::PiSkillDeploymentService::reconcile_all(
&self.db,
@@ -810,13 +970,22 @@ impl ProxyService {
// process that already loaded the old gateway projection therefore
// cannot enter a catalog whose directory ownership is in flight.
let epoch = self.pi_runtime.begin_mutation().await;
if let Err(error) = self.restore_pi_direct_projection_at(&old_models_path) {
let _ = self.pi_runtime.republish_current(epoch).await;
return Err(error);
}
let old_direct_receipt = match self.restore_pi_direct_projection_at(&old_models_path) {
Ok(receipt) => receipt,
Err(error) => {
let _ = self.pi_runtime.republish_current(epoch).await;
return Err(error);
}
};
if let Err(error) = crate::settings::update_settings(next) {
let runtime_restored = self.reconcile_pi_runtime_at_epoch(epoch).await;
let expected_old_direct = old_direct_receipt.attempted_snapshot();
let runtime_restored = self
.reconcile_pi_runtime_at_epoch_with_native_precondition(
epoch,
Some(&expected_old_direct),
)
.await;
return Err(AppError::Config(if runtime_restored.is_ok() {
format!(
"failed to save the new Pi directory; the previous gateway projection was restored: {error}"
@@ -836,19 +1005,24 @@ impl ProxyService {
.db
.save_prompt_selection(AppType::Pi.as_str(), &previous_prompts)
.is_ok();
let new_native_restored = crate::pi_config::document::restore_pi_provider_values(
&new_models_path,
&new_native_before,
)
.is_ok();
let skills_restored = settings_restored
&& crate::services::skill_deployment::PiSkillDeploymentService::reconcile_all(
&self.db,
)
.is_ok();
let runtime_restored = self.reconcile_pi_runtime_at_epoch(epoch).await.is_ok();
let runtime_restored = if settings_restored {
let expected_old_direct = old_direct_receipt.attempted_snapshot();
self.reconcile_pi_runtime_at_epoch_with_native_precondition(
epoch,
Some(&expected_old_direct),
)
.await
.is_ok()
} else {
self.pi_runtime.republish_current(epoch).await.is_ok()
};
return Err(AppError::Config(format!(
"failed to reconcile Pi prompts in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, native={new_native_restored}, skills={skills_restored}, gateway={runtime_restored}"
"failed to reconcile Pi prompts in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, gateway={runtime_restored}"
)));
}
@@ -860,45 +1034,54 @@ impl ProxyService {
.db
.save_prompt_selection(AppType::Pi.as_str(), &previous_prompts)
.is_ok();
let new_native_restored = crate::pi_config::document::restore_pi_provider_values(
&new_models_path,
&new_native_before,
)
.is_ok();
let skills_restored = settings_restored
&& crate::services::skill_deployment::PiSkillDeploymentService::reconcile_all(
&self.db,
)
.is_ok();
let runtime_restored = self.reconcile_pi_runtime_at_epoch(epoch).await.is_ok();
let runtime_restored = if settings_restored {
let expected_old_direct = old_direct_receipt.attempted_snapshot();
self.reconcile_pi_runtime_at_epoch_with_native_precondition(
epoch,
Some(&expected_old_direct),
)
.await
.is_ok()
} else {
self.pi_runtime.republish_current(epoch).await.is_ok()
};
return Err(AppError::Config(format!(
"failed to reconcile Pi Skills in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, native={new_native_restored}, skills={skills_restored}, gateway={runtime_restored}"
"failed to reconcile Pi Skills in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, gateway={runtime_restored}"
)));
}
if let Err(error) = self.reconcile_pi_runtime_at_epoch(epoch).await {
if let Err(error) = self
.reconcile_pi_runtime_at_epoch_with_native_precondition(epoch, Some(&new_native_before))
.await
{
let settings_restored = crate::settings::update_settings(existing.clone()).is_ok();
let prompts_restored = self
.db
.save_prompt_selection(AppType::Pi.as_str(), &previous_prompts)
.is_ok();
let new_native_restored = crate::pi_config::document::restore_pi_provider_values(
&new_models_path,
&new_native_before,
)
.is_ok();
let skills_restored = settings_restored
&& crate::services::skill_deployment::PiSkillDeploymentService::reconcile_all(
&self.db,
)
.is_ok();
let old_gateway_restored = if settings_restored {
self.reconcile_pi_runtime_at_epoch(epoch).await.is_ok()
let expected_old_direct = old_direct_receipt.attempted_snapshot();
self.reconcile_pi_runtime_at_epoch_with_native_precondition(
epoch,
Some(&expected_old_direct),
)
.await
.is_ok()
} else {
self.pi_runtime.republish_current(epoch).await.is_ok()
};
return Err(AppError::Config(format!(
"failed to publish Pi in the new native directory: {error}; rollback: native={new_native_restored}, settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, old_gateway={old_gateway_restored}"
"failed to publish Pi in the new native directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, old_gateway={old_gateway_restored}"
)));
}
@@ -910,11 +1093,26 @@ impl ProxyService {
return Ok(());
}
let epoch = self.pi_runtime.begin_mutation().await;
if let Err(error) = self.restore_pi_direct_projection() {
let _ = self.pi_runtime.republish_current(epoch).await;
return Err(error);
let direct_receipt = match self.restore_pi_direct_projection() {
Ok(receipt) => receipt,
Err(error) => {
let _ = self.pi_runtime.republish_current(epoch).await;
return Err(error);
}
};
if let Err(error) = self.pi_runtime.close(epoch).await {
let expected_direct = direct_receipt.attempted_snapshot();
let rollback = self
.reconcile_pi_runtime_at_epoch_with_native_precondition(
epoch,
Some(&expected_direct),
)
.await;
return Err(AppError::Config(format!(
"failed to close Pi admission after direct projection: {error}; gateway rollback={rollback:?}"
)));
}
self.pi_runtime.close(epoch).await
Ok(())
}
/// Enter the portable-import boundary while the caller holds Pi's switch
@@ -958,16 +1156,32 @@ impl ProxyService {
pub(crate) async fn rotate_pi_gateway_token(&self) -> Result<(), AppError> {
let _guard = self.switch_locks.lock_for_app(AppType::Pi.as_str()).await;
let previous = crate::settings::get_or_create_pi_gateway_token()?;
let takeover_enabled = crate::settings::pi_takeover_enabled();
let native_before = if takeover_enabled {
let models_path = crate::pi_config::native::get_pi_models_path()?;
let gateway = self.current_pi_gateway_projection()?;
Some(self.preflight_pi_owned_projection_at(&models_path, &gateway)?)
} else {
None
};
crate::settings::reset_pi_gateway_token()?;
if !crate::settings::pi_takeover_enabled() {
if !takeover_enabled {
return Ok(());
}
let native_before = native_before.expect("enabled takeover has a native preflight");
let epoch = self.pi_runtime.begin_mutation().await;
if let Err(error) = self.reconcile_pi_runtime_at_epoch(epoch).await {
if let Err(error) = self
.reconcile_pi_runtime_at_epoch_with_native_precondition(epoch, Some(&native_before))
.await
{
let token_restored = crate::settings::replace_pi_gateway_token(previous);
let runtime_restored = if token_restored.is_ok() {
let rollback_epoch = self.pi_runtime.begin_mutation().await;
self.reconcile_pi_runtime_at_epoch(rollback_epoch).await
self.reconcile_pi_runtime_at_epoch_with_native_precondition(
rollback_epoch,
Some(&native_before),
)
.await
} else {
Err(AppError::Config(
"failed to restore the previous Pi gateway credential".to_string(),
@@ -1561,7 +1775,7 @@ impl ProxyService {
let direct_projection = self.restore_pi_direct_projection();
match direct_projection {
Ok(()) => {
Ok(_) => {
let epoch = self.pi_runtime.begin_mutation().await;
let admission_closed = self.pi_runtime.close(epoch).await;
return Err(if admission_closed.is_ok() {
@@ -1597,16 +1811,25 @@ impl ProxyService {
return Ok(());
}
let epoch = self.pi_runtime.begin_mutation().await;
if let Err(error) = self.restore_pi_direct_projection() {
let _ = self.pi_runtime.republish_current(epoch).await;
return Err(format!(
"failed to restore Pi's direct native projection: {error}"
));
}
let direct_receipt = match self.restore_pi_direct_projection() {
Ok(receipt) => receipt,
Err(error) => {
let _ = self.pi_runtime.republish_current(epoch).await;
return Err(format!(
"failed to restore Pi's direct native projection: {error}"
));
}
};
if let Err(error) = crate::settings::set_pi_takeover_enabled(false) {
// Desired state did not change; rebuild the gateway projection and
// restore admission so the native file cannot be left lying.
let _ = self.reconcile_pi_runtime_at_epoch(epoch).await;
let expected_direct = direct_receipt.attempted_snapshot();
let _ = self
.reconcile_pi_runtime_at_epoch_with_native_precondition(
epoch,
Some(&expected_direct),
)
.await;
return Err(format!("failed to persist Pi takeover disable: {error}"));
}
@@ -4164,16 +4387,17 @@ mod tests {
"apiKey": "native-key",
"models": [{"id": "native-model"}]
});
let expected_gateway = serde_json::json!({
"api": "openai-responses",
"baseUrl": "http://127.0.0.1:15721/pi/route",
"apiKey": "gateway-token",
"models": [{"id": "model-a"}]
});
std::fs::write(
&path,
serde_json::to_vec_pretty(&serde_json::json!({
"providers": {
"managed-pi": {
"api": "openai-responses",
"baseUrl": "http://127.0.0.1:15721/pi/route",
"apiKey": "gateway-token",
"models": [{"id": "model-a"}]
},
"managed-pi": expected_gateway.clone(),
"native": unowned.clone()
}
}))
@@ -4183,7 +4407,10 @@ mod tests {
state
.proxy_service
.restore_pi_direct_projection_at(&path)
.restore_pi_direct_projection_at_with_expected_gateway(
&path,
&indexmap::IndexMap::from([("managed-pi".to_string(), Some(expected_gateway))]),
)
.expect("restore direct Pi projection");
let restored: Value =
@@ -4452,6 +4679,91 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn changing_pi_directory_does_not_overwrite_an_exact_key_changed_after_preflight() {
let home = TempHome::new();
crate::settings::reload_settings().expect("reload isolated settings");
let old_dir = home.dir.path().join("old-pi");
let new_dir = home.dir.path().join("new-pi");
std::fs::create_dir_all(&old_dir).expect("old Pi directory");
std::fs::create_dir_all(&new_dir).expect("new Pi directory");
let direct = json!({
"name": "Managed Pi",
"api": "openai-responses",
"baseUrl": "https://managed.example/v1",
"apiKey": "managed-key",
"models": [{"id": "model-a", "name": "Model A"}]
});
let external = json!({
"name": "External Pi",
"api": "openai-responses",
"baseUrl": "https://external.example/v1",
"apiKey": "external-key",
"models": [{"id": "external-model", "name": "External"}]
});
let new_models_path = new_dir.join("models.json");
std::fs::write(
&new_models_path,
serde_json::to_vec(&json!({"providers": {}})).expect("serialize direct document"),
)
.expect("seed direct document");
let mut settings = crate::settings::get_settings();
settings.pi_config_dir = Some(old_dir.to_string_lossy().into_owned());
settings.pi_takeover_enabled = false;
crate::settings::update_settings(settings).expect("old Pi settings");
let db = Arc::new(Database::memory().expect("database"));
db.create_pi_catalog_provider(
NewProviderAggregate::from_input(
AppType::Pi.as_str(),
ProviderMutationInput {
id: "managed-pi".to_string(),
name: "Managed Pi".to_string(),
settings_config: direct,
website_url: None,
category: None,
created_at: None,
sort_index: Some(0),
notes: None,
meta: None,
icon: Some("pi".to_string()),
icon_color: None,
in_failover_queue: false,
},
)
.expect("aggregate"),
"managed-pi",
)
.expect("managed provider");
crate::pi_config::shared_file::replace_before_next_compare_exchange(
&new_models_path,
&serde_json::to_vec(&json!({"providers": {"managed-pi": external.clone()}}))
.expect("serialize external document"),
);
let service = ProxyService::new(db);
let switch_guard = service.lock_switch_for_app(AppType::Pi.as_str()).await;
let existing = crate::settings::get_settings();
let mut next = existing.clone();
next.pi_config_dir = Some(new_dir.to_string_lossy().into_owned());
let error = service
.replace_settings_with_pi_directory_boundary_under_lock(&switch_guard, &existing, next)
.await
.expect_err("the post-preflight native edit must win");
assert!(error.to_string().contains("changed since"));
assert_eq!(
crate::settings::get_settings().pi_config_dir,
existing.pi_config_dir
);
let live: Value = serde_json::from_slice(
&std::fs::read(&new_models_path).expect("read external native document"),
)
.expect("parse external native document");
assert_eq!(live.pointer("/providers/managed-pi"), Some(&external));
}
#[tokio::test]
#[serial]
async fn changing_pi_directory_without_takeover_reconciles_missing_agents_truth() {
@@ -4544,6 +4856,143 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn pi_gateway_returns_the_last_real_retryable_upstream_response_when_no_later_send_occurs(
) {
let home = TempHome::new();
crate::settings::reload_settings().expect("reload isolated settings");
let pi_dir = home.dir.path().join("pi");
std::fs::create_dir_all(&pi_dir).expect("Pi directory");
let mut settings = crate::settings::get_settings();
settings.pi_config_dir = Some(pi_dir.to_string_lossy().into_owned());
settings.pi_takeover_enabled = false;
crate::settings::update_settings(settings).expect("Pi settings");
let upstream_hits = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let hits = upstream_hits.clone();
let upstream = axum::Router::new().fallback(axum::routing::any(move || {
let hits = hits.clone();
async move {
hits.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
let mut response = axum::response::Response::builder()
.status(http::StatusCode::TOO_MANY_REQUESTS)
.header("x-upstream-marker", "real-429")
.header(http::header::CONTENT_TYPE, "application/json")
.body(axum::body::Body::from(
r#"{"error":{"message":"upstream rate limit"}}"#,
))
.expect("upstream response");
response
.headers_mut()
.insert("retry-after", http::HeaderValue::from_static("17"));
response
}
}));
let upstream_listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
.await
.expect("upstream listener");
let upstream_address = upstream_listener.local_addr().expect("upstream address");
let upstream_task = tokio::spawn(async move {
axum::serve(upstream_listener, upstream)
.await
.expect("upstream server");
});
let db = Arc::new(Database::memory().expect("database"));
use_ephemeral_proxy_port(&db).await;
let direct = json!({
"name": "Retryable upstream",
"api": "openai-responses",
"baseUrl": format!("http://{upstream_address}/v1"),
"apiKey": "upstream-key",
"models": [{"id": "model-a", "name": "Model A"}]
});
db.create_pi_catalog_provider(
NewProviderAggregate::from_input(
AppType::Pi.as_str(),
ProviderMutationInput {
id: "retryable-upstream".to_string(),
name: "Retryable upstream".to_string(),
settings_config: direct.clone(),
website_url: None,
category: None,
created_at: None,
sort_index: Some(0),
notes: None,
meta: None,
icon: Some("pi".to_string()),
icon_color: None,
in_failover_queue: false,
},
)
.expect("aggregate"),
"retryable-upstream",
)
.expect("managed provider");
std::fs::write(
pi_dir.join("models.json"),
serde_json::to_vec(&json!({
"providers": {"retryable-upstream": direct}
}))
.expect("serialize native catalog"),
)
.expect("native catalog");
let service = ProxyService::new(db);
service
.set_takeover_for_app(AppType::Pi.as_str(), true)
.await
.expect("enable Pi gateway");
let projected: Value = serde_json::from_slice(
&std::fs::read(pi_dir.join("models.json")).expect("projected catalog"),
)
.expect("parse projected catalog");
let base_url = projected
.pointer("/providers/retryable-upstream/baseUrl")
.and_then(Value::as_str)
.expect("gateway base URL");
let gateway_token = projected
.pointer("/providers/retryable-upstream/apiKey")
.and_then(Value::as_str)
.expect("gateway token");
let response = reqwest::Client::new()
.post(format!("{base_url}/responses"))
.bearer_auth(gateway_token)
.json(&json!({"model": "model-a", "input": "hello"}))
.send()
.await
.expect("gateway response");
let status = response.status();
let marker = response
.headers()
.get("x-upstream-marker")
.and_then(|value| value.to_str().ok())
.map(str::to_string);
let retry_after = response
.headers()
.get("retry-after")
.and_then(|value| value.to_str().ok())
.map(str::to_string);
let body = response.text().await.expect("upstream body");
upstream_task.abort();
assert_eq!(status, reqwest::StatusCode::TOO_MANY_REQUESTS);
assert_eq!(marker.as_deref(), Some("real-429"));
assert_eq!(retry_after.as_deref(), Some("17"));
assert_eq!(body, r#"{"error":{"message":"upstream rate limit"}}"#);
assert_eq!(
upstream_hits.load(std::sync::atomic::Ordering::SeqCst),
1,
"local candidate exhaustion must not invent a phantom retry"
);
service
.set_takeover_for_app(AppType::Pi.as_str(), false)
.await
.expect("disable Pi gateway");
}
#[tokio::test]
#[serial]
async fn failed_listener_rebind_recovers_pi_on_the_previous_listener() {
@@ -4716,6 +5165,75 @@ mod tests {
.expect("explicit disable clears desired state");
}
#[tokio::test]
#[serial]
async fn listener_absence_does_not_authorize_overwriting_a_non_direct_native_key() {
let home = TempHome::new();
crate::settings::reload_settings().expect("reload isolated settings");
let pi_dir = home.dir.path().join("pi");
std::fs::create_dir_all(&pi_dir).expect("Pi directory");
let mut settings = crate::settings::get_settings();
settings.pi_config_dir = Some(pi_dir.to_string_lossy().into_owned());
settings.pi_takeover_enabled = true;
crate::settings::update_settings(settings).expect("Pi settings");
let db = Arc::new(Database::memory().expect("database"));
let direct = json!({
"name": "Managed Pi",
"api": "openai-responses",
"baseUrl": "https://managed.example/v1",
"apiKey": "managed-key",
"models": [{"id": "model-a", "name": "Model A"}]
});
db.create_pi_catalog_provider(
NewProviderAggregate::from_input(
AppType::Pi.as_str(),
ProviderMutationInput {
id: "managed-pi".to_string(),
name: "Managed Pi".to_string(),
settings_config: direct,
website_url: None,
category: None,
created_at: None,
sort_index: Some(0),
notes: None,
meta: None,
icon: Some("pi".to_string()),
icon_color: None,
in_failover_queue: false,
},
)
.expect("aggregate"),
"managed-pi",
)
.expect("managed provider");
let external = json!({
"name": "External edit",
"api": "openai-responses",
"baseUrl": "https://external.example/v1",
"apiKey": "external-key",
"models": [{"id": "external-model"}]
});
let native_bytes = serde_json::to_vec(&json!({
"providers": {"managed-pi": external}
}))
.expect("serialize external catalog");
let models_path = pi_dir.join("models.json");
std::fs::write(&models_path, &native_bytes).expect("external native catalog");
let service = ProxyService::new(db);
let error = service
.set_takeover_for_app(AppType::Pi.as_str(), false)
.await
.expect_err("no listener is not ownership evidence");
assert!(error.contains("not already in its direct projection"));
assert!(crate::settings::pi_takeover_enabled());
assert_eq!(
std::fs::read(&models_path).expect("external catalog remains"),
native_bytes
);
}
fn seed_codex_model_template() {
let codex_dir = crate::codex_config::get_codex_config_dir();
std::fs::create_dir_all(&codex_dir).expect("create codex dir");