mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-08-04 11:43:57 +08:00
fix(pi): harden gateway publication and retry responses
This commit is contained in:
+596
-78
@@ -611,6 +611,15 @@ impl ProxyService {
|
||||
pub(crate) async fn reconcile_pi_runtime_at_epoch(
|
||||
&self,
|
||||
catalog_epoch: u64,
|
||||
) -> Result<Vec<String>, AppError> {
|
||||
self.reconcile_pi_runtime_at_epoch_with_native_precondition(catalog_epoch, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn reconcile_pi_runtime_at_epoch_with_native_precondition(
|
||||
&self,
|
||||
catalog_epoch: u64,
|
||||
expected_native: Option<&crate::pi_config::document::PiProviderValuesSnapshot>,
|
||||
) -> Result<Vec<String>, AppError> {
|
||||
#[cfg(test)]
|
||||
if self.fail_next_pi_reconcile.swap(false, Ordering::AcqRel) {
|
||||
@@ -642,11 +651,54 @@ impl ProxyService {
|
||||
token,
|
||||
app_config,
|
||||
)?;
|
||||
crate::pi_config::document::apply_pi_provider_patch(
|
||||
&crate::pi_config::native::get_pi_models_path()?,
|
||||
&build.projection_patch,
|
||||
)?;
|
||||
self.pi_runtime.publish(build.snapshot).await?;
|
||||
let models_path = crate::pi_config::native::get_pi_models_path()?;
|
||||
let native_receipt = if build.projection_patch.is_empty() {
|
||||
None
|
||||
} else {
|
||||
let expected = match expected_native {
|
||||
Some(expected) => {
|
||||
crate::pi_config::document::PiProviderValuesSnapshot {
|
||||
file_existed: expected.file_existed,
|
||||
values: build
|
||||
.projection_patch
|
||||
.keys()
|
||||
.map(|provider_key| {
|
||||
expected
|
||||
.values
|
||||
.get(provider_key)
|
||||
.cloned()
|
||||
.map(|value| (provider_key.clone(), value))
|
||||
.ok_or_else(|| {
|
||||
AppError::Config(format!(
|
||||
"Pi runtime provider key '{provider_key}' was not included in the catalog preflight"
|
||||
))
|
||||
})
|
||||
})
|
||||
.collect::<Result<_, _>>()?,
|
||||
}
|
||||
}
|
||||
None => self.preflight_pi_owned_projection_at(
|
||||
&models_path,
|
||||
&build.projection_patch,
|
||||
)?,
|
||||
};
|
||||
Some(
|
||||
crate::pi_config::document::apply_pi_provider_patch_with_receipt(
|
||||
&models_path,
|
||||
&expected,
|
||||
&build.projection_patch,
|
||||
)?,
|
||||
)
|
||||
};
|
||||
if let Err(error) = self.pi_runtime.publish(build.snapshot).await {
|
||||
let rollback = native_receipt.as_ref().map_or(
|
||||
Ok(()),
|
||||
crate::pi_config::document::PiProviderPatchReceipt::rollback,
|
||||
);
|
||||
return Err(AppError::Config(format!(
|
||||
"failed to publish Pi runtime after native projection: {error}; native rollback={rollback:?}"
|
||||
)));
|
||||
}
|
||||
Ok(build.direct_only_provider_ids)
|
||||
}
|
||||
|
||||
@@ -692,12 +744,126 @@ impl ProxyService {
|
||||
fn restore_pi_direct_projection_at(
|
||||
&self,
|
||||
models_path: &std::path::Path,
|
||||
) -> Result<(), AppError> {
|
||||
let patch = direct_pi_projection_patch(self.db.as_ref())?;
|
||||
crate::pi_config::document::apply_pi_provider_patch(models_path, &patch)
|
||||
) -> Result<crate::pi_config::document::PiProviderPatchReceipt, AppError> {
|
||||
if self
|
||||
.pi_listener
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.is_none()
|
||||
{
|
||||
return self.confirm_pi_projection_is_already_direct_at(models_path);
|
||||
}
|
||||
let gateway = self.current_pi_gateway_projection()?;
|
||||
self.restore_pi_direct_projection_at_with_expected_gateway(models_path, &gateway)
|
||||
}
|
||||
|
||||
fn restore_pi_direct_projection(&self) -> Result<(), AppError> {
|
||||
fn confirm_pi_projection_is_already_direct_at(
|
||||
&self,
|
||||
models_path: &std::path::Path,
|
||||
) -> Result<crate::pi_config::document::PiProviderPatchReceipt, AppError> {
|
||||
let direct = direct_pi_projection_patch(self.db.as_ref())?;
|
||||
let before = crate::pi_config::document::snapshot_pi_provider_values(
|
||||
models_path,
|
||||
direct.keys().cloned(),
|
||||
)?;
|
||||
if let Some(provider_key) = direct.iter().find_map(|(provider_key, expected)| {
|
||||
(before.values.get(provider_key) != Some(expected)).then_some(provider_key)
|
||||
}) {
|
||||
return Err(AppError::Conflict(format!(
|
||||
"Pi listener is unavailable and provider key '{provider_key}' is not already in its direct projection"
|
||||
)));
|
||||
}
|
||||
crate::pi_config::document::apply_pi_provider_patch_with_receipt(
|
||||
models_path,
|
||||
&before,
|
||||
&direct,
|
||||
)
|
||||
}
|
||||
|
||||
fn current_pi_gateway_projection(
|
||||
&self,
|
||||
) -> Result<indexmap::IndexMap<String, Option<Value>>, AppError> {
|
||||
let listener = self
|
||||
.pi_listener
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.clone()
|
||||
.ok_or_else(|| {
|
||||
AppError::Conflict(
|
||||
"cannot restore Pi direct projection without its active listener identity"
|
||||
.to_string(),
|
||||
)
|
||||
})?;
|
||||
let gateway_token = crate::settings::get_pi_gateway_token().or_else(|settings_error| {
|
||||
self.pi_runtime
|
||||
.retained_gateway_token(listener.server_generation)
|
||||
.ok_or(settings_error)
|
||||
})?;
|
||||
let gateway = build_pi_runtime(
|
||||
self.db.as_ref(),
|
||||
listener.server_generation,
|
||||
0,
|
||||
&listener.gateway_origin,
|
||||
gateway_token,
|
||||
crate::settings::get_pi_app_proxy_config(),
|
||||
)?
|
||||
.projection_patch;
|
||||
Ok(gateway)
|
||||
}
|
||||
|
||||
fn preflight_pi_owned_projection_at(
|
||||
&self,
|
||||
models_path: &std::path::Path,
|
||||
expected_gateway: &indexmap::IndexMap<String, Option<Value>>,
|
||||
) -> Result<crate::pi_config::document::PiProviderValuesSnapshot, AppError> {
|
||||
let direct = direct_pi_projection_patch(self.db.as_ref())?;
|
||||
let direct_keys = direct.keys().collect::<std::collections::BTreeSet<_>>();
|
||||
let gateway_keys = expected_gateway
|
||||
.keys()
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
if direct_keys != gateway_keys {
|
||||
return Err(AppError::Conflict(
|
||||
"Pi direct and gateway projections cover different exact-key ownership".to_string(),
|
||||
));
|
||||
}
|
||||
let before = crate::pi_config::document::snapshot_pi_provider_values(
|
||||
models_path,
|
||||
direct.keys().cloned(),
|
||||
)?;
|
||||
for (provider_key, direct_value) in &direct {
|
||||
let observed = before
|
||||
.values
|
||||
.get(provider_key)
|
||||
.expect("every owned key was preflighted");
|
||||
let gateway_value = expected_gateway
|
||||
.get(provider_key)
|
||||
.expect("gateway ownership keys were checked");
|
||||
if observed != direct_value && observed != gateway_value {
|
||||
return Err(AppError::Conflict(format!(
|
||||
"Pi provider key '{provider_key}' changed outside CC Switch"
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(before)
|
||||
}
|
||||
|
||||
fn restore_pi_direct_projection_at_with_expected_gateway(
|
||||
&self,
|
||||
models_path: &std::path::Path,
|
||||
expected_gateway: &indexmap::IndexMap<String, Option<Value>>,
|
||||
) -> Result<crate::pi_config::document::PiProviderPatchReceipt, AppError> {
|
||||
let direct = direct_pi_projection_patch(self.db.as_ref())?;
|
||||
let before = self.preflight_pi_owned_projection_at(models_path, expected_gateway)?;
|
||||
crate::pi_config::document::apply_pi_provider_patch_with_receipt(
|
||||
models_path,
|
||||
&before,
|
||||
&direct,
|
||||
)
|
||||
}
|
||||
|
||||
fn restore_pi_direct_projection(
|
||||
&self,
|
||||
) -> Result<crate::pi_config::document::PiProviderPatchReceipt, AppError> {
|
||||
let models_path = crate::pi_config::native::get_pi_models_path()?;
|
||||
self.restore_pi_direct_projection_at(&models_path)
|
||||
}
|
||||
@@ -751,19 +917,21 @@ impl ProxyService {
|
||||
|
||||
if !existing.pi_takeover_enabled {
|
||||
crate::settings::update_settings(next)?;
|
||||
if let Err(error) =
|
||||
crate::pi_config::document::apply_pi_provider_patch(&new_models_path, &direct_patch)
|
||||
{
|
||||
let settings_restored = crate::settings::update_settings(existing.clone()).is_ok();
|
||||
let new_native_restored = crate::pi_config::document::restore_pi_provider_values(
|
||||
let native_receipt =
|
||||
match crate::pi_config::document::apply_pi_provider_patch_with_receipt(
|
||||
&new_models_path,
|
||||
&new_native_before,
|
||||
)
|
||||
.is_ok();
|
||||
return Err(AppError::Config(format!(
|
||||
"failed to publish managed Pi providers in the new directory: {error}; rollback: settings={settings_restored}, native={new_native_restored}"
|
||||
&direct_patch,
|
||||
) {
|
||||
Ok(receipt) => receipt,
|
||||
Err(error) => {
|
||||
let settings_restored =
|
||||
crate::settings::update_settings(existing.clone()).is_ok();
|
||||
return Err(AppError::Config(format!(
|
||||
"failed to publish managed Pi providers in the new directory: {error}; rollback: settings={settings_restored}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
};
|
||||
if let Err(error) =
|
||||
PromptService::reconcile_pi_native_under_guard(self.db.as_ref(), &prompt_guard)
|
||||
{
|
||||
@@ -772,11 +940,7 @@ impl ProxyService {
|
||||
.db
|
||||
.save_prompt_selection(AppType::Pi.as_str(), &previous_prompts)
|
||||
.is_ok();
|
||||
let new_native_restored = crate::pi_config::document::restore_pi_provider_values(
|
||||
&new_models_path,
|
||||
&new_native_before,
|
||||
)
|
||||
.is_ok();
|
||||
let new_native_restored = native_receipt.rollback().is_ok();
|
||||
return Err(AppError::Config(format!(
|
||||
"failed to reconcile Pi prompts in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, native={new_native_restored}"
|
||||
)));
|
||||
@@ -789,11 +953,7 @@ impl ProxyService {
|
||||
.db
|
||||
.save_prompt_selection(AppType::Pi.as_str(), &previous_prompts)
|
||||
.is_ok();
|
||||
let new_native_restored = crate::pi_config::document::restore_pi_provider_values(
|
||||
&new_models_path,
|
||||
&new_native_before,
|
||||
)
|
||||
.is_ok();
|
||||
let new_native_restored = native_receipt.rollback().is_ok();
|
||||
let skills_restored = settings_restored
|
||||
&& crate::services::skill_deployment::PiSkillDeploymentService::reconcile_all(
|
||||
&self.db,
|
||||
@@ -810,13 +970,22 @@ impl ProxyService {
|
||||
// process that already loaded the old gateway projection therefore
|
||||
// cannot enter a catalog whose directory ownership is in flight.
|
||||
let epoch = self.pi_runtime.begin_mutation().await;
|
||||
if let Err(error) = self.restore_pi_direct_projection_at(&old_models_path) {
|
||||
let _ = self.pi_runtime.republish_current(epoch).await;
|
||||
return Err(error);
|
||||
}
|
||||
let old_direct_receipt = match self.restore_pi_direct_projection_at(&old_models_path) {
|
||||
Ok(receipt) => receipt,
|
||||
Err(error) => {
|
||||
let _ = self.pi_runtime.republish_current(epoch).await;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(error) = crate::settings::update_settings(next) {
|
||||
let runtime_restored = self.reconcile_pi_runtime_at_epoch(epoch).await;
|
||||
let expected_old_direct = old_direct_receipt.attempted_snapshot();
|
||||
let runtime_restored = self
|
||||
.reconcile_pi_runtime_at_epoch_with_native_precondition(
|
||||
epoch,
|
||||
Some(&expected_old_direct),
|
||||
)
|
||||
.await;
|
||||
return Err(AppError::Config(if runtime_restored.is_ok() {
|
||||
format!(
|
||||
"failed to save the new Pi directory; the previous gateway projection was restored: {error}"
|
||||
@@ -836,19 +1005,24 @@ impl ProxyService {
|
||||
.db
|
||||
.save_prompt_selection(AppType::Pi.as_str(), &previous_prompts)
|
||||
.is_ok();
|
||||
let new_native_restored = crate::pi_config::document::restore_pi_provider_values(
|
||||
&new_models_path,
|
||||
&new_native_before,
|
||||
)
|
||||
.is_ok();
|
||||
let skills_restored = settings_restored
|
||||
&& crate::services::skill_deployment::PiSkillDeploymentService::reconcile_all(
|
||||
&self.db,
|
||||
)
|
||||
.is_ok();
|
||||
let runtime_restored = self.reconcile_pi_runtime_at_epoch(epoch).await.is_ok();
|
||||
let runtime_restored = if settings_restored {
|
||||
let expected_old_direct = old_direct_receipt.attempted_snapshot();
|
||||
self.reconcile_pi_runtime_at_epoch_with_native_precondition(
|
||||
epoch,
|
||||
Some(&expected_old_direct),
|
||||
)
|
||||
.await
|
||||
.is_ok()
|
||||
} else {
|
||||
self.pi_runtime.republish_current(epoch).await.is_ok()
|
||||
};
|
||||
return Err(AppError::Config(format!(
|
||||
"failed to reconcile Pi prompts in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, native={new_native_restored}, skills={skills_restored}, gateway={runtime_restored}"
|
||||
"failed to reconcile Pi prompts in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, gateway={runtime_restored}"
|
||||
)));
|
||||
}
|
||||
|
||||
@@ -860,45 +1034,54 @@ impl ProxyService {
|
||||
.db
|
||||
.save_prompt_selection(AppType::Pi.as_str(), &previous_prompts)
|
||||
.is_ok();
|
||||
let new_native_restored = crate::pi_config::document::restore_pi_provider_values(
|
||||
&new_models_path,
|
||||
&new_native_before,
|
||||
)
|
||||
.is_ok();
|
||||
let skills_restored = settings_restored
|
||||
&& crate::services::skill_deployment::PiSkillDeploymentService::reconcile_all(
|
||||
&self.db,
|
||||
)
|
||||
.is_ok();
|
||||
let runtime_restored = self.reconcile_pi_runtime_at_epoch(epoch).await.is_ok();
|
||||
let runtime_restored = if settings_restored {
|
||||
let expected_old_direct = old_direct_receipt.attempted_snapshot();
|
||||
self.reconcile_pi_runtime_at_epoch_with_native_precondition(
|
||||
epoch,
|
||||
Some(&expected_old_direct),
|
||||
)
|
||||
.await
|
||||
.is_ok()
|
||||
} else {
|
||||
self.pi_runtime.republish_current(epoch).await.is_ok()
|
||||
};
|
||||
return Err(AppError::Config(format!(
|
||||
"failed to reconcile Pi Skills in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, native={new_native_restored}, skills={skills_restored}, gateway={runtime_restored}"
|
||||
"failed to reconcile Pi Skills in the new directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, gateway={runtime_restored}"
|
||||
)));
|
||||
}
|
||||
|
||||
if let Err(error) = self.reconcile_pi_runtime_at_epoch(epoch).await {
|
||||
if let Err(error) = self
|
||||
.reconcile_pi_runtime_at_epoch_with_native_precondition(epoch, Some(&new_native_before))
|
||||
.await
|
||||
{
|
||||
let settings_restored = crate::settings::update_settings(existing.clone()).is_ok();
|
||||
let prompts_restored = self
|
||||
.db
|
||||
.save_prompt_selection(AppType::Pi.as_str(), &previous_prompts)
|
||||
.is_ok();
|
||||
let new_native_restored = crate::pi_config::document::restore_pi_provider_values(
|
||||
&new_models_path,
|
||||
&new_native_before,
|
||||
)
|
||||
.is_ok();
|
||||
let skills_restored = settings_restored
|
||||
&& crate::services::skill_deployment::PiSkillDeploymentService::reconcile_all(
|
||||
&self.db,
|
||||
)
|
||||
.is_ok();
|
||||
let old_gateway_restored = if settings_restored {
|
||||
self.reconcile_pi_runtime_at_epoch(epoch).await.is_ok()
|
||||
let expected_old_direct = old_direct_receipt.attempted_snapshot();
|
||||
self.reconcile_pi_runtime_at_epoch_with_native_precondition(
|
||||
epoch,
|
||||
Some(&expected_old_direct),
|
||||
)
|
||||
.await
|
||||
.is_ok()
|
||||
} else {
|
||||
self.pi_runtime.republish_current(epoch).await.is_ok()
|
||||
};
|
||||
return Err(AppError::Config(format!(
|
||||
"failed to publish Pi in the new native directory: {error}; rollback: native={new_native_restored}, settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, old_gateway={old_gateway_restored}"
|
||||
"failed to publish Pi in the new native directory: {error}; rollback: settings={settings_restored}, prompts={prompts_restored}, skills={skills_restored}, old_gateway={old_gateway_restored}"
|
||||
)));
|
||||
}
|
||||
|
||||
@@ -910,11 +1093,26 @@ impl ProxyService {
|
||||
return Ok(());
|
||||
}
|
||||
let epoch = self.pi_runtime.begin_mutation().await;
|
||||
if let Err(error) = self.restore_pi_direct_projection() {
|
||||
let _ = self.pi_runtime.republish_current(epoch).await;
|
||||
return Err(error);
|
||||
let direct_receipt = match self.restore_pi_direct_projection() {
|
||||
Ok(receipt) => receipt,
|
||||
Err(error) => {
|
||||
let _ = self.pi_runtime.republish_current(epoch).await;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
if let Err(error) = self.pi_runtime.close(epoch).await {
|
||||
let expected_direct = direct_receipt.attempted_snapshot();
|
||||
let rollback = self
|
||||
.reconcile_pi_runtime_at_epoch_with_native_precondition(
|
||||
epoch,
|
||||
Some(&expected_direct),
|
||||
)
|
||||
.await;
|
||||
return Err(AppError::Config(format!(
|
||||
"failed to close Pi admission after direct projection: {error}; gateway rollback={rollback:?}"
|
||||
)));
|
||||
}
|
||||
self.pi_runtime.close(epoch).await
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Enter the portable-import boundary while the caller holds Pi's switch
|
||||
@@ -958,16 +1156,32 @@ impl ProxyService {
|
||||
pub(crate) async fn rotate_pi_gateway_token(&self) -> Result<(), AppError> {
|
||||
let _guard = self.switch_locks.lock_for_app(AppType::Pi.as_str()).await;
|
||||
let previous = crate::settings::get_or_create_pi_gateway_token()?;
|
||||
let takeover_enabled = crate::settings::pi_takeover_enabled();
|
||||
let native_before = if takeover_enabled {
|
||||
let models_path = crate::pi_config::native::get_pi_models_path()?;
|
||||
let gateway = self.current_pi_gateway_projection()?;
|
||||
Some(self.preflight_pi_owned_projection_at(&models_path, &gateway)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
crate::settings::reset_pi_gateway_token()?;
|
||||
if !crate::settings::pi_takeover_enabled() {
|
||||
if !takeover_enabled {
|
||||
return Ok(());
|
||||
}
|
||||
let native_before = native_before.expect("enabled takeover has a native preflight");
|
||||
let epoch = self.pi_runtime.begin_mutation().await;
|
||||
if let Err(error) = self.reconcile_pi_runtime_at_epoch(epoch).await {
|
||||
if let Err(error) = self
|
||||
.reconcile_pi_runtime_at_epoch_with_native_precondition(epoch, Some(&native_before))
|
||||
.await
|
||||
{
|
||||
let token_restored = crate::settings::replace_pi_gateway_token(previous);
|
||||
let runtime_restored = if token_restored.is_ok() {
|
||||
let rollback_epoch = self.pi_runtime.begin_mutation().await;
|
||||
self.reconcile_pi_runtime_at_epoch(rollback_epoch).await
|
||||
self.reconcile_pi_runtime_at_epoch_with_native_precondition(
|
||||
rollback_epoch,
|
||||
Some(&native_before),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
Err(AppError::Config(
|
||||
"failed to restore the previous Pi gateway credential".to_string(),
|
||||
@@ -1561,7 +1775,7 @@ impl ProxyService {
|
||||
|
||||
let direct_projection = self.restore_pi_direct_projection();
|
||||
match direct_projection {
|
||||
Ok(()) => {
|
||||
Ok(_) => {
|
||||
let epoch = self.pi_runtime.begin_mutation().await;
|
||||
let admission_closed = self.pi_runtime.close(epoch).await;
|
||||
return Err(if admission_closed.is_ok() {
|
||||
@@ -1597,16 +1811,25 @@ impl ProxyService {
|
||||
return Ok(());
|
||||
}
|
||||
let epoch = self.pi_runtime.begin_mutation().await;
|
||||
if let Err(error) = self.restore_pi_direct_projection() {
|
||||
let _ = self.pi_runtime.republish_current(epoch).await;
|
||||
return Err(format!(
|
||||
"failed to restore Pi's direct native projection: {error}"
|
||||
));
|
||||
}
|
||||
let direct_receipt = match self.restore_pi_direct_projection() {
|
||||
Ok(receipt) => receipt,
|
||||
Err(error) => {
|
||||
let _ = self.pi_runtime.republish_current(epoch).await;
|
||||
return Err(format!(
|
||||
"failed to restore Pi's direct native projection: {error}"
|
||||
));
|
||||
}
|
||||
};
|
||||
if let Err(error) = crate::settings::set_pi_takeover_enabled(false) {
|
||||
// Desired state did not change; rebuild the gateway projection and
|
||||
// restore admission so the native file cannot be left lying.
|
||||
let _ = self.reconcile_pi_runtime_at_epoch(epoch).await;
|
||||
let expected_direct = direct_receipt.attempted_snapshot();
|
||||
let _ = self
|
||||
.reconcile_pi_runtime_at_epoch_with_native_precondition(
|
||||
epoch,
|
||||
Some(&expected_direct),
|
||||
)
|
||||
.await;
|
||||
return Err(format!("failed to persist Pi takeover disable: {error}"));
|
||||
}
|
||||
|
||||
@@ -4164,16 +4387,17 @@ mod tests {
|
||||
"apiKey": "native-key",
|
||||
"models": [{"id": "native-model"}]
|
||||
});
|
||||
let expected_gateway = serde_json::json!({
|
||||
"api": "openai-responses",
|
||||
"baseUrl": "http://127.0.0.1:15721/pi/route",
|
||||
"apiKey": "gateway-token",
|
||||
"models": [{"id": "model-a"}]
|
||||
});
|
||||
std::fs::write(
|
||||
&path,
|
||||
serde_json::to_vec_pretty(&serde_json::json!({
|
||||
"providers": {
|
||||
"managed-pi": {
|
||||
"api": "openai-responses",
|
||||
"baseUrl": "http://127.0.0.1:15721/pi/route",
|
||||
"apiKey": "gateway-token",
|
||||
"models": [{"id": "model-a"}]
|
||||
},
|
||||
"managed-pi": expected_gateway.clone(),
|
||||
"native": unowned.clone()
|
||||
}
|
||||
}))
|
||||
@@ -4183,7 +4407,10 @@ mod tests {
|
||||
|
||||
state
|
||||
.proxy_service
|
||||
.restore_pi_direct_projection_at(&path)
|
||||
.restore_pi_direct_projection_at_with_expected_gateway(
|
||||
&path,
|
||||
&indexmap::IndexMap::from([("managed-pi".to_string(), Some(expected_gateway))]),
|
||||
)
|
||||
.expect("restore direct Pi projection");
|
||||
|
||||
let restored: Value =
|
||||
@@ -4452,6 +4679,91 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn changing_pi_directory_does_not_overwrite_an_exact_key_changed_after_preflight() {
|
||||
let home = TempHome::new();
|
||||
crate::settings::reload_settings().expect("reload isolated settings");
|
||||
let old_dir = home.dir.path().join("old-pi");
|
||||
let new_dir = home.dir.path().join("new-pi");
|
||||
std::fs::create_dir_all(&old_dir).expect("old Pi directory");
|
||||
std::fs::create_dir_all(&new_dir).expect("new Pi directory");
|
||||
|
||||
let direct = json!({
|
||||
"name": "Managed Pi",
|
||||
"api": "openai-responses",
|
||||
"baseUrl": "https://managed.example/v1",
|
||||
"apiKey": "managed-key",
|
||||
"models": [{"id": "model-a", "name": "Model A"}]
|
||||
});
|
||||
let external = json!({
|
||||
"name": "External Pi",
|
||||
"api": "openai-responses",
|
||||
"baseUrl": "https://external.example/v1",
|
||||
"apiKey": "external-key",
|
||||
"models": [{"id": "external-model", "name": "External"}]
|
||||
});
|
||||
let new_models_path = new_dir.join("models.json");
|
||||
std::fs::write(
|
||||
&new_models_path,
|
||||
serde_json::to_vec(&json!({"providers": {}})).expect("serialize direct document"),
|
||||
)
|
||||
.expect("seed direct document");
|
||||
|
||||
let mut settings = crate::settings::get_settings();
|
||||
settings.pi_config_dir = Some(old_dir.to_string_lossy().into_owned());
|
||||
settings.pi_takeover_enabled = false;
|
||||
crate::settings::update_settings(settings).expect("old Pi settings");
|
||||
let db = Arc::new(Database::memory().expect("database"));
|
||||
db.create_pi_catalog_provider(
|
||||
NewProviderAggregate::from_input(
|
||||
AppType::Pi.as_str(),
|
||||
ProviderMutationInput {
|
||||
id: "managed-pi".to_string(),
|
||||
name: "Managed Pi".to_string(),
|
||||
settings_config: direct,
|
||||
website_url: None,
|
||||
category: None,
|
||||
created_at: None,
|
||||
sort_index: Some(0),
|
||||
notes: None,
|
||||
meta: None,
|
||||
icon: Some("pi".to_string()),
|
||||
icon_color: None,
|
||||
in_failover_queue: false,
|
||||
},
|
||||
)
|
||||
.expect("aggregate"),
|
||||
"managed-pi",
|
||||
)
|
||||
.expect("managed provider");
|
||||
crate::pi_config::shared_file::replace_before_next_compare_exchange(
|
||||
&new_models_path,
|
||||
&serde_json::to_vec(&json!({"providers": {"managed-pi": external.clone()}}))
|
||||
.expect("serialize external document"),
|
||||
);
|
||||
|
||||
let service = ProxyService::new(db);
|
||||
let switch_guard = service.lock_switch_for_app(AppType::Pi.as_str()).await;
|
||||
let existing = crate::settings::get_settings();
|
||||
let mut next = existing.clone();
|
||||
next.pi_config_dir = Some(new_dir.to_string_lossy().into_owned());
|
||||
let error = service
|
||||
.replace_settings_with_pi_directory_boundary_under_lock(&switch_guard, &existing, next)
|
||||
.await
|
||||
.expect_err("the post-preflight native edit must win");
|
||||
assert!(error.to_string().contains("changed since"));
|
||||
assert_eq!(
|
||||
crate::settings::get_settings().pi_config_dir,
|
||||
existing.pi_config_dir
|
||||
);
|
||||
let live: Value = serde_json::from_slice(
|
||||
&std::fs::read(&new_models_path).expect("read external native document"),
|
||||
)
|
||||
.expect("parse external native document");
|
||||
assert_eq!(live.pointer("/providers/managed-pi"), Some(&external));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn changing_pi_directory_without_takeover_reconciles_missing_agents_truth() {
|
||||
@@ -4544,6 +4856,143 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn pi_gateway_returns_the_last_real_retryable_upstream_response_when_no_later_send_occurs(
|
||||
) {
|
||||
let home = TempHome::new();
|
||||
crate::settings::reload_settings().expect("reload isolated settings");
|
||||
let pi_dir = home.dir.path().join("pi");
|
||||
std::fs::create_dir_all(&pi_dir).expect("Pi directory");
|
||||
let mut settings = crate::settings::get_settings();
|
||||
settings.pi_config_dir = Some(pi_dir.to_string_lossy().into_owned());
|
||||
settings.pi_takeover_enabled = false;
|
||||
crate::settings::update_settings(settings).expect("Pi settings");
|
||||
|
||||
let upstream_hits = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let hits = upstream_hits.clone();
|
||||
let upstream = axum::Router::new().fallback(axum::routing::any(move || {
|
||||
let hits = hits.clone();
|
||||
async move {
|
||||
hits.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
let mut response = axum::response::Response::builder()
|
||||
.status(http::StatusCode::TOO_MANY_REQUESTS)
|
||||
.header("x-upstream-marker", "real-429")
|
||||
.header(http::header::CONTENT_TYPE, "application/json")
|
||||
.body(axum::body::Body::from(
|
||||
r#"{"error":{"message":"upstream rate limit"}}"#,
|
||||
))
|
||||
.expect("upstream response");
|
||||
response
|
||||
.headers_mut()
|
||||
.insert("retry-after", http::HeaderValue::from_static("17"));
|
||||
response
|
||||
}
|
||||
}));
|
||||
let upstream_listener = tokio::net::TcpListener::bind(("127.0.0.1", 0))
|
||||
.await
|
||||
.expect("upstream listener");
|
||||
let upstream_address = upstream_listener.local_addr().expect("upstream address");
|
||||
let upstream_task = tokio::spawn(async move {
|
||||
axum::serve(upstream_listener, upstream)
|
||||
.await
|
||||
.expect("upstream server");
|
||||
});
|
||||
|
||||
let db = Arc::new(Database::memory().expect("database"));
|
||||
use_ephemeral_proxy_port(&db).await;
|
||||
let direct = json!({
|
||||
"name": "Retryable upstream",
|
||||
"api": "openai-responses",
|
||||
"baseUrl": format!("http://{upstream_address}/v1"),
|
||||
"apiKey": "upstream-key",
|
||||
"models": [{"id": "model-a", "name": "Model A"}]
|
||||
});
|
||||
db.create_pi_catalog_provider(
|
||||
NewProviderAggregate::from_input(
|
||||
AppType::Pi.as_str(),
|
||||
ProviderMutationInput {
|
||||
id: "retryable-upstream".to_string(),
|
||||
name: "Retryable upstream".to_string(),
|
||||
settings_config: direct.clone(),
|
||||
website_url: None,
|
||||
category: None,
|
||||
created_at: None,
|
||||
sort_index: Some(0),
|
||||
notes: None,
|
||||
meta: None,
|
||||
icon: Some("pi".to_string()),
|
||||
icon_color: None,
|
||||
in_failover_queue: false,
|
||||
},
|
||||
)
|
||||
.expect("aggregate"),
|
||||
"retryable-upstream",
|
||||
)
|
||||
.expect("managed provider");
|
||||
std::fs::write(
|
||||
pi_dir.join("models.json"),
|
||||
serde_json::to_vec(&json!({
|
||||
"providers": {"retryable-upstream": direct}
|
||||
}))
|
||||
.expect("serialize native catalog"),
|
||||
)
|
||||
.expect("native catalog");
|
||||
|
||||
let service = ProxyService::new(db);
|
||||
service
|
||||
.set_takeover_for_app(AppType::Pi.as_str(), true)
|
||||
.await
|
||||
.expect("enable Pi gateway");
|
||||
let projected: Value = serde_json::from_slice(
|
||||
&std::fs::read(pi_dir.join("models.json")).expect("projected catalog"),
|
||||
)
|
||||
.expect("parse projected catalog");
|
||||
let base_url = projected
|
||||
.pointer("/providers/retryable-upstream/baseUrl")
|
||||
.and_then(Value::as_str)
|
||||
.expect("gateway base URL");
|
||||
let gateway_token = projected
|
||||
.pointer("/providers/retryable-upstream/apiKey")
|
||||
.and_then(Value::as_str)
|
||||
.expect("gateway token");
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
.post(format!("{base_url}/responses"))
|
||||
.bearer_auth(gateway_token)
|
||||
.json(&json!({"model": "model-a", "input": "hello"}))
|
||||
.send()
|
||||
.await
|
||||
.expect("gateway response");
|
||||
let status = response.status();
|
||||
let marker = response
|
||||
.headers()
|
||||
.get("x-upstream-marker")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_string);
|
||||
let retry_after = response
|
||||
.headers()
|
||||
.get("retry-after")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_string);
|
||||
let body = response.text().await.expect("upstream body");
|
||||
upstream_task.abort();
|
||||
|
||||
assert_eq!(status, reqwest::StatusCode::TOO_MANY_REQUESTS);
|
||||
assert_eq!(marker.as_deref(), Some("real-429"));
|
||||
assert_eq!(retry_after.as_deref(), Some("17"));
|
||||
assert_eq!(body, r#"{"error":{"message":"upstream rate limit"}}"#);
|
||||
assert_eq!(
|
||||
upstream_hits.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"local candidate exhaustion must not invent a phantom retry"
|
||||
);
|
||||
service
|
||||
.set_takeover_for_app(AppType::Pi.as_str(), false)
|
||||
.await
|
||||
.expect("disable Pi gateway");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn failed_listener_rebind_recovers_pi_on_the_previous_listener() {
|
||||
@@ -4716,6 +5165,75 @@ mod tests {
|
||||
.expect("explicit disable clears desired state");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn listener_absence_does_not_authorize_overwriting_a_non_direct_native_key() {
|
||||
let home = TempHome::new();
|
||||
crate::settings::reload_settings().expect("reload isolated settings");
|
||||
let pi_dir = home.dir.path().join("pi");
|
||||
std::fs::create_dir_all(&pi_dir).expect("Pi directory");
|
||||
let mut settings = crate::settings::get_settings();
|
||||
settings.pi_config_dir = Some(pi_dir.to_string_lossy().into_owned());
|
||||
settings.pi_takeover_enabled = true;
|
||||
crate::settings::update_settings(settings).expect("Pi settings");
|
||||
|
||||
let db = Arc::new(Database::memory().expect("database"));
|
||||
let direct = json!({
|
||||
"name": "Managed Pi",
|
||||
"api": "openai-responses",
|
||||
"baseUrl": "https://managed.example/v1",
|
||||
"apiKey": "managed-key",
|
||||
"models": [{"id": "model-a", "name": "Model A"}]
|
||||
});
|
||||
db.create_pi_catalog_provider(
|
||||
NewProviderAggregate::from_input(
|
||||
AppType::Pi.as_str(),
|
||||
ProviderMutationInput {
|
||||
id: "managed-pi".to_string(),
|
||||
name: "Managed Pi".to_string(),
|
||||
settings_config: direct,
|
||||
website_url: None,
|
||||
category: None,
|
||||
created_at: None,
|
||||
sort_index: Some(0),
|
||||
notes: None,
|
||||
meta: None,
|
||||
icon: Some("pi".to_string()),
|
||||
icon_color: None,
|
||||
in_failover_queue: false,
|
||||
},
|
||||
)
|
||||
.expect("aggregate"),
|
||||
"managed-pi",
|
||||
)
|
||||
.expect("managed provider");
|
||||
let external = json!({
|
||||
"name": "External edit",
|
||||
"api": "openai-responses",
|
||||
"baseUrl": "https://external.example/v1",
|
||||
"apiKey": "external-key",
|
||||
"models": [{"id": "external-model"}]
|
||||
});
|
||||
let native_bytes = serde_json::to_vec(&json!({
|
||||
"providers": {"managed-pi": external}
|
||||
}))
|
||||
.expect("serialize external catalog");
|
||||
let models_path = pi_dir.join("models.json");
|
||||
std::fs::write(&models_path, &native_bytes).expect("external native catalog");
|
||||
|
||||
let service = ProxyService::new(db);
|
||||
let error = service
|
||||
.set_takeover_for_app(AppType::Pi.as_str(), false)
|
||||
.await
|
||||
.expect_err("no listener is not ownership evidence");
|
||||
assert!(error.contains("not already in its direct projection"));
|
||||
assert!(crate::settings::pi_takeover_enabled());
|
||||
assert_eq!(
|
||||
std::fs::read(&models_path).expect("external catalog remains"),
|
||||
native_bytes
|
||||
);
|
||||
}
|
||||
|
||||
fn seed_codex_model_template() {
|
||||
let codex_dir = crate::codex_config::get_codex_config_dir();
|
||||
std::fs::create_dir_all(&codex_dir).expect("create codex dir");
|
||||
|
||||
Reference in New Issue
Block a user