feat: unify Codex third-party providers into stable "custom" history bucket

Codex filters resume history by `model_provider`, so switching between
provider-specific ids like `rightcode` and `aihubmix` made past sessions
appear to vanish. Collapse all third-party providers into a single
stable bucket so cross-switch history stays visible.

- Normalize live `model_provider` to "custom" on every Codex write
  (reserved built-in ids like openai/ollama are preserved).
- Add device-level one-shot migration that rewrites historical JSONL
  session files and the `state_5.sqlite` threads table from legacy
  provider ids into the "custom" bucket. Backs up originals under
  `~/.cc-switch/backups/codex-history-provider-migration-v1/` and uses
  the SQLite Backup API for the state DB.
- Record completion in `settings.json` under `localMigrations` so the
  migration is strictly idempotent across launches.
- Update Codex provider preset templates to emit `model_provider = "custom"`
  out of the box.
This commit is contained in:
Jason
2026-05-20 17:10:38 +08:00
parent 2a4651a21e
commit b44f83f7c5
11 changed files with 771 additions and 121 deletions
+559
View File
@@ -0,0 +1,559 @@
//! Codex 第三方历史会话归桶迁移。
//!
//! 只迁移本机 `~/.codex` 历史数据;完成标记写入设备级 `settings.json`
//! 失败时不写标记,下一次启动自动重试。
use crate::codex_config::{
get_codex_config_dir, is_custom_codex_model_provider_id, read_codex_config_text,
stable_codex_model_provider_id_from_config, CC_SWITCH_CODEX_MODEL_PROVIDER_ID,
};
use crate::config::{atomic_write, copy_file, get_app_config_dir};
use crate::database::{is_official_seed_id, Database};
use crate::error::AppError;
use crate::settings::CodexThirdPartyHistoryProviderBucketMigration;
use chrono::{Local, Utc};
use rusqlite::{backup::Backup, params_from_iter, Connection};
use serde_json::Value;
use std::collections::{BTreeSet, HashSet};
use std::fs;
use std::hash::{Hash, Hasher};
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime};
use toml_edit::DocumentMut;
const MIGRATION_NAME: &str = "codex-history-provider-migration-v1";
const CODEX_STATE_DB_FILENAME: &str = "state_5.sqlite";
#[derive(Debug, Clone, Default)]
pub struct CodexHistoryProviderBucketMigrationOutcome {
pub source_provider_ids: Vec<String>,
pub migrated_jsonl_files: usize,
pub migrated_state_rows: usize,
pub skipped_reason: Option<String>,
}
pub fn maybe_migrate_codex_third_party_history_provider_bucket(
db: &Database,
) -> Result<CodexHistoryProviderBucketMigrationOutcome, AppError> {
if crate::settings::is_codex_third_party_history_provider_bucket_migrated() {
return Ok(CodexHistoryProviderBucketMigrationOutcome {
skipped_reason: Some("already_migrated".to_string()),
..Default::default()
});
}
let source_provider_ids = collect_source_model_provider_ids(db)?;
if source_provider_ids.is_empty() {
crate::settings::mark_codex_third_party_history_provider_bucket_migrated(
CodexThirdPartyHistoryProviderBucketMigration {
completed_at: Utc::now().to_rfc3339(),
target_provider_id: CC_SWITCH_CODEX_MODEL_PROVIDER_ID.to_string(),
source_provider_ids: Vec::new(),
migrated_jsonl_files: 0,
migrated_state_rows: 0,
},
)?;
return Ok(CodexHistoryProviderBucketMigrationOutcome {
skipped_reason: Some("no_third_party_provider_ids".to_string()),
..Default::default()
});
}
let backup_root = migration_backup_root();
let codex_dir = get_codex_config_dir();
let migrated_jsonl_files =
migrate_codex_jsonl_files(&codex_dir, &source_provider_ids, &backup_root)?;
let migrated_state_rows =
migrate_codex_state_dbs(&codex_dir, &source_provider_ids, &backup_root)?;
let source_provider_ids_vec: Vec<String> = source_provider_ids.iter().cloned().collect();
crate::settings::mark_codex_third_party_history_provider_bucket_migrated(
CodexThirdPartyHistoryProviderBucketMigration {
completed_at: Utc::now().to_rfc3339(),
target_provider_id: CC_SWITCH_CODEX_MODEL_PROVIDER_ID.to_string(),
source_provider_ids: source_provider_ids_vec.clone(),
migrated_jsonl_files,
migrated_state_rows,
},
)?;
Ok(CodexHistoryProviderBucketMigrationOutcome {
source_provider_ids: source_provider_ids_vec,
migrated_jsonl_files,
migrated_state_rows,
skipped_reason: None,
})
}
fn collect_source_model_provider_ids(db: &Database) -> Result<BTreeSet<String>, AppError> {
let providers = db.get_all_providers("codex")?;
let mut ids = BTreeSet::new();
for provider in providers.values() {
if provider.category.as_deref() == Some("official")
|| is_official_seed_id(&provider.id)
|| provider.is_codex_oauth()
{
continue;
}
if is_migratable_source_id(&provider.id) {
ids.insert(provider.id.clone());
}
let Some(config_text) = provider
.settings_config
.get("config")
.and_then(|value| value.as_str())
else {
continue;
};
if let Some(provider_id) = stable_codex_model_provider_id_from_config(config_text) {
if is_migratable_source_id(&provider_id) {
ids.insert(provider_id);
}
}
}
Ok(ids)
}
fn is_migratable_source_id(provider_id: &str) -> bool {
let trimmed = provider_id.trim();
!trimmed.is_empty()
&& trimmed != CC_SWITCH_CODEX_MODEL_PROVIDER_ID
&& is_custom_codex_model_provider_id(trimmed)
}
fn migration_backup_root() -> PathBuf {
get_app_config_dir()
.join("backups")
.join(MIGRATION_NAME)
.join(Local::now().format("%Y%m%d_%H%M%S").to_string())
}
fn migrate_codex_jsonl_files(
codex_dir: &Path,
source_provider_ids: &BTreeSet<String>,
backup_root: &Path,
) -> Result<usize, AppError> {
let mut files = Vec::new();
collect_jsonl_files(&codex_dir.join("sessions"), &mut files, 0, 8);
collect_jsonl_files(&codex_dir.join("archived_sessions"), &mut files, 0, 4);
let source_provider_ids: HashSet<String> = source_provider_ids.iter().cloned().collect();
let mut migrated = 0;
for file_path in files {
if rewrite_codex_session_file_for_provider_bucket(
&file_path,
codex_dir,
&source_provider_ids,
backup_root,
)? {
migrated += 1;
}
}
Ok(migrated)
}
fn collect_jsonl_files(dir: &Path, files: &mut Vec<PathBuf>, depth: u8, max_depth: u8) {
if depth > max_depth || !dir.is_dir() {
return;
}
let entries = match fs::read_dir(dir) {
Ok(entries) => entries,
Err(err) => {
log::debug!(
"Failed to read Codex session directory {}: {err}",
dir.display()
);
return;
}
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
collect_jsonl_files(&path, files, depth + 1, max_depth);
} else if path.extension().and_then(|ext| ext.to_str()) == Some("jsonl") {
files.push(path);
}
}
}
fn rewrite_codex_session_file_for_provider_bucket(
path: &Path,
codex_dir: &Path,
source_provider_ids: &HashSet<String>,
backup_root: &Path,
) -> Result<bool, AppError> {
let metadata_before = fs::metadata(path).map_err(|e| AppError::io(path, e))?;
let modified_before = metadata_before.modified().ok();
let len_before = metadata_before.len();
let content = fs::read_to_string(path).map_err(|e| AppError::io(path, e))?;
let mut rewritten = String::with_capacity(content.len());
let mut changed = false;
for segment in content.split_inclusive('\n') {
let (line, newline) = segment
.strip_suffix('\n')
.map(|line| (line, "\n"))
.unwrap_or((segment, ""));
if let Some(next_line) = rewrite_codex_session_meta_line(line, source_provider_ids) {
rewritten.push_str(&next_line);
changed = true;
} else {
rewritten.push_str(line);
}
rewritten.push_str(newline);
}
if !changed {
return Ok(false);
}
ensure_codex_session_file_unchanged(path, modified_before, len_before)?;
backup_codex_jsonl_file(path, codex_dir, backup_root)?;
ensure_codex_session_file_unchanged(path, modified_before, len_before)?;
atomic_write(path, rewritten.as_bytes())?;
Ok(true)
}
fn ensure_codex_session_file_unchanged(
path: &Path,
modified_before: Option<SystemTime>,
len_before: u64,
) -> Result<(), AppError> {
let metadata_after = fs::metadata(path).map_err(|e| AppError::io(path, e))?;
if metadata_after.modified().ok() != modified_before || metadata_after.len() != len_before {
return Err(AppError::Message(format!(
"Codex session file changed during migration: {}",
path.display()
)));
}
Ok(())
}
fn rewrite_codex_session_meta_line(
line: &str,
source_provider_ids: &HashSet<String>,
) -> Option<String> {
if !line.contains("\"session_meta\"") || !line.contains("\"model_provider\"") {
return None;
}
let mut value: Value = serde_json::from_str(line).ok()?;
if value.get("type").and_then(Value::as_str) != Some("session_meta") {
return None;
}
let payload = value.get_mut("payload")?.as_object_mut()?;
let current_provider = payload.get("model_provider")?.as_str()?;
if !source_provider_ids.contains(current_provider) {
return None;
}
payload.insert(
"model_provider".to_string(),
Value::String(CC_SWITCH_CODEX_MODEL_PROVIDER_ID.to_string()),
);
serde_json::to_string(&value).ok()
}
fn migrate_codex_state_dbs(
codex_dir: &Path,
source_provider_ids: &BTreeSet<String>,
backup_root: &Path,
) -> Result<usize, AppError> {
let config_text = read_codex_config_text().unwrap_or_default();
let mut migrated = 0;
for db_path in codex_state_db_paths(codex_dir, &config_text) {
migrated += migrate_codex_state_db_provider_bucket(
&db_path,
codex_dir,
source_provider_ids,
backup_root,
)?;
}
Ok(migrated)
}
fn codex_state_db_paths(codex_dir: &Path, config_text: &str) -> Vec<PathBuf> {
let mut paths = vec![codex_dir.join(CODEX_STATE_DB_FILENAME)];
if let Some(sqlite_home) = sqlite_home_from_codex_config(config_text) {
let db_path = sqlite_home.join(CODEX_STATE_DB_FILENAME);
if !paths.contains(&db_path) {
paths.push(db_path);
}
}
paths
}
fn sqlite_home_from_codex_config(config_text: &str) -> Option<PathBuf> {
let doc = config_text.parse::<DocumentMut>().ok()?;
let raw = doc.get("sqlite_home")?.as_str()?.trim();
if raw.is_empty() {
return None;
}
Some(resolve_user_path(raw))
}
fn resolve_user_path(raw: &str) -> PathBuf {
if raw == "~" {
return crate::config::get_home_dir();
}
if let Some(rest) = raw.strip_prefix("~/") {
return crate::config::get_home_dir().join(rest);
}
if let Some(rest) = raw.strip_prefix("~\\") {
return crate::config::get_home_dir().join(rest);
}
PathBuf::from(raw)
}
fn migrate_codex_state_db_provider_bucket(
db_path: &Path,
codex_dir: &Path,
source_provider_ids: &BTreeSet<String>,
backup_root: &Path,
) -> Result<usize, AppError> {
if !db_path.exists() || source_provider_ids.is_empty() {
return Ok(0);
}
let conn = Connection::open(db_path)
.map_err(|e| AppError::Database(format!("打开 Codex state DB 失败: {e}")))?;
conn.busy_timeout(Duration::from_secs(5))
.map_err(|e| AppError::Database(format!("设置 Codex state DB busy_timeout 失败: {e}")))?;
if !Database::table_exists(&conn, "threads")?
|| !Database::has_column(&conn, "threads", "model_provider")?
{
return Ok(0);
}
let placeholders = placeholders(source_provider_ids.len());
let count_sql =
format!("SELECT COUNT(*) FROM threads WHERE model_provider IN ({placeholders})");
let matching_rows: i64 = conn
.query_row(
&count_sql,
params_from_iter(source_provider_ids.iter()),
|row| row.get(0),
)
.map_err(|e| AppError::Database(format!("统计 Codex state DB 待迁移行失败: {e}")))?;
if matching_rows == 0 {
return Ok(0);
}
backup_codex_state_db(db_path, codex_dir, backup_root, &conn)?;
let update_sql =
format!("UPDATE threads SET model_provider = ? WHERE model_provider IN ({placeholders})");
let mut values = Vec::with_capacity(source_provider_ids.len() + 1);
values.push(CC_SWITCH_CODEX_MODEL_PROVIDER_ID.to_string());
values.extend(source_provider_ids.iter().cloned());
let changed = conn
.execute(&update_sql, params_from_iter(values.iter()))
.map_err(|e| AppError::Database(format!("迁移 Codex state DB provider 失败: {e}")))?;
Ok(changed)
}
fn placeholders(count: usize) -> String {
std::iter::repeat_n("?", count)
.collect::<Vec<_>>()
.join(", ")
}
fn backup_codex_jsonl_file(
path: &Path,
codex_dir: &Path,
backup_root: &Path,
) -> Result<(), AppError> {
let backup_path = backup_root
.join("jsonl")
.join(relative_backup_path(path, codex_dir));
copy_existing_file(path, &backup_path)
}
fn backup_codex_state_db(
db_path: &Path,
codex_dir: &Path,
backup_root: &Path,
source_conn: &Connection,
) -> Result<(), AppError> {
let backup_path = backup_root
.join("state")
.join(relative_backup_path(db_path, codex_dir));
if let Some(parent) = backup_path.parent() {
fs::create_dir_all(parent).map_err(|e| AppError::io(parent, e))?;
}
let mut backup_conn = Connection::open(&backup_path)
.map_err(|e| AppError::Database(format!("创建 Codex state DB 备份失败: {e}")))?;
let backup = Backup::new(source_conn, &mut backup_conn)
.map_err(|e| AppError::Database(format!("初始化 Codex state DB 备份失败: {e}")))?;
backup
.run_to_completion(5, Duration::from_millis(25), None)
.map_err(|e| AppError::Database(format!("写入 Codex state DB 备份失败: {e}")))?;
Ok(())
}
fn copy_existing_file(source: &Path, target: &Path) -> Result<(), AppError> {
if let Some(parent) = target.parent() {
fs::create_dir_all(parent).map_err(|e| AppError::io(parent, e))?;
}
copy_file(source, target)
}
fn relative_backup_path(path: &Path, root: &Path) -> PathBuf {
if let Ok(relative) = path.strip_prefix(root) {
return relative.to_path_buf();
}
let mut hasher = std::collections::hash_map::DefaultHasher::new();
path.hash(&mut hasher);
let hash = hasher.finish();
let file_name = path
.file_name()
.map(|name| name.to_string_lossy().to_string())
.unwrap_or_else(|| "file".to_string());
PathBuf::from("external").join(format!("{hash:016x}-{file_name}"))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::provider::Provider;
use tempfile::tempdir;
fn source_ids(values: &[&str]) -> BTreeSet<String> {
values.iter().map(|value| value.to_string()).collect()
}
#[test]
fn rewrites_only_codex_session_meta_provider_ids() {
let dir = tempdir().expect("tempdir");
let codex_dir = dir.path().join(".codex");
let backup_root = dir.path().join("backup");
let session_dir = codex_dir.join("sessions/2026/05/20");
fs::create_dir_all(&session_dir).expect("create session dir");
let path = session_dir.join("rollout-test.jsonl");
fs::write(
&path,
concat!(
"{\"type\":\"session_meta\",\"payload\":{\"id\":\"s1\",\"model_provider\":\"rightcode\"}}\n",
"{\"type\":\"response_item\",\"payload\":{\"type\":\"message\",\"role\":\"user\",\"content\":\"hi\"}}\n"
),
)
.expect("write session");
let changed = rewrite_codex_session_file_for_provider_bucket(
&path,
&codex_dir,
&HashSet::from(["rightcode".to_string()]),
&backup_root,
)
.expect("rewrite");
assert!(changed);
let next = fs::read_to_string(&path).expect("read rewritten");
assert!(next.contains("\"model_provider\":\"custom\""));
assert!(backup_root
.join("jsonl/sessions/2026/05/20/rollout-test.jsonl")
.exists());
}
#[test]
fn updates_codex_state_db_thread_provider_ids() {
let dir = tempdir().expect("tempdir");
let codex_dir = dir.path().join(".codex");
fs::create_dir_all(&codex_dir).expect("create codex dir");
let db_path = codex_dir.join(CODEX_STATE_DB_FILENAME);
let conn = Connection::open(&db_path).expect("open db");
conn.execute_batch(
"CREATE TABLE threads (
id TEXT PRIMARY KEY,
model_provider TEXT NOT NULL
);
INSERT INTO threads (id, model_provider) VALUES
('a', 'rightcode'),
('b', 'openai'),
('c', 'aihubmix');",
)
.expect("seed db");
drop(conn);
let backup_root = dir.path().join("backup");
let changed = migrate_codex_state_db_provider_bucket(
&db_path,
&codex_dir,
&source_ids(&["rightcode", "aihubmix"]),
&backup_root,
)
.expect("migrate state db");
assert_eq!(changed, 2);
let conn = Connection::open(&db_path).expect("reopen db");
let custom_count: i64 = conn
.query_row(
"SELECT COUNT(*) FROM threads WHERE model_provider = 'custom'",
[],
|row| row.get(0),
)
.expect("count custom");
let openai_count: i64 = conn
.query_row(
"SELECT COUNT(*) FROM threads WHERE model_provider = 'openai'",
[],
|row| row.get(0),
)
.expect("count openai");
assert_eq!(custom_count, 2);
assert_eq!(openai_count, 1);
let backup_path = backup_root.join("state").join(CODEX_STATE_DB_FILENAME);
let backup_conn = Connection::open(&backup_path).expect("open backup db");
let backed_up_source_count: i64 = backup_conn
.query_row(
"SELECT COUNT(*) FROM threads WHERE model_provider IN ('rightcode', 'aihubmix')",
[],
|row| row.get(0),
)
.expect("count backed up source providers");
assert_eq!(backed_up_source_count, 2);
}
#[test]
fn collects_third_party_provider_ids_from_codex_providers() {
let db = Database::memory().expect("memory db");
let third_party = Provider::with_id(
"rightcode".to_string(),
"RightCode".to_string(),
serde_json::json!({
"auth": {},
"config": "model_provider = \"aihubmix\"\n\n[model_providers.aihubmix]\nname = \"AIHubMix\"\nbase_url = \"https://example.com/v1\""
}),
None,
);
let mut official = Provider::with_id(
"codex-official".to_string(),
"OpenAI Official".to_string(),
serde_json::json!({"auth": {}, "config": "model_provider = \"openai\""}),
None,
);
official.category = Some("official".to_string());
db.save_provider("codex", &third_party)
.expect("save third-party");
db.save_provider("codex", &official).expect("save official");
let ids = collect_source_model_provider_ids(&db).expect("collect ids");
assert!(ids.contains("rightcode"));
assert!(ids.contains("aihubmix"));
assert!(!ids.contains("openai"));
assert!(!ids.contains("codex-official"));
}
}