From ac52c851bfc4854ed0ff8914595a41c41427e495 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 13 Jul 2026 16:27:51 +0800 Subject: [PATCH] fix(codex): infer image capability for generated catalogs and resync takeover live on save MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mapped GPT models were rejected by Codex clients with "model does not support image inputs". Two root causes: - Catalog entries for native-Responses/Anthropic providers cloned a template whose input_modalities defaulted to ["text"], so every mapped model was advertised text-only. model_catalog_json replaces Codex's built-in model table wholesale, and both the TUI and the IDE extension block images pre-send when the current model is found without "image". - Editing the current Codex provider during proxy takeover only refreshed the DB backup, so removing the mapping left a stale model_catalog_json pointer (and its text-only catalog file) active in live config. Changes: - New shared model_capabilities module: explicit row declaration first, then a confirmed text-only registry (exact tail match only — prefix matching removed, variants enumerated so future -vl/-vision models fail open), everything else unknown. - Catalog generation writes input_modalities from that inference for all tool profiles: unknown models fail open to ["text","image"]; only confirmed text-only models are advertised as ["text"], giving users a clear client-side prompt instead of silent image stripping. - Live catalog reverse-import collapses modalities that match current inference, so registry corrections are not frozen into hidden row overrides and the rectifier's heuristic opt-out keeps working. - Saving the current Codex provider while takeover owns live now re-projects the live config (mirrors the hot-switch path), so mapping edits and removals take effect immediately. - Media rectifier delegates to the shared module; its preflight toggle is documented (4 locales) as proxy-request-only, never affecting catalog capability declarations. --- CHANGELOG.md | 6 + src-tauri/src/codex_config.rs | 172 +++++++++-- src-tauri/src/lib.rs | 1 + src-tauri/src/model_capabilities.rs | 276 ++++++++++++++++++ src-tauri/src/proxy/media_sanitizer.rs | 167 ++--------- src-tauri/src/proxy/types.rs | 8 +- .../codex_native_responses_template.json | 3 +- src-tauri/src/services/provider/mod.rs | 160 +++++++++- src/config/codexProviderPresets.ts | 5 +- src/i18n/locales/en.json | 4 +- src/i18n/locales/ja.json | 4 +- src/i18n/locales/zh-TW.json | 4 +- src/i18n/locales/zh.json | 4 +- src/types.ts | 6 +- 14 files changed, 627 insertions(+), 193 deletions(-) create mode 100644 src-tauri/src/model_capabilities.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bc54d941..5be62964c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to CC Switch will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed + +- **Codex Image Capabilities Are Inferred Without a User Toggle**: Generated Codex model catalogs now advertise only models in CC Switch's confirmed, exact text-only registry as `input_modalities = ["text"]`; GPT, aliases, new suffix variants, and all unknown models fail open to `["text", "image"]`. The rectifier's “Text-Only Model Preflight” switch continues to control only proactive proxy request-body replacement and does not change Codex's catalog declaration. Live catalog reverse-import also collapses inferred modalities instead of persisting them as hidden row overrides, so a later registry correction or a model's multimodal upgrade takes effect automatically; only declarations that differ from inference survive a DB-missing/import round-trip. + ## [3.16.5] - 2026-07-01 Development since v3.16.4 reworks the Codex native-Responses path — restoring a generated model catalog for proxy-less direct-connect, decoupling model mapping from the local-routing toggle, and adding a host/model-prefix blacklist that disables Codex's built-in web_search on gateways that reject it — alongside a broad wave of new provider presets (Qiniu and Code0.ai across all seven apps, the FennoAI/ZetaAPI/TeamoRouter/NekoCode partners, and the non-partner Amux), Claude Sonnet 5 pricing plus a default-tier bump to it, a categorized two-level session view with group-level batch selection, live auto-sync of the shared Claude common config on switch, and a run of credential-safety, tool-detection, Doubao model-id, branding, and icon-size fixes. diff --git a/src-tauri/src/codex_config.rs b/src-tauri/src/codex_config.rs index 4a4ed8785..ef8646302 100644 --- a/src-tauri/src/codex_config.rs +++ b/src-tauri/src/codex_config.rs @@ -6,6 +6,7 @@ use crate::config::{ write_json_file, write_text_file, }; use crate::error::AppError; +use crate::model_capabilities::{image_input_capability_from_modalities, ImageInputCapability}; use serde_json::{json, Value}; use std::fs; use std::process::Command; @@ -437,6 +438,17 @@ fn extract_codex_top_level_u64(config_text: &str, field: &str) -> Option { .filter(|value| *value > 0) } +fn codex_catalog_input_modalities( + model: &str, + declared_modalities: Option<&[String]>, +) -> Vec { + let modalities = match image_input_capability_from_modalities(model, declared_modalities) { + ImageInputCapability::Unsupported => &["text"][..], + ImageInputCapability::Supported | ImageInputCapability::Unknown => &["text", "image"][..], + }; + modalities.iter().map(|item| (*item).to_string()).collect() +} + fn codex_catalog_model_entry( template: &Value, spec: &CodexCatalogModelSpec, @@ -459,6 +471,18 @@ fn codex_catalog_model_entry( entry_obj.insert("availability_nux".to_string(), Value::Null); entry_obj.insert("upgrade".to_string(), Value::Null); + // Image support is a model capability, not a tool-profile capability. + // Trust hidden preset metadata first, then the confirmed text-only registry; + // every unknown model fails open so GPT/relay aliases are never declared + // text-only merely because a template had a conservative default. + entry_obj.insert( + "input_modalities".to_string(), + json!(codex_catalog_input_modalities( + &spec.model, + spec.input_modalities.as_deref(), + )), + ); + if profile != CodexCatalogToolProfile::ProxyChat { // Native `/responses` and Anthropic gateways reject / drop Codex's freeform // `apply_patch` (type=="custom") tool. Strip any key that would make Codex @@ -491,9 +515,6 @@ fn codex_catalog_model_entry( if let Some(parallel) = spec.supports_parallel_tool_calls { entry_obj.insert("supports_parallel_tool_calls".to_string(), json!(parallel)); } - if let Some(modalities) = &spec.input_modalities { - entry_obj.insert("input_modalities".to_string(), json!(modalities)); - } } entry @@ -507,8 +528,9 @@ struct CodexCatalogModelSpec { /// Per-row override for the native template's `supports_parallel_tool_calls` /// (e.g. MiniMax=true, MiMo=false). Only consulted for `NativeResponses`. supports_parallel_tool_calls: Option, - /// Per-row override for the native template's `input_modalities` - /// (e.g. `["text","image"]`). Only consulted for `NativeResponses`. + /// Hidden per-row capability declaration from built-in provider metadata. + /// When omitted, all catalog profiles consult the shared text-only model + /// registry and otherwise default to `["text", "image"]`. input_modalities: Option>, /// Per-row override for the native template's `base_instructions` (the /// model identity / system preamble). Carries each vendor's OFFICIAL value @@ -1000,7 +1022,7 @@ pub fn prepare_codex_config_text_with_model_catalog( /// Reverse of `prepare_codex_config_text_with_model_catalog`: read the /// cc-switch–maintained catalog file referenced by `~/.codex/config.toml` and /// convert it back into the simplified shape the frontend table uses: -/// `{ "models": [{ "model", "displayName"?, "contextWindow"? }, ...] }`. +/// `{ "models": [{ "model", "displayName"?, "contextWindow"?, hidden overrides... }, ...] }`. /// /// We only reverse-parse catalogs whose `model_catalog_json` path is the /// cc-switch–generated file (identified by filename @@ -1008,14 +1030,14 @@ pub fn prepare_codex_config_text_with_model_catalog( /// left alone — surfacing its richer structure as the simplified table would /// be a downgrade we can't safely round-trip. /// -/// `displayName` and `contextWindow` are omitted from the returned entry when -/// the on-disk value matches the fallback that +/// `displayName`, `contextWindow`, and `inputModalities` are omitted from the +/// returned entry when the on-disk value matches the fallback that /// `codex_model_catalog_from_settings` injects for unset inputs (slug for -/// display_name, `model_context_window` or 128_000 for context_window). This -/// preserves the "user left it blank" intent across round-trip; an unavoidable -/// edge case is that a user-typed value that happens to equal the fallback -/// will also collapse to blank, but the next save writes the same fallback so -/// behavior stays consistent. +/// display_name, `model_context_window` or 128_000 for context_window, and the +/// shared confirmed-text-only inference for input modalities). This preserves +/// the "user left it blank" intent across round-trip; an unavoidable edge case +/// is that a user-typed value that happens to equal the fallback also collapses +/// to blank, but the next save writes the same fallback so behavior is stable. /// /// All failure modes (missing file, parse error, no `model_catalog_json`, /// entries without `slug`) collapse to `Ok(None)` so callers can treat this @@ -1070,9 +1092,8 @@ pub(crate) fn resolve_cc_switch_catalog_path( } /// Pure reverse-parsing core: convert Codex catalog JSON text back into the -/// frontend's simplified `{ models: [{ model, displayName?, contextWindow? }] }` -/// shape. Returns `None` when the catalog is unparseable, has no `models` -/// array, or yields zero valid entries. +/// frontend's simplified model-mapping shape. Returns `None` when the catalog +/// is unparseable, has no `models` array, or yields zero valid entries. fn build_simplified_catalog_from_texts(config_text: &str, catalog_text: &str) -> Option { let catalog: Value = serde_json::from_str(catalog_text).ok()?; let models = catalog.get("models").and_then(|m| m.as_array())?; @@ -1112,8 +1133,7 @@ fn build_simplified_catalog_from_texts(config_text: &str, catalog_text: &str) -> } // Preserve native-profile per-row overrides so a DB-SSOT-missing - // fallback round-trip doesn't silently drop them (they are ignored by - // the ProxyChat profile, so carrying them is harmless). + // fallback round-trip doesn't silently drop them. if let Some(parallel) = entry .get("supports_parallel_tool_calls") .and_then(|v| v.as_bool()) @@ -1126,7 +1146,8 @@ fn build_simplified_catalog_from_texts(config_text: &str, catalog_text: &str) -> .filter_map(|m| m.as_str()) .map(str::to_string) .collect(); - if !mods.is_empty() { + let inferred = codex_catalog_input_modalities(model, None); + if !mods.is_empty() && mods != inferred { obj.insert("inputModalities".to_string(), json!(mods)); } } @@ -2797,6 +2818,85 @@ base_url = "https://production.api/v1" ); } + #[test] + fn catalog_infers_image_input_independently_of_tool_profile() { + // Start from a deliberately text-only template to prove that every + // profile overwrites template defaults with shared capability logic. + let template = json!({ + "input_modalities": ["text"], + "apply_patch_tool_type": "freeform" + }); + let specs = vec![ + CodexCatalogModelSpec { + model: "gpt-5.4".to_string(), + display_name: "GPT 5.4".to_string(), + context_window: 128_000, + supports_parallel_tool_calls: None, + input_modalities: None, + base_instructions: None, + }, + CodexCatalogModelSpec { + model: "deepseek/deepseek-v4-pro".to_string(), + display_name: "DeepSeek V4 Pro".to_string(), + context_window: 128_000, + supports_parallel_tool_calls: None, + input_modalities: None, + base_instructions: None, + }, + CodexCatalogModelSpec { + model: "glm-5.2v".to_string(), + display_name: "GLM 5.2V".to_string(), + context_window: 128_000, + supports_parallel_tool_calls: None, + input_modalities: None, + base_instructions: None, + }, + CodexCatalogModelSpec { + model: "deepseek-v4-flash".to_string(), + display_name: "Explicit Visual Override".to_string(), + context_window: 128_000, + supports_parallel_tool_calls: None, + input_modalities: Some(vec!["text".to_string(), "image".to_string()]), + base_instructions: None, + }, + CodexCatalogModelSpec { + model: "custom-text-alias".to_string(), + display_name: "Explicit Text Override".to_string(), + context_window: 128_000, + supports_parallel_tool_calls: None, + input_modalities: Some(vec!["text".to_string()]), + base_instructions: None, + }, + ]; + + for profile in [ + CodexCatalogToolProfile::ProxyChat, + CodexCatalogToolProfile::NativeResponses, + CodexCatalogToolProfile::Anthropic, + ] { + let catalog = codex_model_catalog_from_specs(&specs, &template, profile); + let models = catalog["models"].as_array().expect("models array"); + let modalities = |slug: &str| { + models + .iter() + .find(|entry| entry["slug"] == slug) + .and_then(|entry| entry.get("input_modalities")) + .cloned() + .unwrap_or(Value::Null) + }; + + assert_eq!(modalities("gpt-5.4"), json!(["text", "image"])); + assert_eq!(modalities("deepseek/deepseek-v4-pro"), json!(["text"])); + assert_eq!(modalities("glm-5.2v"), json!(["text", "image"])); + assert_eq!( + modalities("deepseek-v4-flash"), + json!(["text", "image"]), + "explicit provider metadata must override the text-only registry" + ); + assert_eq!(modalities("custom-text-alias"), json!(["text"])); + } + } + #[test] fn native_responses_catalog_always_carries_base_instructions() { // Regression guard for the "missing field `base_instructions`" parse @@ -3146,6 +3246,40 @@ web_search = "disabled" ); } + #[test] + fn build_simplified_catalog_squashes_inferred_modalities_and_keeps_overrides() { + let catalog = r#"{ + "models": [ + { "slug": "gpt-5.4", "input_modalities": ["text", "image"] }, + { "slug": "deepseek-v4-pro", "input_modalities": ["text"] }, + { "slug": "gpt-text-override", "input_modalities": ["text"] }, + { "slug": "deepseek-v4-flash", "input_modalities": ["text", "image"] } + ] + }"#; + + let result = build_simplified_catalog_from_texts("", catalog).expect("entries"); + let models = result.get("models").unwrap().as_array().unwrap(); + + assert!( + models[0].get("inputModalities").is_none(), + "GPT text+image is inferred and must not become a sticky hidden override" + ); + assert!( + models[1].get("inputModalities").is_none(), + "confirmed text-only capability is inferred and must remain registry-driven" + ); + assert_eq!( + models[2].get("inputModalities"), + Some(&json!(["text"])), + "an unknown model explicitly forced to text-only must round-trip" + ); + assert_eq!( + models[3].get("inputModalities"), + Some(&json!(["text", "image"])), + "an explicit image override for a registered text-only model must round-trip" + ); + } + #[test] fn build_simplified_catalog_returns_none_when_unparseable() { assert!(build_simplified_catalog_from_texts("", "not json").is_none()); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 86a78b234..b2b214c1a 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -20,6 +20,7 @@ mod lightweight; #[cfg(target_os = "linux")] mod linux_fix; mod mcp; +mod model_capabilities; mod openclaw_config; mod opencode_config; mod panic_hook; diff --git a/src-tauri/src/model_capabilities.rs b/src-tauri/src/model_capabilities.rs new file mode 100644 index 000000000..2070204ae --- /dev/null +++ b/src-tauri/src/model_capabilities.rs @@ -0,0 +1,276 @@ +use serde_json::Value; + +/// Image-input capability shared by Codex catalog generation and proxy request +/// rectification. +/// +/// `Unknown` is intentionally distinct from `Supported`: callers may choose +/// different execution policies without duplicating the model-name registry. +/// The Codex catalog treats unknown models as image-capable (fail open), while +/// the media rectifier leaves their request bodies untouched. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ImageInputCapability { + Supported, + Unsupported, + Unknown, +} + +/// Resolve image-input capability from an explicit declaration first, then the +/// confirmed text-only model registry when the caller enables registry lookup. +pub(crate) fn resolve_image_input_capability( + model: &str, + declared_support: Option, + use_confirmed_registry: bool, +) -> ImageInputCapability { + match declared_support { + Some(true) => ImageInputCapability::Supported, + Some(false) => ImageInputCapability::Unsupported, + None if use_confirmed_registry && is_confirmed_text_only_model(model) => { + ImageInputCapability::Unsupported + } + None => ImageInputCapability::Unknown, + } +} + +/// Resolve a model's image-input capability from the provider settings shapes +/// accepted by the proxy (`modelCatalog.models`, `modelCatalog`, or `models`). +pub(crate) fn image_input_capability_from_settings( + settings: &Value, + model: &str, + use_confirmed_registry: bool, +) -> ImageInputCapability { + resolve_image_input_capability( + model, + declared_model_image_support(settings, model), + use_confirmed_registry, + ) +} + +/// Convert a catalog row's explicit modality list into the shared capability +/// representation, falling back to the text-only registry when omitted. +pub(crate) fn image_input_capability_from_modalities( + model: &str, + modalities: Option<&[String]>, +) -> ImageInputCapability { + let declared_support = modalities.map(|items| { + items + .iter() + .any(|item| item.trim().eq_ignore_ascii_case("image")) + }); + resolve_image_input_capability(model, declared_support, true) +} + +/// Models that CC Switch is willing to advertise to clients as text-only. +/// +/// This registry is deliberately exact and fail-open. A new suffix is not +/// inherited automatically: it remains image-capable until its capability is +/// confirmed, preventing a future `-vision`/`-vl` variant from being blocked by +/// the Codex client before a request can reach the proxy. +pub(crate) fn is_confirmed_text_only_model(model: &str) -> bool { + let normalized = normalize_model_id(model); + let tail = normalized.rsplit('/').next().unwrap_or(normalized.as_str()); + + const CONFIRMED_TAILS: &[&str] = &[ + "ark-code-latest", + "deepseek-chat", + "deepseek-reasoner", + "deepseek-v4-flash", + "deepseek-v4-pro", + "glm-5.1", + // Exact rather than prefix matching: GLM visual models use a `v` + // suffix (for example glm-5.2v), which must remain image-capable. + "glm-5.2", + "kat-coder", + "kat-coder-pro", + "kat-coder-pro v1", + "kat-coder-pro v2", + "kat-coder-pro-v1", + "kat-coder-pro-v2", + "ling-2.5-1t", + "longcat-2.0", + "longcat-flash-chat", + "minimax-m2.7", + "minimax-m2.7-highspeed", + "mimo-v2.5-pro", + "qwen3-coder-480b", + "qwen3-coder-480b-a35b-instruct", + "qwen3-coder-flash", + "qwen3-coder-next", + "qwen3-coder-plus", + "step-3.5-flash", + "step-3.5-flash-2603", + "us.deepseek.r1-v1", + ]; + + CONFIRMED_TAILS.contains(&tail) +} + +fn declared_model_image_support(settings: &Value, model: &str) -> Option { + [ + settings + .get("modelCatalog") + .and_then(|catalog| catalog.get("models")), + settings.get("modelCatalog"), + settings.get("models"), + ] + .into_iter() + .flatten() + .find_map(|value| declared_model_image_support_in_value(value, model)) +} + +fn declared_model_image_support_in_value(value: &Value, model: &str) -> Option { + if let Some(models) = value.as_array() { + return models.iter().find_map(|entry| { + model_entry_matches(entry, None, model).then(|| explicit_image_support(entry))? + }); + } + + let object = value.as_object()?; + object.iter().find_map(|(key, entry)| { + model_entry_matches(entry, Some(key), model).then(|| explicit_image_support(entry))? + }) +} + +fn explicit_image_support(entry: &Value) -> Option { + if let Some(value) = entry + .get("supportsImage") + .or_else(|| entry.get("supports_image")) + .or_else(|| entry.get("vision")) + .and_then(Value::as_bool) + { + return Some(value); + } + + [ + entry.get("input"), + entry.pointer("/modalities/input"), + entry.get("input_modalities"), + entry.get("inputModalities"), + ] + .into_iter() + .flatten() + .find_map(input_modalities_support_image) +} + +fn input_modalities_support_image(value: &Value) -> Option { + let modalities = value.as_array()?; + Some(modalities.iter().any(|item| { + item.as_str() + .map(str::trim) + .is_some_and(|item| item.eq_ignore_ascii_case("image")) + })) +} + +fn model_entry_matches(entry: &Value, key: Option<&str>, model: &str) -> bool { + key.is_some_and(|key| model_ids_match(key, model)) + || ["model", "id", "name"] + .into_iter() + .filter_map(|field| entry.get(field).and_then(Value::as_str)) + .any(|candidate| model_ids_match(candidate, model)) +} + +fn model_ids_match(candidate: &str, model: &str) -> bool { + let candidate = normalize_model_id(candidate); + let model = normalize_model_id(model); + if candidate.is_empty() || model.is_empty() { + return false; + } + if candidate == model { + return true; + } + + let candidate_tail = candidate.rsplit('/').next().unwrap_or(candidate.as_str()); + let model_tail = model.rsplit('/').next().unwrap_or(model.as_str()); + candidate_tail == model_tail || candidate == model_tail || candidate_tail == model +} + +fn normalize_model_id(value: &str) -> String { + let mut normalized = value + .trim() + .trim_start_matches("models/") + .trim() + .to_ascii_lowercase(); + if let Some(stripped) = + normalized.strip_suffix(crate::claude_desktop_config::ONE_M_CONTEXT_MARKER) + { + normalized = stripped.trim().to_string(); + } + normalized +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn gpt_and_unknown_models_remain_unknown_without_declarations() { + for model in ["gpt-5.4", "gpt-5.5", "gpt-5.6-sol", "custom-alias"] { + assert_eq!( + resolve_image_input_capability(model, None, true), + ImageInputCapability::Unknown, + "{model} must fail open" + ); + } + } + + #[test] + fn confirmed_text_only_registry_normalizes_namespaces_and_context_markers() { + assert!(is_confirmed_text_only_model("deepseek/deepseek-v4-pro")); + assert!(is_confirmed_text_only_model("GLM-5.2[1M]")); + assert!(is_confirmed_text_only_model("qwen/qwen3-coder-plus")); + assert!(is_confirmed_text_only_model( + "Qwen/Qwen3-Coder-480B-A35B-Instruct" + )); + assert!(is_confirmed_text_only_model("MiniMax-M2.7-Highspeed")); + assert!(is_confirmed_text_only_model("step-3.5-flash-2603")); + assert!(!is_confirmed_text_only_model("glm-5.2v")); + } + + #[test] + fn unconfirmed_family_suffixes_fail_open() { + for model in [ + "minimax-m2.7-vision", + "qwen3-coder-ultra", + "qwen3-coder-vl", + "step-3.5-flash-vision", + ] { + assert!( + !is_confirmed_text_only_model(model), + "unconfirmed variant {model} must not be hard-gated" + ); + } + } + + #[test] + fn explicit_capability_overrides_the_registry() { + assert_eq!( + resolve_image_input_capability("deepseek-v4-pro", Some(true), true), + ImageInputCapability::Supported + ); + assert_eq!( + resolve_image_input_capability("gpt-5.4", Some(false), true), + ImageInputCapability::Unsupported + ); + } + + #[test] + fn provider_settings_support_multiple_capability_shapes() { + let settings = json!({ + "modelCatalog": { + "models": [ + { "model": "vision", "modalities": { "input": ["text", "image"] } }, + { "model": "text", "inputModalities": ["text"] } + ] + } + }); + + assert_eq!( + image_input_capability_from_settings(&settings, "vision", true), + ImageInputCapability::Supported + ); + assert_eq!( + image_input_capability_from_settings(&settings, "text", true), + ImageInputCapability::Unsupported + ); + } +} diff --git a/src-tauri/src/proxy/media_sanitizer.rs b/src-tauri/src/proxy/media_sanitizer.rs index 880b18fa4..ec1b4ea0c 100644 --- a/src-tauri/src/proxy/media_sanitizer.rs +++ b/src-tauri/src/proxy/media_sanitizer.rs @@ -1,3 +1,6 @@ +#[cfg(test)] +use crate::model_capabilities::is_confirmed_text_only_model as confirmed_text_only_model; +use crate::model_capabilities::{image_input_capability_from_settings, ImageInputCapability}; use crate::provider::Provider; use crate::proxy::error::ProxyError; use serde_json::{json, Value}; @@ -9,9 +12,9 @@ pub const UNSUPPORTED_IMAGE_MARKER: &str = "[Unsupported Image]"; /// Two paths, both reached only when the caller's media-fallback switch is on: /// - explicit capability from the provider config (modelCatalog / modalities) is /// always trusted — it is declaration-driven, never a guess; -/// - the curated `known_text_only_model` list is a heuristic *prediction* and only -/// runs when `allow_heuristic` is true, so a mislabeled multimodal model cannot -/// have its images silently stripped when the user opts out. +/// - the confirmed text-only registry is used for proactive replacement only +/// when `allow_heuristic` is true. This switch controls silent request-body +/// mutation, not the capability truth advertised by the Codex model catalog. pub fn replace_images_for_text_only_model( body: &mut Value, provider: &Provider, @@ -27,13 +30,9 @@ pub fn replace_images_for_text_only_model( .map(str::trim) .unwrap_or(""); - match explicit_model_image_support(provider, model) { - Some(true) => return 0, - Some(false) => return replace_images_in_body(body), - None => {} - } - - if !allow_heuristic || !known_text_only_model(model) { + if image_input_capability_from_settings(&provider.settings_config, model, allow_heuristic) + != ImageInputCapability::Unsupported + { return 0; } @@ -237,95 +236,6 @@ fn replace_image_block_with_text_marker(block: &mut Value, text_type: &str) { } } -fn explicit_model_image_support(provider: &Provider, model: &str) -> Option { - let settings = &provider.settings_config; - [ - settings - .get("modelCatalog") - .and_then(|catalog| catalog.get("models")), - settings.get("modelCatalog"), - settings.get("models"), - ] - .into_iter() - .flatten() - .find_map(|value| explicit_model_image_support_in_value(value, model)) -} - -fn known_text_only_model(model: &str) -> bool { - let normalized = normalize_model_id(model); - let tail = normalized.rsplit('/').next().unwrap_or(normalized.as_str()); - - const EXACT_TAILS: &[&str] = &[ - "ark-code-latest", - "deepseek-chat", - "deepseek-reasoner", - "deepseek-v4-flash", - "deepseek-v4-pro", - "glm-5.1", - // 精确匹配而非 TAIL_PREFIXES：智谱视觉版沿用 4v/5v 命名（glm-5.2v）， - // 前缀匹配会误剥未来多模态变体的图片。 - "glm-5.2", - "kat-coder", - "kat-coder-pro", - "kat-coder-pro v1", - "kat-coder-pro v2", - "kat-coder-pro-v1", - "kat-coder-pro-v2", - "ling-2.5-1t", - "longcat-2.0", - "longcat-flash-chat", - "mimo-v2.5-pro", - "us.deepseek.r1-v1", - ]; - - const TAIL_PREFIXES: &[&str] = &["minimax-m2.7", "qwen3-coder", "step-3.5-flash"]; - - EXACT_TAILS.contains(&tail) || TAIL_PREFIXES.iter().any(|prefix| tail.starts_with(prefix)) -} - -fn explicit_model_image_support_in_value(value: &Value, model: &str) -> Option { - if let Some(models) = value.as_array() { - return models.iter().find_map(|entry| { - model_entry_matches(entry, None, model).then(|| explicit_image_support(entry))? - }); - } - - let object = value.as_object()?; - object.iter().find_map(|(key, entry)| { - model_entry_matches(entry, Some(key), model).then(|| explicit_image_support(entry))? - }) -} - -fn explicit_image_support(entry: &Value) -> Option { - if let Some(value) = entry - .get("supportsImage") - .or_else(|| entry.get("supports_image")) - .or_else(|| entry.get("vision")) - .and_then(Value::as_bool) - { - return Some(value); - } - - [ - entry.get("input"), - entry.pointer("/modalities/input"), - entry.get("input_modalities"), - entry.get("inputModalities"), - ] - .into_iter() - .flatten() - .find_map(input_modalities_support_image) -} - -fn input_modalities_support_image(value: &Value) -> Option { - let modalities = value.as_array()?; - Some(modalities.iter().any(|item| { - item.as_str() - .map(str::trim) - .is_some_and(|item| item.eq_ignore_ascii_case("image")) - })) -} - fn extract_error_text(body: &str) -> String { if let Ok(value) = serde_json::from_str::(body) { let candidates = [ @@ -350,43 +260,6 @@ fn extract_error_text(body: &str) -> String { body.to_string() } -fn model_entry_matches(entry: &Value, key: Option<&str>, model: &str) -> bool { - key.is_some_and(|key| model_ids_match(key, model)) - || ["model", "id", "name"] - .into_iter() - .filter_map(|field| entry.get(field).and_then(Value::as_str)) - .any(|candidate| model_ids_match(candidate, model)) -} - -fn model_ids_match(candidate: &str, model: &str) -> bool { - let candidate = normalize_model_id(candidate); - let model = normalize_model_id(model); - if candidate.is_empty() || model.is_empty() { - return false; - } - if candidate == model { - return true; - } - - let candidate_tail = candidate.rsplit('/').next().unwrap_or(candidate.as_str()); - let model_tail = model.rsplit('/').next().unwrap_or(model.as_str()); - candidate_tail == model_tail || candidate == model_tail || candidate_tail == model -} - -fn normalize_model_id(value: &str) -> String { - let mut normalized = value - .trim() - .trim_start_matches("models/") - .trim() - .to_ascii_lowercase(); - if let Some(stripped) = - normalized.strip_suffix(crate::claude_desktop_config::ONE_M_CONTEXT_MARKER) - { - normalized = stripped.trim().to_string(); - } - normalized -} - #[cfg(test)] mod tests { use super::*; @@ -431,7 +304,7 @@ mod tests { } #[test] - fn known_text_only_models_replace_images_before_send() { + fn confirmed_text_only_models_replace_images_before_send() { let provider = provider(json!({})); let mut body = json!({ "model": "deepseek/deepseek-v4-pro", @@ -453,7 +326,7 @@ mod tests { } #[test] - fn known_text_only_models_replace_chat_image_url_before_send() { + fn confirmed_text_only_models_replace_chat_image_url_before_send() { let provider = provider(json!({})); let mut body = json!({ "model": "deepseek-v4-flash", @@ -477,7 +350,7 @@ mod tests { } #[test] - fn known_text_only_models_replace_codex_input_image_before_send() { + fn confirmed_text_only_models_replace_codex_input_image_before_send() { let provider = provider(json!({})); let mut body = json!({ "model": "deepseek-v4-flash", @@ -504,9 +377,9 @@ mod tests { fn longcat_models_are_classified_text_only() { // LongCat-2.0 (like the retired Flash Chat) is a text-only model; the // preset ships it in mixed case, so the classifier must normalize first. - assert!(known_text_only_model("LongCat-2.0")); - assert!(known_text_only_model("longcat/LongCat-2.0")); - assert!(known_text_only_model("LongCat-Flash-Chat")); + assert!(confirmed_text_only_model("LongCat-2.0")); + assert!(confirmed_text_only_model("longcat/LongCat-2.0")); + assert!(confirmed_text_only_model("LongCat-Flash-Chat")); } #[test] @@ -662,7 +535,7 @@ mod tests { } #[test] - fn known_text_only_prefixes_replace_images_before_send() { + fn confirmed_text_only_variant_replaces_images_before_send() { let provider = provider(json!({})); let mut body = json!({ "model": "therouter/qwen/qwen3-coder-480b", @@ -761,11 +634,11 @@ mod tests { fn glm_52_is_classified_text_only() { // issue #5025：火山 Coding Plan 的 GLM 5.2 是纯文本端点， // 映射链 glm-5.2[1M] 归一化后尾部为 glm-5.2。 - assert!(known_text_only_model("glm-5.2")); - assert!(known_text_only_model("GLM-5.2[1M]")); - assert!(known_text_only_model("zai-org/GLM-5.2")); + assert!(confirmed_text_only_model("glm-5.2")); + assert!(confirmed_text_only_model("GLM-5.2[1M]")); + assert!(confirmed_text_only_model("zai-org/GLM-5.2")); // 未来视觉版（智谱 4v/5v 命名惯例）不能被误判为纯文本。 - assert!(!known_text_only_model("glm-5.2v")); + assert!(!confirmed_text_only_model("glm-5.2v")); } #[test] diff --git a/src-tauri/src/proxy/types.rs b/src-tauri/src/proxy/types.rs index f2dfc5800..6dc5bc659 100644 --- a/src-tauri/src/proxy/types.rs +++ b/src-tauri/src/proxy/types.rs @@ -219,11 +219,11 @@ pub struct RectifierConfig { /// 让对话不中断。总开关，管辖「显式声明 text-only」与「上游报错后兜底」两条事实驱动路径。 #[serde(default = "default_true")] pub request_media_fallback: bool, - /// 请求整流：启发式 text-only 模型名匹配（默认开启） + /// 请求整流：确认纯文本注册表的发送前降级（默认开启） /// - /// 在模型未声明能力时，按内置模型名列表预测性地剥离图片（发送前）。 - /// 受 request_media_fallback 管辖；单独关闭后仅保留「显式声明」与「上游兜底」， - /// 避免内置列表把多模态模型误判成 text-only 而静默剥图。 + /// 在模型未声明能力时，按内置的确认纯文本注册表预先剥离图片。 + /// 受 request_media_fallback 管辖；单独关闭只停用代理的注册表预判， + /// 仍保留「显式声明」与「上游兜底」，且不改变 Codex 模型目录声明。 #[serde(default = "default_true")] pub request_media_heuristic: bool, } diff --git a/src-tauri/src/resources/codex_native_responses_template.json b/src-tauri/src/resources/codex_native_responses_template.json index a513fe1db..644a5123b 100644 --- a/src-tauri/src/resources/codex_native_responses_template.json +++ b/src-tauri/src/resources/codex_native_responses_template.json @@ -32,7 +32,8 @@ "effective_context_window_percent": 95, "experimental_supported_tools": [], "input_modalities": [ - "text" + "text", + "image" ], "supports_search_tool": false } diff --git a/src-tauri/src/services/provider/mod.rs b/src-tauri/src/services/provider/mod.rs index 9dcf52948..98398734e 100644 --- a/src-tauri/src/services/provider/mod.rs +++ b/src-tauri/src/services/provider/mod.rs @@ -1108,6 +1108,135 @@ command = "legacy-cmd" ); } + #[tokio::test] + #[serial] + async fn update_current_codex_provider_refreshes_and_clears_catalog_during_takeover() { + let _home = TempHome::new(); + crate::settings::reload_settings().expect("reload settings"); + + let db = Arc::new(Database::memory().expect("init db")); + let state = AppState::new(db.clone()); + + let mut original = Provider::with_id( + "p1".into(), + "Codex A".into(), + json!({ + "auth": { "OPENAI_API_KEY": "token-a" }, + "config": r#"model_provider = "custom" +model = "old-model" + +[model_providers.custom] +name = "Codex A" +base_url = "https://api.a.example/v1" +wire_api = "responses" +requires_openai_auth = true +"#, + "modelCatalog": { + "models": [{ "model": "old-model" }] + } + }), + None, + ); + original.meta = Some(ProviderMeta { + api_format: Some("openai_responses".into()), + ..Default::default() + }); + db.save_provider("codex", &original).expect("save provider"); + db.set_current_provider("codex", "p1") + .expect("set current provider"); + crate::settings::set_current_provider(&AppType::Codex, Some("p1")) + .expect("set local current provider"); + + db.update_proxy_config(ProxyConfig { + live_takeover_active: true, + listen_port: 0, + ..Default::default() + }) + .await + .expect("update proxy config"); + { + let mut config = db + .get_proxy_config_for_app("codex") + .await + .expect("get app proxy config"); + config.enabled = true; + db.update_proxy_config_for_app(config) + .await + .expect("enable Codex proxy config"); + } + db.save_live_backup( + "codex", + &serde_json::to_string(&original.settings_config).expect("serialize backup"), + ) + .await + .expect("seed live backup"); + + state + .proxy_service + .start() + .await + .expect("start proxy service"); + state + .proxy_service + .sync_codex_live_from_provider_while_proxy_active(&original) + .await + .expect("seed taken-over Codex live config"); + assert!( + state + .proxy_service + .detect_takeover_in_live_config_for_app(&AppType::Codex), + "seeded Codex live config should be recognized as takeover-owned" + ); + + let mut updated = original.clone(); + updated.settings_config["config"] = json!( + r#"model_provider = "custom" +model = "gpt-5.4" + +[model_providers.custom] +name = "Codex A" +base_url = "https://api.updated.example/v1" +wire_api = "responses" +requires_openai_auth = true +"# + ); + updated.settings_config["modelCatalog"] = json!({ + "models": [{ "model": "gpt-5.4", "displayName": "GPT 5.4" }] + }); + + ProviderService::update(&state, AppType::Codex, None, updated.clone()) + .expect("update current Codex provider mapping"); + + let catalog_path = crate::codex_config::get_codex_model_catalog_path(); + let catalog: Value = read_json_file(&catalog_path).expect("read generated catalog"); + assert_eq!(catalog["models"][0]["slug"], "gpt-5.4"); + assert_eq!( + catalog["models"][0]["input_modalities"], + json!(["text", "image"]), + "unknown/GPT models must fail open to image input" + ); + let live_config = fs::read_to_string(crate::codex_config::get_codex_config_path()) + .expect("read Codex config.toml"); + assert!(live_config.contains("model_catalog_json")); + + updated.settings_config["modelCatalog"] = json!({ "models": [] }); + ProviderService::update(&state, AppType::Codex, None, updated) + .expect("remove current Codex provider mapping"); + + let live_config = fs::read_to_string(crate::codex_config::get_codex_config_path()) + .expect("read Codex config.toml after mapping removal"); + assert!( + !live_config.contains("model_catalog_json"), + "removing mappings during takeover must clear the stale catalog pointer" + ); + + state + .proxy_service + .stop() + .await + .expect("stop proxy service"); + } + #[cfg(any(target_os = "macos", windows))] #[tokio::test] #[serial] @@ -2191,15 +2320,28 @@ impl ProviderService { .map_err(|e| AppError::Message(format!("更新 Live 备份失败: {e}")))?; } - if matches!(app_type, AppType::Claude) - && futures::executor::block_on(state.proxy_service.is_running()) - { - futures::executor::block_on( - state - .proxy_service - .sync_claude_live_from_provider_while_proxy_active(&provider), - ) - .map_err(|e| AppError::Message(format!("同步 Claude Live 配置失败: {e}")))?; + if futures::executor::block_on(state.proxy_service.is_running()) { + if matches!(app_type, AppType::Claude) { + futures::executor::block_on( + state + .proxy_service + .sync_claude_live_from_provider_while_proxy_active(&provider), + ) + .map_err(|e| { + AppError::Message(format!("同步 Claude Live 配置失败: {e}")) + })?; + } else if live_taken_over && matches!(app_type, AppType::Codex) { + // Codex model mappings are projected into a generated + // model_catalog_json file. Refresh takeover-owned Live + // immediately so adding/removing mappings cannot leave + // the previous catalog pointer and capabilities active. + futures::executor::block_on( + state + .proxy_service + .sync_codex_live_from_provider_while_proxy_active(&provider), + ) + .map_err(|e| AppError::Message(format!("同步 Codex Live 配置失败: {e}")))?; + } } } else { write_live_with_common_config(state.db.as_ref(), &app_type, &provider)?; diff --git a/src/config/codexProviderPresets.ts b/src/config/codexProviderPresets.ts index c498a821b..461200764 100644 --- a/src/config/codexProviderPresets.ts +++ b/src/config/codexProviderPresets.ts @@ -80,8 +80,9 @@ function modelCatalog( displayName?: string; contextWindow?: number; // Native Responses (direct) overrides for the generated - // model-catalogs.json; omit to inherit the native template defaults - // (supports_parallel_tool_calls=false, input_modalities=["text"]). + // model-catalogs.json. Omitted input modalities are inferred by the + // backend: confirmed text-only models stay text-only; everything else + // defaults to text+image. supportsParallelToolCalls?: boolean; inputModalities?: string[]; // Vendor's OFFICIAL base_instructions; omit to inherit the neutral diff --git a/src/i18n/locales/en.json b/src/i18n/locales/en.json index 8ddcb7d08..e5653879b 100644 --- a/src/i18n/locales/en.json +++ b/src/i18n/locales/en.json @@ -382,8 +382,8 @@ "thinkingBudgetDescription": "When an Anthropic-type provider returns budget_tokens constraint errors (such as at least 1024), automatically normalizes thinking to enabled, sets thinking budget to 32000, and raises max_tokens to 64000 if needed, then retries once", "mediaFallback": "Unsupported Image Fallback", "mediaFallbackDescription": "When a provider does not support image input (it declares text-only capability, or the upstream returns an unsupported-image error), automatically replaces image blocks with an [Unsupported Image] marker so the conversation is not interrupted", - "mediaHeuristic": "Heuristic Text-Only Model Detection", - "mediaHeuristicDescription": "Predictively strips images before sending based on a built-in model-name list. When off, only the declared-text-only and upstream-error fallback paths remain, preventing the built-in list from misclassifying a multimodal model as text-only and silently dropping images" + "mediaHeuristic": "Text-Only Model Preflight", + "mediaHeuristicDescription": "Before sending, strips images for models in the built-in confirmed text-only registry. Turning this off only disables the proxy's registry-based preflight; declared text-only handling and upstream-error fallback remain, and Codex model-catalog capability declarations are unchanged" }, "optimizer": { "title": "Bedrock Request Optimizer", diff --git a/src/i18n/locales/ja.json b/src/i18n/locales/ja.json index 5e5494663..df6cb7787 100644 --- a/src/i18n/locales/ja.json +++ b/src/i18n/locales/ja.json @@ -382,8 +382,8 @@ "thinkingBudgetDescription": "Anthropic タイプのプロバイダーが budget_tokens 制約エラー（例: 1024 以上）を返した場合、thinking を enabled に正規化し、thinking 予算を 32000 に設定し、必要に応じて max_tokens を 64000 に引き上げて 1 回リトライします", "mediaFallback": "非対応画像フォールバック", "mediaFallbackDescription": "プロバイダーが画像入力に対応していない場合（テキストのみの能力が宣言されている、または上流が画像非対応エラーを返した場合）、画像ブロックを [Unsupported Image] マーカーに自動置換し、会話を中断させません", - "mediaHeuristic": "ヒューリスティックなテキスト専用モデル判定", - "mediaHeuristicDescription": "送信前に内蔵モデル名リストに基づいて画像を予測的に削除します。オフにすると「宣言済みテキスト専用」と「上流エラー時のフォールバック」のみが残り、内蔵リストがマルチモーダルモデルをテキスト専用と誤判定して画像を静かに削除するのを防ぎます" + "mediaHeuristic": "テキスト専用モデルの事前判定", + "mediaHeuristicDescription": "送信前に、内蔵の確認済みテキスト専用モデル登録に基づいて画像を除去します。オフにしても無効になるのはプロキシの登録ベース事前処理だけで、宣言済みテキスト専用モデルの処理と上流エラー時のフォールバックは残り、Codex モデルカタログの能力宣言は変わりません" }, "optimizer": { "title": "Bedrock リクエストオプティマイザー", diff --git a/src/i18n/locales/zh-TW.json b/src/i18n/locales/zh-TW.json index 513f40e9b..9b4adeed5 100644 --- a/src/i18n/locales/zh-TW.json +++ b/src/i18n/locales/zh-TW.json @@ -382,8 +382,8 @@ "thinkingBudgetDescription": "當 Anthropic 類型供應商回傳 budget_tokens 限制錯誤（如至少 1024）時，自動將 thinking 規範為 enabled，並將 budget 設為 32000，同時在需要時將 max_tokens 設為 64000，然後重試一次", "mediaFallback": "不支援圖片降級", "mediaFallbackDescription": "當供應商不支援圖片輸入時（已宣告純文字能力，或上游回傳不支援圖片的錯誤），自動將圖片區塊替換為 [Unsupported Image] 標記，使對話不中斷", - "mediaHeuristic": "啟發式純文字模型辨識", - "mediaHeuristicDescription": "傳送前依內建模型名稱清單預測性地移除圖片。關閉後僅保留「已宣告純文字」與「上游錯誤兜底」兩條事實驅動路徑，避免內建清單將多模態模型誤判為純文字而靜默丟失圖片" + "mediaHeuristic": "純文字模型預判", + "mediaHeuristicDescription": "傳送前依內建的已確認純文字模型登錄表移除圖片。關閉只會停用代理的登錄表預判；已宣告純文字模型的處理與上游錯誤兜底仍會保留，且不會改變 Codex 模型目錄的能力宣告" }, "optimizer": { "title": "Bedrock 請求最佳化工具", diff --git a/src/i18n/locales/zh.json b/src/i18n/locales/zh.json index 0398e58dc..d7cb2e15d 100644 --- a/src/i18n/locales/zh.json +++ b/src/i18n/locales/zh.json @@ -382,8 +382,8 @@ "thinkingBudgetDescription": "当 Anthropic 类型供应商返回 budget_tokens 约束错误（如至少 1024）时，自动将 thinking 规范为 enabled 并将 budget 设为 32000，同时在需要时将 max_tokens 设为 64000，然后重试一次", "mediaFallback": "不支持图片降级", "mediaFallbackDescription": "当供应商不支持图片输入时（已声明纯文本能力，或上游返回不支持图片的错误），自动将图片块替换为 [Unsupported Image] 标记，使对话不中断", - "mediaHeuristic": "启发式纯文本模型识别", - "mediaHeuristicDescription": "发送前按内置模型名列表预测性剥离图片。关闭后仅保留「已声明纯文本」与「上游报错兜底」两条事实驱动路径，避免内置列表把多模态模型误判为纯文本而静默丢图" + "mediaHeuristic": "纯文本模型预判", + "mediaHeuristicDescription": "发送前根据内置的已确认纯文本模型注册表剥离图片。关闭仅停用代理的注册表预判；已声明纯文本模型的处理和上游报错兜底仍保留，且不会改变 Codex 模型目录的能力声明" }, "optimizer": { "title": "Bedrock 请求优化器", diff --git a/src/types.ts b/src/types.ts index 0f9eccb50..afb99def3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -260,9 +260,9 @@ export interface CodexCatalogModel { model: string; displayName?: string; contextWindow?: string | number; - // Native Responses (direct) profile overrides for the generated - // model-catalogs.json. Ignored by the chat/proxy profile. - // e.g. MiniMax: supportsParallelToolCalls=true, inputModalities=["text","image"]. + // Hidden provider capability metadata for the generated model catalog. + // supportsParallelToolCalls is native-profile-only; inputModalities wins over + // automatic text-only model detection for every profile. supportsParallelToolCalls?: boolean; inputModalities?: string[]; // Vendor's OFFICIAL base_instructions (model identity / system preamble).