mirror of
https://github.com/farion1231/cc-switch.git
synced 2026-07-24 12:44:18 +08:00
Handle inline thinking in Codex Chat conversion
- Split leading <think> blocks from Chat content into Responses reasoning output. - Keep assistant text output free of inline thinking tags. - Support streamed inline thinking blocks before normal text deltas. - Add regression coverage for MiniMax-style inline thinking responses.
This commit is contained in:
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
use super::transform_codex_chat::{
|
use super::transform_codex_chat::{
|
||||||
chat_usage_to_responses_usage, response_id_from_chat_id, response_status_from_finish_reason,
|
chat_usage_to_responses_usage, response_id_from_chat_id, response_status_from_finish_reason,
|
||||||
|
split_leading_think_block, strip_leading_think_open_tag,
|
||||||
};
|
};
|
||||||
use crate::proxy::sse::{strip_sse_field, take_sse_block};
|
use crate::proxy::sse::{strip_sse_field, take_sse_block};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
@@ -27,6 +28,20 @@ struct ReasoningItemState {
|
|||||||
done: bool,
|
done: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||||
|
enum InlineThinkMode {
|
||||||
|
#[default]
|
||||||
|
Detecting,
|
||||||
|
Reasoning,
|
||||||
|
Text,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct InlineThinkState {
|
||||||
|
mode: InlineThinkMode,
|
||||||
|
buffer: String,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
struct ToolCallState {
|
struct ToolCallState {
|
||||||
output_index: Option<u32>,
|
output_index: Option<u32>,
|
||||||
@@ -48,6 +63,7 @@ struct ChatToResponsesState {
|
|||||||
next_output_index: u32,
|
next_output_index: u32,
|
||||||
text: TextItemState,
|
text: TextItemState,
|
||||||
reasoning: ReasoningItemState,
|
reasoning: ReasoningItemState,
|
||||||
|
inline_think: InlineThinkState,
|
||||||
tools: BTreeMap<usize, ToolCallState>,
|
tools: BTreeMap<usize, ToolCallState>,
|
||||||
output_items: Vec<(u32, Value)>,
|
output_items: Vec<(u32, Value)>,
|
||||||
latest_usage: Option<Value>,
|
latest_usage: Option<Value>,
|
||||||
@@ -65,6 +81,7 @@ impl Default for ChatToResponsesState {
|
|||||||
next_output_index: 0,
|
next_output_index: 0,
|
||||||
text: TextItemState::default(),
|
text: TextItemState::default(),
|
||||||
reasoning: ReasoningItemState::default(),
|
reasoning: ReasoningItemState::default(),
|
||||||
|
inline_think: InlineThinkState::default(),
|
||||||
tools: BTreeMap::new(),
|
tools: BTreeMap::new(),
|
||||||
output_items: Vec::new(),
|
output_items: Vec::new(),
|
||||||
latest_usage: None,
|
latest_usage: None,
|
||||||
@@ -110,12 +127,12 @@ impl ChatToResponsesState {
|
|||||||
|
|
||||||
if let Some(content) = delta.get("content").and_then(|v| v.as_str()) {
|
if let Some(content) = delta.get("content").and_then(|v| v.as_str()) {
|
||||||
if !content.is_empty() {
|
if !content.is_empty() {
|
||||||
events.extend(self.finalize_reasoning());
|
events.extend(self.push_content_delta(content));
|
||||||
events.extend(self.push_text_delta(content));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(tool_calls) = delta.get("tool_calls").and_then(|v| v.as_array()) {
|
if let Some(tool_calls) = delta.get("tool_calls").and_then(|v| v.as_array()) {
|
||||||
|
events.extend(self.flush_inline_think_at_boundary());
|
||||||
events.extend(self.finalize_reasoning());
|
events.extend(self.finalize_reasoning());
|
||||||
for tool_call in tool_calls {
|
for tool_call in tool_calls {
|
||||||
events.extend(self.push_tool_call_delta(tool_call));
|
events.extend(self.push_tool_call_delta(tool_call));
|
||||||
@@ -130,6 +147,98 @@ impl ChatToResponsesState {
|
|||||||
events
|
events
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn push_content_delta(&mut self, delta: &str) -> Vec<Bytes> {
|
||||||
|
match self.inline_think.mode {
|
||||||
|
InlineThinkMode::Text => {
|
||||||
|
let mut events = self.finalize_reasoning();
|
||||||
|
events.extend(self.push_text_delta(delta));
|
||||||
|
events
|
||||||
|
}
|
||||||
|
InlineThinkMode::Detecting => {
|
||||||
|
self.inline_think.buffer.push_str(delta);
|
||||||
|
match leading_think_prefix_decision(&self.inline_think.buffer) {
|
||||||
|
ThinkPrefixDecision::NeedMore => Vec::new(),
|
||||||
|
ThinkPrefixDecision::Reasoning => {
|
||||||
|
self.inline_think.mode = InlineThinkMode::Reasoning;
|
||||||
|
self.drain_complete_inline_think()
|
||||||
|
}
|
||||||
|
ThinkPrefixDecision::Text => {
|
||||||
|
self.inline_think.mode = InlineThinkMode::Text;
|
||||||
|
let text = std::mem::take(&mut self.inline_think.buffer);
|
||||||
|
let mut events = self.finalize_reasoning();
|
||||||
|
events.extend(self.push_text_delta(&text));
|
||||||
|
events
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
InlineThinkMode::Reasoning => {
|
||||||
|
self.inline_think.buffer.push_str(delta);
|
||||||
|
self.drain_complete_inline_think()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drain_complete_inline_think(&mut self) -> Vec<Bytes> {
|
||||||
|
let Some((reasoning, answer)) = split_leading_think_block(&self.inline_think.buffer) else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
|
||||||
|
self.inline_think.mode = InlineThinkMode::Text;
|
||||||
|
self.inline_think.buffer.clear();
|
||||||
|
|
||||||
|
let mut events = Vec::new();
|
||||||
|
if !reasoning.is_empty() {
|
||||||
|
events.extend(self.push_reasoning_delta(&reasoning));
|
||||||
|
events.extend(self.finalize_reasoning());
|
||||||
|
}
|
||||||
|
if !answer.is_empty() {
|
||||||
|
events.extend(self.push_text_delta(&answer));
|
||||||
|
}
|
||||||
|
|
||||||
|
events
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush_inline_think_at_boundary(&mut self) -> Vec<Bytes> {
|
||||||
|
match self.inline_think.mode {
|
||||||
|
InlineThinkMode::Text => Vec::new(),
|
||||||
|
InlineThinkMode::Detecting => {
|
||||||
|
self.inline_think.mode = InlineThinkMode::Text;
|
||||||
|
let text = std::mem::take(&mut self.inline_think.buffer);
|
||||||
|
if text.is_empty() {
|
||||||
|
Vec::new()
|
||||||
|
} else {
|
||||||
|
let mut events = self.finalize_reasoning();
|
||||||
|
events.extend(self.push_text_delta(&text));
|
||||||
|
events
|
||||||
|
}
|
||||||
|
}
|
||||||
|
InlineThinkMode::Reasoning => {
|
||||||
|
let buffered = std::mem::take(&mut self.inline_think.buffer);
|
||||||
|
self.inline_think.mode = InlineThinkMode::Text;
|
||||||
|
if let Some((reasoning, answer)) = split_leading_think_block(&buffered) {
|
||||||
|
let mut events = Vec::new();
|
||||||
|
if !reasoning.is_empty() {
|
||||||
|
events.extend(self.push_reasoning_delta(&reasoning));
|
||||||
|
events.extend(self.finalize_reasoning());
|
||||||
|
}
|
||||||
|
if !answer.is_empty() {
|
||||||
|
events.extend(self.push_text_delta(&answer));
|
||||||
|
}
|
||||||
|
return events;
|
||||||
|
}
|
||||||
|
|
||||||
|
let reasoning = strip_leading_think_open_tag(&buffered).unwrap_or(buffered);
|
||||||
|
if reasoning.is_empty() {
|
||||||
|
Vec::new()
|
||||||
|
} else {
|
||||||
|
let mut events = self.push_reasoning_delta(&reasoning);
|
||||||
|
events.extend(self.finalize_reasoning());
|
||||||
|
events
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn ensure_response_started(&mut self) -> Vec<Bytes> {
|
fn ensure_response_started(&mut self) -> Vec<Bytes> {
|
||||||
if self.response_started {
|
if self.response_started {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
@@ -375,6 +484,7 @@ impl ChatToResponsesState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let mut events = self.ensure_response_started();
|
let mut events = self.ensure_response_started();
|
||||||
|
events.extend(self.flush_inline_think_at_boundary());
|
||||||
events.extend(self.finalize_reasoning());
|
events.extend(self.finalize_reasoning());
|
||||||
events.extend(self.finalize_text());
|
events.extend(self.finalize_text());
|
||||||
events.extend(self.finalize_tools());
|
events.extend(self.finalize_tools());
|
||||||
@@ -663,6 +773,29 @@ fn chat_delta_reasoning_text(delta: &Value) -> Option<String> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
enum ThinkPrefixDecision {
|
||||||
|
NeedMore,
|
||||||
|
Reasoning,
|
||||||
|
Text,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn leading_think_prefix_decision(buffer: &str) -> ThinkPrefixDecision {
|
||||||
|
let trimmed = buffer.trim_start();
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
return ThinkPrefixDecision::NeedMore;
|
||||||
|
}
|
||||||
|
|
||||||
|
if trimmed.starts_with("<think>") {
|
||||||
|
return ThinkPrefixDecision::Reasoning;
|
||||||
|
}
|
||||||
|
|
||||||
|
if "<think>".starts_with(trimmed) {
|
||||||
|
return ThinkPrefixDecision::NeedMore;
|
||||||
|
}
|
||||||
|
|
||||||
|
ThinkPrefixDecision::Text
|
||||||
|
}
|
||||||
|
|
||||||
/// Create a stream that converts Chat Completions SSE chunks into Responses SSE events.
|
/// Create a stream that converts Chat Completions SSE chunks into Responses SSE events.
|
||||||
pub fn create_responses_sse_stream_from_chat<E: std::error::Error + Send + 'static>(
|
pub fn create_responses_sse_stream_from_chat<E: std::error::Error + Send + 'static>(
|
||||||
stream: impl Stream<Item = Result<Bytes, E>> + Send + 'static,
|
stream: impl Stream<Item = Result<Bytes, E>> + Send + 'static,
|
||||||
@@ -832,6 +965,24 @@ mod tests {
|
|||||||
assert!(reasoning_pos < message_pos);
|
assert!(reasoning_pos < message_pos);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn converts_inline_think_chat_sse_to_reasoning_without_leaking_tags() {
|
||||||
|
let output = collect(vec![
|
||||||
|
"data: {\"id\":\"chatcmpl_minimax\",\"created\":123,\"model\":\"MiniMax-M2.7\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"<think>\\nNeed\"}}]}\n\n",
|
||||||
|
"data: {\"id\":\"chatcmpl_minimax\",\"created\":123,\"model\":\"MiniMax-M2.7\",\"choices\":[{\"delta\":{\"content\":\" context.</think>\\n\\npong\"},\"finish_reason\":\"stop\"}]}\n\n",
|
||||||
|
"data: {\"id\":\"chatcmpl_minimax\",\"created\":123,\"model\":\"MiniMax-M2.7\",\"choices\":[],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":6,\"total_tokens\":10,\"completion_tokens_details\":{\"reasoning_tokens\":3}}}\n\n",
|
||||||
|
])
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert!(output.contains("event: response.reasoning_summary_text.delta"));
|
||||||
|
assert!(output.contains("Need context."));
|
||||||
|
assert!(output.contains("\"text\":\"pong\""));
|
||||||
|
assert!(output.contains("\"reasoning_tokens\":3"));
|
||||||
|
assert!(!output.contains("<think>"));
|
||||||
|
assert!(!output.contains("</think>"));
|
||||||
|
assert!(output.contains("event: response.completed"));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn converts_tool_call_chat_sse_to_responses_sse() {
|
async fn converts_tool_call_chat_sse_to_responses_sse() {
|
||||||
let output = collect(vec![
|
let output = collect(vec![
|
||||||
|
|||||||
@@ -23,6 +23,8 @@ const EXTRA_CHAT_PASSTHROUGH_FIELDS: &[&str] = &[
|
|||||||
"top_logprobs",
|
"top_logprobs",
|
||||||
"user",
|
"user",
|
||||||
];
|
];
|
||||||
|
const THINK_OPEN_TAG: &str = "<think>";
|
||||||
|
const THINK_CLOSE_TAG: &str = "</think>";
|
||||||
|
|
||||||
/// Convert an OpenAI Responses request into an OpenAI Chat Completions request.
|
/// Convert an OpenAI Responses request into an OpenAI Chat Completions request.
|
||||||
pub fn responses_to_chat_completions(body: Value) -> Result<Value, ProxyError> {
|
pub fn responses_to_chat_completions(body: Value) -> Result<Value, ProxyError> {
|
||||||
@@ -421,11 +423,20 @@ fn chat_reasoning_text(message: &Value) -> Option<String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let reasoning = message.get("reasoning")?;
|
if let Some(reasoning) = message.get("reasoning") {
|
||||||
for key in ["content", "text", "summary"] {
|
for key in ["content", "text", "summary"] {
|
||||||
if let Some(text) = reasoning.get(key).and_then(|v| v.as_str()) {
|
if let Some(text) = reasoning.get(key).and_then(|v| v.as_str()) {
|
||||||
if !text.is_empty() {
|
if !text.is_empty() {
|
||||||
return Some(text.to_string());
|
return Some(text.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(content) = message.get("content").and_then(|v| v.as_str()) {
|
||||||
|
if let Some((reasoning, _answer)) = split_leading_think_block(content) {
|
||||||
|
if !reasoning.is_empty() {
|
||||||
|
return Some(reasoning);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -437,6 +448,9 @@ fn chat_message_to_response_output_item(message: &Value, response_id: &str) -> O
|
|||||||
let mut content = Vec::new();
|
let mut content = Vec::new();
|
||||||
|
|
||||||
if let Some(text) = message.get("content").and_then(|v| v.as_str()) {
|
if let Some(text) = message.get("content").and_then(|v| v.as_str()) {
|
||||||
|
let text = split_leading_think_block(text)
|
||||||
|
.map(|(_reasoning, answer)| answer)
|
||||||
|
.unwrap_or_else(|| text.to_string());
|
||||||
if !text.is_empty() {
|
if !text.is_empty() {
|
||||||
content.push(json!({
|
content.push(json!({
|
||||||
"type": "output_text",
|
"type": "output_text",
|
||||||
@@ -496,6 +510,36 @@ fn chat_message_to_response_output_item(message: &Value, response_id: &str) -> O
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn split_leading_think_block(text: &str) -> Option<(String, String)> {
|
||||||
|
let leading_ws_len = text.len() - text.trim_start().len();
|
||||||
|
let after_ws = &text[leading_ws_len..];
|
||||||
|
if !after_ws.starts_with(THINK_OPEN_TAG) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let body_start = leading_ws_len + THINK_OPEN_TAG.len();
|
||||||
|
let close_relative = text[body_start..].find(THINK_CLOSE_TAG)?;
|
||||||
|
let close_start = body_start + close_relative;
|
||||||
|
let answer_start = close_start + THINK_CLOSE_TAG.len();
|
||||||
|
|
||||||
|
Some((
|
||||||
|
text[body_start..close_start].trim().to_string(),
|
||||||
|
strip_think_answer_separator(&text[answer_start..]).to_string(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn strip_leading_think_open_tag(text: &str) -> Option<String> {
|
||||||
|
let leading_ws_len = text.len() - text.trim_start().len();
|
||||||
|
let after_ws = &text[leading_ws_len..];
|
||||||
|
after_ws
|
||||||
|
.strip_prefix(THINK_OPEN_TAG)
|
||||||
|
.map(|value| value.trim().to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn strip_think_answer_separator(text: &str) -> &str {
|
||||||
|
text.trim_start_matches(['\r', '\n', '\t', ' '])
|
||||||
|
}
|
||||||
|
|
||||||
fn chat_tool_calls_to_response_output_items(message: &Value) -> Vec<Value> {
|
fn chat_tool_calls_to_response_output_items(message: &Value) -> Vec<Value> {
|
||||||
let mut output = Vec::new();
|
let mut output = Vec::new();
|
||||||
|
|
||||||
@@ -788,6 +832,43 @@ mod tests {
|
|||||||
assert_eq!(result["usage"]["input_tokens_details"]["cached_tokens"], 3);
|
assert_eq!(result["usage"]["input_tokens_details"]["cached_tokens"], 3);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chat_response_to_responses_splits_inline_think_content() {
|
||||||
|
let input = json!({
|
||||||
|
"id": "chatcmpl_think",
|
||||||
|
"object": "chat.completion",
|
||||||
|
"created": 123,
|
||||||
|
"model": "MiniMax-M2.7",
|
||||||
|
"choices": [{
|
||||||
|
"message": {
|
||||||
|
"role": "assistant",
|
||||||
|
"content": "<think>\nI should answer with pong.\n</think>\n\npong"
|
||||||
|
},
|
||||||
|
"finish_reason": "stop"
|
||||||
|
}],
|
||||||
|
"usage": {
|
||||||
|
"prompt_tokens": 10,
|
||||||
|
"completion_tokens": 20,
|
||||||
|
"total_tokens": 30,
|
||||||
|
"completion_tokens_details": {"reasoning_tokens": 18}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let result = chat_completion_to_response(input).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(result["output"][0]["type"], "reasoning");
|
||||||
|
assert_eq!(
|
||||||
|
result["output"][0]["summary"][0]["text"],
|
||||||
|
"I should answer with pong."
|
||||||
|
);
|
||||||
|
assert_eq!(result["output"][1]["type"], "message");
|
||||||
|
assert_eq!(result["output"][1]["content"][0]["text"], "pong");
|
||||||
|
assert_eq!(
|
||||||
|
result["usage"]["output_tokens_details"]["reasoning_tokens"],
|
||||||
|
18
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn chat_response_length_maps_to_incomplete_response() {
|
fn chat_response_length_maps_to_incomplete_response() {
|
||||||
let input = json!({
|
let input = json!({
|
||||||
|
|||||||
Reference in New Issue
Block a user