fix: handle UTF-8 multi-byte characters split across stream chunk boundaries (#1923)

* fix: handle UTF-8 multi-byte characters split across stream chunk boundaries

Replace String::from_utf8_lossy with append_utf8_safe in all four SSE
streaming paths. When a multi-byte UTF-8 character (e.g. Chinese, emoji)
is split across TCP chunk boundaries, from_utf8_lossy silently replaces
the incomplete halves with U+FFFD (�). This caused intermittent garbled
output in Claude Code when using the Copilot reverse proxy, because the
format conversion streams reconstruct SSE events from the corrupted buffer.

The new append_utf8_safe function preserves incomplete trailing bytes in
a remainder buffer and merges them with the next chunk before decoding,
ensuring characters are never split during UTF-8 conversion.

Fixes: intermittent U+FFFD replacement characters in Claude Code output
via Copilot proxy (not reproducible with direct Copilot connections like
opencode because they pass through raw bytes without format conversion).

* style: fix cargo fmt formatting in UTF-8 boundary tests

---------

Co-authored-by: Cod1ng <codingts@gmail.com>
Co-authored-by: encodets <encodets@gmail.com>

src-tauri/src/proxy/providers/streaming.rs

@@ -93,6 +93,7 @@ pub fn create_anthropic_sse_stream<E: std::error::Error + Send + 'static>(
 ) -> impl Stream<Item = Result<Bytes, std::io::Error>> + Send {
     async_stream::stream! {
         let mut buffer = String::new();
+        let mut utf8_remainder: Vec<u8> = Vec::new();
         let mut message_id = None;
         let mut current_model = None;
         let mut next_content_index: u32 = 0;
@@ -107,8 +108,7 @@ pub fn create_anthropic_sse_stream<E: std::error::Error + Send + 'static>(
         while let Some(chunk) = stream.next().await {
             match chunk {
                 Ok(bytes) => {
-                    let text = String::from_utf8_lossy(&bytes);
-                    buffer.push_str(&text);
+                    crate::proxy::sse::append_utf8_safe(&mut buffer, &mut utf8_remainder, &bytes);
 
                     while let Some(pos) = buffer.find("\n\n") {
                         let line = buffer[..pos].to_string();
@@ -750,4 +750,45 @@ mod tests {
         assert!(deltas.contains(&"{\"a\":"));
         assert!(deltas.contains(&"1}"));
     }
+
+    #[tokio::test]
+    async fn test_streaming_chinese_split_across_chunks_no_replacement_chars() {
+        // "你好" split across two TCP chunks inside a streaming text delta.
+        // Before the fix, from_utf8_lossy would produce U+FFFD for each half.
+        let full = concat!(
+            "data: {\"id\":\"chatcmpl_3\",\"model\":\"gpt-4o\",\"choices\":[{\"delta\":{\"content\":\"你好\"}}]}\n\n",
+            "data: {\"id\":\"chatcmpl_3\",\"model\":\"gpt-4o\",\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":2}}\n\n",
+            "data: [DONE]\n\n"
+        );
+        let bytes = full.as_bytes();
+
+        // Find "你" in the byte stream and split inside it
+        let ni_start = bytes.windows(3).position(|w| w == "你".as_bytes()).unwrap();
+        let split_point = ni_start + 1; // split after first byte of "你"
+
+        let chunk1 = Bytes::from(bytes[..split_point].to_vec());
+        let chunk2 = Bytes::from(bytes[split_point..].to_vec());
+
+        let upstream = stream::iter(vec![
+            Ok::<_, std::io::Error>(chunk1),
+            Ok::<_, std::io::Error>(chunk2),
+        ]);
+        let converted = create_anthropic_sse_stream(upstream);
+        let chunks: Vec<_> = converted.collect().await;
+
+        let merged = chunks
+            .into_iter()
+            .map(|chunk| String::from_utf8_lossy(chunk.unwrap().as_ref()).to_string())
+            .collect::<String>();
+
+        // Must contain the original Chinese characters, not replacement chars
+        assert!(
+            merged.contains("你好"),
+            "expected '你好' in output, got replacement chars (U+FFFD)"
+        );
+        assert!(
+            !merged.contains('\u{FFFD}'),
+            "output must not contain U+FFFD replacement characters"
+        );
+    }
 }

src-tauri/src/proxy/providers/streaming_responses.rs

@@ -101,6 +101,7 @@ pub fn create_anthropic_sse_stream_from_responses<E: std::error::Error + Send +
 ) -> impl Stream<Item = Result<Bytes, std::io::Error>> + Send {
     async_stream::stream! {
         let mut buffer = String::new();
+        let mut utf8_remainder: Vec<u8> = Vec::new();
         let mut message_id: Option<String> = None;
         let mut current_model: Option<String> = None;
         let mut has_sent_message_start = false;
@@ -118,8 +119,7 @@ pub fn create_anthropic_sse_stream_from_responses<E: std::error::Error + Send +
         while let Some(chunk) = stream.next().await {
             match chunk {
                 Ok(bytes) => {
-                    let text = String::from_utf8_lossy(&bytes);
-                    buffer.push_str(&text);
+                    crate::proxy::sse::append_utf8_safe(&mut buffer, &mut utf8_remainder, &bytes);
 
                     // SSE 事件由 \n\n 分隔
                     while let Some(pos) = buffer.find("\n\n") {
@@ -1029,4 +1029,45 @@ mod tests {
         assert_eq!(text_stops, 1);
         assert_eq!(text_deltas, vec!["你".to_string(), "好".to_string()]);
     }
+
+    #[tokio::test]
+    async fn test_streaming_responses_chinese_split_across_chunks_no_replacement_chars() {
+        // Chinese text delta split across two TCP chunks.
+        let full = concat!(
+            "event: response.created\n",
+            "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_cn\",\"model\":\"gpt-4o\",\"usage\":{\"input_tokens\":5,\"output_tokens\":0}}}\n\n",
+            "event: response.output_text.delta\n",
+            "data: {\"type\":\"response.output_text.delta\",\"delta\":\"你好世界\"}\n\n",
+            "event: response.completed\n",
+            "data: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":5,\"output_tokens\":4}}}\n\n"
+        );
+        let bytes = full.as_bytes();
+
+        // Find "你" and split inside it
+        let ni_start = bytes.windows(3).position(|w| w == "你".as_bytes()).unwrap();
+        let split_point = ni_start + 2; // split after second byte of "你"
+
+        let chunk1 = Bytes::from(bytes[..split_point].to_vec());
+        let chunk2 = Bytes::from(bytes[split_point..].to_vec());
+
+        let upstream = stream::iter(vec![
+            Ok::<_, std::io::Error>(chunk1),
+            Ok::<_, std::io::Error>(chunk2),
+        ]);
+        let converted = create_anthropic_sse_stream_from_responses(upstream);
+        let chunks: Vec<_> = converted.collect().await;
+        let merged = chunks
+            .into_iter()
+            .map(|c| String::from_utf8_lossy(c.unwrap().as_ref()).to_string())
+            .collect::<String>();
+
+        assert!(
+            merged.contains("你好世界"),
+            "expected '你好世界' in output, got replacement chars (U+FFFD)"
+        );
+        assert!(
+            !merged.contains('\u{FFFD}'),
+            "output must not contain U+FFFD replacement characters"
+        );
+    }
 }