add streaming support (#1)

Signed-off-by: Jintao Zhang <zhangjintao9020@gmail.com>

server/index.js

@@ -15,9 +15,18 @@ app.use(express.json());
 
 app.post('/api/chat', async (req, res) => {
   const { messages, apiEndpoint, apiKey, model } = req.body;
+  
+  console.log('Received chat request with:', {
+    apiEndpoint,
+    model,
+    messageCount: messages.length
+  });
 
   try {
-    const response = await fetch(`${apiEndpoint}/v1/chat/completions`, {
+    const apiUrl = `${apiEndpoint}/v1/chat/completions`;
+    console.log('Calling API endpoint:', apiUrl);
+    
+    const response = await fetch(apiUrl, {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
@@ -25,17 +34,42 @@ app.post('/api/chat', async (req, res) => {
       },
       body: JSON.stringify({
         model: model,
-        messages: messages
+        messages: messages,
+        stream: true
       })
     });
 
+    console.log('API response status:', response.status);
+    console.log('API response headers:', Object.fromEntries(response.headers.entries()));
+
     if (!response.ok) {
       const errorData = await response.text();
+      console.error('API error:', {
+        status: response.status,
+        error: errorData
+      });
       throw new Error(`API error: ${response.status} - ${errorData}`);
     }
 
-    const data = await response.json();
-    res.json(data);
+    // Set headers for streaming
+    res.setHeader('Content-Type', 'text/event-stream');
+    res.setHeader('Cache-Control', 'no-cache');
+    res.setHeader('Connection', 'keep-alive');
+
+    // Set SSE headers
+    res.writeHead(200, {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache',
+      'Connection': 'keep-alive'
+    });
+
+    // Pipe the API response to the client
+    response.body.pipe(res).on('error', (err) => {
+      console.error('Stream error:', err);
+      res.status(500).end();
+    }).on('end', () => {
+      console.log('Stream completed successfully');
+    });
   } catch (error) {
     console.error('Error:', error);
     res.status(500).json({ error: error.message });
@@ -54,4 +88,4 @@ if (process.env.NODE_ENV === 'production') {
 
 app.listen(port, () => {
   console.log(`Server running at http://localhost:${port}`);
-}); 
+});

src/components/ChatWindow.jsx

@@ -4,6 +4,8 @@ function ChatWindow({ chat, settings, onUpdateChat }) {
   const [input, setInput] = useState('');
   const [isLoading, setIsLoading] = useState(false);
   const [collapsedThinks, setCollapsedThinks] = useState(new Set());
+  const [partialResponse, setPartialResponse] = useState('');
+  const [streamController, setStreamController] = useState(null);
 
   // 添加解析消息的函数
   const parseMessage = (content) => {
@@ -29,10 +31,15 @@ function ChatWindow({ chat, settings, onUpdateChat }) {
   const handleSendMessage = async () => {
     if (!input.trim()) return;
 
+    // Cancel any ongoing stream
+    if (streamController) {
+      streamController.abort();
+    }
+
     const newMessage = {
       role: 'user',
       content: input,
-      timestamp: Date.now()  // 确保添加时间戳
+      timestamp: Date.now()
     };
 
     const updatedChat = {
@@ -41,8 +48,12 @@ function ChatWindow({ chat, settings, onUpdateChat }) {
     };
     onUpdateChat(updatedChat);
     setInput('');
+    setPartialResponse('');
+
+    // Create new AbortController for this stream
+    const controller = new AbortController();
+    setStreamController(controller);
 
-    // 发送API请求
     setIsLoading(true);
     try {
       const response = await fetch('/api/chat', {
@@ -59,6 +70,7 @@ function ChatWindow({ chat, settings, onUpdateChat }) {
           model: settings.model,
           apiEndpoint: settings.apiEndpoint,
         }),
+        signal: controller.signal
       });
 
       if (!response.ok) {
@@ -66,25 +78,50 @@ function ChatWindow({ chat, settings, onUpdateChat }) {
         throw new Error(`HTTP error! status: ${response.status}, message: ${errorData}`);
       }
 
-      const data = await response.json();
-      
-      // 添加AI回复
-      const aiMessage = {
+      const reader = response.body.getReader();
+      const decoder = new TextDecoder();
+      let aiMessage = {
         role: 'assistant',
-        content: data.choices[0].message.content,
+        content: '',
         timestamp: Date.now()
       };
 
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+
+        const chunk = decoder.decode(value);
+        const lines = chunk.split('\n').filter(line => line.trim() !== '');
+        
+        for (const line of lines) {
+          const message = line.replace(/^data: /, '');
+          if (message === '[DONE]') break;
+
+          try {
+            const parsed = JSON.parse(message);
+            const content = parsed.choices[0].delta.content || '';
+            aiMessage.content += content;
+            setPartialResponse(aiMessage.content);
+          } catch (error) {
+            console.error('Error parsing chunk:', error);
+          }
+        }
+      }
+
       const finalChat = {
         ...updatedChat,
         messages: [...updatedChat.messages, aiMessage]
       };
       onUpdateChat(finalChat);
     } catch (error) {
-      console.error('Failed to send message:', error);
-      alert(`Failed to send message: ${error.message}`);
+      if (error.name !== 'AbortError') {
+        console.error('Failed to send message:', error);
+        alert(`Failed to send message: ${error.message}`);
+      }
     } finally {
       setIsLoading(false);
+      setPartialResponse('');
+      setStreamController(null);
     }
   };
 
@@ -129,7 +166,10 @@ function ChatWindow({ chat, settings, onUpdateChat }) {
         {isLoading && (
           <div className="message assistant">
             <div className="message-content">
-              <div className="loading-indicator">Thinking...</div>
+              <div className="content-text">
+                {partialResponse}
+                <span className="loading-cursor">|</span>
+              </div>
             </div>
           </div>
         )}
@@ -153,4 +193,4 @@ function ChatWindow({ chat, settings, onUpdateChat }) {
   );
 }
 
-export default ChatWindow; 
+export default ChatWindow;

vite.config.js

@@ -14,9 +14,9 @@ export default defineConfig({
         }
       },
       '/api': {
-        target: 'http://localhost:3600',
+        target: 'http://localhost:7860',
         changeOrigin: true
       }
     }
   }
-}) 
+})