Update main.py

main.py

@@ -59,6 +59,22 @@ async def completion(request: ChatCompletionRequestV0, response_mode=None):
     response = llm(request.prompt)
     return response
 
+async def generate_response(chat_chunks, llm):
+    for chat_chunk in chat_chunks:
+        response = {
+            'choices': [
+                {
+                    'message': {
+                        'role': 'system',
+                        'content': llm.detokenize(chat_chunk)
+                    },
+                    'finish_reason': 'stop' if llm.is_eos_token(chat_chunk) else 'unknown'
+                }
+            ]
+        }
+        yield f"data: {json.dumps(response)}\n\n"
+    yield "event: done\ndata: {}\n\n"
+
 @app.post("/v1/chat/completions")
 async def chat(request: ChatCompletionRequest):
     combined_messages = ' '.join([message.content for message in request.messages])