feat: text to speech for AI response

requirements.txt

@@ -12,4 +12,5 @@ langgraph-swarm
 langchain-google-genai
 python-dotenv
 loguru
-python-multipart
+python-multipart
+deepgram-sdk

src/agents/lesson_practice_2/prompt.py

@@ -18,6 +18,7 @@ I'm **WISE**, your friendly English conversation partner! I create natural, enga
 - **Maximum English practice** for user
 
 ### Immediate Handoff to Teaching Agent When:
+- If the user initiates the conversation in a language other than English
 - User speaks Vietnamese or requests Vietnamese explanation
 - User asks "How do I say...?" or "What does... mean?"
 - User makes same error 3+ times 

src/agents/role_play/flow.py

@@ -9,7 +9,7 @@ class RolePlayAgent:
         pass
 
     @staticmethod
-    def route_to_active_agent(state: State) -> str:
+    def route_to_active_agent(state: State):
         if state["active_agent"] == "Roleplay Agent":
             return "Roleplay Agent"
         elif state["active_agent"] == "Guiding Agent":

src/agents/role_play/prompt.py

@@ -11,6 +11,8 @@ You are **{your_role}** in an English learning conversation. Create authentic, e
 
 ## LANGUAGE DECISION MATRIX
 
+## CRITICAL LANGUAGE RULE:
+**IF USER SPEAKS ANY LANGUAGE OTHER THAN ENGLISH → IMMEDIATELY HAND OFF TO GUIDING AGENT**
 **CRITICAL RULE: If user uses Vietnamese at any point, immediately handoff to Guiding Agent.**
 
 ### ✅ CONTINUE ROLEPLAY:
@@ -19,6 +21,7 @@ You are **{your_role}** in an English learning conversation. Create authentic, e
 - Communication intent is clear in English
 
 ### ❌ HANDOFF TO GUIDING AGENT:
+- If the user initiates the conversation in a language other than English
 - User speaks primarily Vietnamese or non-English
 - User speaks <30% English 
 - User asks for language help in ANY language

src/agents/role_play/tools.py

@@ -11,6 +11,6 @@ def function_name(
     """
     logger.info(f"Received input: {input}")
     # Thực hiện các thao tác cần thiết với input
-    result = f"Processed: {input}"
+    result: str = f"Processed: {input}"
     logger.info(f"Returning result: {result}")
     return result

src/apis/routes/chat_route.py

@@ -9,6 +9,7 @@ from fastapi import (
 from fastapi.responses import JSONResponse, StreamingResponse
 from src.utils.logger import logger
 from src.agents.role_play.flow import role_play_agent
+from src.services.tts_service import tts_service
 from pydantic import BaseModel, Field
 from typing import Dict, Any, Optional
 from src.agents.role_play.scenarios import get_scenarios
@@ -144,6 +145,7 @@ async def roleplay_stream(
     ),
     text_message: Optional[str] = Form(None, description="Text message from user"),
     audio_file: Optional[UploadFile] = File(None, description="Audio file from user"),
+    audio: bool = Form(False, description="Whether to return TTS audio response"),
 ):
     """Send a message (text or audio) to the roleplay agent with streaming response"""
     logger.info(f"Received streaming roleplay request: {session_id}")
@@ -214,6 +216,7 @@ async def roleplay_stream(
 
     async def generate_stream():
         """Generator function for streaming responses"""
+        accumulated_content = ""
         try:
             input_graph = {
                 "messages": [message],
@@ -242,6 +245,9 @@ async def roleplay_stream(
                         content = getattr(message_chunk, 'content', '')
                         
                     if content:
+                        # Accumulate content for TTS
+                        accumulated_content += content
+                        
                         # Create SSE-formatted response
                         response_data = {
                             "type": "message_chunk",
@@ -257,8 +263,30 @@ async def roleplay_stream(
                         # Small delay to prevent overwhelming the client
                         await asyncio.sleep(0.01)
             
-            # Send completion signal
-            completion_data = {"type": "completion", "content": ""}
+            # Generate TTS audio if requested
+            audio_data = None
+            if audio and accumulated_content.strip():
+                try:
+                    logger.info(f"Generating TTS for accumulated content: {len(accumulated_content)} chars")
+                    audio_result = await tts_service.text_to_speech(accumulated_content)
+                    if audio_result:
+                        audio_data = {
+                            "audio_data": audio_result["audio_data"],
+                            "mime_type": audio_result["mime_type"],
+                            "format": audio_result["format"]
+                        }
+                        logger.info("TTS audio generated successfully")
+                    else:
+                        logger.warning("TTS generation failed")
+                except Exception as tts_error:
+                    logger.error(f"TTS generation error: {str(tts_error)}")
+            
+            # Send completion signal with optional audio
+            completion_data = {
+                "type": "completion", 
+                "content": "",
+                "audio": audio_data
+            }
             yield f"data: {json.dumps(completion_data)}\n\n"
             
         except Exception as e:

src/apis/routes/lesson_route.py

@@ -10,6 +10,7 @@ from fastapi import (
 )
 from fastapi.responses import JSONResponse, StreamingResponse
 from src.utils.logger import logger
+from src.services.tts_service import tts_service
 from pydantic import BaseModel, Field
 from typing import List, Dict, Any, Optional
 from src.agents.lesson_practice.flow import lesson_practice_agent
@@ -367,6 +368,7 @@ async def chat_v2_stream(
     ),
     text_message: Optional[str] = Form(None, description="Text message from user"),
     audio_file: Optional[UploadFile] = File(None, description="Audio file from user"),
+    audio: bool = Form(False, description="Whether to return TTS audio response"),
 ):
     """Send a message (text or audio) to the lesson practice v2 agent with streaming response"""
     logger.info(f"Received streaming lesson practice v2 request: {session_id}")
@@ -437,6 +439,7 @@ async def chat_v2_stream(
 
     async def generate_stream():
         """Generator function for streaming responses"""
+        accumulated_content = ""
         try:
             input_graph = {
                 "messages": [message],
@@ -465,6 +468,9 @@ async def chat_v2_stream(
                         content = getattr(message_chunk, 'content', '')
                         
                     if content:
+                        # Accumulate content for TTS
+                        accumulated_content += content
+                        
                         # Create SSE-formatted response
                         response_data = {
                             "type": "message_chunk",
@@ -480,8 +486,30 @@ async def chat_v2_stream(
                         # Small delay to prevent overwhelming the client
                         await asyncio.sleep(0.01)
             
-            # Send completion signal
-            completion_data = {"type": "completion", "content": ""}
+            # Generate TTS audio if requested
+            audio_data = None
+            if audio and accumulated_content.strip():
+                try:
+                    logger.info(f"Generating TTS for lesson v2 content: {len(accumulated_content)} chars")
+                    audio_result = await tts_service.text_to_speech(accumulated_content)
+                    if audio_result:
+                        audio_data = {
+                            "audio_data": audio_result["audio_data"],
+                            "mime_type": audio_result["mime_type"],
+                            "format": audio_result["format"]
+                        }
+                        logger.info("Lesson v2 TTS audio generated successfully")
+                    else:
+                        logger.warning("Lesson v2 TTS generation failed")
+                except Exception as tts_error:
+                    logger.error(f"Lesson v2 TTS generation error: {str(tts_error)}")
+            
+            # Send completion signal with optional audio
+            completion_data = {
+                "type": "completion", 
+                "content": "",
+                "audio": audio_data
+            }
             yield f"data: {json.dumps(completion_data)}\n\n"
             
         except Exception as e:

src/services/tts_service.py

@@ -0,0 +1,114 @@
+"""
+Text-to-Speech (TTS) Service using Deepgram API
+"""
+
+import requests
+import os
+import base64
+from src.utils.logger import logger
+from typing import Optional
+
+class TTSService:
+    """Service for handling text-to-speech conversion using Deepgram API"""
+    
+    def __init__(self):
+        self.api_key = os.getenv("YOUR_DEEPGRAM_API_KEY")
+        self.base_url = "https://api.deepgram.com/v1/speak"
+        self.default_model = "aura-2-thalia-en"
+        
+        if not self.api_key:
+            logger.error("Deepgram API key not found in environment variables")
+            raise ValueError("Deepgram API key is required")
+    
+    async def text_to_speech(
+        self, 
+        text: str, 
+        model: Optional[str] = None,
+        format: str = "mp3"
+    ) -> Optional[dict]:
+        """
+        Convert text to speech using Deepgram API
+        
+        Args:
+            text (str): The text to convert to speech
+            model (str): The TTS model to use (default: aura-2-thalia-en)
+            format (str): Audio format (default: mp3)
+            
+        Returns:
+            dict: Contains audio data and metadata, or None if failed
+        """
+        try:
+            if not text or not text.strip():
+                logger.warning("Empty text provided for TTS conversion")
+                return None
+                
+            # Clean and prepare text
+            cleaned_text = text.strip()
+            if len(cleaned_text) > 2000:  # Limit text length for TTS
+                cleaned_text = cleaned_text[:2000] + "..."
+                logger.warning(f"Text truncated to 2000 characters for TTS")
+            
+            # Prepare request
+            url = self.base_url
+            querystring = {"model": model or self.default_model}
+            payload = {"text": cleaned_text}
+            headers = {
+                "Authorization": f"Token {self.api_key}",
+                "Content-Type": "application/json"
+            }
+            
+            logger.info(f"Converting text to speech: {cleaned_text[:100]}...")
+            
+            # Make request to Deepgram API
+            response = requests.post(
+                url, 
+                json=payload, 
+                headers=headers, 
+                params=querystring,
+                timeout=30
+            )
+            
+            if response.status_code == 200:
+                # Encode audio data as base64
+                audio_data = response.content
+                audio_base64 = base64.b64encode(audio_data).decode('utf-8')
+                
+                # Determine MIME type based on format
+                mime_type = f"audio/{format}"
+                if format == "mp3":
+                    mime_type = "audio/mpeg"
+                elif format == "wav":
+                    mime_type = "audio/wav"
+                
+                result = {
+                    "audio_data": audio_base64,
+                    "mime_type": mime_type,
+                    "format": format,
+                    "text": cleaned_text,
+                    "model": model or self.default_model,
+                    "size_bytes": len(audio_data)
+                }
+                
+                logger.info(f"TTS conversion successful: {len(audio_data)} bytes")
+                return result
+                
+            else:
+                logger.error(f"Deepgram TTS API error: {response.status_code} - {response.text}")
+                return None
+                
+        except requests.exceptions.Timeout:
+            logger.error("TTS request timed out")
+            return None
+        except requests.exceptions.RequestException as e:
+            logger.error(f"TTS request failed: {str(e)}")
+            return None
+        except Exception as e:
+            logger.error(f"Unexpected error in TTS conversion: {str(e)}")
+            return None
+    
+    def is_available(self) -> bool:
+        """Check if TTS service is available"""
+        return bool(self.api_key)
+
+# Global TTS service instance
+tts_service = TTSService()