Update app.py

app.py

@@ -109,12 +109,11 @@ MODEL_CONTEXT_SIZES = {
         "mistralai/Mistral-7B-Instruct-v0.3": 32768,
         "microsoft/phi-3-mini-4k-instruct": 4096,
         "microsoft/Phi-3.5-mini-instruct": 4096,
-        "microsoft/Phi-3-mini-128k-instruct": 131072, # Added Phi-3 128k
+        "microsoft/Phi-3-mini-128k-instruct": 131072,
         "HuggingFaceH4/zephyr-7b-beta": 8192,
         "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO": 32768,
         "google/gemma-2-2b-it": 2048,
         "microsoft/phi-2": 2048,
-        # Add other model contexts here
     }
 }
 
@@ -522,14 +521,28 @@ def send_to_model_impl(prompt, model_selection, hf_model_choice, hf_custom_model
         elif model_selection == "GLHF API":
             if not glhf_api_key:
                 return "Error: GLHF API key required", None
-            summary = send_to_glhf(
-                prompt,
-                glhf_model == "Use HuggingFace Model",
-                hf_custom_model if hf_model_choice == "Custom Model" else model_registry.hf_models[hf_model_choice],
-                glhf_custom_model,
-                glhf_api_key,
-                use_rate_limits
-            )
+                
+            # Determine the actual model ID to use
+            if glhf_model == "Use HuggingFace Model":
+                model_id = f"hf:{hf_custom_model if hf_model_choice == 'Custom Model' else model_registry.hf_models[hf_model_choice]}"
+            else:
+                model_id = f"hf:{glhf_custom_model}"
+                
+            summary = send_to_glhf(prompt, glhf_api_key, model_id, use_rate_limits)
+            
+            if not summary:
+                return "Error: No response from model", None
+                
+            if not isinstance(summary, str):
+                return "Error: Invalid response type from model", None
+                
+            # Create download file for valid responses
+            if not summary.startswith("Error"):
+                with tempfile.NamedTemporaryFile(delete=False, mode='w', suffix='.txt') as f:
+                    f.write(summary)
+                    return summary, f.name
+                    
+            return summary, None
         
         else:
             return "Error: Invalid model selection", None
@@ -583,7 +596,7 @@ def send_to_hf_inference(prompt: str, model_name: str, api_key: str = None, use_
 
 def send_to_glhf(prompt: str, use_hf_model: bool, model_name: str, custom_model: str,
                 api_key: str, use_rate_limit: bool = False) -> str:
-    """Send prompt to GLHF API with model selection and proper stream handling."""
+    """Send prompt to GLHF API with model selection."""
     def _send():
         try:
             import openai
@@ -594,42 +607,22 @@ def send_to_glhf(prompt: str, use_hf_model: bool, model_name: str, custom_model:
 
             model_id = f"hf:{model_name if use_hf_model else custom_model}"
             
-            try:
-                # First try without streaming
-                completion = client.chat.completions.create(
-                    stream=False,
-                    model=model_id,
-                    messages=[
-                        {"role": "system", "content": "You are a helpful assistant."},
-                        {"role": "user", "content": prompt}
-                    ],
-                )
-                return completion.choices[0].message.content
-            except Exception as non_stream_error:
-                logging.warning(f"Non-streaming GLHF failed, trying streaming: {non_stream_error}")
-                
-                # Fallback to streaming if needed
-                completion = client.chat.completions.create(
-                    stream=True,
-                    model=model_id,
-                    messages=[
-                        {"role": "system", "content": "You are a helpful assistant."},
-                        {"role": "user", "content": prompt}
-                    ],
-                )
+            # For GLHF, always use streaming for reliability
+            completion = client.chat.completions.create(
+                stream=True,
+                model=model_id,
+                messages=[
+                    {"role": "system", "content": "You are a helpful assistant."},
+                    {"role": "user", "content": prompt}
+                ],
+            )
 
-                response_text = []
-                try:
-                    for chunk in completion:
-                        if chunk.choices and chunk.choices[0].delta.content is not None:
-                            response_text.append(chunk.choices[0].delta.content)
-                except Exception as stream_error:
-                    if response_text:  # If we got partial response, return it
-                        logging.warning(f"Streaming interrupted but got partial response: {stream_error}")
-                        return "".join(response_text)
-                    raise  # Re-raise if we got nothing
+            response_text = []
+            for chunk in completion:
+                if chunk.choices[0].delta.content is not None:
+                    response_text.append(chunk.choices[0].delta.content)
                     
-                return "".join(response_text)
+            return "".join(response_text)
 
         except Exception as e:
             logging.error(f"GLHF API error: {e}")
@@ -702,41 +695,27 @@ def send_to_cohere(prompt: str, api_key: str = None, model: str = None, use_rate
 
     return apply_rate_limit(_send, 16) if use_rate_limit else _send()
 
-def send_to_glhf(prompt: str, use_hf_model: bool, model_name: str, custom_model: str,
-                api_key: str, use_rate_limit: bool = False) -> str:
-    """Send prompt to GLHF API with model selection."""
+def send_to_groq(prompt: str, model_name: str, api_key: str, use_rate_limit: bool = False) -> str:
+    """Send prompt to Groq API."""
     def _send():
         try:
-            import openai
-            client = openai.OpenAI(
-                api_key=api_key,
-                base_url="https://glhf.chat/api/openai/v1",
-            )
-
-            model_id = f"hf:{model_name if use_hf_model else custom_model}"
-            
-            # For GLHF, always use streaming for reliability
-            completion = client.chat.completions.create(
-                stream=True,
-                model=model_id,
-                messages=[
-                    {"role": "system", "content": "You are a helpful assistant."},
-                    {"role": "user", "content": prompt}
-                ],
+            client = Groq(api_key=api_key)
+            response = client.chat.completions.create(
+                model=model_name,
+                messages=[{
+                    "role": "user", 
+                    "content": prompt
+                }],
+                temperature=0.7,
+                max_tokens=500,
+                top_p=0.95
             )
-
-            response_text = []
-            for chunk in completion:
-                if chunk.choices[0].delta.content is not None:
-                    response_text.append(chunk.choices[0].delta.content)
-                    
-            return "".join(response_text)
-
+            return response.choices[0].message.content
         except Exception as e:
-            logging.error(f"GLHF API error: {e}")
-            return f"Error with GLHF API: {str(e)}"
+            logging.error(f"Groq API error: {e}")
+            return f"Error with Groq API: {str(e)}"
 
-    return apply_rate_limit(_send, 384) if use_rate_limit else _send()
+    return apply_rate_limit(_send, 4) if use_rate_limit else _send()
 
 def estimate_tokens(text: str) -> int:
     """Rough token estimation: ~4 characters per token on average"""
@@ -1342,20 +1321,20 @@ with gr.Blocks(css="""
     send_to_model_btn.click(
         fn=send_to_model,
         inputs=[
-            generated_prompt,
-            model_choice,
-            hf_model,
-            hf_custom_model,
-            hf_api_key,
-            groq_model,
-            groq_api_key,
-            openai_api_key,
-            openai_model,
-            cohere_api_key,
-            cohere_model,
-            glhf_api_key,
-            glhf_model,
-            glhf_custom_model
+            generated_prompt,    # prompt
+            model_choice,        # model_selection
+            hf_model,           # hf_model_choice
+            hf_custom_model,    # hf_custom_model
+            hf_api_key,         # hf_api_key
+            groq_model,         # groq_model_choice
+            groq_api_key,       # groq_api_key
+            openai_api_key,     # openai_api_key
+            openai_model,       # openai_model_choice
+            cohere_api_key,     # cohere_api_key
+            cohere_model,       # cohere_model
+            glhf_api_key,       # glhf_api_key
+            glhf_model,         # glhf_model
+            glhf_custom_model   # glhf_custom_model
         ],
         outputs=[summary_output, download_summary]
     )