Update app.py

app.py

@@ -12,6 +12,8 @@ import logging
 import webbrowser
 from huggingface_hub import InferenceClient
 from typing import Dict, List, Optional, Tuple
+from functools import wraps
+import threading
 import time
 from groq import Groq  # Import the Groq client
 
@@ -63,7 +65,6 @@ MODEL_CONTEXT_SIZES = {
         "microsoft/Phi-3-mini-128k-instruct": 131072, # Added Phi-3 128k
         "HuggingFaceH4/zephyr-7b-beta": 8192,
         "deepseek-ai/DeepSeek-Coder-V2-Instruct": 8192,
-        "meta-llama/Llama-3-8b-Instruct": 8192,
         "mistralai/Mistral-7B-Instruct-v0.3": 32768,
         "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO": 32768,
         "microsoft/Phi-3.5-mini-instruct": 4096,
@@ -103,30 +104,60 @@ MODEL_CONTEXT_SIZES = {
         "command-light-nightly": 4096,
         "c4ai-aya-expanse-8b": 8192,
         "c4ai-aya-expanse-32b": 131072,
+    },
+    "GLHF API": {
+        "mistralai/Mistral-7B-Instruct-v0.3": 32768,
+        "microsoft/phi-3-mini-4k-instruct": 4096,
+        "microsoft/Phi-3-mini-128k-instruct": 131072, # Added Phi-3 128k
+        "HuggingFaceH4/zephyr-7b-beta": 8192,
+        "mistralai/Mistral-7B-Instruct-v0.3": 32768,
+        "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO": 32768,
+        "microsoft/Phi-3.5-mini-instruct": 4096,
+        "google/gemma-2-2b-it": 2048,
+        "microsoft/phi-2": 2048,
+        # Add other model contexts here
     }
 }
 
+class RateLimit:
+    def __init__(self, calls_per_min):
+        self.calls_per_min = calls_per_min
+        self.calls = []
+        self.lock = threading.Lock()
+        
+    def __call__(self, func):
+        @wraps(func)
+        def wrapped(*args, **kwargs):
+            with self.lock:
+                now = time.time()
+                # Remove old calls
+                self.calls = [call for call in self.calls if call > now - 60]
+                
+                if len(self.calls) >= self.calls_per_min:
+                    sleep_time = self.calls[0] - (now - 60)
+                    if sleep_time > 0:
+                        time.sleep(sleep_time)
+                        
+                self.calls.append(now)
+                return func(*args, **kwargs)
+        return wrapped
+
 class ModelRegistry:
    def __init__(self):
        # HuggingFace Models
        self.hf_models = {
-            "Phi-3 Mini 4K": "microsoft/phi-3-mini-4k-instruct",
-            "Phi-3 Mini 128k": "microsoft/Phi-3-mini-128k-instruct", # Added
-            "Zephyr 7B Beta": "HuggingFaceH4/zephyr-7b-beta",
-            "DeepSeek Coder V2": "deepseek-ai/DeepSeek-Coder-V2-Instruct",
-            "Meta Llama 3.1 8B": "meta-llama/Llama-3-8b-Instruct",
-            "Meta Llama 3.1 70B": "meta-llama/Meta-Llama-3.1-70B-Instruct",
-            "Mixtral 7B": "mistralai/Mistral-7B-Instruct-v0.3",
-            "Nous-Hermes": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
-            "Cohere Command R+": "CohereForAI/c4ai-command-r-plus",
-            "Aya 23-35B": "CohereForAI/aya-23-35B",
-            "Phi-3.5 Mini": "microsoft/Phi-3.5-mini-instruct", # Added
-            "SmolLM2 1.7B": "HuggingFaceTB/SmolLM2-1.7B-Instruct", # Added
-            "Gemma 2 2B": "google/gemma-2-2b-it", # Added
-            "GPT2": "openai-community/gpt2", # Added
-            "Phi-2": "microsoft/phi-2", # Added
-            "TinyLlama 1.1B": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", # Added
-            "Custom Model": ""  # Keep for custom models
+            "Mixtral 7B": "mistralai/Mistral-7B-Instruct-v0.3",  # works well
+            "Nous-Hermes": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",  # works well
+            "Zephyr 7B": "HuggingFaceH4/zephyr-7b-beta",  # works
+            "Phi-3.5 Mini": "microsoft/Phi-3.5-mini-instruct",  # works but poor results
+            "Gemma 2 2B": "google/gemma-2-2b-it",  # works but often busy
+            "GPT2": "openai-community/gpt2",  # works with token limits
+            "Phi-2": "microsoft/phi-2",  # works with token limits
+            "TinyLlama 1.1B": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",  # works with token limits
+            "DeepSeek Coder V2 (Pro)": "deepseek-ai/DeepSeek-Coder-V2-Instruct",  # needs API key
+            "Meta Llama 3.1 70B (Pro)": "meta-llama/Meta-Llama-3.1-70B-Instruct",  # needs API key
+            "Aya 23-35B (Pro)": "CohereForAI/aya-23-35B",  # needs API key
+            "Custom Model": ""
         }
        
        # Default Groq Models
@@ -369,6 +400,16 @@ def send_to_model(prompt, model_selection, hf_model_choice, hf_custom_model, hf_
             return "Error: Groq API key required", None
         elif model_selection == "OpenAI ChatGPT" and not openai_api_key:
             return "Error: OpenAI API key required", None
+        elif model_selection == "GLHF API":
+            if not glhf_api_key:
+                return "Error: GLHF API key required", None
+            if glhf_model == "Use HuggingFace Model":
+                model_id = hf_custom_model if hf_model_choice == "Custom Model" else model_registry.hf_models[hf_model_choice]
+                summary = send_to_glhf(prompt, True, model_id, "", glhf_api_key)
+            else:
+                if not glhf_custom_model.strip():
+                    return "Error: Custom model ID required", None
+                summary = send_to_glhf(prompt, False, "", glhf_custom_model.strip(), glhf_api_key)
             
         # Call implementation with error handling
         try:
@@ -408,8 +449,9 @@ def send_to_model(prompt, model_selection, hf_model_choice, hf_custom_model, hf_
         logging.info("send to model completed.")
 
 def send_to_model_impl(prompt, model_selection, hf_model_choice, hf_custom_model, hf_api_key,
-                      groq_model_choice, groq_api_key, openai_api_key, openai_model_choice):
-    """Implementation of model sending with improved error handling."""
+                      groq_model_choice, groq_api_key, openai_api_key, openai_model_choice,
+                      cohere_api_key=None, cohere_model=None, glhf_api_key=None):
+    """Implementation of model sending with all providers."""
     logging.info("send to model impl commencing...")
     
     try:
@@ -417,12 +459,10 @@ def send_to_model_impl(prompt, model_selection, hf_model_choice, hf_custom_model
             return "Text copied to clipboard. Use paste for processing.", None
 
         if model_selection == "HuggingFace Inference":
-            # First try without API key
             model_id = hf_custom_model if hf_model_choice == "Custom Model" else model_registry.hf_models[hf_model_choice]
             summary = send_to_hf_inference(prompt, model_id)
-            if summary.startswith("Error"):
-                if hf_api_key:  # If first try failed and we have an API key, try with it
-                    summary = send_to_hf_inference(prompt, model_id, hf_api_key)
+            if summary.startswith("Error") and hf_api_key:
+                summary = send_to_hf_inference(prompt, model_id, hf_api_key)
             
         elif model_selection == "Groq API":
             summary = send_to_groq(prompt, groq_model_choice, groq_api_key)
@@ -431,7 +471,13 @@ def send_to_model_impl(prompt, model_selection, hf_model_choice, hf_custom_model
             summary = send_to_openai(prompt, openai_api_key, model=openai_model_choice)
             
         elif model_selection == "Cohere API":
-            summary = send_to_cohere(prompt)
+            summary = send_to_cohere(prompt, cohere_api_key, cohere_model)
+            
+        elif model_selection == "GLHF API":
+            if not glhf_api_key:
+                return "Error: GLHF API key required", None
+            model_id = hf_custom_model if hf_model_choice == "Custom Model" else model_registry.hf_models[hf_model_choice]
+            summary = send_to_glhf(prompt, model_id, glhf_api_key)
         
         else:
             return "Error: Invalid model selection", None
@@ -458,6 +504,11 @@ def send_to_model_impl(prompt, model_selection, hf_model_choice, hf_custom_model
 def send_to_hf_inference(prompt: str, model_name: str, api_key: str = None) -> str:
     """Send prompt to HuggingFace Inference API with optional authentication."""
     try:
+        # Check token limits first
+        is_within_limits, error_msg = check_token_limits(prompt, model_name)
+        if not is_within_limits:
+            return error_msg
+        
         client = InferenceClient(token=api_key) if api_key else InferenceClient()
         response = client.text_generation(
             prompt,
@@ -556,26 +607,92 @@ def send_to_openai(prompt: str, api_key: str, model: str = "gpt-3.5-turbo") -> s
         logging.error(f"OpenAI API error: {e}")
         raise  # Re-raise to be handled by caller
     
+@RateLimit(calls_per_min=16)  # 80% of 20 calls/min
 def send_to_cohere(prompt: str, api_key: str = None) -> str:
-    """Send prompt to Cohere API with optional authentication."""
+    """Send prompt to Cohere API with V2 and V1 fallback."""
     try:
-        import cohere
-        client = cohere.Client(api_key) if api_key else cohere.Client()
-        
-        response = client.chat(
-            message=prompt,
-            temperature=0.7,
-            max_tokens=500,
-        )
-        
-        if hasattr(response, 'text'):
+        # Try V2 first
+        try:
+            import cohere
+            client = cohere.ClientV2(api_key) if api_key else cohere.ClientV2()
+            response = client.chat(
+                model="command-r-plus-08-2024",  # Using latest model
+                messages=[{
+                    "role": "user",
+                    "content": prompt
+                }],
+                temperature=0.7,
+            )
+            return response.message.content[0].text
+        except Exception as v2_error:
+            logging.warning(f"Cohere V2 failed, trying V1: {v2_error}")
+            
+            # Fallback to V1
+            client = cohere.Client(api_key) if api_key else cohere.Client()
+            response = client.chat(
+                message=prompt,
+                temperature=0.7,
+                max_tokens=500,
+            )
             return response.text
-        else:
-            return "Error: No response text from Cohere"
             
     except Exception as e:
         logging.error(f"Cohere API error: {e}")
-        return f"Error with Cohere API: {str(e)}"  # Return error message instead of raising
+        return f"Error with Cohere API: {str(e)}"
+
+@RateLimit(calls_per_min=384)  # 80% of 480/8hours = 60/hour = 1/min
+def send_to_glhf(prompt: str, use_hf_model: bool, model_name: str, custom_model: str, api_key: str) -> str:
+    """Send prompt to GLHF API with model selection."""
+    try:
+        import openai
+        
+        client = openai.OpenAI(
+            api_key=api_key,
+            base_url="https://glhf.chat/api/openai/v1",
+        )
+
+        # Select model based on user choice
+        model_id = f"hf:{model_name if use_hf_model else custom_model}"
+        
+        # Always use streaming for reliability
+        completion = client.chat.completions.create(
+            stream=True,
+            model=model_id,
+            messages=[
+                {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": prompt}
+            ],
+        )
+
+        response_text = []
+        for chunk in completion:
+            if chunk.choices[0].delta.content is not None:
+                response_text.append(chunk.choices[0].delta.content)
+                
+        return "".join(response_text)
+
+    except Exception as e:
+        logging.error(f"GLHF API error: {e}")
+        return f"Error with GLHF API: {str(e)}"
+
+def estimate_tokens(text: str) -> int:
+    """Rough token estimation: ~4 characters per token on average"""
+    return len(text) // 4
+
+def check_token_limits(prompt: str, model_name: str) -> tuple[bool, str]:
+    """Check if prompt might exceed model's token limits."""
+    token_limited_models = {
+        "openai-community/gpt2": 1500,  # 2048 - buffer
+        "microsoft/phi-2": 1500,
+        "TinyLlama/TinyLlama-1.1B-Chat-v1.0": 1500
+    }
+    
+    if model_name in token_limited_models:
+        estimated_tokens = estimate_tokens(prompt)
+        max_tokens = token_limited_models[model_name]
+        if estimated_tokens > max_tokens:
+            return False, f"Prompt too long (estimated {estimated_tokens} tokens). This model supports max {max_tokens} tokens."
+    return True, ""
 
 def copy_text_js(element_id: str) -> str:
     return f"""function() {{
@@ -808,6 +925,50 @@ with gr.Blocks(css="""
                         type="password"
                     )
                     groq_refresh_btn = gr.Button("🔄 Refresh Groq Models")  # Add refresh button
+                
+               with gr.Column(visible=False) as glhf_options:
+                    glhf_api_key = gr.Textbox(
+                        label="🔑 GLHF API Key",
+                        type="password"
+                    )
+                    glhf_model = gr.Radio(
+                        choices=["Use HuggingFace Model", "Custom Model"],
+                        value="Use HuggingFace Model",
+                        label="Model Selection"
+                    )
+                    glhf_custom_model = gr.Textbox(
+                        label="Custom Model ID (owner/model format)",
+                        placeholder="e.g., mistralai/Mixtral-8x7B-Instruct-v0.2",
+                        visible=False
+                    )
+
+               # Add visibility toggle
+               def toggle_glhf_custom_model(choice):
+                     return gr.update(visible=choice == "Custom Model")
+
+               glhf_model.change(
+                    toggle_glhf_custom_model,
+                    inputs=[glhf_model],
+                    outputs=[glhf_custom_model]
+                )
+
+               # Update Cohere options container
+               with gr.Column(visible=False) as cohere_options:
+                    cohere_api_key = gr.Textbox(
+                        label="🔑 Cohere API Key (optional - needed for some models)",
+                        type="password"
+                    )
+                    cohere_model = gr.Dropdown(
+                        choices=[
+                            "command-r-plus-08-2024",
+                            "command-r-plus-04-2024",
+                            "command-r",
+                            "command",
+                            "command-light"
+                        ],
+                        value="command-r-plus-08-2024",
+                        label="Cohere Model"
+                    )
                     
                send_to_model_btn = gr.Button("🚀 Send to Model", variant="primary")
                open_chatgpt_button = gr.Button("🌐 Open ChatGPT")
@@ -861,7 +1022,9 @@ with gr.Blocks(css="""
         return (
             gr.update(visible=choice == "HuggingFace Inference"),
             gr.update(visible=choice == "Groq API"),
-            gr.update(visible=choice == "OpenAI ChatGPT")
+            gr.update(visible=choice == "OpenAI ChatGPT"),
+            gr.update(visible=choice == "Cohere API"),
+            gr.update(visible=choice == "GLHF API")
         )
 
     def refresh_groq_models_list():
@@ -1093,7 +1256,12 @@ with gr.Blocks(css="""
             groq_model,
             groq_api_key,
             openai_api_key,
-            openai_model
+            openai_model,
+            cohere_api_key,
+            cohere_model,
+            glhf_api_key,
+            glhf_model,
+            glhf_custom_model
         ],
         outputs=[summary_output, download_summary]
     )