Update main.py

main.py

@@ -2,11 +2,11 @@ import httpx
 from fastapi import FastAPI, Request, HTTPException
 from fastapi.responses import StreamingResponse
 import json
-import asyncio
+import random # Import the random library
 
 app = FastAPI()
 
-# List of API URLs, with the primary first and fallbacks after
+# List of API URLs to be randomized
 API_URLS = [
     "https://api.deepinfra.com/v1/openai/chat/completions",
     "https://stage.api.deepinfra.com/v1/openai/chat/completions",
@@ -15,15 +15,14 @@ API_URLS = [
 @app.post("/v1/openai/chat/completions")
 async def proxy_deepinfra(request: Request):
     """
-    Proxies chat completion requests to the DeepInfra API with fallback support.
+    Proxies chat completion requests to the DeepInfra API.
+    It randomizes the order of API URLs and uses the next as a fallback.
     """
     try:
         body = await request.json()
     except json.JSONDecodeError:
         raise HTTPException(status_code=400, detail="Invalid JSON in request body")
 
-    # The original curl command headers can be used as a base
-    # and overridden by headers from the incoming request if needed.
     headers = {
         'sec-ch-ua-platform': request.headers.get('sec-ch-ua-platform', '"Windows"'),
         'Referer': request.headers.get('Referer', 'https://deepinfra.com/'),
@@ -35,24 +34,29 @@ async def proxy_deepinfra(request: Request):
         'Content-Type': request.headers.get('Content-Type', 'application/json'),
     }
 
+    # Create a randomized list of URLs for this specific request
+    shuffled_urls = random.sample(API_URLS, len(API_URLS))
+
     async def stream_generator():
         last_error = None
-        for url in API_URLS:
+        for url in shuffled_urls:
+            print(f"Attempting to connect to: {url}")
             try:
                 async with httpx.AsyncClient() as client:
-                    async with client.stream("POST", url, headers=headers, json=body, timeout=30.0) as response:
-                        # Raise an exception for bad status codes to trigger the fallback
+                    # Setting timeout=None disables client-side timeouts
+                    async with client.stream("POST", url, headers=headers, json=body, timeout=None) as response:
                         response.raise_for_status()
+                        # If successful, print the URL being used
+                        print(f"Successfully connected. Streaming from: {url}")
                         async for chunk in response.aiter_bytes():
                             yield chunk
-                        # If we successfully stream the response, we're done.
-                        return
+                        return # Exit after successful stream
             except (httpx.RequestError, httpx.HTTPStatusError) as e:
-                # Store the last error and try the next URL
                 last_error = e
-                print(f"Error with URL {url}: {e}. Trying next fallback.")
+                print(f"Failed to connect to {url}: {e}. Trying next URL.")
                 continue
-        # If all URLs failed, raise an exception based on the last error.
+
+        # This part is reached only if all URLs in the shuffled list failed
         if last_error:
             raise HTTPException(status_code=502, detail=f"All API endpoints failed. Last error: {last_error}")