Create query.md

query.md

@@ -0,0 +1,384 @@
+```python
+import requests
+import json
+
+# Build model mapping
+original_models = [
+    # OpenAI Models
+    "gpt-3.5-turbo",
+    "gpt-3.5-turbo-202201",
+    "gpt-4o",
+    "gpt-4o-2024-05-13",
+    "o1-preview",
+
+    # Claude Models
+    "claude",
+    "claude-3-5-sonnet",
+    "claude-sonnet-3.5",
+    "claude-3-5-sonnet-20240620",
+
+    # Meta/LLaMA Models
+    "@cf/meta/llama-2-7b-chat-fp16",
+    "@cf/meta/llama-2-7b-chat-int8",
+    "@cf/meta/llama-3-8b-instruct",
+    "@cf/meta/llama-3.1-8b-instruct",
+    "@cf/meta-llama/llama-2-7b-chat-hf-lora",
+    "llama-3.1-405b",
+    "llama-3.1-70b",
+    "llama-3.1-8b",
+    "meta-llama/Llama-2-7b-chat-hf",
+    "meta-llama/Llama-3.1-70B-Instruct",
+    "meta-llama/Llama-3.1-8B-Instruct",
+    "meta-llama/Llama-3.2-11B-Vision-Instruct",
+    "meta-llama/Llama-3.2-1B-Instruct",
+    "meta-llama/Llama-3.2-3B-Instruct",
+    "meta-llama/Llama-3.2-90B-Vision-Instruct",
+    "meta-llama/Llama-Guard-3-8B",
+    "meta-llama/Meta-Llama-3-70B-Instruct",
+    "meta-llama/Meta-Llama-3-8B-Instruct",
+    "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo",
+    "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo",
+
+    # Mistral Models
+    "mistral",
+    "mistral-large",
+    "@cf/mistral/mistral-7b-instruct-v0.1",
+    "@cf/mistral/mistral-7b-instruct-v0.2-lora",
+    "@hf/mistralai/mistral-7b-instruct-v0.2",
+    "mistralai/Mistral-7B-Instruct-v0.2",
+    "mistralai/Mistral-7B-Instruct-v0.3",
+    "mistralai/Mixtral-8x22B-Instruct-v0.1",
+    "mistralai/Mixtral-8x7B-Instruct-v0.1",
+
+    # Qwen Models
+    "@cf/qwen/qwen1.5-0.5b-chat",
+    "@cf/qwen/qwen1.5-1.8b-chat",
+    "@cf/qwen/qwen1.5-7b-chat-awq",
+    "@cf/qwen/qwen1.5-14b-chat-awq",
+    "Qwen/Qwen2.5-3B-Instruct",
+    "Qwen/Qwen2.5-72B-Instruct",
+    "Qwen/Qwen2.5-Coder-32B-Instruct",
+
+    # Google/Gemini Models
+    "@cf/google/gemma-2b-it-lora",
+    "@cf/google/gemma-7b-it-lora",
+    "@hf/google/gemma-7b-it",
+    "google/gemma-1.1-2b-it",
+    "google/gemma-1.1-7b-it",
+    "gemini-pro",
+    "gemini-1.5-pro",
+    "gemini-1.5-pro-latest",
+    "gemini-1.5-flash",
+
+    # Cohere Models
+    "c4ai-aya-23-35b",
+    "c4ai-aya-23-8b",
+    "command",
+    "command-light",
+    "command-light-nightly",
+    "command-nightly",
+    "command-r",
+    "command-r-08-2024",
+    "command-r-plus",
+    "command-r-plus-08-2024",
+    "rerank-english-v2.0",
+    "rerank-english-v3.0",
+    "rerank-multilingual-v2.0",
+    "rerank-multilingual-v3.0",
+
+    # Microsoft Models
+    "@cf/microsoft/phi-2",
+    "microsoft/DialoGPT-medium",
+    "microsoft/Phi-3-medium-4k-instruct",
+    "microsoft/Phi-3-mini-4k-instruct",
+    "microsoft/Phi-3.5-mini-instruct",
+    "microsoft/WizardLM-2-8x22B",
+
+    # Yi Models
+    "01-ai/Yi-1.5-34B-Chat",
+    "01-ai/Yi-34B-Chat",
+]
+
+# Create mapping from simplified model names to original model names
+model_mapping = {}
+simplified_models = []
+
+for original_model in original_models:
+    simplified_name = original_model.split('/')[-1]
+    if simplified_name in model_mapping:
+        # Conflict detected, handle as per instructions
+        print(f"Conflict detected for model name '{simplified_name}'. Excluding '{original_model}' from available models.")
+        continue
+    model_mapping[simplified_name] = original_model
+    simplified_models.append(simplified_name)
+
+def generate(
+    model,
+    messages,
+    temperature=0.7,
+    top_p=1.0,
+    n=1,
+    stream=False,
+    stop=None,
+    max_tokens=None,
+    presence_penalty=0.0,
+    frequency_penalty=0.0,
+    logit_bias=None,
+    user=None,
+    timeout=30,
+):
+    """
+    Generates a chat completion using the provided model and messages.
+    """
+    # Use the simplified model names
+    models = simplified_models
+
+    if model not in models:
+        raise ValueError(f"Invalid model: {model}. Choose from: {', '.join(models)}")
+    
+    # Map simplified model name to original model name
+    original_model = model_mapping[model]
+
+    api_endpoint = "https://chat.typegpt.net/api/openai/v1/chat/completions"
+    
+    headers = {
+        "authority": "chat.typegpt.net",
+        "accept": "application/json, text/event-stream",
+        "accept-language": "en-US,en;q=0.9",
+        "content-type": "application/json",
+        "origin": "https://chat.typegpt.net",
+        "referer": "https://chat.typegpt.net/",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+    }
+    
+    # Payload
+    payload = {
+        "messages": messages,
+        "stream": stream,
+        "model": original_model,
+        "temperature": temperature,
+        "presence_penalty": presence_penalty,
+        "frequency_penalty": frequency_penalty,
+        "top_p": top_p,
+    }
+
+    # Only include max_tokens if it's not None
+    if max_tokens is not None:
+        payload["max_tokens"] = max_tokens
+
+    # Only include 'stop' if it's not None
+    if stop is not None:
+        payload["stop"] = stop
+
+    # Check if logit_bias is provided
+    if logit_bias is not None:
+        payload["logit_bias"] = logit_bias
+
+    # Include 'user' if provided
+    if user is not None:
+        payload["user"] = user
+
+    # Start the request
+    session = requests.Session()
+    response = session.post(
+        api_endpoint, headers=headers, json=payload, stream=stream, timeout=timeout
+    )
+    
+    if not response.ok:
+        raise Exception(f"Failed to generate response - ({response.status_code}, {response.reason}) - {response.text}")
+    
+    def stream_response():
+        for line in response.iter_lines():
+            if line:
+                line = line.decode("utf-8")
+                if line.startswith("data: "):
+                    line = line[6:]  # Remove "data: " prefix
+                    if line.strip() == "[DONE]":
+                        break
+                    try:
+                        data = json.loads(line)
+                        yield data
+                    except json.JSONDecodeError:
+                        continue
+
+    if stream:
+        return stream_response()
+    else:
+        return response.json()
+
+if __name__ == "__main__":
+    # Example usage
+    # model = "claude-3-5-sonnet-20240620"
+    # model = "qwen1.5-0.5b-chat"
+    # model = "llama-2-7b-chat-fp16"
+    model = "gpt-3.5-turbo"
+    messages = [
+        {"role": "system", "content": "Be Detailed"},
+        {"role": "user", "content": "What is the knowledge cut off? Be specific and also specify the month, year and date. If not sure, then provide approximate."}
+    ]
+
+    # try:
+    #     # For non-streamed response
+    #     response = generate(
+    #         model=model,
+    #         messages=messages,
+    #         temperature=0.5,
+    #         max_tokens=4000,
+    #         stream=False  # Change to True for streaming
+    #     )
+    #     if 'choices' in response:
+    #         reply = response['choices'][0]['message']['content']
+    #         print(reply)
+    #     else:
+    #         print("No response received.")
+    # except Exception as e:
+    #     print(e)
+
+
+    try:
+        # For streamed response
+        response = generate(
+            model=model,
+            messages=messages,
+            temperature=0.5,
+            max_tokens=4000,
+            stream=True,  # Change to False for non-streamed response
+        )
+        for data in response:
+            if 'choices' in data:
+                reply = data['choices'][0]['delta']['content']
+                print(reply, end="", flush=True)
+            else:
+                print("No response received.")
+    except Exception as e:
+        print(e)
+```
+
+```python
+from fastapi import FastAPI, Request, Response
+from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi.middleware.cors import CORSMiddleware
+import uvicorn
+import asyncio
+import json
+import requests
+
+from TYPEGPT.typegpt_api import generate, model_mapping, simplified_models
+from api_info import developer_info
+
+app = FastAPI()
+
+# Set up CORS middleware if needed
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+@app.get("/health_check")
+async def health_check():
+    return {"status": "OK"}
+
+@app.get("/models")
+async def get_models():
+    # Retrieve models from TypeGPT API and forward the response
+    api_endpoint = "https://chat.typegpt.net/api/openai/v1/models"
+    try:
+        response = requests.get(api_endpoint)
+        # return response.text
+        return JSONResponse(content=response.json(), status_code=response.status_code)
+    except Exception as e:
+        return JSONResponse(content={"error": str(e)}, status_code=500)
+
+@app.post("/chat/completions")
+async def chat_completions(request: Request):
+    # Receive the JSON payload
+    try:
+        body = await request.json()
+    except Exception as e:
+        return JSONResponse(content={"error": "Invalid JSON payload"}, status_code=400)
+
+    # Extract parameters
+    model = body.get("model")
+    messages = body.get("messages")
+    temperature = body.get("temperature", 0.7)
+    top_p = body.get("top_p", 1.0)
+    n = body.get("n", 1)
+    stream = body.get("stream", False)
+    stop = body.get("stop")
+    max_tokens = body.get("max_tokens")
+    presence_penalty = body.get("presence_penalty", 0.0)
+    frequency_penalty = body.get("frequency_penalty", 0.0)
+    logit_bias = body.get("logit_bias")
+    user = body.get("user")
+    timeout = 30  # or set based on your preference
+
+    # Validate required parameters
+    if not model:
+        return JSONResponse(content={"error": "The 'model' parameter is required."}, status_code=400)
+    if not messages:
+        return JSONResponse(content={"error": "The 'messages' parameter is required."}, status_code=400)
+
+    # Call the generate function
+    try:
+        if stream:
+            async def generate_stream():
+                response = generate(
+                    model=model,
+                    messages=messages,
+                    temperature=temperature,
+                    top_p=top_p,
+                    n=n,
+                    stream=True,
+                    stop=stop,
+                    max_tokens=max_tokens,
+                    presence_penalty=presence_penalty,
+                    frequency_penalty=frequency_penalty,
+                    logit_bias=logit_bias,
+                    user=user,
+                    timeout=timeout,
+                )
+                
+                for chunk in response:
+                    yield f"data: {json.dumps(chunk)}\n\n"
+                yield "data: [DONE]\n\n"
+
+            return StreamingResponse(
+                generate_stream(),
+                media_type="text/event-stream",
+                headers={
+                    "Cache-Control": "no-cache",
+                    "Connection": "keep-alive",
+                    "Transfer-Encoding": "chunked"
+                }
+            )
+        else:
+            response = generate(
+                model=model,
+                messages=messages,
+                temperature=temperature,
+                top_p=top_p,
+                n=n,
+                stream=False,
+                stop=stop,
+                max_tokens=max_tokens,
+                presence_penalty=presence_penalty,
+                frequency_penalty=frequency_penalty,
+                logit_bias=logit_bias,
+                user=user,
+                timeout=timeout,
+            )
+            return JSONResponse(content=response)
+    except Exception as e:
+        return JSONResponse(content={"error": str(e)}, status_code=500)
+
+@app.get("/developer_info")
+async def get_developer_info():
+    return JSONResponse(content=developer_info)
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)
+```