Update main.py

main.py

@@ -6,12 +6,13 @@ import prompt_style
 
 
 model_id = "failspy/Meta-Llama-3-8B-Instruct-abliterated-v3"
-client = InferenceClient(model_id)
+client = None
 
 class Item(BaseModel):
     prompt: str
     history: list
     system_prompt: str
+    token:str
     temperature: float = 0.6
     max_new_tokens: int = 1024
     top_p: float = 0.95
@@ -44,6 +45,8 @@ def generate(item: Item):
         seed=item.seed,
     )
 
+    if not client:
+        client = InferenceClient(token=item.token, model=model_id)
     formatted_prompt = format_prompt(item)
     stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
     output = ""