fixed bug on running openllm

.env.example

@@ -7,7 +7,7 @@ LLM_MODEL_TYPE=huggingface
 # LLM_MODEL_TYPE=stablelm
 # LLM_MODEL_TYPE=openllm
 
-OPENLLM_SERVER_URL=
+OPENLLM_SERVER_URL=http://localhost:64300
 
 OPENAI_API_KEY=
 

Makefile

@@ -25,6 +25,9 @@ else
 	openllm start llama --model-id meta-llama/Llama-2-7b-chat-hf --port=${PORT}
 endif
 
+openllm-cpu:
+	CUDA_VISIBLE_DEVICES="" openllm start llama --model-id meta-llama/Llama-2-7b-chat-hf
+
 ingest:
 	python ingest.py
 

app_modules/llm_loader.py

@@ -161,7 +161,7 @@ class LLMLoader:
                 print(f"               server url: {server_url}")
                 self.llm = OpenLLM(
                     server_url=server_url,
-                    callbacks=callbacks,
+                    # callbacks=callbacks,
                     verbose=True,
                 )
             elif self.llm_model_type.startswith("gpt4all"):