Update run.py

run.py

@@ -164,20 +164,20 @@ else:
   import os
   import requests
   import subprocess
-  modelPath="/home/af/gguf/models/discolm_german_7b_v1.Q4_0.gguf"
+  modelPath="/home/af/gguf/models/Discolm_german_7b_v1.Q4_0.gguf"
   if(os.path.exists(modelPath)==False):
     #url="https://huggingface.co/TheBloke/WizardLM-13B-V1.2-GGUF/resolve/main/wizardlm-13b-v1.2.Q4_0.gguf"
-    #url="https://huggingface.co/TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF/resolve/main/mixtral-8x7b-instruct-v0.1.Q4_0.gguf?download=true"
+    url="https://huggingface.co/TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF/resolve/main/mixtral-8x7b-instruct-v0.1.Q4_0.gguf?download=true"
     #url="https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_0.gguf?download=true"
-    url="https://huggingface.co/TheBloke/DiscoLM_German_7b_v1-GGUF/resolve/main/discolm_german_7b_v1.Q4_0.gguf?download=true"
+    #url="https://huggingface.co/TheBloke/DiscoLM_German_7b_v1-GGUF/resolve/main/discolm_german_7b_v1.Q4_0.gguf?download=true"
     response = requests.get(url)
-    with open("./model.gguf", mode="wb") as file:
+    with open("./Mixtral-8x7b-instruct.gguf", mode="wb") as file:
       file.write(response.content)
     print("Model downloaded")  
-    modelPath="./model.gguf"
+    modelPath="./Mixtral-8x7b-instruct.gguf"
   print(modelPath)
   n="20" 
-  if("mixtral-8x7b-instruct" in modelPath): n="0" # mixtral seems to cause problems here...
+  if("Mixtral-8x7b-instruct" in modelPath): n="0" # mixtral seems to cause problems here...
   command = ["python3", "-m", "llama_cpp.server", "--model", modelPath, "--host", "0.0.0.0", "--port", "2600", "--n_threads", "8", "--n_gpu_layers", n]
   subprocess.Popen(command)
   print("Server ready!")
@@ -189,32 +189,28 @@ else:
 def extend_prompt(message="", history=None, system=None, RAGAddon=None, system2=None, zeichenlimit=None,historylimit=4): #float("Inf")
   startOfString=""
   if zeichenlimit is None: zeichenlimit=1000000000 # :-)
-  template0=" [INST]{system}\n  [/INST] </s>" if onPrem else "<s> [INST] {system} [/INST] </s>" 
+  template0=" [INST]{system}\n  [/INST] </s>" 
   template1=" [INST] {message} [/INST]"
   template2=" {response}</s>"
-  if("gemma-" in modelPath): # https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1
+  if("Gemma-" in modelPath): # https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1
     template0="<start_of_turn>user{system}</end_of_turn>" 
     template1="<start_of_turn>user{message}</end_of_turn><start_of_turn>model"
     template2="{response}</end_of_turn>"
-  if("mixtral-8x7b-instruct" in modelPath): # https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1
+  if("Mixtral-8x7b-instruct" in modelPath): # https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1
     startOfString="<s>"
-    template0=" [INST]{system}\n  [/INST] </s>" if onPrem else "<s> [INST]{system}\n  [/INST] </s>" 
+    template0=" [INST]{system}\n  [/INST] </s>"  
     template1=" [INST] {message} [/INST]"
     template2=" {response}</s>"
   if("Mistral-7B-Instruct" in modelPath): #https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2
     startOfString="<s>"
-    template0="[INST]{system}\n [/INST]</s>" if onPrem else "<s>[INST]{system}\n [/INST]</s>" 
+    template0="[INST]{system}\n [/INST]</s>"
     template1="[INST] {message} [/INST]"
     template2=" {response}</s>"
-  if("openchat-3.5" in modelPath): #https://huggingface.co/TheBloke/openchat-3.5-0106-GGUF
+  if("Openchat-3.5" in modelPath): #https://huggingface.co/TheBloke/openchat-3.5-0106-GGUF
     template0="GPT4 Correct User: {system}<|end_of_turn|>GPT4 Correct Assistant: Okay.<|end_of_turn|>"
     template1="GPT4 Correct User: {message}<|end_of_turn|>GPT4 Correct Assistant: "
     template2="{response}<|end_of_turn|>"
-  if("SauerkrautLM-7b-HerO" in modelPath):  #https://huggingface.co/VAGOsolutions/SauerkrautLM-7b-HerO
-    template0="<|im_start|>system\n{system}<|im_end|>\n"
-    template1="<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
-    template2="{response}<|im_end|>\n"
-  if("discolm_german_7b" in modelPath): #https://huggingface.co/DiscoResearch/DiscoLM_German_7b_v1
+  if(("Discolm_german_7b" in modelPath) or ("SauerkrautLM-7b-HerO" in modelPath)):  #https://huggingface.co/VAGOsolutions/SauerkrautLM-7b-HerO
     template0="<|im_start|>system\n{system}<|im_end|>\n"
     template1="<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
     template2="{response}<|im_end|>\n"
@@ -222,7 +218,7 @@ def extend_prompt(message="", history=None, system=None, RAGAddon=None, system2=
     template0="{system} " #<s>
     template1="USER: {message} ASSISTANT: "
     template2="{response}</s>"
-  if("phi-2" in modelPath): #https://huggingface.co/TheBloke/phi-2-GGUF
+  if("Phi-2" in modelPath): #https://huggingface.co/TheBloke/phi-2-GGUF
     template0="Instruct: {system}\nOutput: Okay.\n"
     template1="Instruct: {message}\nOutput:"
     template2="{response}\n"