Try llava-v1.6-mistral-7b.Q5_K_M.gguf

Dockerfile

@@ -26,8 +26,8 @@ ENV PATH="/usr/local/cuda/bin:$PATH" \
 WORKDIR /app
 
 # Download ggml and mmproj models from HuggingFace
-RUN wget https://huggingface.co/mys/ggml_bakllava-1/resolve/main/ggml-model-q4_k.gguf && \
-    wget https://huggingface.co/mys/ggml_bakllava-1/resolve/main/mmproj-model-f16.gguf 
+RUN wget https://huggingface.co/cjpais/llava-1.6-mistral-7b-gguf/resolve/main/llava-v1.6-mistral-7b.Q5_K_M.gguf && \
+    wget https://huggingface.co/cjpais/llava-1.6-mistral-7b-gguf/resolve/main/mmproj-model-f16.gguf 
 
 # Clone and build llava-server with CUDA support
 RUN git clone https://github.com/ggerganov/llama.cpp.git && \
@@ -39,10 +39,10 @@ RUN git clone https://github.com/ggerganov/llama.cpp.git && \
 # Create a non-root user for security reasons
 RUN useradd -m -u 1000 user && \
     mkdir -p /home/user/app && \
-    cp /app/ggml-model-q4_k.gguf /home/user/app && \
+    cp /app/llava-v1.6-mistral-7b.Q5_K_M.gguf /home/user/app && \
     cp /app/mmproj-model-f16.gguf /home/user/app
 
-RUN chown user:user /home/user/app/ggml-model-q4_k.gguf && \
+RUN chown user:user /home/user/app/llava-v1.6-mistral-7b.Q5_K_M.gguf && \
     chown user:user /home/user/app/mmproj-model-f16.gguf
 
 USER user
@@ -54,4 +54,4 @@ WORKDIR $HOME/app
 EXPOSE 8080
 
 # Start the llava-server with models
-CMD ["/app/llama.cpp/server", "--model", "ggml-model-q4_k.gguf", "--mmproj", "mmproj-model-f16.gguf", "--threads", "4", "--host", "0.0.0.0"]
+CMD ["/app/llama.cpp/server", "--model", "llava-v1.6-mistral-7b.Q5_K_M.gguf", "--mmproj", "mmproj-model-f16.gguf", "--threads", "6", "--host", "0.0.0.0"]