init inference

app.py

@@ -1,75 +1,66 @@
+import os
+import subprocess
 import gradio as gr
-from llama_cpp import Llama
 from huggingface_hub import hf_hub_download
 
-# Model identifier from Hugging Face
-adapter_repo = "Mat17892/llama_lora_gguf"  # Hugging Face model ID
-
-# Download the GGUF file from Hugging Face
-lora_adapter_path = hf_hub_download(repo_id=adapter_repo, filename="llama_lora_adapter.gguf")
-
-from huggingface_hub import hf_hub_download
+# Hugging Face repository IDs
+base_model_repo = "unsloth/Llama-3.2-3B-Instruct-GGUF"
+adapter_repo = "Mat17892/llama_lora_gguf"
 
 # Download the base model GGUF file
-base_model_repo = "unsloth/Llama-3.2-3B-Instruct-GGUF"
+print("Downloading base model...")
 base_model_path = hf_hub_download(repo_id=base_model_repo, filename="Llama-3.2-3B-Instruct-Q8_0.gguf")
 
+# Download the LoRA adapter GGUF file
+print("Downloading LoRA adapter...")
+lora_adapter_path = hf_hub_download(repo_id=adapter_repo, filename="llama_lora_adapter.gguf")
 
-# Load the base model
-print("Loading base model...")
-llm = Llama(model_path=base_model_path, n_ctx=2048, n_threads=8)
-
-# Apply the LoRA adapter
-print("Applying LoRA adapter...")
-llm.load_adapter(adapter_path=lora_adapter_path)
-
-print("Model ready with LoRA adapter!")
-
-# Chat function
-def chat_with_model(user_input, chat_history):
-    """
-    Process user input and generate a response from the model.
-    :param user_input: User's input string
-    :param chat_history: List of [user_message, ai_response] pairs
-    :return: Updated chat history
-    """
-    # Construct the prompt from chat history
+# Function to run `llama-cli` with base model and adapter
+def run_llama_cli(prompt):
+    print("Running inference with llama-cli...")
+    cmd = [
+        "./llama-cli",
+        "-c", "2048",  # Context length
+        "-cnv",        # Enable conversational mode
+        "-m", base_model_path,
+        "--lora", lora_adapter_path,
+        "--prompt", prompt,
+    ]
+    try:
+        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = process.communicate()
+
+        if process.returncode != 0:
+            print("Error during inference:")
+            print(stderr.decode())
+            return "Error: Could not generate response."
+
+        return stdout.decode().strip()
+    except Exception as e:
+        print(f"Exception occurred: {e}")
+        return "Error: Could not generate response."
+
+# Gradio interface
+def chatbot_fn(user_input, chat_history):
+    # Build the full chat history as the prompt
     prompt = ""
     for user, ai in chat_history:
         prompt += f"User: {user}\nAI: {ai}\n"
-    prompt += f"User: {user_input}\nAI:"  # Add the latest user input
+    prompt += f"User: {user_input}\nAI:"  # Add latest user input
 
-    # Generate response from the model
-    raw_response = llm(prompt)["choices"][0]["text"].strip()
+    # Generate response using llama-cli
+    response = run_llama_cli(prompt)
 
-    # Clean the response (remove extra tags, if any)
-    response = raw_response.split("User:")[0].strip()
-
-    # Update chat history with the new turn
+    # Update chat history
     chat_history.append((user_input, response))
     return chat_history, chat_history
 
-
-# Gradio UI
+# Build the Gradio UI
 with gr.Blocks() as demo:
-    gr.Markdown("# 🦙 LLaMA GGUF Chatbot")
-    chatbot = gr.Chatbot(label="Chat with the GGUF Model")
+    gr.Markdown("# 🦙 LLaMA Chatbot with Base Model and LoRA Adapter")
+    chatbot = gr.Chatbot(label="Chat with the Model")
 
     with gr.Row():
         with gr.Column(scale=4):
             user_input = gr.Textbox(label="Your Message", placeholder="Type a message...")
-        with gr.Column(scale=1):
-            submit_btn = gr.Button("Send")
-
-    chat_history = gr.State([])
-
-    # Link components
-    submit_btn.click(
-        chat_with_model,
-        inputs=[user_input, chat_history],
-        outputs=[chatbot, chat_history],
-        show_progress=True,
-    )
-
-# Launch the app
-demo.launch()
+        with gr.Column(scale=