Update app.py

app.py

@@ -10,6 +10,9 @@ import time
 import datetime
 import requests, json
 
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
+
 loop = asyncio.get_event_loop()
 # Monkey patch
 def get_types(cls_set: List[Type], component: str):
@@ -30,43 +33,25 @@ def get_types(cls_set: List[Type], component: str):
     return docset, types
 routes.get_types = get_types
 
-
-q = queue.Queue()
-
-arrr = []
-
-from huggingface_hub import hf_hub_download
 hf_hub_download(repo_id='StarFox7/Llama-2-ko-7B-chat-gguf', filename='Llama-2-ko-7B-chat-gguf-q4_0.bin', local_dir='./')
 
+llm = Llama(model_path = 'Llama-2-ko-7B-chat-gguf-q4_0.bin',
+            n_ctx=2048,
+      )
 # App code
-def res(x, id, cdata, url):    
-    global q
-
-    arr = [x, id, str(cdata.split(",", 1)[0]), url]
-    q.put(arr)
-
-    arrr.append(x)
-    print("\n_Done\n\n")
-    return "Done"
+def res(x):    
+    output = llm(f"다음은 A와 B의 역할극이야. 너는 B야. A와 대화하고 있어. 친구에게 친근하고 간결하게 잘 대답해줘.\n\n### A:\n{x}\n\n### B:\n", max_tokens=100, stop=["###"], echo=True)
+    return output['choices'][0]['text']
 
-def rese(x):    
-
-    print(f"{arrr}")
-    return "Done"
     
 with gr.Blocks() as demo:
     count = 0
     aa = gr.Interface(
-      fn=res,
-      inputs=["text","text", "text", "text"],
-      outputs="text",
-      description="call",
-    )
-
-    bb = gr.Interface(
-      fn=rese,
+      fn=chat,
       inputs=["text"],
       outputs="text",
       description="call",
     )
+
+
     demo.queue(max_size=32).launch(enable_queue=True)