Update app.py

app.py

@@ -6,7 +6,7 @@ import torch
 import random
 import time
 import re
-from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig, TextStreamer
 
 
 # Set an environment variable
@@ -16,9 +16,10 @@ zero = torch.Tensor([0]).cuda()
 print(zero.device) # <-- 'cpu' 🤔
 
 
-model_id = 'FINGU-AI/Finance-OrpoMistral-7B'              #attn_implementation="flash_attention_2",
+model_id = 'FINGU-AI/Qwen-Orpo-v1'              #attn_implementation="flash_attention_2",
 model = AutoModelForCausalLM.from_pretrained(model_id,attn_implementation="sdpa",  torch_dtype= torch.bfloat16)
 tokenizer = AutoTokenizer.from_pretrained(model_id)
+streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
 model.to('cuda')
 
 # terminators = [
@@ -38,21 +39,25 @@ generation_params = {
 @spaces.GPU
 def inference(query):
     messages = [
-    {"role": "system", "content": """You are a friendly chatbot who always responds in the style of a trader."""},
+    {"role": "system", "content": """You are ai trader, invester helpfull assistant."""},
     {"role": "user", "content": f"{query}"}, 
 ]
 
     tokenized_chat = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").to("cuda")
 
-    outputs = model.generate(tokenized_chat, **generation_params)
-    decoded_outputs = tokenizer.batch_decode(outputs)
-    assistant_response = decoded_outputs[0].split("Assistant:")[-1].strip()
-    return assistant_response
+    outputs = model.generate(tokenized_chat, **generation_params, streamer=streamer)
+    return outputs
 
+examples = ['How can options strategies such as straddles, strangles, and spreads be used to hedge against market volatility?',
+           'How do changes in interest rates, inflation, and GDP growth impact stock and bond markets?',
+           'What are the key components and strategies involved in developing an effective algorithmic trading system?',
+           'How can investors integrate environmental, social, and governance (ESG) factors into their investment decisions to achieve both financial returns and social impact?',
+           'How do geopolitical events such as trade wars, political instability, and international conflicts affect global financial markets?',
+           'How does blockchain technology have the potential to disrupt financial markets and investment practices?']
 
 def response(message, history):
     text = inference(message)
     for i in range(len(text)):
         time.sleep(0.01)
         yield text[: i + 1]
-gr.ChatInterface(response).launch()
+gr.ChatInterface(response,examples=examples).launch()