Update app.py

app.py

@@ -4,6 +4,7 @@ import numpy as np
 import re
 import tempfile
 from datetime import datetime
+from transformers import AutoModelForCausalLM, AutoTokenizer
 from langchain_community.document_loaders import PDFPlumberLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.embeddings import HuggingFaceEmbeddings
@@ -18,46 +19,8 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
 from langchain.llms.huggingface_pipeline import HuggingFacePipeline
 from huggingface_hub import login
 
-#### Model Testing ###########
-print(f"-- Model test started")
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-model_name = "Qwen/Qwen2.5-0.5B-Instruct"
-
-model = AutoModelForCausalLM.from_pretrained(
-    model_name,
-)
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-
-prompt = "Give me a short introduction to large language model."
-messages = [
-    {"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."},
-    {"role": "user", "content": prompt}
-]
-text = tokenizer.apply_chat_template(
-    messages,
-    tokenize=False,
-    add_generation_prompt=True
-)
-print(f"-- Model Invoking")
-model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
-
-generated_ids = model.generate(
-    **model_inputs,
-    max_new_tokens=512
-)
-generated_ids = [
-    output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
-]
-
-response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
-print(f"-- Model testresponse{model_inputs}")
-
-##########################
-
 
 # Load the model and tokenizer
-# model_name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
 model_name= "Qwen/Qwen2.5-0.5B-Instruct"
 
 # Initialize classifier once for input guardrail
@@ -122,34 +85,16 @@ if uploaded_files:
         tokenizer = AutoTokenizer.from_pretrained(
             model_name, 
             trust_remote_code=True,
-            padding_side="left"  # Important for some models
         )
         model = AutoModelForCausalLM.from_pretrained(
             model_name, 
             trust_remote_code=True,
         )
-        
-        # Create pipeline with generation parameters
-        pipeline_llm = pipeline(
-            "text-generation",
-            model=model,
-            tokenizer=tokenizer,
-            max_new_tokens=1024,
-            temperature=0.3,
-            top_p=0.95,
-            repetition_penalty=1.15,
-            return_full_text=False  # Important for response formatting
-        )
-        
-        llm = HuggingFacePipeline(pipeline=pipeline_llm)
-
-        llm_sample_resp = llm("Explain what is Retrieval Augmented Generation (RAG)?")
-        print(f"-- llmsampleresponse:{llm_sample_resp}")
-        
+              
         
         # Update prompt template
         PROMPT_TEMPLATE = """
-        <|system|>
+        <|User|>
         You are a senior financial analyst. Analyze these financial reports:
         1. Compare key metrics between documents
         2. Identify trends across reporting periods
@@ -168,7 +113,6 @@ if uploaded_files:
             template=PROMPT_TEMPLATE,
             input_variables=["context", "question"]
         )
-        llm_chain = LLMChain(llm=llm, prompt=qa_prompt)
 
     # Interactive Q&A Interface
     st.header("🔍 Cross-Document Financial Inquiry")
@@ -215,11 +159,35 @@ if uploaded_files:
             # Response Generation
             context = "\n".join([doc.page_content for doc in filtered_docs])
             print(f"-- Retrieved context:{context}")
+
+            # prompt
+            prompt = qa_prompt.format(context=context, question=user_input)
+            ####
+            # Generation
+            messages = [
+                {"role": "system", "content": "You are Financial assistant."},
+                {"role": "user", "content": prompt}
+            ]
+            text = tokenizer.apply_chat_template(
+                messages,
+                tokenize=False,
+                add_generation_prompt=True
+            )
+            
+            model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
             
-            analysis = llm_chain.run(
-                context=context, 
-                question=user_input
+            print(f"-- Model Invoking")
+            generated_ids = model.generate(
+                **model_inputs,
+                max_new_tokens=512
             )
+            generated_ids = [
+                output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+            ]
+            
+            analysis = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+
+            ###
             print(f"Analysis result:{analysis}")
             
             # Response Cleaning