Update app.py

app.py

@@ -463,34 +463,6 @@ def optimize_vocabulary(texts, vocab_size=10000, min_frequency=2):
     tokenizer.train_from_iterator(optimized_texts, trainer)
     
     return tokenizer, optimized_texts
-
-# New preprocessing function
-def optimize_query(query, llm_model, chunks, embedding_model, vector_store_type, search_type, top_k):
-    # Use a HuggingFace model for text generation
-    #model_id = "google/flan-t5-large"
-    #tokenizer = AutoTokenizer.from_pretrained(model_id)
-    #model = AutoModelForCausalLM.from_pretrained(model_id)
-    #pipe = pipeline(
-    #    "text-generation", model=model, tokenizer=tokenizer, max_new_tokens=512
-    #)
-    #llm = HuggingFacePipeline(pipeline=pipe)
-
-    #llm = HuggingFacePipeline(pipeline(model="HuggingFaceH4/zephyr-7b-beta"))
-    
-    
-    # Create a temporary vector store for query optimization
-    temp_vector_store = get_vector_store(vector_store_type, chunks, embedding_model)
-    
-    # Create a retriever with the temporary vector store
-    temp_retriever = get_retriever(temp_vector_store, search_type, {"k": top_k})
-    
-    multi_query_retriever = MultiQueryRetriever.from_llm(
-        retriever=temp_retriever,
-        llm=llm
-    )
-    optimized_queries = multi_query_retriever.generate_queries(query)
-    return optimized_queries
-
     
 # New postprocessing function
 def rerank_results(results, query, reranker):