hash-map commited on
Commit
4fb008b
·
verified ·
1 Parent(s): 2aeb5e7

Update rag.py

Browse files
Files changed (1) hide show
  1. rag.py +11 -0
rag.py CHANGED
@@ -3,6 +3,17 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
3
  from langchain.embeddings import HuggingFaceEmbeddings
4
  from langchain.vectorstores import FAISS
5
  from langchain.llms import Ollama
 
 
 
 
 
 
 
 
 
 
 
6
  db = FAISS.load_local(
7
  folder_path="./",
8
  embeddings=HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2"),
 
3
  from langchain.embeddings import HuggingFaceEmbeddings
4
  from langchain.vectorstores import FAISS
5
  from langchain.llms import Ollama
6
+
7
+ loader = DirectoryLoader('.', glob="all_dialogues.txt")
8
+ docs = loader.load()
9
+
10
+ text_splitter = RecursiveCharacterTextSplitter(
11
+ chunk_size=1000, chunk_overlap=200
12
+ )
13
+ texts = text_splitter.split_documents(docs)
14
+ embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
15
+
16
+
17
  db = FAISS.load_local(
18
  folder_path="./",
19
  embeddings=HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2"),