Update app.py
Browse files
app.py
CHANGED
|
@@ -84,6 +84,9 @@ qa_embeddings = semantic_model.encode(qa_questions, convert_to_tensor=True)
|
|
| 84 |
#-------------------------bm25---------------------------------
|
| 85 |
from rank_bm25 import BM25Okapi
|
| 86 |
from nltk.tokenize import word_tokenize
|
|
|
|
|
|
|
|
|
|
| 87 |
|
| 88 |
def rerank_with_bm25(docs, query):
|
| 89 |
tokenized_docs = [word_tokenize(doc['content'].lower()) for doc in docs]
|
|
|
|
| 84 |
#-------------------------bm25---------------------------------
|
| 85 |
from rank_bm25 import BM25Okapi
|
| 86 |
from nltk.tokenize import word_tokenize
|
| 87 |
+
import nltk
|
| 88 |
+
nltk.download('punkt')
|
| 89 |
+
|
| 90 |
|
| 91 |
def rerank_with_bm25(docs, query):
|
| 92 |
tokenized_docs = [word_tokenize(doc['content'].lower()) for doc in docs]
|