Spaces:
Running
Running
remove fuzzy
Browse files
app.py
CHANGED
|
@@ -6,13 +6,9 @@ from sentence_transformers import SentenceTransformer
|
|
| 6 |
from qdrant_client import QdrantClient
|
| 7 |
from qdrant_client.models import Filter, FieldCondition, MatchValue
|
| 8 |
import os
|
| 9 |
-
from rapidfuzz import process, fuzz
|
| 10 |
from pythainlp.tokenize import word_tokenize
|
| 11 |
from pyairtable import Table
|
| 12 |
from pyairtable import Api
|
| 13 |
-
import pickle
|
| 14 |
-
import re
|
| 15 |
-
import unicodedata
|
| 16 |
|
| 17 |
|
| 18 |
qdrant_client = QdrantClient(
|
|
@@ -52,41 +48,6 @@ model_config = {
|
|
| 52 |
# Global memory to hold feedback state
|
| 53 |
latest_query_result = {"query": "", "result": "", "model": "", "raw_query": "", "time": ""}
|
| 54 |
|
| 55 |
-
with open("keyword_whitelist.pkl", "rb") as f:
|
| 56 |
-
keyword_whitelist = pickle.load(f)
|
| 57 |
-
|
| 58 |
-
def smart_tokenize(query: str) -> list:
|
| 59 |
-
tokens = word_tokenize(query, engine="newmm")
|
| 60 |
-
if len("".join(tokens)) < len(query) * 0.7: # ตัดคำขาดเกินไป
|
| 61 |
-
return query.strip().split()
|
| 62 |
-
return tokens
|
| 63 |
-
|
| 64 |
-
def normalize_and_clean_thai(text: str) -> str:
|
| 65 |
-
text = unicodedata.normalize("NFC", text)
|
| 66 |
-
text = text.replace("เแ", "แ").replace("เเ", "แ")
|
| 67 |
-
return text
|
| 68 |
-
|
| 69 |
-
def correct_query_smart(query: str, whitelist=None, threshold=70) -> str:
|
| 70 |
-
query_norm = normalize_and_clean_thai(query)
|
| 71 |
-
tokens = query_norm.strip().split()
|
| 72 |
-
|
| 73 |
-
# ถ้า token เดียว → fuzzy ตรงไปที่คำเต็มเลย
|
| 74 |
-
if len(tokens) == 1:
|
| 75 |
-
match, score, _ = process.extractOne(tokens[0].lower(), whitelist, scorer=fuzz.token_sort_ratio)
|
| 76 |
-
return match if score >= threshold else query_norm
|
| 77 |
-
|
| 78 |
-
# token หลายคำ → ลองแก้ทีละคำ
|
| 79 |
-
corrected = []
|
| 80 |
-
for word in tokens:
|
| 81 |
-
word_lower = word.lower()
|
| 82 |
-
if word_lower in whitelist:
|
| 83 |
-
corrected.append(word)
|
| 84 |
-
else:
|
| 85 |
-
match, score, _ = process.extractOne(word_lower, whitelist, scorer=fuzz.token_sort_ratio)
|
| 86 |
-
corrected.append(match if score >= threshold else word)
|
| 87 |
-
|
| 88 |
-
return " ".join(corrected)
|
| 89 |
-
|
| 90 |
# 🌟 Main search function
|
| 91 |
def search_product(query, model_name):
|
| 92 |
start_time = time.time()
|
|
@@ -94,7 +55,7 @@ def search_product(query, model_name):
|
|
| 94 |
return "<p>❌ ไม่พบโมเดล</p>"
|
| 95 |
|
| 96 |
latest_query_result["raw_query"] = query
|
| 97 |
-
corrected_query =
|
| 98 |
|
| 99 |
query_embed = model_config[model_name]["func"](corrected_query)
|
| 100 |
collection_name = model_config[model_name]["collection"]
|
|
|
|
| 6 |
from qdrant_client import QdrantClient
|
| 7 |
from qdrant_client.models import Filter, FieldCondition, MatchValue
|
| 8 |
import os
|
|
|
|
| 9 |
from pythainlp.tokenize import word_tokenize
|
| 10 |
from pyairtable import Table
|
| 11 |
from pyairtable import Api
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
|
| 14 |
qdrant_client = QdrantClient(
|
|
|
|
| 48 |
# Global memory to hold feedback state
|
| 49 |
latest_query_result = {"query": "", "result": "", "model": "", "raw_query": "", "time": ""}
|
| 50 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
# 🌟 Main search function
|
| 52 |
def search_product(query, model_name):
|
| 53 |
start_time = time.time()
|
|
|
|
| 55 |
return "<p>❌ ไม่พบโมเดล</p>"
|
| 56 |
|
| 57 |
latest_query_result["raw_query"] = query
|
| 58 |
+
corrected_query = query
|
| 59 |
|
| 60 |
query_embed = model_config[model_name]["func"](corrected_query)
|
| 61 |
collection_name = model_config[model_name]["collection"]
|