Update app.py
Browse files
app.py
CHANGED
|
@@ -6,8 +6,10 @@ import torch.nn.functional as F
|
|
| 6 |
import faiss
|
| 7 |
import numpy as np
|
| 8 |
import matplotlib.pyplot as plt
|
|
|
|
|
|
|
| 9 |
import os
|
| 10 |
-
|
| 11 |
# Load Models
|
| 12 |
lang_detect_model = AutoModelForSequenceClassification.from_pretrained("papluca/xlm-roberta-base-language-detection")
|
| 13 |
lang_detect_tokenizer = AutoTokenizer.from_pretrained("papluca/xlm-roberta-base-language-detection")
|
|
@@ -31,14 +33,23 @@ xlm_to_nllb = {
|
|
| 31 |
"sa": "san_Deva"
|
| 32 |
}
|
| 33 |
|
| 34 |
-
#
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
|
| 41 |
-
# Language
|
| 42 |
def detect_language(text):
|
| 43 |
inputs = lang_detect_tokenizer(text, return_tensors="pt", truncation=True, padding=True)
|
| 44 |
with torch.no_grad():
|
|
@@ -47,24 +58,7 @@ def detect_language(text):
|
|
| 47 |
pred = torch.argmax(probs, dim=1).item()
|
| 48 |
return id2lang[pred]
|
| 49 |
|
| 50 |
-
|
| 51 |
-
detected_lang = detect_language(input_text)
|
| 52 |
-
print(f"\n🔍 Detected Language Code: {detected_lang}")
|
| 53 |
-
else:
|
| 54 |
-
print("🚫 Empty input text. Exiting.")
|
| 55 |
-
raise SystemExit
|
| 56 |
-
|
| 57 |
-
# Choose target language
|
| 58 |
-
print("\n🌐 Available Output Languages:")
|
| 59 |
-
for code, lang in nllb_langs.items():
|
| 60 |
-
print(f"{code} → {lang}")
|
| 61 |
-
|
| 62 |
-
target_code = input("\n🔤 Enter target language code (e.g., eng_Latn): ").strip()
|
| 63 |
-
if target_code not in nllb_langs:
|
| 64 |
-
print("❌ Invalid code. Defaulting to English (eng_Latn).")
|
| 65 |
-
target_code = "eng_Latn"
|
| 66 |
-
|
| 67 |
-
# Translation
|
| 68 |
def translate(text, src_code, tgt_code):
|
| 69 |
trans_tokenizer.src_lang = src_code
|
| 70 |
encoded = trans_tokenizer(text, return_tensors="pt", truncation=True, padding=True)
|
|
@@ -73,93 +67,15 @@ def translate(text, src_code, tgt_code):
|
|
| 73 |
generated = trans_model.generate(**encoded, forced_bos_token_id=target_lang_id)
|
| 74 |
return trans_tokenizer.decode(generated[0], skip_special_tokens=True)
|
| 75 |
except:
|
| 76 |
-
print("❌ Translation failed.")
|
| 77 |
return ""
|
| 78 |
|
| 79 |
-
|
| 80 |
-
print(f"\n📜 Text to Translate:\n{input_text}\n")
|
| 81 |
-
print(f"🌍 Source Language: {src_nllb} → Target Language: {target_code}")
|
| 82 |
-
|
| 83 |
-
translated_text = translate(input_text, src_nllb, target_code)
|
| 84 |
-
# Output translated text
|
| 85 |
-
if translated_text.strip():
|
| 86 |
-
print("\n✅ Translation Complete!\n")
|
| 87 |
-
print("🔸 Translated Text:\n")
|
| 88 |
-
print(translated_text)
|
| 89 |
-
|
| 90 |
-
with open("translated_output.txt", "w", encoding="utf-8") as f:
|
| 91 |
-
f.write(translated_text)
|
| 92 |
-
files.download("translated_output.txt")
|
| 93 |
-
else:
|
| 94 |
-
print("❌ No translated text produced.")
|
| 95 |
-
raise SystemExit
|
| 96 |
-
#Create Corpus and FAISS Index
|
| 97 |
-
corpus = [
|
| 98 |
-
"धर्म एव हतो हन्ति धर्मो रक्षति रक्षितः",
|
| 99 |
-
"Dharma when destroyed, destroys; when protected, protects.",
|
| 100 |
-
"The moon affects tides and mood, according to Jyotisha",
|
| 101 |
-
"One should eat according to the season – Rituacharya",
|
| 102 |
-
"Balance of Tridosha is health – Ayurveda principle",
|
| 103 |
-
"Ethics in Mahabharata reflect situational dharma",
|
| 104 |
-
"Meditation improves memory and mental clarity",
|
| 105 |
-
"Jyotisha links planetary motion with life patterns"
|
| 106 |
-
]
|
| 107 |
-
|
| 108 |
-
corpus_embeddings = embed_model.encode(corpus, convert_to_numpy=True)
|
| 109 |
-
dimension = corpus_embeddings.shape[1]
|
| 110 |
-
index = faiss.IndexFlatL2(dimension)
|
| 111 |
-
index.add(corpus_embeddings)
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
# Semantic Search Function
|
| 115 |
def search_semantic(query, top_k=3):
|
| 116 |
query_embedding = embed_model.encode([query])
|
| 117 |
distances, indices = index.search(query_embedding, top_k)
|
| 118 |
return [(corpus[i], float(distances[0][idx])) for idx, i in enumerate(indices[0])]
|
| 119 |
|
| 120 |
-
#
|
| 121 |
-
print("\n🔎 Searching for similar Sanskrit knowledge...")
|
| 122 |
-
results = search_semantic(translated_text)
|
| 123 |
-
|
| 124 |
-
print("\n🔍 Top Semantic Matches:")
|
| 125 |
-
for i, (text, score) in enumerate(results, 1):
|
| 126 |
-
print(f"\n{i}. {text}\n Similarity Score: {score:.4f}")
|
| 127 |
-
|
| 128 |
-
# Visualize Semantic Scores
|
| 129 |
-
labels = [f"{i+1}. Match {i+1}" for i in range(len(results))]
|
| 130 |
-
scores = [score for _, score in results]
|
| 131 |
-
|
| 132 |
-
plt.figure(figsize=(10, 6))
|
| 133 |
-
bars = plt.barh(labels, scores, color="skyblue")
|
| 134 |
-
|
| 135 |
-
plt.xlabel("Similarity Score", fontsize=12)
|
| 136 |
-
plt.title("Top Semantic Matches", fontsize=14)
|
| 137 |
-
plt.gca().invert_yaxis()
|
| 138 |
-
|
| 139 |
-
for bar in bars:
|
| 140 |
-
plt.text(bar.get_width() + 0.5, bar.get_y() + 0.25, f"{bar.get_width():.2f}", fontsize=10)
|
| 141 |
-
|
| 142 |
-
plt.tight_layout()
|
| 143 |
-
plt.savefig("semantic_similarity_plot.png")
|
| 144 |
-
plt.show()
|
| 145 |
-
|
| 146 |
-
files.download("semantic_similarity_plot.png")
|
| 147 |
-
|
| 148 |
-
# BLEU Score Evaluation
|
| 149 |
-
from sacrebleu import corpus_bleu
|
| 150 |
-
|
| 151 |
-
reference = input("📘 Enter correct human translation (for BLEU evaluation): ").strip()
|
| 152 |
-
if reference:
|
| 153 |
-
bleu = corpus_bleu([translated_text], [[reference]])
|
| 154 |
-
print(f"\n📏 BLEU Score: {bleu.score:.2f}")
|
| 155 |
-
else:
|
| 156 |
-
print("ℹ️ BLEU evaluation skipped (no reference entered).")
|
| 157 |
-
|
| 158 |
-
# ✅ Gradio App Interface
|
| 159 |
-
import gradio as gr
|
| 160 |
-
import matplotlib.pyplot as plt
|
| 161 |
-
from sacrebleu import corpus_bleu
|
| 162 |
-
|
| 163 |
def full_pipeline(user_input_text, target_lang_code, human_ref=""):
|
| 164 |
if not user_input_text.strip():
|
| 165 |
return "⚠️ Empty input", "", [], "", ""
|
|
@@ -174,6 +90,7 @@ def full_pipeline(user_input_text, target_lang_code, human_ref=""):
|
|
| 174 |
sem_results = search_semantic(translated)
|
| 175 |
result_list = [f"{i+1}. {txt} (Score: {score:.2f})" for i, (txt, score) in enumerate(sem_results)]
|
| 176 |
|
|
|
|
| 177 |
labels = [f"{i+1}" for i in range(len(sem_results))]
|
| 178 |
scores = [score for _, score in sem_results]
|
| 179 |
plt.figure(figsize=(6, 4))
|
|
@@ -195,7 +112,7 @@ def full_pipeline(user_input_text, target_lang_code, human_ref=""):
|
|
| 195 |
|
| 196 |
return detected_lang, translated, result_list, plot_path, bleu_score
|
| 197 |
|
| 198 |
-
#
|
| 199 |
gr.Interface(
|
| 200 |
fn=full_pipeline,
|
| 201 |
inputs=[
|
|
@@ -212,4 +129,4 @@ gr.Interface(
|
|
| 212 |
],
|
| 213 |
title="🌍 Multilingual Translator + Semantic Search",
|
| 214 |
description="Detects language → Translates → Finds related Sanskrit concepts → BLEU optional."
|
| 215 |
-
).launch(
|
|
|
|
| 6 |
import faiss
|
| 7 |
import numpy as np
|
| 8 |
import matplotlib.pyplot as plt
|
| 9 |
+
import gradio as gr
|
| 10 |
+
from sacrebleu import corpus_bleu
|
| 11 |
import os
|
| 12 |
+
|
| 13 |
# Load Models
|
| 14 |
lang_detect_model = AutoModelForSequenceClassification.from_pretrained("papluca/xlm-roberta-base-language-detection")
|
| 15 |
lang_detect_tokenizer = AutoTokenizer.from_pretrained("papluca/xlm-roberta-base-language-detection")
|
|
|
|
| 33 |
"sa": "san_Deva"
|
| 34 |
}
|
| 35 |
|
| 36 |
+
# Static Corpus
|
| 37 |
+
corpus = [
|
| 38 |
+
"धर्म एव हतो हन्ति धर्मो रक्षति रक्षितः",
|
| 39 |
+
"Dharma when destroyed, destroys; when protected, protects.",
|
| 40 |
+
"The moon affects tides and mood, according to Jyotisha",
|
| 41 |
+
"One should eat according to the season – Rituacharya",
|
| 42 |
+
"Balance of Tridosha is health – Ayurveda principle",
|
| 43 |
+
"Ethics in Mahabharata reflect situational dharma",
|
| 44 |
+
"Meditation improves memory and mental clarity",
|
| 45 |
+
"Jyotisha links planetary motion with life patterns"
|
| 46 |
+
]
|
| 47 |
+
corpus_embeddings = embed_model.encode(corpus, convert_to_numpy=True)
|
| 48 |
+
dimension = corpus_embeddings.shape[1]
|
| 49 |
+
index = faiss.IndexFlatL2(dimension)
|
| 50 |
+
index.add(corpus_embeddings)
|
| 51 |
|
| 52 |
+
# Detect Language
|
| 53 |
def detect_language(text):
|
| 54 |
inputs = lang_detect_tokenizer(text, return_tensors="pt", truncation=True, padding=True)
|
| 55 |
with torch.no_grad():
|
|
|
|
| 58 |
pred = torch.argmax(probs, dim=1).item()
|
| 59 |
return id2lang[pred]
|
| 60 |
|
| 61 |
+
# Translate
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
def translate(text, src_code, tgt_code):
|
| 63 |
trans_tokenizer.src_lang = src_code
|
| 64 |
encoded = trans_tokenizer(text, return_tensors="pt", truncation=True, padding=True)
|
|
|
|
| 67 |
generated = trans_model.generate(**encoded, forced_bos_token_id=target_lang_id)
|
| 68 |
return trans_tokenizer.decode(generated[0], skip_special_tokens=True)
|
| 69 |
except:
|
|
|
|
| 70 |
return ""
|
| 71 |
|
| 72 |
+
# Semantic Search
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
def search_semantic(query, top_k=3):
|
| 74 |
query_embedding = embed_model.encode([query])
|
| 75 |
distances, indices = index.search(query_embedding, top_k)
|
| 76 |
return [(corpus[i], float(distances[0][idx])) for idx, i in enumerate(indices[0])]
|
| 77 |
|
| 78 |
+
# Full pipeline for Gradio
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
def full_pipeline(user_input_text, target_lang_code, human_ref=""):
|
| 80 |
if not user_input_text.strip():
|
| 81 |
return "⚠️ Empty input", "", [], "", ""
|
|
|
|
| 90 |
sem_results = search_semantic(translated)
|
| 91 |
result_list = [f"{i+1}. {txt} (Score: {score:.2f})" for i, (txt, score) in enumerate(sem_results)]
|
| 92 |
|
| 93 |
+
# Plot
|
| 94 |
labels = [f"{i+1}" for i in range(len(sem_results))]
|
| 95 |
scores = [score for _, score in sem_results]
|
| 96 |
plt.figure(figsize=(6, 4))
|
|
|
|
| 112 |
|
| 113 |
return detected_lang, translated, result_list, plot_path, bleu_score
|
| 114 |
|
| 115 |
+
# Gradio App
|
| 116 |
gr.Interface(
|
| 117 |
fn=full_pipeline,
|
| 118 |
inputs=[
|
|
|
|
| 129 |
],
|
| 130 |
title="🌍 Multilingual Translator + Semantic Search",
|
| 131 |
description="Detects language → Translates → Finds related Sanskrit concepts → BLEU optional."
|
| 132 |
+
).launch()
|