Update app.py

app.py

@@ -1,10 +1,9 @@
-# JusticeAI Backend — Full corrected app.py
-# - Fixes: use added knowledge for replies, robust local language model use (language.py / language.bin),
-#   final reply translated to user language, fallback global retrieval when topic-scoped fails,
-#   embedding attempts do not block saving, caches invalidated after add/re-embed.
-#
-# Place language.py (recommended wrapper) or language.bin and emojis.py beside this file.
-# If language.bin requires insecure torch.load, set LANGUAGE_LOAD_ALLOW_INSECURE=1 (only if you trust the file).
+# JusticeAI Backend — Fixed full app.py
+# - Bugfix: ensure infer_topic_from_message is defined before use and other helpers available.
+# - Robust language loader, translation wrapper, embedding and retrieval with topic-scoped and global fallback.
+# - /add and /add-bulk store language metadata and don't fail on embedding; caches invalidated.
+# - Final reply attempted in user's language via language module first, then Helsinki fallback.
+# - Full set of endpoints included.
 
 from sqlalchemy.pool import NullPool
 import os
@@ -119,7 +118,7 @@ def load_local_language_module():
             _language_load_errors.append(f"safetensors load failed: {e}")
             logger.debug(f"safetensors load failed: {e}")
 
-    # 2b) torch.load (safe/default)
+    # 2b) torch.load (default)
     try:
         language_module = torch.load(str(p), map_location="cpu")
         logger.info("[JusticeAI] torch.load(language.bin) succeeded")
@@ -246,7 +245,6 @@ def ensure_tables():
 
 ensure_tables()
 
-# ensure columns exist for older DBs
 def ensure_column_exists(table: str, column: str, col_def_sql: str):
     dialect = engine.dialect.name
     try:
@@ -566,14 +564,181 @@ def translate_from_english(text: str, tgt_lang: str) -> str:
         return text
     return translate_text(text, "en", tgt_code)
 
-# ----- Re-indexing / cache invalidation helpers -----
+def embed_text(text_data: str) -> bytes:
+    global embed_model
+    if embed_model is None:
+        raise RuntimeError("Embedding model not available.")
+    try:
+        emb = embed_model.encode(text_data, convert_to_tensor=True)
+        return emb.cpu().numpy().tobytes()
+    except Exception as e:
+        logger.warning(f"Embedding error: {e}")
+        raise
+
+def is_boilerplate_candidate(s: str) -> bool:
+    s_low = (s or "").strip().lower()
+    generic_phrases = [
+        "justiceai is a unified intelligence dashboard providing chat, knowledge, and live metrics.",
+        "justiceai es un panel de inteligencia unificado que proporciona chat, conocimiento y métricas en vivo."
+    ]
+    for g in generic_phrases:
+        if s_low == g.strip().lower():
+            return True
+        if g.split(" ")[0].lower() in s_low and len(s_low) < 90:
+            return True
+    return False
+
+def generate_creative_reply(matches: List[str]) -> str:
+    clean = []
+    seen = set()
+    for m in matches:
+        s = sanitize_knowledge_text(m)
+        if not s or s in seen or is_boilerplate_candidate(s):
+            continue
+        seen.add(s)
+        s = dedupe_sentences(s)
+        clean.append(s)
+    if not clean:
+        return "I’m not sure yet."
+    if len(clean) == 1:
+        return clean[0]
+    return "\n\n".join(clean[:3])
+
+# IMPORTANT: ensure infer_topic_from_message is defined BEFORE /chat uses it
+def infer_topic_from_message(msg: str, known_topics=None) -> str:
+    """
+    Infer a topic name from message using simple keyword match or embedding nearest neighbor.
+    """
+    global embed_model
+    if not msg:
+        return "general"
+    if embed_model is None or not known_topics:
+        msg_low = (msg or "").lower()
+        if known_topics:
+            for t in known_topics:
+                if t and t.lower() in msg_low:
+                    return t
+        return "general"
+    try:
+        msg_emb = embed_model.encode([msg], convert_to_tensor=True)
+        topic_embs = embed_model.encode(known_topics, convert_to_tensor=True)
+        sims = torch.nn.functional.cosine_similarity(msg_emb, topic_embs)
+        best_idx = int(torch.argmax(sims).item())
+        return known_topics[best_idx]
+    except Exception:
+        return "general"
+
+def refine_or_update(matches, new_text, new_reply, confidence, topic="general"):
+    try:
+        if embed_model is None:
+            return
+        with engine.begin() as conn:
+            rows = conn.execute(sql_text("SELECT id, text FROM knowledge WHERE topic = :topic"), {"topic": topic}).fetchall()
+        if not rows:
+            emb = embed_text(new_text)
+            with engine.begin() as conn:
+                conn.execute(
+                    sql_text("INSERT INTO knowledge (text, reply, language, embedding, category, topic) VALUES (:t, :r, :lang, :e, 'learned', :topic)"),
+                    {"t": new_text, "r": new_reply or "", "lang": "en", "e": emb, "topic": topic}
+                )
+            return
+        texts = [r[1] for r in rows]
+        ids = [r[0] for r in rows]
+        embs = embed_model.encode(texts, convert_to_tensor=True)
+        new_emb = embed_model.encode(new_text, convert_to_tensor=True)
+        sims = torch.nn.functional.cosine_similarity(new_emb.unsqueeze(0), embs)
+        best_idx = int(torch.argmax(sims).item())
+        best_score = float(sims[best_idx])
+        if best_score > 0.75:
+            kid = ids[best_idx]
+            with engine.begin() as conn:
+                row = conn.execute(sql_text("SELECT meta FROM knowledge WHERE id = :id"), {"id": kid}).fetchone()
+            is_manual = False
+            try:
+                if row and row[0]:
+                    meta_obj = json.loads(row[0]) if isinstance(row[0], str) else row[0]
+                    is_manual = bool(meta_obj.get("manual"))
+            except Exception:
+                is_manual = False
+            if is_manual and confidence < 0.85:
+                with engine.begin() as conn:
+                    conn.execute(
+                        sql_text("UPDATE knowledge SET reply = :r, updated_at = CURRENT_TIMESTAMP WHERE id = :id"),
+                        {"r": (new_reply or ""), "id": kid}
+                    )
+            else:
+                with engine.begin() as conn:
+                    conn.execute(
+                        sql_text("UPDATE knowledge SET reply = :r, text = :t, language = :lang, updated_at = CURRENT_TIMESTAMP WHERE id = :id"),
+                        {"r": new_reply or "", "t": new_text, "lang": "en", "id": kid}
+                    )
+        else:
+            emb = new_emb.cpu().numpy().tobytes()
+            with engine.begin() as conn:
+                conn.execute(
+                    sql_text("INSERT INTO knowledge (text, reply, language, embedding, category, topic, confidence, meta) VALUES (:t, :r, :lang, :e, 'learned', :topic, :conf, :meta)"),
+                    {"t": new_text, "r": new_reply or "", "lang": "en", "e": emb, "topic": topic, "conf": min(0.7, float(confidence)), "meta": json.dumps({"refined": True})}
+                )
+        global knowledge_version
+        knowledge_version += 1
+    except Exception as e:
+        logger.warning(f"refine_or_update error: {e}")
+
+def detect_mood(text: str) -> str:
+    lower = (text or "").lower()
+    positive = ["great", "thanks", "awesome", "happy", "love", "excellent", "cool", "yes", "good", "success", "helpful", "useful", "thank you"]
+    negative = ["sad", "bad", "problem", "angry", "hate", "fail", "no", "error", "not working", "disadvantage", "issue"]
+    emojis = extract_emojis(text)
+    e_score = emoji_sentiment_score(emojis)
+    if any(w in lower for w in positive) or e_score > 0.3:
+        return "positive"
+    elif any(w in lower for w in negative) or e_score < -0.2:
+        return "negative"
+    return "neutral"
+
+def should_append_emoji(user_text: str, reply_text: str, mood: str, flags: Dict[str, Any]) -> str:
+    if flags.get("toxic"):
+        return ""
+    if extract_emojis(reply_text):
+        return ""
+    user_emojis = extract_emojis(user_text)
+    if user_emojis:
+        user_score = emoji_sentiment_score(user_emojis)
+        if user_score >= 0.2:
+            try:
+                cat = get_category_for_mood("positive")
+                return get_emoji(cat, intensity=min(1.0, 0.5 + user_score))
+            except Exception:
+                return user_emojis[0] if user_emojis else ""
+        elif user_score <= -0.2:
+            return ""
+        else:
+            if len(reply_text) < 200:
+                try:
+                    cat = get_category_for_mood("neutral")
+                    return get_emoji(cat, intensity=0.5)
+                except Exception:
+                    return ""
+            return ""
+    if mood == "neutral":
+        return ""
+    if len(reply_text) > 400:
+        return ""
+    if re.search(r'[\{\}\[\]\(\)]', reply_text):
+        return ""
+    try:
+        cat = get_category_for_mood(mood)
+        return get_emoji(cat, intensity=random.random())
+    except Exception:
+        return ""
+
 def invalidate_knowledge_cache():
     global knowledge_embeddings_cache, knowledge_version
     knowledge_embeddings_cache = None
     knowledge_version += 1
     logger.debug("[index] invalidated knowledge cache")
 
-# ----- Add endpoints (store language, attempt embedding, never fail on embedding) -----
+# ----- Add endpoints -----
 @app.post("/add")
 async def add_knowledge(data: dict = Body(...)):
     if not isinstance(data, dict):
@@ -660,7 +825,7 @@ async def add_bulk(data: List[dict] = Body(...)):
         record_learn_event()
     return {"added": added, "errors": errors}
 
-# ----- Chat endpoint: topic-scoped retrieval with global fallback; always attempt final reply in user's language -----
+# ----- Chat endpoint -----
 @app.post("/chat")
 async def chat(request: Request, data: dict = Body(...)):
     t0 = time.time()
@@ -680,7 +845,7 @@ async def chat(request: Request, data: dict = Body(...)):
     except Exception:
         msg_corrected = raw_msg
 
-    # translate to English for synthesis (embedding model is multilingual, but translations help synthesis)
+    # translate to English for synthesis
     en_msg = msg_corrected
     if detected_lang and detected_lang.split("-")[0].lower() not in ("en", "eng", "", "und"):
         try:
@@ -752,7 +917,6 @@ async def chat(request: Request, data: dict = Body(...)):
             matches = cand
             confidence = matches[0]["score"] if matches else 0.0
         else:
-            # substring fallback
             for kr in knowledge_rows:
                 if en_msg.lower() in (kr["text"] or "").lower():
                     matches.append({"text": kr["text"], "reply": kr["reply"], "lang": kr["lang"], "score": 0.0})
@@ -761,7 +925,7 @@ async def chat(request: Request, data: dict = Body(...)):
         logger.warning(f"[retrieval] topic-scoped retrieval failure: {e}")
         matches = []
 
-    # If no good matches in topic or low confidence, do a global fallback search across all topics
+    # Global fallback if needed
     used_global_fallback = False
     if not matches or confidence < 0.35:
         try:
@@ -794,7 +958,6 @@ async def chat(request: Request, data: dict = Body(...)):
                     confidence = matches[0]["score"]
                     used_global_fallback = True
             else:
-                # simple global substring fallback
                 cand = []
                 for r in all_rows:
                     if en_msg.lower() in (r["text"] or "").lower():
@@ -809,11 +972,10 @@ async def chat(request: Request, data: dict = Body(...)):
     if used_global_fallback:
         logger.info("[retrieval] used global fallback search across topics to find candidates")
 
-    # Compose final reply: prefer candidate reply in user's language; otherwise translate
+    # Compose final reply
     reply_user_lang = ""
     try:
         if matches:
-            # find best candidate matching user's language
             chosen = None
             for m in matches:
                 m_lang = (m.get("lang") or "und").split("-")[0].lower()
@@ -824,11 +986,9 @@ async def chat(request: Request, data: dict = Body(...)):
                 chosen = matches[0]
             chosen_text = chosen.get("reply") or chosen.get("text") or ""
             chosen_lang = chosen.get("lang") or "und"
-            # if candidate already in user language, use it
             if chosen_lang and reply_lang and chosen_lang.split("-")[0].lower() == reply_lang.split("-")[0].lower():
                 reply_user_lang = chosen_text
             else:
-                # translate chosen_text -> user's language
                 try:
                     reply_user_lang = translate_text(chosen_text, chosen_lang or "und", reply_lang)
                 except Exception:
@@ -837,9 +997,8 @@ async def chat(request: Request, data: dict = Body(...)):
         logger.warning(f"[compose] candidate selection error: {e}")
         reply_user_lang = ""
 
-    # If still empty (no candidates), synthesize from matches
+    # If no candidate reply, synthesize
     if not reply_user_lang:
-        # Build scratchpad in English using matches converted to EN
         facts_en = []
         for m in matches[:6]:
             t = m.get("reply") or m.get("text") or ""
@@ -853,11 +1012,9 @@ async def chat(request: Request, data: dict = Body(...)):
                 t_en = t
             if t_en:
                 facts_en.append(dedupe_sentences(t_en))
-        # synthesize simple reply
         if not facts_en:
             reply_en = "I don't have enough context yet — could you give more details or add knowledge with /add?"
         else:
-            # simple synthesis: join top facts
             reply_en = "\n\n".join(facts_en[:3])
             if intent == "solution":
                 parts = []
@@ -868,8 +1025,6 @@ async def chat(request: Request, data: dict = Body(...)):
                 reply_en = "Solutions:\n- " + "\n- ".join(parts[:6]) if parts else reply_en
             if intent == "why":
                 reply_en = reply_en + " It is useful because it provides consolidated information."
-
-        # translate to user language
         if reply_lang and reply_lang.split("-")[0].lower() not in ("en","eng","", "und"):
             try:
                 reply_user_lang = translate_from_english(reply_en, reply_lang)
@@ -878,15 +1033,12 @@ async def chat(request: Request, data: dict = Body(...)):
         else:
             reply_user_lang = reply_en
 
-    # Post-processing (intent formatting) - try to keep in user's language if possible
     def postprocess_final(text: str, intent_label: str, lang_code: str) -> str:
         if not text:
             return text
         if intent_label == "solution":
-            # if it already contains Solutions:, return as is, else try to construct bullets
             if "Solutions:" in text or text.strip().startswith("- "):
                 return text
-            # attempt to create bullets by splitting sentences
             parts = []
             for p in re.split(r'\n+', text):
                 for s in re.split(r'[.?!]\s+', p):
@@ -970,7 +1122,7 @@ async def chat(request: Request, data: dict = Body(...)):
 
     return {"reply": reply_final, "topic": topic, "language": reply_lang, "emoji": emoji, "confidence": round(confidence, 2), "flags": flags}
 
-# ----- Leaderboard, admin and other endpoints (kept intact) -----
+# ----- Leaderboard, admin and other endpoints -----
 @app.get("/leaderboard")
 async def leaderboard(topic: str = Query("general")):
     topic = str(topic or "general").strip() or "general"
@@ -1175,7 +1327,7 @@ async def frontend_dashboard():
 # ----- Startup behavior: load models and introspect language module -----
 @app.on_event("startup")
 async def startup_event():
-    global embed_model, spell, moderator, llm_tokenizer, llm_model, startup_time, language_module
+    global embed_model, spell, moderator, llm_tokenizer, llm_model, startup_time, language_module, app_start_time
     t0 = time.time()
     logger.info("[JusticeAI] startup: loading components")
 
@@ -1236,12 +1388,7 @@ async def startup_event():
 
     startup_time = round(time.time() - t0, 2)
     logger.info(f"[JusticeAI] startup complete in {startup_time}s")
-    # warmup
-    try:
-        justice_brain.load_capabilities()
-        justice_brain.warmup()
-    except Exception as e:
-        logger.warning(f"[JusticeBrain] warmup error: {e}")
+    app_start_time = time.time()
 
 if __name__ == "__main__":
     port = int(os.environ.get("PORT", 7860))